def Mark(
    x=None, text='', color='',
    bold=True, underline=False):
  """ This function prints an object x and adds a description text.
      It is useful for for debugging. """

  start = ''
  end = ''
  if color != '' or bold or underline:
    end='\033[0m'
  colorDict = {
      '': '',
      None: '',
      'purple' : '\033[95m',
      'cyan' : '\033[96m',
      'darkcyan' : '\033[36m',
      'blue' : '\033[94m',
      'green' : '\033[92m',
      'yellow' : '\033[93m',
      'red' : '\033[91m',
      'bold' : '\033[1m',
      'underline' : '\033[4m',
      'end' : '\033[0m'}
  if bold:
    start = start + colorDict['bold']
  if underline:
    start = start + colorDict['underline']
  start = start + colorDict[color]

  print("*** " + start + text + end)
  if x is not None:
    print(x)

'''
## examples
Mark(2*10, 'xxx', bold=False, underline=True)
Mark(2*10, 'xxx', bold=False, underline=True, color='red')
Mark(x='', text='xxx', bold=True, underline=True, color='green')
'''

## These are the default functions to communicate with OS
# re-write them if needed
FileExists = os.path.exists
OpenFile = open
ListDir = os.listdir

def CustomMarkFcn(fn=None, logTime=True, color=''):

  """ This functions returns a custom function which prints x,
      with description text.
      It also saves x in a file if fn is not None. """

  fileExists = False
  if fn != None:
    fileExists = FileExists(fn)
  if fileExists:
    appendWrite = 'a' # append if already exists
  else:
    appendWrite = 'w' # make a new file if not

  # define the Marking Fcn here
  def F(x=None, text='', color=color, bold=True, underline=False):
    timeStr = str(datetime.datetime.now())[:19]
    if fn is not None:
      orig_stdout = sys.stdout
      f = OpenFile(fn, appendWrite)
      sys.stdout = f
      if logTime:
        Mark(text='This was run at this time:' + timeStr,
             bold=False, underline=False, color='')
      Mark(x=x, text=text, color='', bold=False, underline=False)
      f.close()
      sys.stdout = orig_stdout
    if logTime:
      Mark(text='This was run at this time:' + timeStr)
    Mark(x=x, text=text, color=color, bold=bold, underline=underline)

  return F

'''
fn = 'log.txt'
CustomMark = CustomMarkFcn(fn=fn)
CustomMark(x=2, text='NO')
'''

# to print a function definition
PrintFcnContent = inspect.getsourcelines

## mapping a dictionary via map
#def MapDict(f, dic):
#  return dict(map(lambda (k,v): (k, f(v)), dic.iteritems()))

def BitOr(x):
  """bitwise OR: same as BIT_OR in SQL."""
  return functools.reduce(lambda a,b: (a|b), x)

def Signif(n):
  """ Builds a function for rounding up to n number of significant digits."""

  def F(x):
    if math.isnan(x):
      return x
    if x == 0:
      return 0
    out = round(np.absolute(x),
                -int(math.floor(math.log10(np.absolute(x))) + (-n+1)))
    if x < 0:
      out = -out
    return out

  return F

### Reading / Writing  Data

## read csv
def ReadCsv(
    fn,
    sep=',',
    nrows=None,
    typeDict={},
    header='infer',
    engine='c',
    error_bad_lines=False,
    printLog=False):

  with OpenFile(fn, 'r') as f:
    df = pd.read_csv(
        f, sep=sep, nrows=nrows, dtype=typeDict, header=header,
        engine=engine, error_bad_lines=error_bad_lines)
  if printLog:
    print(fn + ' was read.')
  return df

## write csv (or tsv)
def WriteCsv(
    fn,
    df,
    sep=',',
    append=False,
    index=False,
    printLog=False):

  wa = 'w'
  header = list(df.columns)

  if append:
    wa = 'a'
    header = False

  with OpenFile(fn, wa) as f:
    df.to_csv(f, sep=sep, index=index, mode=wa, header=header)

  if printLog:
    print(fn + ' was written.')

  return None

## reads multiple data files according to a pattern given.
## Filters them and then row binds them.
## the pattern is given in three lists: prefix, middle, suffix
def ReadMultipleDf(
    prefix,
    middle,
    suffix,
    ReadF=ReadCsv,
    DfFilterF=None):

  n = max([len(prefix), len(middle), len(suffix)])

  def FillList(x):
    if len(x) < n:
      x = x*n
      x = x[:n]
    return x

  prefix = FillList(prefix)
  suffix = FillList(suffix)
  middle = FillList(middle)
  df = pd.DataFrame({'prefix': prefix, 'middle': middle, 'suffix': suffix})
  fileList = (df['prefix'] + df['middle'] + df['suffix']).values
  #dfList = list()
  for i in range(len(fileList)):
    f = fileList[i]
    df = ReadF(f)
    if DfFilterF != None:
      df = DfFilterF(df)
    if i == 0:
      dfAll = df
    else:
      dfAll = dfAll.append(df, ignore_index=True)
  return dfAll

## Read all files in a dir with same columns
# and concatenating them
def ReadDirData(
    path, ListDirF=ListDir, ReadF=ReadCsv,
    WriteF=WriteCsv, writeFn=None, DfFilterF=None):

  print(path)
  fileList = ListDirF(path)
  print(fileList)
  #dfList = list()
  outDf = None
  for i in range(len(fileList)):
    f = path + fileList[i]
    print("*** opening: " + f)
    df = ReadF(f)
    print("data shape for this partition:")
    print(df.shape)

    if DfFilterF != None:
      df = DfFilterF(df)
      print("data shape for this partition after filtering:")
      print(df.shape)

    ## we either row bind data or we write data if writeFn is not None
    if writeFn == None:
      if i == 0:
        outDf = df
      else:
        outDf = outDf.append(df, ignore_index=True)
    else:
      if i == 0:
        WriteF(fn=writeFn, df=df, sep=',', append=False)
      else:
        WriteF(fn=writeFn, df=df, sep=',', append=True)

  print("First rows of data:")
  print(df.iloc[:5])

  return outDf

## Read all files in a dir with same columns
# and concatenating them
def ReadDirData_parallel(
    path, ListDirF=ListDir, ReadF=ReadCsv,
    WriteF=WriteCsv, writeFn=None,
    DfFilterF=None, returnDfDict=False,
    limitFileNum=None):

  print(path)
  fileList = ListDirF(path)
  print(fileList)

  if limitFileNum is not None:
    k = min(limitFileNum, len(fileList))
    fileList = fileList[:k]

  outDf = None

  dfDict = {}

  def F(i):
    f = path + fileList[i]
    Mark(text="opening: partition " + str(i) + '; ' + f)
    df = ReadF(f)
    Mark(df.shape, text="data shape for partition " + str(i))

    if DfFilterF != None:
      df = DfFilterF(df)
      Mark(
          df.shape,
          text="data shape for partition " + str(i) + " after filtering:")

    dfDict[i] = df

    return None

  [F(x) for x in range(len(fileList))]

  if returnDfDict:
    return dfDict

  ## we either row bind data or we write data if writeFn is not None
  if writeFn is None:
    '''
    for i in range(len(fileList)):
      if i == 0:
        outDf = dfDict[i]
      else:
        outDf = outDf.append(dfDict[i], ignore_index=True)
    '''
    outDf = pd.concat(dfDict.values())
  else:
    for i in range(len(fileList)):
      if i == 0:
        WriteF(fn=writeFn, df=dfDict[i], sep=',', append=False)
      else:
        WriteF(fn=writeFn, df=dfDict[i], sep=',', append=True)

  Mark(outDf.iloc[:10], text="First rows of data:")

  return outDf

def Write_shardedData_parallel(
    df, fnPrefix, path, fnExten=".csv",
    partitionCol=None,
    shardNum=100, WriteF=WriteCsv,
    limitFileNum=None):

  """ write sharded data wrt a partition column
  the data is written in parallel for speed purposes
  also at read time we can read data faster"""

  if partitionCol is None:
    partitionCol = "dummy_col"
    df["dummy_col"] = range(len(df))

  def Bucket(s):
    return int(hashlib.sha1(str(s)).hexdigest(), 16) % (shardNum)

  df["shard"] = df[partitionCol].map(Bucket)

  if partitionCol is None:
    del df["dummy_col"]

  def Write(bucket):
    df0 = df[df["shard"] == bucket]
    fn = path + fnPrefix + "_" + str(bucket) + ".csv"
    WriteF(fn=fn, df=df0, sep=',', append=False)
    print(fn + " was written")

  buckets = list(set(df["shard"].values))

  if limitFileNum is not None:
    k = min(limitFileNum, len(buckets))
    buckets = buckets[:k]

  [Write(bucket) for bucket in buckets]

  return None

"""
df = GenUsageDf_forTesting()
path = ""
Write_shardedData_parallel(
    df=df, fnPrefix="test", path=path, fnExten=".csv",
    partitionCol="user_id",
    WriteF=WriteCsv)

"""

############### Part 1: Data frame and data wrangling functions

## generate a data frame manually for testing data frame functions
# and usage metrics
def GenUsageDf_forTesting():

  df = pd.DataFrame(columns=[
      'country', 'user_id', 'expt', 'date', 'time',
      'end_time', 'prod', 'form_factor'])

  df.loc[len(df)] = ['US', '0', 'base', '2017-04-12', '2017-04-12 00:03:00',
                     '2017-04-12 00:04:00', 'PresFeat', 'COMP']
  df.loc[len(df)] = ['US', '0', 'base', '2017-04-12', '2017-04-12 00:04:01',
                     '2017-04-12 00:05:03', 'photoFeat', 'COMP']
  df.loc[len(df)] = ['US', '0', 'base', '2017-04-12', '2017-04-12 00:05:05',
                     '2017-04-12 00:06:04', 'PresFeat', 'PHN']
  df.loc[len(df)] = ['US', '0', 'base', '2017-04-12', '2017-04-12 00:06:05',
                     '2017-04-12 00:06:08', 'PresFeat', 'PHN']
  df.loc[len(df)] = ['US', '0', 'base', '2017-04-12', '2017-04-12 00:06:30',
                     '2017-04-12 00:06:45', 'exploreFeat', 'COMP']
  df.loc[len(df)] = ['JP', '1', 'base', '2017-04-12', '2017-04-12 00:07:00',
                     '2017-04-12 00:07:50', 'editingFeat', 'PHN']
  df.loc[len(df)] = ['JP', '1', 'base', '2017-04-12', '2017-04-12 00:14:00',
                     '2017-04-12 00:14:10', 'photoFeat', 'COMP']
  df.loc[len(df)] = ['JP', '1', 'base', '2017-04-12', '2017-04-12 00:16:00',
                     '2017-04-12 00:17:09', 'locFeat', 'COMP']
  df.loc[len(df)] = ['JP', '1', 'base',  '2017-04-12', '2017-04-12 00:18:00',
                     '2017-04-12 00:18:30', 'browsingFeat', 'COMP']
  df.loc[len(df)] = ['JP', '1', 'base', '2017-04-12', '2017-04-12 00:20:00',
                     '2017-04-12 00:21:00', 'locFeat', 'COMP']
  df.loc[len(df)] = ['JP', '1', 'base', '2017-04-12', '2017-04-12 00:22:00',
                     '2017-04-12 00:22:00', 'browsingFeat', 'PHN']
  df.loc[len(df)] = ['JP', '1', 'base', '2017-04-12', '2017-04-12 00:03:00',
                     '2017-04-12 00:04:00', 'PresFeat', 'COMP']
  df.loc[len(df)] = ['JP', '1', 'base', '2017-04-12', '2017-04-12 00:04:01',
                     '2017-04-12 00:05:03', 'photoFeat', 'COMP']
  df.loc[len(df)] = ['JP', '1', 'base', '2017-04-12', '2017-04-12 00:05:05',
                     '2017-04-12 00:06:04', 'PresFeat', 'PHN']
  df.loc[len(df)] = ['US', '2', 'base', '2017-04-12', '2017-04-12 00:06:05',
                     '2017-04-12 00:06:08', 'PresFeat', 'PHN']
  df.loc[len(df)] = ['US', '2', 'base', '2017-04-12', '2017-04-12 00:06:30',
                     '2017-04-12 00:06:45', 'exploreFeat', 'COMP']
  df.loc[len(df)] = ['US', '2', 'base', '2017-04-12', '2017-04-12 00:07:00',
                     '2017-04-12 00:07:50', 'editingFeat', 'PHN']
  df.loc[len(df)] = ['JP', '3', 'test', '2017-04-12', '2017-04-12 00:14:00',
                     '2017-04-12 00:14:10', 'photoFeat', 'COMP']
  df.loc[len(df)] = ['JP', '3', 'test', '2017-04-12', '2017-04-12 00:14:20',
                     '2017-04-12 00:18:59', 'locFeat', 'COMP']
  df.loc[len(df)] = ['JP', '3', 'test', '2017-04-12', '2017-04-12 00:19:00',
                     '2017-04-12 00:20:00', 'locFeat', 'COMP']
  df.loc[len(df)] = ['JP', '3', 'test', '2017-04-12', '2017-04-12 00:20:20',
                     '2017-04-12 00:22:00', 'browsingFeat', 'PHN']
  df.loc[len(df)] = ['US', '4', 'test', '2017-04-14', '2017-04-14 00:03:10',
                     '2017-04-14 00:04:00', 'PresFeat', 'COMP']
  df.loc[len(df)] = ['US', '4', 'test', '2017-04-14', '2017-04-14 00:04:10',
                     '2017-04-14 00:05:03', 'photoFeat', 'COMP']
  df.loc[len(df)] = ['US', '4', 'test', '2017-04-14', '2017-04-14 00:05:15',
                     '2017-04-14 00:06:04', 'PresFeat', 'PHN']
  df.loc[len(df)] = ['US', '4', 'test', '2017-04-14', '2017-04-14 00:06:01',
                     '2017-04-14 00:06:08', 'PresFeat', 'PHN']
  df.loc[len(df)] = ['US', '4', 'test', '2017-04-14', '2017-04-14 00:06:35',
                     '2017-04-14 00:06:45', 'PresFeat', 'COMP']
  df.loc[len(df)] = ['US', '5', 'test', '2017-04-14', '2017-04-14 00:03:07',
                     '2017-04-14 00:04:00', 'PresFeat', 'COMP']
  df.loc[len(df)] = ['US', '5', 'test', '2017-04-14', '2017-04-14 00:04:04',
                     '2017-04-14 00:05:03', 'photoFeat', 'COMP']
  df.loc[len(df)] = ['US', '5', 'test', '2017-04-14', '2017-04-14 00:05:04',
                     '2017-04-14 00:06:04', 'PresFeat', 'PHN']
  df.loc[len(df)] = ['US', '5', 'test', '2017-04-14', '2017-04-14 00:06:03',
                     '2017-04-14 00:06:08', 'PresFeat', 'PHN']
  df.loc[len(df)] = ['US', '5', 'test', '2017-04-14', '2017-04-14 00:06:28',
                     '2017-04-14 00:06:45', 'PresFeat', 'COMP']
  df.loc[len(df)] = ['JP', '6', 'test', '2017-04-14', '2017-04-14 00:14:01',
                     '2017-04-14 00:14:10', 'photoFeat', 'COMP']
  df.loc[len(df)] = ['JP', '6', 'test', '2017-04-14', '2017-04-14 00:14:19',
                     '2017-04-14 00:18:59', 'locFeat', 'COMP']
  df.loc[len(df)] = ['JP', '6', 'test', '2017-04-14', '2017-04-14 00:19:10',
                     '2017-04-14 00:20:00', 'locFeat', 'COMP']
  df.loc[len(df)] = ['JP', '6', 'test', '2017-04-14', '2017-04-14 00:20:11',
                     '2017-04-14 00:22:00', 'browsingFeat', 'PHN']
  df.loc[len(df)] = ['JP', '7', 'base', '2017-04-15', '2017-04-15 00:14:11',
                     '2017-04-15 00:14:10', 'photoFeat', 'COMP']
  df.loc[len(df)] = ['JP', '7', 'base', '2017-04-15', '2017-04-15 00:14:22',
                     '2017-04-15 00:18:59', 'locFeat', 'COMP']
  df.loc[len(df)] = ['JP', '7', 'base', '2017-04-15', '2017-04-15 00:19:57',
                     '2017-04-15 00:20:00', 'locFeat', 'COMP']
  df.loc[len(df)] = ['JP', '7', 'base', '2017-04-15', '2017-04-15 00:21:56',
                     '2017-04-15 00:22:00', 'browsingFeat', 'PHN']
  df.loc[len(df)] = ['FR', '8', 'base', '2017-04-12', '2017-04-12 00:03:00',
                     '2017-04-12 00:04:00', 'PresFeat', 'COMP']
  df.loc[len(df)] = ['FR', '8', 'base', '2017-04-12', '2017-04-12 00:04:01',
                     '2017-04-12 00:05:03', 'photoFeat', 'COMP']
  df.loc[len(df)] = ['FR', '8', 'base', '2017-04-12', '2017-04-12 00:05:05',
                     '2017-04-12 00:06:04', 'PresFeat', 'PHN']
  df.loc[len(df)] = ['FR', '8', 'base', '2017-04-12', '2017-04-12 00:06:05',
                     '2017-04-12 00:06:08', 'PresFeat', 'PHN']
  df.loc[len(df)] = ['FR', '8', 'base', '2017-04-12', '2017-04-12 00:06:30',
                     '2017-04-12 00:06:45', 'exploreFeat', 'COMP']
  df.loc[len(df)] = ['FR', '9', 'test', '2017-04-15', '2017-04-15 00:14:11',
                     '2017-04-15 00:14:10', 'photoFeat', 'COMP']
  df.loc[len(df)] = ['FR', '9', 'test', '2017-04-15', '2017-04-15 00:14:22',
                     '2017-04-15 00:18:59', 'locFeat', 'COMP']
  df.loc[len(df)] = ['FR', '9', 'test', '2017-04-15', '2017-04-15 00:19:57',
                     '2017-04-15 00:20:00', 'locFeat', 'COMP']
  df.loc[len(df)] = ['FR', '9', 'test', '2017-04-15', '2017-04-15 00:21:56',
                     '2017-04-15 00:22:00', 'browsingFeat', 'PHN']
  df.loc[len(df)] = ['NG', '10', 'test', '2017-04-16', '2017-04-15 00:21:56',
                     '2017-04-15 00:22:00', 'StorageFeat', 'PHN']
  df.loc[len(df)] = ['IR', '11', 'test', '2017-04-12', '2017-04-15 00:21:56',
                     '2017-04-15 00:22:00', 'browsingFeat', 'PHN']
  df.loc[len(df)] = ['IR', '12', 'base', '2017-04-16', '2017-04-15 00:21:56',
                     '2017-04-15 00:22:00', 'watchFeat', 'PHN']
  df.loc[len(df)] = ['IR', '13', 'base', '2017-04-12', '2017-04-12 00:03:00',
                     '2017-04-12 00:04:00', 'PresFeat', 'COMP']
  df.loc[len(df)] = ['RU', '14', 'base', '2017-04-12', '2017-04-12 00:03:00',
                     '2017-04-12 00:04:00', 'PresFeat', 'COMP']
  df.loc[len(df)] = ['RU', '15', 'base', '2017-04-13', '2017-04-13 00:03:00',
                     '2017-04-13 00:04:00', 'PresFeat', 'COMP']
  df.loc[len(df)] = ['RU', '16', 'base', '2017-04-14', '2017-04-14 00:03:00',
                     '2017-04-14 00:04:00', 'PresFeat', 'COMP']

  df['user_id'] = 'id' + df['user_id']
  def F(x):
    return(datetime.datetime.strptime(x[:19], "%Y-%m-%d %H:%M:%S"))

  for col in ['time', 'end_time']:
    df[col] = df[col].map(F)

  df['duration'] = (df['end_time'] - df['time']) / np.timedelta64(1, 's')
  df['value'] = np.random.uniform(low=1.0, high=5.0, size=df.shape[0])

  return df

def BuildCondInd(df, condDict):

  """ subsets a df according to values given in the dict: condDict
  the data columns are given in the dictionary keys
  the possible values (a list of values) for each column are
  given in the dict values """

  cols = condDict.keys()
  n = df.shape[0]
  ind = pd.Series([True] * n)
  for i in range(len(cols)):
    col = cols[i]
    valueList = condDict[col]
    if valueList != None and valueList != []:
      ind0 = (df[col].isin(valueList))
      ind = ind * ind0

  return ind

'''
df = pd.DataFrame({
    'a':[2, 1, 3, 2, 2, 2],
    'b':['A', 'A', 'B', 'C', 'C', 'C'],
  'c':['11','22','22','22', '22', '22']})
ind = BuildCondInd(df=df, condDict={'a':[1, 2], 'b':['A', 'B']})
df[ind]

ind = BuildCondInd(df=df, condDict={'a':[1, 2], 'b':None})
df[ind]
'''

## get the sub df immediately
def SubDf_withCond(df, condDict, resetIndex=True):

  if (condDict is None) or (len(condDict) == 0):
    return df

  df = df.reset_index(drop=True)
  ind = BuildCondInd(df=df, condDict=condDict)
  df2 = df[ind].copy()

  if resetIndex:
    df2 = df2.reset_index(drop=True)

  return df2

## subset df based on regex filters on string columns
# every column is given in a key and the value is a regex
def BuildRegexInd(df, regDict):

  cols = regDict.keys()
  n = df.shape[0]
  ind = pd.Series([True]*n)
  for i in range(len(cols)):
    col = cols[i]
    valueList = regDict[col]
    if valueList != None and valueList != []:
      ind0 = pd.Series([False] * n)
      for value in valueList:
        ind0 = ind0 + df[col].map(str).str.contains(value)

    ind = ind * ind0

    return ind

'''
df = pd.DataFrame(
    {'a':[24, 12, 63, 2, 3312, 2],
     'b':['A', 'A', 'BBAA', 'CD', 'CE', 'CF'],
     'c':['11','22','22','23', '22', '22']})

ind = BuildRegexInd(df=df, regDict={'a':['1', '2'], 'b':['A', 'B']})
Mark(df[ind])

ind = BuildRegexInd(df=df, regDict={'a':['1', '3'], 'b':None})
Mark(df[ind])

ind = BuildRegexInd(df=df, regDict={'b':['B', 'C'], 'b':['^(?:(?!CE).)*$']})
Mark(df[ind])

## column b does not include CE but it includes A or B.
ind = BuildRegexInd(df=df, regDict={'b':['^(?!.*CE).*B.*$', '^(?!.*CE).*A.*$']})
Mark(df[ind])
'''

## check for two strings regex
def Regex_includesBothStr(s1, s2):

  out = '^(?=.*' + s1 + ')(?=.*' + s2 + ').*$'

  return out

'''
reg = Regex_includesBothStr(' cat ', ' dog ')
print(reg)
print(pd.Series(['cat-dog', ' cat hates dog ', 'tiger']).str.contains(reg))
'''

## rehashing a column (col)
# the input is a dictionary of data frames with that column
# we make sure the rehashing is fixed across data frames
def RehashCol_dfDict(dfDict, col, newCol=None, omitCol=False):

  if newCol == None:
    newCol = col + '_hashed'
  dfNames = dfDict.keys()
  values = []
  for key in dfNames:
    df0 = dfDict[key]
    values0 = df0[col].values
    values = list(set(values + list(values0)))
  dfHash = pd.DataFrame({col: values, 'tempCol': range(len(values))})
  newDfDict = {}
  for key in dfNames:
    df0 = dfDict[key]
    dfNew = pd.merge(df0, dfHash, on=[col], how='left')
    dfNew[newCol] = dfNew['tempCol']
    del dfNew['tempCol']
    if omitCol:
      del dfNew[col]
    newDfDict[key] = dfNew

  return newDfDict

# it converts a float or string date to datetime
def FloatOrStr_toDate(x, format="%Y%m%d"):

  if (x == None) or (x == 'nan') or (x == np.nan):
    return pd.NaT
  if (type(x).__name__ == 'float') and math.isnan(x):
    return pd.NaT
  s = str(x)
  if s == 'nan':
    return pd.NaT
  import re
  s = re.sub('_', '', s)
  s = re.sub('-', '', s)
  s = re.sub(':', '', s)
  s = s[:8]

  return datetime.datetime.strptime(s, format)

## convert to datetime
def ConvertToDateTime(x, dateTimeFormat="%Y-%m-%d %H:%M:%S", strLen=19):

  if (x == None) or (x == 'nan') or (x == np.nan):
    return pd.NaT
  if (type(x).__name__ == 'float') and math.isnan(x):
    return pd.NaT
  s = str(x)
  if s == 'nan':
    return pd.NaT

  return datetime.datetime.strptime(x[:strLen], dateTimeFormat)

## also lets define a function generator version for easy mapping
def ConvertToDateTimeFcn(dateTimeFormat="%Y-%m-%d %H:%M:%S", strLen=19):

  def F(x):
    return ConvertToDateTime(x, dateTimeFormat=dateTimeFormat, strLen=strLen)

  return F

## convert weekday returned by isoweekday() to string
def WeekDayToStr(x):

    d = {1:'Mon', 2:'Tue', 3:'Wed', 4:'Thu', 5:'Fri', 6:'Sat', 7:'Sun'}
    if x in d.keys():
      return d[x]

    return 'nan'

'''
x = datetime.datetime(2017, 01, 01)
u = x.isoweekday()
WeekDayToStr(u)
'''

## assigns object types to string
# and assign "nan" to missing
def PrepareDf(df):

  # colTypes = [str(df[col].dtype) for col in df.columns]
  for col in df.columns:

    if str(df[col].dtype) == "object":
      df[col].fillna("nan", inplace=True)
      df[col] = df[col].astype(str)

  return df

# variable type
def Type(x):

  return type(x).__name__

## short hashing
def ShortHash(s, length=8):

  s = str(s).encode('utf-8')
  hasher = hashlib.sha1(s)
  return base64.urlsafe_b64encode(hasher.digest()[:length])

"""
ShortHash("asas")
"""

## integrates columns (integCols) out from a data frame.
# It uses integFcn for integration
# it only keeps valueCols for integration
def IntegOutDf(df, integFcn, integOutCols, valueCols=None):

  cols = list(df.columns)

  if valueCols == None:
    valueCols = list(filter(lambda x: x not in integOutCols, cols))

  gCols = list(filter(lambda x: x not in (integOutCols + valueCols), cols))
  if len(gCols) > 0:
    cols = gCols + valueCols
  df = df[cols]

  if len(gCols) == 0:
    gCols = ['tempCol']
    df['tempCol'] = 1

  g = df.groupby(by=gCols)
  dfAgg = g.aggregate(integFcn)
  dfAgg = dfAgg.reset_index()

  if 'tempCol' in dfAgg.columns:
    del dfAgg['tempCol']

  return dfAgg

'''
size =  10
df = pd.DataFrame({
    'categ1':np.random.choice(
        a=['a', 'b', 'c', 'd', 'e'], size=size, replace=True),
    'categ2':np.random.choice(a=['A', 'B'], size=size, replace=True),
    'col1':np.random.uniform(low=0.0, high=100.0, size=size),
    'col2':np.random.uniform(low=0.0, high=100.0, size=size)
    })
print(df)
IntegOutDf(df, integFcn=sum, integOutCols=['categ1'], valueCols=['col1', 'col2'])
'''

## aggregates df with different agg fcns for multiple columns
# gCols is not needed since it will be assume to be
# (all columns - the columns being aggregated)
def AggWithDict(df, aggDict, gCols=None):

  cols = list(df.columns)
  valueCols = aggDict.keys()

  if gCols == None:
    gCols = list(filter(lambda x: x not in (valueCols), cols))

  g = df.groupby(gCols)
  dfAgg = g.aggregate(aggDict)
  dfAgg = dfAgg.reset_index()

  return dfAgg

'''
size =  10
df = pd.DataFrame({
    'categ1':np.random.choice(
        a=['a', 'b', 'c', 'd', 'e'],
        size=size,
        replace=True),
    'categ2':np.random.choice(a=['A', 'B'], size=size, replace=True),
    'col1':np.random.uniform(low=0.0, high=100.0, size=size),
    'col2':np.random.uniform(low=0.0, high=100.0, size=size)
    })

df = df.sort_values(['categ2', 'categ1', 'col1', 'col2'])

print(df)
aggDf0 = AggWithDict(df=df, aggDict={'col1':sum, 'col2':min})
aggDf1 = AggWithDict(df=df, aggDict={'col1':sum, 'col2':min}, gCols=['categ2'])
print(aggDf0)
print(aggDf1)
'''

## find rows which have repeated values on some cols
def FindRepRows(df, cols):
  return pd.concat(g for _, g in df.groupby(cols) if len(g) > 1)

## slice df by sliceCol and with given values in sliceValues
def DfSliceDict(df, sliceCol, sliceValues=None):

  if sliceValues == None:
    sliceValues = list(set(df[sliceCol].values))
  dfDict = {}
  for i in range(len(sliceValues)):
    v = sliceValues[i]
    dfDict[v] = df[df[sliceCol] == v]

  return dfDict

## merge dfDict
def MergeDfDict(dfDict, onCols, how='outer', naFill=None):

  keys = dfDict.keys()
  for i in range(len(keys)):
    key = keys[i]
    df0 = dfDict[key]
    cols = list(df0.columns)
    valueCols = list(filter(lambda x: x not in (onCols), cols))
    df0 = df0[onCols + valueCols]
    df0.columns = onCols + [(s + '_' + key) for s in valueCols]

    if i == 0:
      outDf = df0
    else:
      outDf = pd.merge(outDf, df0, how=how, on=onCols)

  if naFill != None:
    outDf = outDf.fillna(naFill)

  return outDf

'''
def GenDf(size):
  df = pd.DataFrame({
      'categ1':np.random.choice(
          a=['a', 'b', 'c', 'd', 'e'], size=size, replace=True),
      'categ2':np.random.choice(a=['A', 'B'], size=size, replace=True),
      'col1':np.random.uniform(low=0.0, high=100.0, size=size),
      'col2':np.random.uniform(low=0.0, high=100.0, size=size)
      })
  df = df.sort_values(['categ2', 'categ1', 'col1', 'col2'])
  return(df)
size = 5
dfDict = {'US':GenDf(size), 'IN':GenDf(size), 'GER':GenDf(size)}
MergeDfDict(dfDict=dfDict, onCols=['categ1', 'categ2'], how='outer', naFill=0)
'''

## split data based on values of a column: col
def SplitDfByCol(df, col):
  #create unique list of device names
  uniqueNames = df[col].unique()
  #create a data frame dictionary to store your data frames
  dfDict = {elem : pd.DataFrame for elem in uniqueNames}
  for key in dfDict.keys():
    dfDict[key] = df[df[col] == key]
  return dfDict

## calculates  value_counts() aka freq for combination of cols
def CombinFreqDf(
    df, cols=None, countColName='cnt', propColName='prop (%)'):

  if Type(df) == "Series":
    df = pd.DataFrame(df)

  if cols == None:
    cols = list(df.columns)

  if len(cols) < 2:
    cols.append('dummy')
    df['dummy'] = 'NA'

  outDf = df[cols].groupby(cols).agg(len).reset_index()
  outDf.columns = list(outDf.columns[:len(outDf.columns)-1]) + [countColName]
  outDf[propColName] = 100.0 * outDf[countColName] / outDf[countColName].sum()
  outDf = outDf.sort_values([countColName], ascending=[0])

  if 'dummy' in cols:
    del outDf['dummy']
  outDf = outDf.reset_index(drop=True)

  return outDf

'''
df0 = pd.DataFrame({
    'app':['fb', 'fb', 'mailingFeat', 'mailingFeat'],
    'party':['1P', '1P', '3P', '3P']})
CombinFreqDf(df=df0, cols=['app', 'party'])
'''

## maps a categorical variable with too many labels to less labels.
def Remap_lowFreqCategs(
    df,
    cols,
    newLabels="nan",
    otherLabelsToReMap=None,
    freqThresh=5,
    propThresh=0.1,
    labelsNumMax=None):

  df2 = df.copy()
  k = len(cols)

  if Type(freqThresh) == 'int':
    freqThresh = [freqThresh] * k

  if Type(propThresh) in ['int', 'float']:
    propThresh = [propThresh] * k

  if Type(newLabels) == 'str':
    newLabels = [newLabels] * k

  if (labelsNumMax is not None) and Type(labelsNumMax) == 'int':
    labelsNumMax = [labelsNumMax] * k

  def GetFreqLabels(i):

    col = cols[i]
    freqDf = CombinFreqDf(df[col])

    ind = (freqDf["cnt"] > freqThresh[i]) & (freqDf["prop (%)"] > propThresh[i])
    freqLabels = list(freqDf.loc[ind][col].values)

    if labelsNumMax is not None:
      maxNum = min(len(freqLabels), labelsNumMax[i])
      freqLabels = freqLabels[0:(maxNum)]

    if otherLabelsToReMap is not None:
      freqLabels = list(set(freqLabels) - set(otherLabelsToReMap))

    return freqLabels

  freqLabelsList = [GetFreqLabels(x) for x in range(k)]
  freqLabelsDict = dict(zip(cols, freqLabelsList))

  def F(df):

    for i in range(len(cols)):
      col = cols[i]
      newLabel = newLabels[i]
      ind = [x not in freqLabelsDict[col] for x in df[col]]

      if max(ind):
        df.loc[ind, col] = newLabel

    return df

  return {"df":F(df2), "F":F, "freqLabelsDict":freqLabelsDict}

## this function works on a data frame with two categorical columns
## one is the  category column
## one is the label column
## for each category it creates a distribution for the labels
def CalcFreqTablePerCateg(df, categCol, valueCol):

  def AggFcnBuild(categValue):
    def F(x):
      return sum(x == categValue)/(1.0)
    return F

  df = df.fillna('NA')
  labels = list(set(df[valueCol]))

  def G(value):
    AggFcn = AggFcnBuild(value)
    dfAgg = df.groupby([categCol])[[valueCol]].agg(lambda x: AggFcn(x))
    dfAgg = dfAgg.reset_index()
    return dfAgg

  value = labels[0]
  dfAgg = G(value)

  for i in range(1, len(labels)):
    value = labels[i]
    dfAgg1 = G(value)
    dfAgg = pd.merge(dfAgg, dfAgg1, how='left', on=[categCol])
  dfAgg.columns = [categCol] + labels

  return {'df': dfAgg, 'labels': labels}

'''
size = 20
df0 = pd.DataFrame({
    'categ':np.random.choice(a=['a', 'b', 'c'], size=size, replace=True),
    'value':np.random.choice(a=['AA', 'BB', 'CC'], size=size, replace=True),
    'col2':np.random.uniform(low=0.0, high=100.0, size=size),
    'col3':np.random.uniform(low=0.0, high=100.0, size=size),
    'col4':np.random.uniform(low=0.0, high=100.0, size=size)})

CalcFreqTablePerCateg(df=df0, categCol='categ', valueCol='value')['df']
'''

##  merges a dict of tables
def MergeTablesDict(tabDict):

  keys = tabDict.keys()
  #print(keys)
  n = len(keys)
  for i in range(n):
    key = keys[i]
    tab = tabDict[key]
    df = PropDfTab(tab)
    df = df[['categ', 'freq', 'prop']]
    df.columns = ['categ', 'freq_' + key, 'prop_' + key]
    if i == 0:
      outDf = df
    else:
      outDf = pd.merge(outDf, df, on=['categ'], how='outer')
      outDf = outDf.reset_index(drop=True)
  outDf = outDf.fillna(value=0)

  return outDf

## creating a single string column using multiple columns (cols)
# and adding that to the data frame
def Concat_stringColsDf(df, cols, colName=None, sepStr='-'):

  x = ''
  if colName == None:
    colName = sepStr.join(cols)
  for i in range(len(cols)):
    col = cols[i]
    x = (x + df[col].map(str))
    if (i < len(cols)-1):
      x = x +'-'
  df[colName] = x

  return df

'''
df = pd.DataFrame({'a':range(3), 'b':['rr',  'gg', 'gg'], 'c':range(3)})
Concat_stringColsDf(df=df, cols=['a', 'b', 'c'], colName=None, sepStr='-')
'''

## flatten a column (listCol) of df with multiple values
def Flatten_RepField(df, listCol, sep=None):

  if sep != None:
    df = df.assign(**{listCol: df[listCol].str.split(',')})
  outDf = pd.DataFrame({
      col: np.repeat(df[col].values, df[listCol].str.len())
      for col in df.columns.difference([listCol])
  }).assign(
      **{listCol: np.concatenate(df[listCol].values)})[df.columns.tolist()]

  return outDf

'''
df = pd.DataFrame({'var1': ['a,b,c', 'd,e,f'], 'var2': [1, 2], 'var3':[5, 6]})
print(df)
Flatten_RepField(df, listCol='var1', sep=',')
'''

### tables p-value
## for a given table of frequencies and for each category calculates
## the total count of other categs (complement)
def TabCategCompl(tab, categCol, freqCol, complementCol=None):

  categs = tab[categCol].values
  s = tab[freqCol].sum()
  complement = list()

  for i in range(tab.shape[0]):
    categ = categs[i]
    tab0 = tab[(tab[categCol] == categ)]
    x = tab0[freqCol].values[0]
    c = s - x
    complement.append(c)

  if complementCol == None:
    complementCol = freqCol + '_compl'
  tab[complementCol] = complement

  return tab

## does above for multiple columns
def TabCategComplMulti(tab, categCol, freqCols):

  complementCols = []

  for i in range(len(freqCols)):
    freqCol = freqCols[i]
    tab = TabCategCompl(tab=tab, categCol=categCol, freqCol=freqCol,
                        complementCol=None)
    complementCol = freqCol + '_compl'
    complementCols.append(complementCol)

  cols = freqCols + complementCols
  tab = tab[[categCol] + cols]

  return tab

## adds a p-value per categ for comparing two frequencies
def TabComplPvalue(tab, categCol, freqCols):

  tab = TabCategComplMulti(tab, categCol, freqCols)
  n = tab.shape[0]
  pvalueList = []

  for i in range(n):
    r = tab.iloc[i]
    d = pd.DataFrame({'col1': [r[1], r[2]], 'col2': [r[3],r[4]]})
    pvalue = scipy.stats.fisher_exact(table=d, alternative='two-sided')[1]
    pvalueList.append(Signif(3)(pvalue))

  tab['p-value'] = pvalueList

  return tab

#### Useful functions for mapping a string column to another string column
## using a pattern string
## function for classification mapping
## it uses patterns to map to categories (general purpose)
def LabelByPattern(
    x, patternDf, patternCol='pattern', categCol='category',
    exactMatch=False):
  # x a series
  # patternDict has patterns and labels
  # remove duplicate rows
  import re
  patternDf = patternDf.drop_duplicates(keep='first')
  patterns = patternDf[patternCol]
  categs = patternDf[categCol]
  y = ['']*len(x)
  outDf = pd.DataFrame({'x':x, 'y':y})

  for i in range(len(patterns)):
    pattern = patterns[i]
    categ = categs[i]
    hasCateg = x.str.contains(pattern)
    if exactMatch:
      hascateg = (x.str == pattern)

    ind = np.where(hasCateg > 0)[0].tolist()
    for j in ind:
      if not bool(re.search(categ, y[j])):
        y[j] = y[j] + categ

  outDf['y'] = y
  outDf.columns = ['signal', categCol]

  return outDf

## label a data frame based on patternDf
# which includes pattern column and category column
def LabelByPatternDf(
      df, signalCol, patternDf, patternCol, categCol,
      newColName='mapped_category'):

  patternDf = patternDf[[patternCol, categCol]]
  patternDf = patternDf.drop_duplicates(keep='first')
  patternDf = patternDf.reset_index(drop=True)
  x = df[signalCol]
  df2 = LabelByPattern(x=x, patternDf=patternDf, patternCol=patternCol,
                       categCol=categCol)
  df[newColName] = df2[categCol]

  return df

######################### Graphical/Plotting Functions ######################
## bar charts for multiple columns (yCols), with different colors
## x axis labels come from the xCol
def BarPlotMultiple(df, xCol, yCols, rotation=45, pltTitle=''):

  x = range(len(df[xCol]))
  colorList = ['r', 'm', 'g', 'y', 'c']
  x = 8*np.array(x)

  for i in range(len(yCols)):
    col = yCols[i]
    x1 = x + 1*i
    plt.bar(x1, df[col], color=colorList[i], alpha=0.6, label=col)

  locs, labels = plt.xticks()
  plt.xticks(x1, df[xCol], rotation=rotation)
  plt.setp(labels, rotation=rotation, fontsize=10)
  plt.title(pltTitle + ': ' + xCol)
  plt.legend()

import matplotlib.scale as mscale
import matplotlib.pyplot as plt
import matplotlib.transforms as mtransforms
import matplotlib.ticker as ticker
import numpy as np

class SquareRootScale(mscale.ScaleBase):
  """
  ScaleBase class for generating square root scale.
  """

  name = 'squareroot'

  def __init__(self, axis, **kwargs):
    mscale.ScaleBase.__init__(self)

  def set_default_locators_and_formatters(self, axis):
    axis.set_major_locator(ticker.AutoLocator())
    axis.set_major_formatter(ticker.ScalarFormatter())
    axis.set_minor_locator(ticker.NullLocator())
    axis.set_minor_formatter(ticker.NullFormatter())

  def limit_range_for_scale(self, vmin, vmax, minpos):
    return  max(0., vmin), vmax

  class SquareRootTransform(mtransforms.Transform):
    input_dims = 1
    output_dims = 1
    is_separable = True

    def transform_non_affine(self, a):
      return np.array(a)**0.5

    def inverted(self):
      return SquareRootScale.InvertedSquareRootTransform()

  class InvertedSquareRootTransform(mtransforms.Transform):
    input_dims = 1
    output_dims = 1
    is_separable = True

    def transform(self, a):
      return np.array(a)**2

    def inverted(self):
      return SquareRootScale.SquareRootTransform()

  def get_transform(self):
    return self.SquareRootTransform()

mscale.register_scale(SquareRootScale)

## compares the freq of usages given in labelCol
# and creates a joint distribution of labelCol given in x-axis
# and compareCol given with colors
# it either shows the joint probability (prop (%)) on y-axis
# or it will show the freq divided by denomConstant if prop == False
# labelCol: categorical var denoted in x-axis
# compareCol: categorical var denoted by colors
# countDistinctCols: what should be counted once, e.g. unit_id will count
# each item with given unit_id once
# prop = True, will calculates proportions, otherwise we divide counts by
# denomConstant,
# and if denomCountCols is given, we use it to count number of items
# and divide by (denomConstant * itemCount)
def PltCompare_bivarCategFreq(
    df, labelCol, compareCol=None, countDistinctCols=None,
    rotation=90, pltTitle='', compareOrder=None, limitNum=None,
    prop=True, denomConstant=1.0, denomCountCols=None,
    newColName="value", yScale=None):

  if countDistinctCols is not None:
    keepCols = [labelCol] + countDistinctCols
    if compareCol is not None:
      keepCols = keepCols + [compareCol]
    if denomCountCols is not None:
      keepCols = keepCols + denomCountCols

    df = df[keepCols].drop_duplicates().reset_index()

  if compareCol is None:
    combinDf = CombinFreqDf(df[labelCol])
  else:
    combinDf = CombinFreqDf(df[[labelCol, compareCol]])
    hue = compareCol

  if limitNum is not None:
    combinDf = combinDf[:limitNum]

  if compareOrder is not None:
    hue_order = compareOrder

  respCol = "prop (%)"

  #Mark(denomConstant, "denomConstant")
  if denomCountCols is not None:
    itemCount = len(df[denomCountCols].drop_duplicates().reset_index())
    denomConstant = 1.0 * denomConstant * itemCount
  #Mark(denomConstant, "denomConstant")

  if prop is False:
    combinDf[newColName] = combinDf["cnt"] / denomConstant
    respCol = newColName

  if compareCol is None:
    sns.barplot(data=combinDf, x=labelCol, y=respCol)
  else:
    sns.barplot(data=combinDf, x=labelCol, hue=hue, y=respCol)

  locs, labels = plt.xticks()
  out = plt.setp(labels, rotation=rotation, fontsize=10)
  plt.legend(loc='upper right')

  if yScale is not None:
    plt.yscale(yScale)

  return combinDf

"""
df = pd.DataFrame({
    "label":["cat", "dog", "cat", "dog", "dog", "cat", "cat", "dog"],
    "gender":["M", "F", "M", "F", "F", "F", "F", "M"]})

PltCompare_bivarCategFreq(
    df=df, labelCol="label", compareCol="gender")

PltCompare_bivarCategFreq(
    df=df, labelCol="label", compareCol="gender",
    prop=False, denomConstant=1.0, newColName="cnt per day")

"""

## make a boxplot for multiple columns Side by Side (Sbs, include mean with a star
def BoxPlt_dfColsSbS(
    df, cols=None, pltTitle='', xlab='', ylab='value',
    boxColors=['darkkhaki', 'royalblue', 'r', 'g', 'y', 'o', 'b'],
    ylim=None):

  from matplotlib.patches import Polygon
  data = []

  if cols is None:
    cols = df.columns
  for i in range(len(cols)):
    col = cols[i]
    data.append(df[col])

  fig, ax1 = plt.subplots(figsize=(10, 6))
  fig.canvas.set_window_title('')
  plt.subplots_adjust(left=0.075, right=0.95, top=0.9, bottom=0.25)

  bp = plt.boxplot(data, notch=0, sym='+', vert=1, whis=1.5)
  plt.setp(bp['boxes'], color='black')
  plt.setp(bp['whiskers'], color='black')
  plt.setp(bp['fliers'], color='red', marker='+')
  # Add a horizontal grid to the plot, but make it very light in color
  # so we can use it for reading data values but not be distracting
  ax1.yaxis.grid(True, linestyle='-', which='major', color='lightgrey',
                 alpha=0.5)

  # Hide these grid behind plot objects
  ax1.set_axisbelow(True)
  ax1.set_title(pltTitle, fontsize=20)
  ax1.set_xlabel(xlab)
  ax1.set_ylabel(ylab)

  # Now fill the boxes with desired colors
  numBoxes = len(data)
  medians = list(range(numBoxes))

  for i in range(numBoxes):
    box = bp['boxes'][i]
    boxX = []
    boxY = []
    for j in range(5):
      boxX.append(box.get_xdata()[j])
      boxY.append(box.get_ydata()[j])
    boxCoords = list(zip(boxX, boxY))
    boxPolygon = Polygon(boxCoords, facecolor=boxColors[i], label=cols[i])
    ax1.add_patch(boxPolygon)
    # Now draw the median lines back over what we just filled in
    med = bp['medians'][i]
    medianX = []
    medianY = []
    for j in range(2):
      medianX.append(med.get_xdata()[j])
      medianY.append(med.get_ydata()[j])
      plt.plot(medianX, medianY, 'k')
      medians[i] = medianY[0]
    # Finally, overplot the sample averages, with horizontal alignment
    # in the center of each box
    plt.plot([np.average(med.get_xdata())], [np.average(data[i])],
             color='w', marker='*', markeredgecolor='k')

  if ylim is not None:
    ax1.set_ylim(ylim)
  plt.legend()

def CustomSortDf(df, col, orderedValues):

  values = set(df[col].values)
  remainingValues = list(set(df[col].values) - set(orderedValues))
  orderedValues = orderedValues + remainingValues

  df2 = pd.DataFrame(
        {col:orderedValues, "dummy_order":range(len(orderedValues))})

  df3 = pd.merge(df, df2, how="left", on=[col])
  df3 = df3.sort_values(["dummy_order"])
  df3 = df3.reset_index(drop=True)

  del df3["dummy_order"]

  return df3

"""
n = 10
df = pd.DataFrame({
    'categ':np.random.choice(a=['a', 'b', 'c', 'd'], size=n, replace=True),
    'col1':np.random.uniform(low=0.0, high=100.0, size=n),
    'col2':np.random.uniform(low=0.0, high=100.0, size=n),
    'col3':np.random.uniform(low=0.0, high=100.0, size=n),
    'col4':np.random.uniform(low=0.0, high=100.0, size=n)})
col = "categ"
orderedValues = ["c", "a", "b"]

CustomSortDf(df=df, col=col, orderedValues=orderedValues)
"""

## it plots all columns wrt index
# it uses colors to compare them side by side.
def PltCols_wrtIndex(
    df, cols=None, categCol=None, pltTitle='', ymin=None,
    ymax=None, yLabel='', xLabel='', colorList=None,
    orderedValues=None, alphaList=None, sciNotation=False,
    ExtraFcn=None, orient='v',
    sizeAlpha=0.75, legendColAlpha=2):

  df2 = df.copy()

  if cols is None:
    cols = list(df2.columns)

  if categCol is not None:
    df2.index = df2[categCol]
    if (categCol in cols):
      cols = list(set(cols) - set([categCol]))
      # cols = cols.remove(categCol)
      # print(cols)
  # Mark(categs)

  if orderedValues is not None:
    df2 = CustomSortDf(df=df2, col=categCol, orderedValues=orderedValues)

  df2.index = df2[categCol]
  categs = df2.index
  num = len(categs)
  x = range(num)

  if colorList is None:
    colorList = [
        'r', 'g', 'm', 'y', 'c', 'darkkhaki', 'royalblue',
        'darkred', 'crimson', 'darkcyan', 'gold', 'lime', 'black',
        'navy', 'deepskyblue', 'k']

  if alphaList is None:
    alphaList = [0.7] * len(cols)
  stretch = 4 * len(cols)
  x = stretch * np.array(x)

  if orient == 'v':
    fig, ax = plt.subplots(figsize=(15*sizeAlpha, 10*sizeAlpha),
      dpi=1200, facecolor='w', edgecolor='k')

    for i in range(len(cols)):
      col = df2[cols[i]]
      plt.bar(x + 2*i, col.values, alpha=alphaList[i], label=cols[i],
              color=colorList[i], width=2, edgecolor='black',
              linewidth=2.0*sizeAlpha)
    plt.title(pltTitle, fontsize=20, fontweight='bold')

    if sciNotation:
      plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0))
    #labels = [item.get_text() for item in ax.get_xticklabels()]
    labels = categs
    ax.set_xticklabels(labels)
    locs, labels = plt.xticks(x, categs)
    plt.setp(labels, rotation=15, fontsize=17*sizeAlpha, fontweight='bold')

    locs2, labels2 = plt.yticks()
    plt.setp(labels2, rotation=0, fontsize=17*sizeAlpha, fontweight='bold')

    ncol = len(cols) / legendColAlpha

    plt.legend(
        loc='best',
        fontsize=17,
        prop={'weight': 'semibold',
              'size': 17 * sizeAlpha},
        ncol=ncol)
    axes = plt.gca()
    axes.set_ylim([ymin, ymax])
    ax.set_xlabel(xLabel)
    ax.set_ylabel(yLabel)
    if ExtraFcn is not None:
      ExtraFcn(ax)

  if orient == 'h':

    fig, ax = plt.subplots(figsize=(10*sizeAlpha, 15*sizeAlpha), dpi=1200,
      facecolor='black', edgecolor='black')
    for i in range(len(cols)):
      col = df2[cols[i]]
      plt.barh(
          x + 2 * (i - 1),
          col.values,
          alpha=alphaList[i],
          label=cols[i],
          color=colorList[i],
          height=2,
          edgecolor='black',
          linewidth=2.0 * sizeAlpha)
    plt.title(pltTitle, fontsize=20*sizeAlpha, fontweight='bold')

    if sciNotation:
      plt.ticklabel_format(
          style='sci', axis='x', scilimits=(0, 0), prop={'weight': 'bold'})
    labels = categs
    ax.set_yticklabels(labels)
    locs, labels = plt.yticks(x, categs)
    plt.setp(labels, rotation=0, fontsize=17*sizeAlpha, fontweight='bold')

    locs2, labels2 = plt.xticks()
    plt.setp(labels2, rotation=20, fontsize=17*sizeAlpha, fontweight='bold')
    ncol = len(cols) / legendColAlpha

    plt.legend(
        loc='best',
        ncol=ncol,
        prop={'weight': 'semibold',
              'size': 17 * sizeAlpha})
    axes = plt.gca()
    axes.set_xlim([ymin, ymax])
    ax.set_xlabel(yLabel)
    ax.set_ylabel(xLabel)
    if ExtraFcn != None:
      ExtraFcn(ax)
    plt.gca().invert_yaxis()

  return {'fig': fig, 'ax': ax}

'''
n = 3
df = pd.DataFrame({
    'categ':np.random.choice(
        a=['a', 'b', 'c', 'd', 'e', 'f'],
        size=n,
        replace=False),
    'col1':np.random.uniform(low=0.0, high=100.0, size=n),
    'col2':np.random.uniform(low=0.0, high=100.0, size=n),
    'col3':np.random.uniform(low=0.0, high=100.0, size=n),
    'col4':np.random.uniform(low=0.0, high=100.0, size=n)})

orderedValues = ["c", "a", "b", "d", "f", "e"]

PltCols_wrtIndex(
    df=df,
    cols=['col1', 'col2', 'col3', 'col4'],
    categCol='categ',
    orderedValues=orderedValues,
    orient='v',
    sciNotation=True)

'''

## this function creates a plot with each bar representing the distribution
# for a category (given in categCol)
# each distribution is defined on a set of labels
# the distributions are given in each column
def Plt_stackedDist_perCateg(
    df, categCol, cols=None, labels=None,
    sortCols=None, figsize=(10, 5), mainText=''):

  import colorsys

  if cols == None:
    cols = list(df.columns[1:len(df.columns)])
  if labels == None:
    labels = cols
  if sortCols == None:
    sortCols = cols

  df = df.sort(sortCols, ascending=False)
  m = len(cols)
  HSV_tuples = [(x*1.0/m, 0.5, 0.5) for x in range(m)]
  RGB_tuples = [colorsys.hsv_to_rgb(*x) for x in HSV_tuples]
  n = df.shape[0]
  x = pd.Series(n*[0])
  bar_locations = np.arange(n)
  fig, ax = plt.subplots(figsize=figsize)

  for i in range(len(cols)):
    col = cols[i]
    y = df[col].values
    ax.bar(bar_locations, y, bottom=x, color=RGB_tuples[i], label=labels[i])
    x = x + y

  plt.legend(loc="best")
  bar_locations2 = np.arange(n) + 0.5
  plt.xticks(bar_locations2, df[categCol].values, rotation='vertical')
  plt.title(mainText)
  print(bar_locations)
  print(df.loc[0].values)
  fig.show()

'''
df0 = pd.DataFrame({'country':['JP', 'US', 'FR'],
                    'col1':np.random.uniform(low=0.0, high=100.0, size=3),
                    'col2':np.random.uniform(low=0.0, high=100.0, size=3),
                    'col3':np.random.uniform(low=0.0, high=100.0, size=3),
                    'col4':np.random.uniform(low=0.0, high=100.0, size=3)})

Plt_stackedDist_perCateg(
    df=df0, categCol='country', cols=['col1', 'col2', 'col3', 'col4'], labels=None,
    sortCols=None, figsize=(10, 5), mainText='')

'''

## compares the values (valueCol) for the index (pivotIndCol)
# for various classes given in compareCol
# first it pivots the data and then plots side by side
def PivotPlotWrt(
    df, pivotIndCol, compareCol, valueCol,
    cols=None, pltTitle='', sizeAlpha=0.75):

  dfPivot = df.pivot(index=pivotIndCol, columns=compareCol, values=valueCol)
  dfPivot = dfPivot.fillna(0)
  dfPivot[pivotIndCol] = dfPivot.index

  if cols is None:
    cols = list(set(df[compareCol].values))

  p = PltCols_wrtIndex(
      df=dfPivot,
      cols=cols,
      categCol=pivotIndCol,
      orient='h',
      pltTitle=pltTitle,
      sizeAlpha=sizeAlpha)

  return {'df':dfPivot, 'plt':p}

## creating quantiles for a continuous variable and removing repetitions
def Qbins(x, num=10):

  qs = list(
      np.percentile(a=x,
                    q=list(100 * np.linspace(
                        start=0,
                        stop=(1 - 1.0/num),
                        num=num))))
  qs = list(set([float("-inf")] + qs + [float("inf")]))
  qs.sort()

  return qs

# cuts uniformly
def Ubins(x, num=10):

  b = np.linspace(start=min(x), stop=max(x), num=num)
  b = [Signif(2)(x) for x in b]
  b = list(set(b))
  b = [float("-inf")] + b + [float("inf")]
  b.sort()

  return b

## cuts using quantiles
def CutQ(x, num=10):
  qs = Qbins(x, num)
  discX = pd.cut(x, bins=qs)
  return(discX)

## make a bar plot
def BarPlot(y, yLabels, ylim=None, pltTitle='', figSize=[5, 5]):

  n = len(y)
  x = pd.Series(n*[0])
  bar_locations = np.arange(n)
  fig, ax = plt.subplots()
  fig.set_size_inches(figSize[0], figSize[1])
  ax.bar(bar_locations, y, bottom=x, color='r')
  plt.legend(loc="best")
  bar_locations2 = np.arange(n) + 0.5
  plt.xticks(bar_locations2, yLabels, rotation='vertical')
  axes = plt.gca()
  axes.set_ylim(ylim)
  plt.title(pltTitle)
  fig.show()

## creates a cut column with NA being an explicit category
def ExplicitNa_cutCol(df, col, cuts, newCol=None):

  if newCol is None:
    newCol = col + '_range'

  df[newCol] = pd.cut(df[col], cuts)
  df[newCol] = df[newCol].cat.add_categories(["NA"])
  df[newCol] = df[newCol].fillna("NA")

  return df

'''
z = np.random.normal(loc=50.0, scale=20.0, size=10)
z = np.insert(z, 0, float('nan'))
df0 = pd.DataFrame({'z':z})
ExplicitNa_cutCol(
    df=df0,
    col='z',
    cuts=[-20, 0, 20, 40, 60, 80, 100, 120, 140, float('inf')], newCol=None)
'''

## order df based on a cut column
def OrderDf_cutCol(df, cutCol, orderCol='order'):

  def F(s):
    x = re.search(r'.*?\((.*),.*', s)
    if x is None:
      return(float('-inf'))
    return(float(x.group(1)))

  df['order'] = df[cutCol].map(F)
  df = df.sort_values('order')

  return df

'''
z = np.random.normal(loc=50.0, scale=20.0, size=10)
z = np.insert(z, 0, float('nan'))
df0 = pd.DataFrame({'z':z})
u = pd.cut(z, [-20, 0, 20, 40, 60, 80, 100, 120, 140, float('inf')])
df0['col'] = u
df0['col'] = df0['col'].cat.add_categories(["NA"])
df0['col'] = df0['col'].fillna("NA")
OrderDf_cutCol(df=df0, cutCol="col")
'''

## for a variable generated with pd.cut it make a barplot
# it orders the labels based on the their values (rather than freq)
def FreqPlot_cutCol(u, pltTitle='', figSize=[5, 5]):

  tab = u.value_counts()
  df0 = pd.DataFrame(tab)
  df0['label'] = list(df0.index)
  df0 = OrderDf_cutCol(df=df0, cutCol='label')
  df0.columns = ['value', 'label', 'order']
  df0 = df0.sort_values('order')
  BarPlot(
      y=df0['value'], yLabels=df0['label'], pltTitle=pltTitle, figSize=figSize)

  return df0

'''
z = np.random.normal(loc=50.0, scale=20.0, size=1000)
u = pd.cut(z, [-20, 0, 20, 40, 60, 80, 100, 120, 140, float('inf')])
FreqPlot_cutCol(u)
'''

def PropDfTab(tab, ylim=None, categCol='categ', pltIt=False, pltTitle=''):

  d = pd.DataFrame(tab)
  d.columns = ['freq']
  e = (100.0 * d.values) / sum(d.values)
  e = [Signif(5)(x) for x in e]
  d['prop'] = e
  d[categCol] = d.index

  if pltIt:
    BarPlot(y=e, yLabels=list(d.index), ylim=ylim, pltTitle=pltTitle)

  return d

## cut continuous var
def CutConti(x, num=10, method='quantile'):

  if method == 'quantile':
    b = Qbins(x, num=num)
  elif (method == 'uniform'):
    b = Ubins(x, num=num)
  z = pd.cut(x, bins=b)

  return z

## gets a continuous var x, partitions the real line based on quantiles
# or bins of x
# then generates a function which assigns levels to any new value/values
def CutContiFcn(
    x,
    num=10,
    method='quantile',
    intervalColName='interval',
    levelColName='level',
    levelsType='int',
    levelPrefix='Lev',
    rawValueColName='raw'):

  if method == 'quantile':
    b = Qbins(x, num=num)
    print(b)
  elif (method == 'uniform'):
    b = Ubins(x, num=num)

  intervals = sorted(set(pd.cut(x + b[1:], bins=b)))

  if ():
    levels = [levelPrefix + str(x) for x in range(len(intervals))]

  Mark(levels)
  levDf = pd.DataFrame({intervalColName:intervals, levelColName:levels})
  Mark(levDf)

  def F(u):
    z = pd.cut(u, bins=b)
    df0 = pd.DataFrame({rawValueColName:u, intervalColName: z})
    outDf = pd.merge(df0, levDf, on=[intervalColName], how='left')
    return(outDf)

  return F

'''
x = [1, 3, 4, 5, 66, 77, 88]
F = CutContiFcn(
    x, num=10, method='quantile', intervalColName='interval',
    levelColName='level', levelPrefix='Lev', rawValueColName='raw')
F(x)
F(x + [5, 1, 3, 100, -1 , 90, 2.2])
'''

## cuts continuous data and creates a bar plot
def CutBarPlot(x, num=10, method='quantile', pltTitle='', figSize=[5, 5]):

  z = CutConti(x, num=num, method=method)
  u = z.value_counts()
  #print(u)
  d = pd.DataFrame(u)
  d = 100.0 * (d / d.sum())
  d = d.sort_index()
  BarPlot(y=d.values, yLabels=d.index, ylim=None, pltTitle=pltTitle,
          figSize=figSize)

## returns a functions which calculates quantiles according to q (which )
def QuantileFcn(q):

  def F(x):
    return(np.percentile(a=x, q=q))

  return F

def Plt_quantilesPerSlice(
    df, sliceCol, respCol, gridNum=100.0, pltTitle=''):

  slices = list(set(df[sliceCol].values))
  outDict = {}

  for sl in slices:
    x = df[df[sliceCol] == sl][respCol].values
    grid = list(
        gridNum * np.linspace(
            start=1.0/float(gridNum),
            stop=(1 - 1.0/float(gridNum)),
            num=int(gridNum)))
    q = QuantileFcn(grid)(x)
    outDict[sl] = q
    plt.plot(grid, q, label=sl)

  plt.legend()
  plt.title(pltTitle)

  return pd.DataFrame(outDict)

'''
df = pd.DataFrame({
  'value':[1, 1, 1, 2, 3, 4, 2, 2, 2],
  'categ':['a', 'a', 'b', 'b', 'b', 'a', 'a', 'b', 'a']})
Plt_quantilesPerSlice(df=df, sliceCol='categ', respCol='value', pltTitle='')
'''

## takes a vector of labels, eg pandas series
# it returns a freq table with props in dataframe format
def GenFreqTable(x, rounding=None):

  freqTab = x.value_counts()
  distbnTab = 100.0 * x.value_counts() / freqTab.sum()
  labels = freqTab.keys()
  freqValues = list(freqTab)
  propValues = list(distbnTab)

  if rounding is not None:
    propValues = [Signif(rounding)(x) for x in propValues]

  outDict = {'label':labels, 'freq':freqValues, 'prop':propValues}
  outDf = pd.DataFrame(outDict)

  return outDf[['label', 'freq', 'prop']]

'''
x = pd.Series(['a', 'a', 'b', 'b', 'c'])
print(GenFreqTable(x))
'''

## builds a categ distbn for each combination after groupby indCols
def CategDistbnDf(df, indCols, categCol, rounding=None):

  def F1(x):
    return tuple(GenFreqTable(x)['label'].values)

  def F2(x):
    return tuple(GenFreqTable(x)['freq'].values)

  def F3(x):
    return tuple(GenFreqTable(x, rounding=4)['prop'].values)

  df = df[indCols + [categCol]].copy()

  df[categCol + '_freq'] = df[categCol]
  df[categCol + '_prop'] = df[categCol]
  g = df.groupby(indCols)
  outDf = g.aggregate({categCol:F1 , categCol + '_freq':F2,
                       categCol + '_prop':F3})
  outDf = outDf.reset_index()

  return outDf[BringElemsToFront(outDf.columns, indCols + [categCol])]

'''
df = pd.DataFrame({
    'user_id':[1, 1, 1, 2, 2, 2, 2, 2, 1, 1],
    'interface':['A', 'A', 'A', 'B', 'B', 'B', 'A', 'A', 'A', 'B'],
    'categ':['a', 'a', 'b', 'b', 'a', 'a', 'a', 'a', 'b', 'b']})

dnDf = CategDistbnDf(
    df=df, indCols=['user_id', 'interface'], categCol='categ', rounding=None)
'''

## compare label distribution across slices
def LabelDistbn_acrossSlice(
    df,
    sliceCol,
    labelCol,
    slices=None,
    orderedValues=None,
    pltIt=True,
    pltTitle='',
    orderIntervals=False,
    sortBySlice=False,
    limitLabelNum=20):

  def F(group):
    return CombinFreqDf(group[[sliceCol, labelCol]])

  g = df.groupby([sliceCol], as_index=False)
  outDf = g.apply(F).reset_index(drop=True)

  if slices is None:
    slices = list(set(df[sliceCol].values))

  horizDf = None
  for s in slices:
    s = str(s)
    subDf = outDf[outDf[sliceCol].map(str) == s][[labelCol, 'cnt', 'prop (%)']]
    subDf.columns = [labelCol, s + '_cnt', s + '_prop (%)']
    #Mark(subDf[:2])
    if horizDf is None:
      horizDf = subDf
      horizDf['total_cnt'] = subDf[s + '_cnt']
    else:
      horizDf = pd.merge(horizDf, subDf, on=labelCol, how='outer')
      horizDf['total_cnt'] = horizDf['total_cnt'] + horizDf[s + '_cnt']
    #Mark(subDf, 'subDf')
    #Mark(horizDf, 'horizDf')

  print(horizDf)
  horizDf = horizDf.sort_values(['total_cnt'], ascending=[0])
  if orderIntervals:
    horizDf = OrderDf_cutCol(df=horizDf, cutCol=labelCol, orderCol='order')
  if sortBySlice:
    horizDf.sort_values([sliceCol])

  if limitLabelNum is not None:
      horizDf = horizDf[:limitLabelNum]
  p = None

  if pltIt:
    p = PltCols_wrtIndex(
      df=horizDf,
      cols=[str(x) + '_prop (%)' for x in slices],
      categCol=labelCol,
      orderedValues=orderedValues,
      orient='h',
      pltTitle=pltTitle)

  return {'outDf':outDf, 'horizDf':horizDf, 'p':p}

'''
df = GenUsageDf_forTesting()
Mark(df[:2])

res = LabelDistbn_acrossSlice(
    df=df, sliceCol='expt', labelCol='prod', pltIt=True)

res['p']

res = LabelDistbn_acrossSlice(
    df=df,
    sliceCol='expt',
    labelCol='prod',
    orderedValues=[],
    pltIt=True)

'''
# make a single label distbn
def LabelDistbn(
    df,
    labelCol,
    orderIntervals=False,
    pltTitle="",
    CustomOrder=None,
    figSize=[10, 8]):

  out = CombinFreqDf(df[[labelCol]])
  del out['cnt']
  out['prop (%)'] = out['prop (%)'].map(Signif(3))

  if orderIntervals:
    out = OrderDf_cutCol(df=out, cutCol=labelCol, orderCol='order')
    del out['order']

  if CustomOrder is not None:
    out = CustomOrder(out)

  fig, ax = plt.subplots();
  fig.set_size_inches(figSize[0], figSize[1])

  plt.bar(range(len(out)), out['prop (%)'])
  plt.xticks(np.array(range(len(out))) + 0.5, out[labelCol], rotation=90)
  plt.grid(False)
  plt.grid(axis='y',  linewidth=1, color='red', alpha=0.5)
  if pltTitle == "":
    pltTitle = labelCol + " distbn"
  plt.title(pltTitle, fontsize=20, fontweight='bold')

  return out

##
def LabelDistbn_perSlice(
  df,
  sliceCol,
  labelCol,
  pltIt=True,
  pltTitle='',
  orderIntervals=False,
  sortBySlice=False,
  labels=None,
  sizeAlpha=0.75):

  def F(group):
    return CombinFreqDf(group[[sliceCol, labelCol]])

  g = df.groupby([labelCol], as_index=False)
  outDf = g.apply(F).reset_index(drop=True)

  if labels is None:
    labels = list(set(df[labelCol].values))

  horizDf = None
  for l in labels:
    l = str(l)
    subDf = outDf[outDf[labelCol].map(str) == l][[sliceCol, 'cnt', 'prop (%)']]
    subDf.columns = [sliceCol, l + '_cnt', l + '_prop (%)']

    if horizDf is None:
      horizDf = subDf
      horizDf['total_cnt'] = subDf[l + '_cnt']
    else:
      horizDf = pd.merge(horizDf, subDf, on=sliceCol, how='outer')
      horizDf = horizDf.fillna(0)
      horizDf['total_cnt'] = horizDf['total_cnt'] + horizDf[l + '_cnt']

  horizDf = horizDf.sort_values(['total_cnt'], ascending=[0])
  if orderIntervals:
    horizDf = OrderDf_cutCol(df=horizDf, cutCol=sliceCol, orderCol='order')

  if sortBySlice:
    horizDf = horizDf.sort_values([sliceCol])

  horizDf = horizDf[:20]
  p = None

  for l in labels:
    horizDf[l + '_%'] = 100 * (horizDf[l + '_cnt'] / horizDf['total_cnt'])

  if pltIt:
    p = PltCols_wrtIndex(
      df=horizDf,
      cols=[str(x) + '_%' for x in labels],
      categCol=sliceCol,
      orient='h',
      pltTitle=pltTitle,
      sizeAlpha=sizeAlpha)

  return {'outDf':outDf, 'horizDf':horizDf, 'p':p}

## interpolate missing categ values wrt certain columns
# condDict will determine which subset of data should be used to make predictions
# replacedValues are the values which need replacement/interpolation
# dropUnassigned is to determine if we should keep rows which remain unassigned around
def InterpCategColWrt(
    df, yCol, xCols, condDict={}, replacedValues=None, dropUnassigned=True):

  df2 = df.copy()
  if len(condDict) > 0:
    ind = BuildCondInd(df=df2, condDict=condDict)
    df2 = df2[ind].copy()
  if replacedValues is not None:
    df2 = df2[~df2[yCol].isin(replacedValues)].copy()

  predDf = CategDistbnDf(df=df2, indCols=xCols, categCol=yCol)
  predDf[yCol + '_pred'] = predDf[yCol].map(lambda x: x[0])
  ind = df[yCol].isin(replacedValues)
  df3 = df[ind].copy()
  df4 = pd.merge(df3, predDf[xCols + [yCol + '_pred']], on=xCols, how='left')
  df.loc[ind, yCol] = df4[yCol + '_pred'].values

  if dropUnassigned:
    df = df.dropna(subset=[yCol])

  return df

'''
df = pd.DataFrame({
    'user_id':[1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3],
    'os':['and', 'and', 'and', 'randSurface', 'randSurface', 'randSurface', 'randSurface', 'and', 'and', 'and', 'randSurface', 'randSurface', 'randSurface', 'randSurface', 'randSurface'],
    'y':[None, 'c', 'b', 'b', 'b', 'b', 'a', 'a', 'a', 'b', 'b', 'nan', 'b', 'b', None],
    'country': ['us', 'us', 'us', 'us', 'us', 'jp', 'us', 'jp', 'us', 'us', 'us', 'us', 'jp', 'us', 'us']})

print(df)

InterpCategColWrt(df=df,
                  yCol='y',
                  xCols=['user_id', 'os'],
                  condDict={'country':['us']},
                  replacedValues=['nan', None],
                  dropUnassigned=False)
'''

## for a df with sliceCols, it groups by sliceCols
# and for each categCols combination,
# it adds a total count column for the valueCol
# for example sliceCols: country, categCols=[sequence, event_1st, event_2nd]
# and valueCol=sequence_count
# we can figure out the total frequency of each sequence in each country
#  as well as the frequency of the first event for the same country (sliceCols)
# we also agg a grand total for the valueCols for each combination of sliceCols
def AddTotalsDf(
    df, categCols, valueCols, sliceCols=[], aggFnDict=sum,
    integOutOther=False):

  ## if there are no sliceCols, we generate a tempCol to be sliceCol
  ## then we delete it at the end
  l = len(sliceCols)
  if l == 0:
    sliceCols = ['tempCol']
    df['tempCol'] = 1

  ## integrates out wrt sliceCols + categCols first.
  ## so other columns will be dropped
  ## and the valueCols will be integrated out across,
  ## when there are repeated sliceCols + categCol even if there is no extra col
  if integOutOther:
    df = df[sliceCols + categCols + valueCols]
    g = df.groupby(sliceCols + categCols)
    df = g.agg(aggFnDict)
    df = df.reset_index()

  df0 = df[sliceCols + categCols + valueCols]
  outDf = df.copy()

  for categCol in categCols:
    g = df0.groupby(sliceCols + [categCol])
    aggDf = g.agg(aggFnDict)
    aggDf= aggDf.reset_index()
    aggDf.columns = (sliceCols +
                     [categCol] +
                     [categCol + '_' + x + '_agg' for x in valueCols])
    outDf = pd.merge(outDf, aggDf, on=sliceCols + [categCol], how='left')

  # add slice (sliceCols slice) totals: same as above but we drop the categCol
  df0 = df[sliceCols + valueCols]
  g = df0.groupby(sliceCols)
  aggDf = g.agg(aggFnDict)
  aggDf= aggDf.reset_index()
  aggDf.columns = sliceCols + [x + '_slice_total' for x in valueCols]
  outDf = pd.merge(outDf, aggDf, on=sliceCols, how='left')

  # reorder the columns
  cols = (sliceCols +
          sorted(categCols) +
          valueCols +
          list(sorted(set(outDf) - set(sliceCols + categCols + valueCols))))
  outDf = outDf[cols]
  ## remove extra column if it was created
  if l == 0:
    del outDf['tempCol']

  return outDf

'''
df = pd.DataFrame({
    'country':['JP', 'JP', 'JP', 'BR', 'BR', 'BR', 'JP', 'JP', 'JP', 'BR'],
    'seq':['a>b', 'a>b', 'b>a', 'b>a', 'a>b', 'a>b', 'a>c', 'a>c', 'b>c', 'c>b'],
    '1st':['a', 'a', 'b', 'b', 'a', 'a', 'a', 'a', 'b', 'c'],
    '2nd':['b', 'b', 'a', 'a', 'b', 'b', 'c', 'c', 'c', 'b'],
    'count':[10, 11, 1, 20, 2, 2, 2, 200, 1, 1],
    'utility':[-10, -11, 1, 20, 2, 2, 2, -200, 1, 1],})

sliceCols = ['country']
categCols = ['seq', '1st', '2nd']
valueCols = ['count', 'utility']
aggFnDict = {'count':sum, 'utility':np.mean}
AddTotalsDf(
    df=df, categCols=categCols, valueCols=valueCols,
    sliceCols=sliceCols, aggFnDict=sum)
AddTotalsDf(
    df=df, categCols=categCols, valueCols=valueCols,
    sliceCols=sliceCols, aggFnDict=aggFnDict)
AddTotalsDf(
    df=df, categCols=categCols, valueCols=valueCols,
    sliceCols=sliceCols, aggFnDict=aggFnDict, integOutOther=True)
AddTotalsDf(
    df=df, categCols=categCols, valueCols=valueCols,
    sliceCols=[], aggFnDict=aggFnDict, integOutOther=True)
'''

## for a data frame with a countCol, we do bootstrap
def BsWithCounts(df, countCol=None):

  if countCol == None:
    n = df.shape[0]
    ind = np.random.choice(a=n, size=n, replace=True, p=None)
    df2 = df.iloc[ind]
    return(df2)

  df = df.reset_index(drop=True)
  rowInd = list(range(len(df)))
  counts = df[countCol].values
  longInd = []

  for a, b in zip(rowInd, counts):
      longInd.extend([a] * b)

  bsLongInd = np.random.choice(
      a=longInd, size=len(longInd), replace=True, p=None)
  bsIndDf = pd.DataFrame(pd.Series(bsLongInd).value_counts())
  bsRowInd = list(bsIndDf.index)
  bsCounts = bsIndDf[0].values
  df2 = df.iloc[bsRowInd]
  df2[countCol] = bsCounts
  df2 = df2.reset_index(drop=True)

  return df2

'''
df = pd.DataFrame({
    'a':['cats', 'horses', 'dogs', 'wolves'],
    'count':[2, 10, 4, 1]})

Mark(df, 'original df')
countCol = 'count'
Mark(BsWithCounts(df, countCol), ' using counts')
Mark(BsWithCounts(df, countCol=None), ' not using counts')
'''

## get a sublist with unique elements
# while preserving order
def UniqueList(l):
  seen = set()
  seen_add = seen.add
  return [x for x in l if not (x in seen or seen_add(x))]

'''
x = [1, 2, 1, 1, 2, 3] + range(100000) + [1, 2, 1, 1, 2, 3]

tic = time.clock()
UniqueList(x)
toc = time.clock()
Mark((toc-tic)*100)

tic = time.clock()
set(x)
toc = time.clock()
Mark((toc-tic)*100)
'''

## bring certain elements (x) of a list (l) to front
# without re-ordering others
def BringElemsToFront(l, subList):

  front = []
  for k in range(len(subList)):
    front = front + [j for i,j in enumerate(l) if j == subList[k]]
  end = [j for i,j in enumerate(l) if not (j in subList)]

  return front + end

'''
BringElemsToFront(l=[1, 2, 3, 1], subList=[1])
BringElemsToFront(l=[1, 2, 3, 1, 4, 5,], subList=[1, 4, 5])
'''
## get a fcn which returns a color grid of size n
def GetColorGridFcn(n):

    '''Returns a function that maps each index in 0, 1, ... N-1 to a distinct
    RGB color.'''
    color_norm  = matplotlib.colors.Normalize(vmin=0, vmax=n-1)
    scalar_map = matplotlib.cm.ScalarMappable(norm=color_norm, cmap='hsv')
    def map_index_to_rgb_color(index):
        return scalar_map.to_rgba(index)

    return map_index_to_rgb_color

'''
def main():
    n = 5
    fig=plt.figure()
    ax=fig.add_subplot(111)
    plt.axis('scaled')
    ax.set_xlim([ 0, n])
    ax.set_ylim([-0.5, 0.5])
    cmap = GetColorGridFcn(n)
    for i in range(n):
        col = cmap(i)
        rect = plt.Rectangle((i, -0.5), 1, 1, facecolor=col)
        ax.add_artist(rect)
    ax.set_yticks([])
    plt.show()

if __name__=='__main__':
    main()
'''

## takes a dictionary of lists to one string
def DictOfLists_toString(
    d,
    dictElemSepr='__',
    listElemSepr='_',
    keyValueSepr=':',
    noneString=''):

  if d == None or d == {}:
    return(noneString)
  keys = d.keys()
  keys = map(str, keys)
  keys.sort()

  out = ''
  for key in keys:
    if (d[key] != None):
      l = [str(x) for x in d[key]]
      value = str(listElemSepr.join(l))
      if out != '':
        out = out + dictElemSepr
      out = out + key + keyValueSepr + value

  return out

'''
d = {'z':[2], 'd':[1], 'e':[2]}
DictOfLists_toString(d)

d = {'z':[2], 'd':[1], 'e':None}
DictOfLists_toString(d)

condDict = {'form_factor':['PHN']}
condDict = {'form_factor':None}
condDict = {'form_factor':['PHN'], 'country':['JP']}
condDict = None
condStr = DictOfLists_toString(condDict, dictElemSepr='__', listElemSepr='_')
'''

## plotting confidence intervals given in each row
# will label the rows in labelCol is given
def PlotCI(df, colUpper, colLower, y=None, col=None, ciHeight=0.5,
           color='grey', labelCol=None, pltLabel=''):

  if y is None:
    y = range(len(df))

  minCiWidth = (df[colUpper] - df[colLower]).min()

  if col is not None:
    ## following was troubling in log scale,
    # the width of the lines were changing in visualization (not desired)
    '''
    p = plt.barh(
        bottom=y,
        width=np.array([minCiWidth]*len(y)),
        left=df[col],
        height = ciHeight,
        color='green',
        alpha=1,
        label=None)
    '''
    for i in range(len(y)):
      plt.plot(
        [df[col].values[i],
        df[col].values[i]],
        [y[i], y[i] + ciHeight],
        color=color,
        linestyle='-',
        alpha=0.7,
        linewidth=4)

      plt.plot(
        [df[col].values[i],
        df[col].values[i]],
        [y[i], y[i] + ciHeight],
        color="black",
        linestyle='-',
        alpha=0.5,
        linewidth=2,
        dashes=[6, 2])

  if int(matplotlib.__version__[0]) < 3:
    p = plt.barh(
        bottom=y,
        width=(df[colUpper]-df[colLower]).values,
        left=df[colLower],
        color=color,
        edgecolor='black',
        height=ciHeight,
        alpha=0.6,
        label=pltLabel)
  else:
    p = plt.barh(
        y=y,
        width=(df[colUpper]-df[colLower]).values,
        left=df[colLower],
        align="edge",
        color=color,
        edgecolor='black',
        height=ciHeight,
        alpha=0.6,
        label=pltLabel)

  if labelCol is not None:
    plt.yticks(y, df[labelCol].values, rotation='vertical');

'''
df0 = pd.DataFrame({'med':[1, 2, 3, 10], 'upper':[2, 5, 6, 12],
                    'lower':[-1, -2, -3, 4], 'categ':['a', 'b', 'c', 'd']})
PlotCI(df=df0, colUpper='upper', colLower='lower', y=None, col='med',
       ciHeight=0.5, color='grey', labelCol='categ', pltLabel='')
'''

## compares the CI's for available labels in labeCol
# we do that for each slice with different color to compare
def PlotCIWrt(
    df, colUpper, colLower, sliceCols, labelCol, col=None,
    ciHeight=0.5, rotation = 0, addVerLines=[], logScale=False,
    lowerLim=None, pltTitle='', figSize=[5, 20]):

  df2 = Concat_stringColsDf(
    df=df.copy(),
    cols=sliceCols,
    colName='slice_comb',
    sepStr='-')

  labelSet = UniqueList(df2[labelCol].values)
  labelIndDf = pd.DataFrame({labelCol: labelSet})
  labelIndDf = labelIndDf.sort_values([labelCol])
  labelIndDf['labelInd'] = range(len(labelSet))

  n = len(labelIndDf)
  ## groupby each slice
  slicesSet = set(df2['slice_comb'])
  g = df2.groupby(['slice_comb'])
  sliceNum = len(g)
  sliceNames = list(g.groups.keys())
  sliceNames.sort()
  ColorFcn = GetColorGridFcn(sliceNum + 2)

  plt.figure(1);
  fig, ax = plt.subplots();
  fig.set_size_inches(figSize[0], figSize[1]*(n/20.0))

  for i in range(sliceNum):
    sliceName = sliceNames[i]
    df3 = g.get_group(sliceName)
    df3 = pd.merge(df3, labelIndDf, on=[labelCol], how='outer')
    df3 = df3.sort_values([labelCol])
    df3 = df3.fillna(0)

    ciHeight = 1.0 / sliceNum
    shift = ciHeight * i
    y = [(float(x) + shift) for x in range(n)]

    assert (len(df3) == len(y)),("len(y) must be the same as merged df (df3)." +
                                 " This might be because of repeated rows in df3")

    PlotCI(
        df=df3, colUpper=colUpper, colLower=colLower, y=y, col=col,
        ciHeight=ciHeight, color=ColorFcn(i + 1), labelCol=labelCol,
        pltLabel=sliceName)

  for j in range(n + 1):
    plt.axhline(y=j, color='grey', alpha=0.95)

  labels = [item.get_text() for item in ax.get_xticklabels()]
  labels = list(labelIndDf[labelCol].values)
  ax.set_yticklabels(labels)
  locs, labels = plt.yticks([(float(x) + 0.5) for x in range(n)], labels)
  plt.setp(labels, rotation=rotation, fontweight='bold', fontsize="large")

  for x in addVerLines:
    plt.axvline(x=x, color='orange', alpha=0.5)

  if logScale:
    plt.xscale('log')

  if len(addVerLines) > 0:
    #labels = [item.get_text() for item in ax.get_xticklabels()]
    #ax.set_xticklabels(map(str, addVerLines))
    ax = plt.gca() # grab the current axis
    ax.set_xticks(addVerLines) # choose which x locations to have ticks
    ax.set_xticklabels(addVerLines) # set the labels to display at those ticks

  #plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3,
  #       ncol=2, mode="expand", borderaxespad=0.)
  plt.legend(
    bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.,
    prop={'weight':'bold', 'size':'large'})
  plt.xlim((lowerLim, None))
  plt.xlim((lowerLim, None))
  plt.title(
      pltTitle, fontname="Times New Roman",fontweight="bold",
      fontsize="x-large")

  return fig
  #plt.show()

'''
## Example
df0 = pd.DataFrame({
    'med':[1, 2, 3, 10, 11, 12, 1, 2],
    'upper':[2, 5, 6, 12, 13, 16, 5, 6],
    'lower':[-1, -2, -3, 4, 2, 2, 1, 2],
    'categ':['a', 'b', 'c', 'd', 'a', 'c', 'd', 'a'],
    'country':['JP', 'JP', 'JP', 'US', 'US', 'US', 'BR', 'BR']})

res = PlotCIWrt(
  df=df0,
  colUpper='upper',
  colLower='lower',
  sliceCols=['country'],
  labelCol='categ',
  col='med',
  ciHeight=0.5,
  rotation = 0,
  pltTitle="WTF is going on?",
  figSize=[10, 30])
'''

## this function will partition a df using keyCol
# for which their row (so maybe involve other columns to check conditions)
# satisfy conditions
# any combination of keys which passes the condition at least once will be
# considered as satisfy
def PartDf_byKeyCols_wrtCond(
    df, keyCols, condDict, passColName='passesCond'):

  keyDfUnique = df[keyCols].drop_duplicates()
  ind = BuildCondInd(df=df, condDict=condDict)
  passDf = df[ind].copy()
  passKeyDf = passDf[keyCols].drop_duplicates()

  passKeyDf[passColName] = True
  keyDfLabeled = pd.merge(keyDfUnique, passKeyDf, on=keyCols, how='left')
  keyDfLabeled = keyDfLabeled.fillna(False)

  dfLabeled = pd.merge(df, keyDfLabeled, on=keyCols, how='left')

  return {'dfLabeled':dfLabeled, 'keyDfLabeled':keyDfLabeled}

'''
df = pd.DataFrame({
    'user_id':[1, 1, 2, 2, 3, 3, 4, 4],
    'device':['pixel', 'sams', 'lg', 'lg', 'sams', 'pixel', 'nex', 'pixel'],
    'country':['us', 'us', 'jp', 'jp', 'kr', 'kr', 'in', 'in']})

outDict = PartDf_byKeyCols_wrtCond(
  df=df, keyCols=['user_id'], condDict={'device':['pixel'],
  'country':['us', 'in']}, passColName='passesCond')

Mark(df)
Mark(outDict['dfLabeled'])
Mark(outDict['keyDfLabeled'])
'''

## create good pandas boxplots
def PandasBoxPlt(
    df, col, by, ylim=None, yscale=None, pltTitle=None, figSize=None):

  # demonstrate how to customize the display different elements:
  boxprops = dict(linestyle='-', linewidth=4, color='k')
  medianprops = dict(linestyle='-', linewidth=4, color='k')

  bp = df.boxplot(
      column=col, by=by,
      showfliers=False, showmeans=True,
      boxprops=boxprops, medianprops=medianprops)

  if yscale is not None:
    plt.yscale(yscale)
  [ax_tmp.set_xlabel('') for ax_tmp in np.asarray(bp).reshape(-1)]
  fig = np.asarray(bp).reshape(-1)[0].get_figure()

  if figSize is not None:
    fig.set_size_inches(figSize[0], figSize[1])

  plt.xticks(rotation=45)
  axes = plt.gca()

  if pltTitle is not None:
    plt.title(pltTitle)

  if ylim is not None:
    axes.set_ylim(ylim)

  return plt.show()

def Plt_compareUsageSet(
    df, unitCol, usageCol, compareCol=None, excludeValues=[],
    mapToOther=["UNKNOWN", "MOBILE_UNKNOWN"], removeOther=True,
    setLabelsNumMax=15, bpPltTitle=None):

  if compareCol is None:
    compareCol = "..."
    df[compareCol] = "..."

  if len(excludeValues) > 0:
    df = df[~df[usageCol].isin(excludeValues)]

  df2 = df[[unitCol, compareCol, usageCol]].copy()

  res = Remap_lowFreqCategs(
      df=df2, cols=[usageCol],  newLabels="OTHER",
      otherLabelsToReMap=(["", "nan"] + mapToOther),
      freqThresh=10, labelsNumMax=30)

  df2 = res["df"]

  if removeOther:
    df2 = df2[df2[usageCol] != "OTHER"]

  g = df2.groupby([unitCol, compareCol], as_index=False)

  dfSet = g.agg({usageCol:lambda x: tuple(sorted(set(x)))})

  res = Remap_lowFreqCategs(
      df=dfSet, cols=[usageCol],  newLabels="OTHER",
      otherLabelsToReMap=["", "nan"],
      freqThresh=5, labelsNumMax=setLabelsNumMax)

  dfSet = res["df"]

  if removeOther:
    dfSet = dfSet[dfSet[usageCol] != "OTHER"]

  pltTitle = usageCol + " set distbn " + " across " + unitCol + "s"
  res = LabelDistbn_acrossSlice(
      df=dfSet, sliceCol=compareCol, labelCol=usageCol,
      pltIt=True, pltTitle=pltTitle)

  dfCount = g.agg({usageCol:lambda x: len(set(x))})

  res["dfCount"] = dfCount

  if bpPltTitle is None:
    bpPltTitle = "# of " + usageCol + " across " + unitCol + "s"
  PandasBoxPlt(
      df=dfCount, col=usageCol, by=compareCol,
      ylim=[0, None], pltTitle=bpPltTitle)

  return res

def BirthYear_toAgeCateg(x, currentYear=None):

  if currentYear is None:
    currentYear = datetime.datetime.now().year

  if x is None or x == "" or x == 0 or math.isnan(x):
    return "other"

  x = float(x)
  age = currentYear - x

  if age <= 17:
    return "<18"

  if age <= 25:
    return "18-25"

  if age <= 35:
    return "26-35"

  if age <= 50:
    return "36-50"

  return ">51"

def BirthYear_toAge(x, currentYear=None, minBirthYear=1940):

  if currentYear is None:
    currentYear = datetime.datetime.now().year

  if x is None or x == "" or x == 0 or math.isnan(x):
    return None

  if x < minBirthYear or x > currentYear:
    return None

  x = float(x)
  return (currentYear - x)

"""
BirthYear_toAgeCateg(1900)
"""

def Plt_compareDensity(
    df, compareCol, valueCol, compareValues=None):

  if compareValues is None:
    compareValues = set(df[compareCol].values)
  # Iterate through the five airlines
  for value in compareValues:
      # Subset to the airline
      subset = df[df[compareCol] == value]

      # Draw the density plot
      sns.distplot(
          subset[valueCol], hist=False, kde=True,
          kde_kws={'linewidth': 3, "alpha": 0.75},
          label=value)

  # Plot formatting
  plt.legend(prop={'size': 8}, title=compareCol)
  plt.title('Compare Density Plot for Multiple ' + compareCol)
  plt.xlabel(valueCol)
  plt.ylabel('Density')

## drops (multiple) ending vowels from a string
def DropEndingVowels(s, minLength=2):

  cond = True
  while cond and len(s) > minLength:
    if s[len(s)-1].lower() in ["a", "o", "e", "u", "i"]:
      s = s[0:(len(s)-1)]
    else:
      cond = False

  return s

def DropEndingChars(s, chars, minLength=2):

  cond = True
  while cond and len(s) > minLength:
    if s[len(s)-1].lower() in chars:
      s = s[0:(len(s)-1)]
    else:
      cond = False

  return s

## abbreviates a string.
# first we abbreviate each word in a string (phrase)
# then we concat them back and abbreviate the whole phrase
def AbbrString(
    s,
    wordLength=6,
    replaceList=["/", "&", " and ", "-", ",", ";"],
    sep="-",
    totalLength=None,
    wordNumLimit=None):

  for char in replaceList:
    s = s.replace(char, " ")
  sList = s.split(" ")
  sList = [s[0:wordLength] for s in sList]
  sList = [x for x in sList if x not in ["", " ", "  ", "    "]]
  sList = [DropEndingVowels(s) for s in sList]
  sList = list(collections.OrderedDict.fromkeys(sList))
  print(sList)

  if wordNumLimit is not None:
    sList = sList[:wordNumLimit]

  s = sep.join(sList)

  if totalLength is not None:
    s = s[0:totalLength]
    s = DropEndingVowels(s)

  s = DropEndingChars(s=s, chars=["/", "&", " and ", "-", sep, " ", ",", ";"])

  return s

"""
s = "language books/"
AbbrString(s, sep="-")
"""
## replace in pandas is slow
def ReplaceValues_dfCols_viaReplace(
    df, cols, values, newValues, newCols=None):

  if newCols is None:
    newCols = cols

  mappingDict = dict(zip(values, newValues))

  df[newCols] = df[cols].replace(mappingDict)
  return df

def ReplaceValues_dfCols(df, cols, values, newValues, newCols=None):

  if newCols is None:
    newCols = cols

  m = pd.Series(newValues, values)
  df[newCols] = df[cols].stack().map(m).unstack()

  return df

"""
import datetime
import pandas as pd
import numpy as np
import string

n = 10000
m = 500

df = pd.DataFrame(
    pd.DataFrame(
        np.random.choice(list(string.letters), n * m * 3) \
          .reshape(3, -1)).sum().values.reshape(n, -1))
cols = [0, 1]
u = np.unique(df[cols])
fromSeries = pd.Series(u)
toSeries = fromSeries + "XXX"

fromValues = fromSeries.values
toValues = toSeries.values

a = datetime.datetime.now()
df0 = ReplaceValues_dfCols(
    df=df.copy(), cols=cols, values=fromValues, newValues=toValues)
b = datetime.datetime.now()

time1 = b-a
print(time1)

a = datetime.datetime.now()
df1 = ReplaceValues_dfCols_viaReplace(
    df=df.copy(), cols=cols, values=fromValues,
    newValues=toValues, newCols=None)
b = datetime.datetime.now()
time2 = b-a
print(time2)

print(time2.total_seconds() / time1.total_seconds())
"""

def AbbrStringCols(
    df, cols, newCols=None, wordLength=6,
    replaceList=["/", "&", " and ", "-"], sep="-",
    totalLength=None, wordNumLimit=None):

  values = np.unique(df[cols])

  def Abbr(s):
    return AbbrString(
        s=s, wordLength=wordLength, replaceList=replaceList,
        sep=sep, totalLength=totalLength, wordNumLimit=wordNumLimit)

  abbrValues = [Abbr(s) for s in values]

  mapDf = pd.DataFrame({"value":values, "abbr_values": abbrValues})

  df = ReplaceValues_dfCols(
      df=df, cols=cols, values=values, newValues=abbrValues, newCols=newCols)

  return {"df":df, "mapDf":mapDf}

"""
df = pd.DataFrame({
    "col":["life is beautiful", "i like mountains", "ok", "cool"]})

#AbbrStringCols(df, cols=["col"])

AbbrStringCols(df=df, cols=["col"], totalLength=10, wordNumLimit=None)

"""

## convert data.frame to code

def ConvertDf_toCode(df):
  s = (
      "df = pd.DataFrame( %s )"
      % (str(df.to_dict()).replace(" nan"," float('nan')")))

  return s

"""
df = pd.DataFrame({"a":[1, 2, 3], "b":[1, 2, 3]})
ConvertDf_toCode(df)
"""


#! /usr/bin/env python3

count = {}
def char_count(str):
    for char in str:
        c = count.get(char)
        if c is None:
            count[char] = 1
        else:
            count[char] += 1

    print(count)

if __name__ == '__main__':
    s = input('Enter a string')
    char_count(s)


from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QMessageBox
import sys
import json
import mainwindow, mystock, recepe, compute

class MyMainWindow(mainwindow.Ui_MainWindow):
    def setupUi(self, mw, database):
        super().setupUi(mw)
        self.tabWidget = QtWidgets.QTabWidget()
        mw.setCentralWidget(self.tabWidget)
        self.ms = QtWidgets.QWidget()
        self.mystock = mystock.MyStock()
        self.mystock.setupUi(self.ms, database["bernard"]["stock"])
        self.tabWidget.addTab(self.ms, "STOCK")
        self.mr = QtWidgets.QWidget()
        self.myRecepe = recepe.Ui_TabRecepe()
        self.myRecepe.setupUi(self.mr)
        self.tabWidget.addTab(self.mr, "Recepe")
        self.mc = QtWidgets.QWidget()
        self.myCompute = compute.Ui_TabCompute()
        self.myCompute.setupUi(self.mc)
        self.tabWidget.addTab(self.mc, "Compute")

if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    mw = QtWidgets.QMainWindow()
    ui = MyMainWindow()
    ui.setupUi(mw)
    mw.show()
    sys.exit(app.exec_())


import torch
from torch import nn
import torch.nn.functional as F
from torch import optim
from torchvision import datasets, transforms, models
from workspace_utils import active_session
from collections import OrderedDict
import numpy as np
from PIL import Image
import argparse
import json

parser = argparse.ArgumentParser(description='Inference for classification')
parser.add_argument('-i','--image_path',type=str, metavar='', required=True, help='path to image to predict e.g. flowers/test/class/image')
parser.add_argument('-t','--top_k', type=int, metavar='', default=1, help='print out the top K classes along with associated probabilities')
parser.add_argument('-c','--category_names', type=str, metavar='', default='cat_to_name.json', help='load a JSON file that maps the class values to other category names')
parser.add_argument('-g','--gpu',action="store_true", default=False, help='choose training the model on a GPU')

args = parser.parse_args()

with open(args.category_names, 'r') as f:
    cat_to_name = json.load(f)

# a function that loads a checkpoint and rebuilds the model

def load_checkpoint(filepath):
    checkpoint = torch.load(filepath)

     # Load the saved file
    checkpoint = torch.load("checkpoint.pth")
    architecture = checkpoint['architecture']
    # Download pretrained model
    model = getattr(models, architecture)(pretrained=True);
    # Freeze parameters so we don't backprop through them
    for param in model.parameters():
        param.requires_grad = False

    hidden_units = checkpoint['hidden_units']
    model.class_to_idx = checkpoint['class_to_idx']
    model.classifier = checkpoint['classifier']
    model.load_state_dict(checkpoint['state_dict'])
    #optimizer = optim.Adam(model.classifier.parameters())
    #optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
    epochs = checkpoint['epochs']

    return model

model = load_checkpoint('checkpoint.pth')

def process_image(image):
    ''' Scales, crops, and normalizes a PIL image for a PyTorch model,
        returns an Numpy array
    '''
    im = Image.open(image)
    size = 256,256
    im.thumbnail(size)
    #Crop
    left = (256-224)/2
    top = (256-224)/2
    right = (left + 224)
    bottom = (top + 224)
    im = im.crop((left, top, right, bottom))

    np_image = np.array(im)
    np_image = np_image / 255
    mean = [0.485, 0.456, 0.406]
    std = [0.229, 0.224, 0.225]
    np_image = (np_image - mean) / std

    np_image = np_image.transpose(2, 0, 1)
    return np_image

# Reversing idx to class
idx_to_class = {}
for key, value in model.class_to_idx.items():
    idx_to_class[value] = key

def predict(image_path, model, topk):
    ''' Predict the class (or classes) of an image using a trained deep learning model.
    '''
    # Use GPU if it's available
    if args.gpu:
        device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    # TODO: Implement the code to predict the class from an image file
    #preprocess image
    image = process_image(image_path)
    image = torch.from_numpy(np.array([image])).float()

    # turn off dropout
    model.eval()

    #Load image and the model to cpu or gpu
    model.to(device)
    image = image.to(device)

    logps = model.forward(image)
    ps = torch.exp(logps)
    top_p, top_class = ps.topk(topk, dim=1)

    top_class = np.array(top_class)[0]
    top_p = np.array(top_p.detach())[0]

    # Mapping index to class
    top_classes = []
    for i in range(len(top_class)):
        top_classes.append(idx_to_class[top_class[i]])

    # Mapping class to flower name
    flower_names = []
    for i in range(len(top_classes)):
        flower_names.append(cat_to_name[top_classes[i]])

    return top_p, flower_names

probs, classes = predict(args.image_path, model, args.top_k)

print(f"class Probability: {probs}")
print(f"flower name: {classes}")


'''Simple Script to Label Item Description'''

#import libraries packages
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import string
import re
import json

#read from csv
data = pd.read_csv('./item.csv')

item = data['Item Description']

#data cleaning (lowercase, remove whitespace, remove numbers, remove punctutions)
item = item.str.lower() #change to lowercase

#keyword for each label
medical_surgical_label = ['revision', 'replacement', 'reposition']
obstet_label = ['abortion', 'extraction', 'transplantation']
placement_label = ['removal', 'dressing', 'compression']
admin_label = ['irrigation', 'introduction']
osteo_label = ['osteopathic']
chiro_label = ['chiropractic']
imaging_label = ['fluoroscopy', 'ct scan', 'computerized tomography', 'ultrasonography']
mental_label = ['crisis', 'management', 'psychological']
radiation_label = ['cesium', 'radiation', 'photon', 'radiosurgery']
nuclear_label = ['nuclear']

#check string contains any keyword above and label accordingly
for i in item.shape:
	for row in medical_surgical_label:
		item[item.str.contains(row)] = 'Medical and Surgical'
	for row in obstet_label:
		item[item.str.contains(row)] = 'Obstetrics'
	for row in placement_label:
		item[item.str.contains(row)] = 'Placement'
	for row in admin_label:
		item[item.str.contains(row)] = 'Administration'
	for row in osteo_label:
		item[item.str.contains(row)] = 'Osteopathic'
	for row in chiro_label:
		item[item.str.contains(row)] = 'Chiropractic'
	for row in imaging_label:
		item[item.str.contains(row)] = 'Imaging'
	for row in mental_label:
		item[item.str.contains(row)] = 'Mental Health'
	for row in radiation_label:
		item[item.str.contains(row)] = 'Radiation Therapy'
	for row in nuclear_label:
		item[item.str.contains(row)] = 'Nuclear Medicine'

print(item)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Sep 13 00:32:57 2018

@author: pablosanchez
"""
import tensorflow as tf
import utils.constants as const
from networks.dense_net import DenseNet

class DeconvNet(object):
    def __init__(self,  width, height, nchannels, reuse, transfer_fct=tf.nn.relu,
                 act_out=tf.nn.sigmoid, drop_rate=0., kinit=tf.contrib.layers.xavier_initializer(),
                 bias_init=tf.constant_initializer(0.0)):

        self.width = width
        self.height = height
        self.nchannels = nchannels

        self.transfer_fct = transfer_fct
        self.act_out = act_out
        self.reuse = reuse
        self.drop_rate = drop_rate

        self.kinit= kinit
        self.bias_init = bias_init

    def build(self, input_):
        raise NotImplementedError

    def deconv_layer(self, input_, filters, k_size, stride, padding, name, act_func=tf.nn.relu):
        deconv = tf.layers.conv2d_transpose(input_,
                                            filters,
                                            k_size,
                                            strides=stride,
                                            padding=padding,
                                            activation=act_func,
                                            kernel_initializer=self.kinit,
                                            bias_initializer=self.bias_init,
                                            name=name,
                                            reuse=self.reuse)
        print('[*] Layer (',deconv.name, ') output shape:', deconv.get_shape().as_list())

        return deconv

class DeconvNet3(DeconvNet):
    def __init__(self, input_, width, height, nchannels, reuse, transfer_fct=tf.nn.relu,
                 act_out=tf.nn.sigmoid, drop_rate=0., kinit=tf.contrib.layers.xavier_initializer(),
                 bias_init=tf.constant_initializer(0.0)):
        super().__init__(width, height, nchannels,reuse, transfer_fct, act_out, drop_rate, kinit, bias_init)

        self.output = self.build(input_)

    def build(self, input_):
        aux_size = self.width//2//2
        aux_size_2 = self.height//2//2
        initial_n_channels = 64
        out_dense_dim = aux_size*aux_size_2*initial_n_channels
        hidden_dim = input_.get_shape()[-1].value*3

        dense = DenseNet(input_=input_,
                         hidden_dim=hidden_dim,
                         output_dim=out_dense_dim,
                         num_layers=2,
                         transfer_fct=self.transfer_fct,
                         act_out=self.transfer_fct,
                         reuse=self.reuse,
                         kinit=self.kinit,
                         bias_init=self.bias_init,
                         drop_rate=self.drop_rate)
        x = dense.output
        x = tf.reshape(x, [-1,aux_size,aux_size_2,initial_n_channels])
        x = self.deconv_layer(input_=x,
                              filters=64,
                              k_size=4,  #[4, 4]
                              stride=2,
                              padding='SAME',
                              name='deconv_1',
                              act_func=self.transfer_fct)

        x = self.deconv_layer(input_=x,
                            filters=32,
                            k_size=4,  #[4, 4]
                            stride=2,
                            padding='SAME',
                            name='deconv_2',
                            act_func=self.transfer_fct)
        x = self.deconv_layer(input_=x,
                            filters=self.nchannels,
                            k_size=4,  #[4, 4]
                            stride=1,
                            padding='SAME',
                            name='deconv_3',
                            act_func=self.act_out)

        return x


import os
import time
import requests
import functools
from concurrent.futures import ThreadPoolExecutor
import click
import yaml

def _get_connection_urls(workload_yaml):
    with open(workload_yaml) as f:
        workload = yaml.safe_load(f)
    uris = workload.get("EnvironmentDetails", {}).get("MongosyncConnectionURIs")
    if not uris:
        raise Exception(
            f"This actor requires setting EnvironmentDetails: MongosyncConnectionURIs to use this script"
        )
    return uris

def poll(workload_yaml, predicate, key):
    """
    Wait for all mongosyncs to reach a certain state (e.g. predicate returns False)
    based on a value returned by the /progress endpoint
    """
    connection_urls = _get_connection_urls(workload_yaml)

    def get_progress():
        res = requests.get(f"{url}/api/v1/progress")
        return res.json()["progress"][key]

    for url in connection_urls:
        info = get_progress()
        while predicate(info):
            time.sleep(1)
            print(f"Polling {url} for {key}, current value = {info}", flush=True)
            info = get_progress()

def _change_one_mongosync_state(route, body, url):
    """
    Change state of a given mongosync running at the provided url
    """
    resp = requests.post(f"{url}{route}", json=body)
    print(resp.json(), flush=True)
    success = resp.json()["success"]
    if not success:
        raise Exception(f"State change failed at route {route}")
    return success

def change_state(workload_yaml, route, body):
    """
    Helper function to change state of mongosync. This must
    send all requests in parallel, as some commands block until
    all instances recieve them
    """

    connection_urls = _get_connection_urls(workload_yaml)

    fn = functools.partial(_change_one_mongosync_state, route, body)
    with ThreadPoolExecutor() as executor:
        futures = []
        # Using executor.map swallows exceptions from the task,
        # using .submit and then accessing the future's .result
        # will cause exceptions to be rethrown
        for url in connection_urls:
            futures.append(executor.submit(fn, url))
        for f in futures:
            f.result()

@click.group(name="MongosyncActor", context_settings=dict(help_option_names=["-h", "--help"]))
def cli():
    pass

@cli.command(
    "start",
    help=("Issue /start to all mongosync processes"),
)
@click.argument("workload_yaml", nargs=1)
def start(workload_yaml):
    change_state(workload_yaml, "/api/v1/start", {"Source": "cluster0", "Destination": "cluster1"})

@cli.command(
    "poll_for_cea",
    help=("Poll all available instances for the CEA stage"),
)
@click.argument("workload_yaml", nargs=1)
def poll_for_cea(workload_yaml):
    poll(workload_yaml, lambda x: x != "change event application", "info")

@cli.command(
    "poll_for_commit_point",
    help=("Wait till all the instances canCommit = true and lagTimeSeconds < 120"),
)
@click.argument("workload_yaml", nargs=1)
def poll_for_commit_point(workload_yaml):
    poll(workload_yaml, lambda x: bool(x) == False, "canCommit") or poll(
        workload_yaml, lambda x: int(x) > 120, "lagTimeSeconds"
    )

@cli.command(
    "drain_writes",
    help=("Wait till all writes have been drained to the destination cluster"),
)
@click.argument("workload_yaml", nargs=1)
def drain_writes(workload_yaml):
    poll(workload_yaml, lambda x: int(x) > 5, "lagTimeSeconds")

@cli.command(
    "commit",
    help=("Commit the migration"),
)
@click.argument("workload_yaml", nargs=1)
def commit(workload_yaml):
    change_state(workload_yaml, "/api/v1/commit", {})

@cli.command(
    "wait_for_commit",
    help=("Wait until all mongosyncs are finished commiting the migration"),
)
@click.argument("workload_yaml", nargs=1)
def wait_for_commit(workload_yaml):
    poll(workload_yaml, lambda x: x != "COMMITTED", "state")

@cli.command(
    "pause",
    help=("Pause the migration"),
)
@click.argument("workload_yaml", nargs=1)
def pause(workload_yaml):
    change_state(workload_yaml, "/api/v1/pause", {})

@cli.command(
    "resume",
    help=("Resume the migration"),
)
@click.argument("workload_yaml", nargs=1)
def resume(workload_yaml):
    change_state(workload_yaml, "/api/v1/resume", {})

if __name__ == "__main__":
    cli()


from numpy import linspace, sin

from chaco.api import ArrayPlotData, Plot
from chaco.tools.api import PanTool, ZoomTool
from enable.api import ComponentEditor
from traits.api import Enum, HasTraits, Instance
from traitsui.api import Item, Group, View

class PlotEditor(HasTraits):

    plot = Instance(Plot)
    plot_type = Enum("scatter", "line")
    orientation = Enum("horizontal", "vertical")
    traits_view = View(
        Item("orientation", label="Orientation"),
        Item("plot", editor=ComponentEditor(), show_label=False),
        width=500,
        height=500,
        resizable=True,
    )

    def __init__(self, *args, **kw):
        HasTraits.__init__(self, *args, **kw)
        # Create the data and the PlotData object
        x = linspace(-14, 14, 100)
        y = sin(x) * x ** 3
        plotdata = ArrayPlotData(x=x, y=y)
        # Create the scatter plot
        plot = Plot(plotdata)
        plot.plot(("x", "y"), type=self.plot_type, color="blue")
        plot.tools.append(PanTool(plot))
        plot.tools.append(ZoomTool(plot))
        self.plot = plot

    def _orientation_changed(self):
        if self.orientation == "vertical":
            self.plot.orientation = "v"
        else:
            self.plot.orientation = "h"
        self.plot.request_redraw()

# ===============================================================================
# demo object that is used by the demo.py application.
# ===============================================================================
class Demo(HasTraits):

    # Scatter plot.
    scatter_plot = Instance(PlotEditor)

    # Line plot.
    line_plot = Instance(PlotEditor)

    traits_view = View(
        Group(Item("@scatter_plot", show_label=False), label="Scatter"),
        Group(Item("@line_plot", show_label=False), label="Line"),
        title="Chaco Plot",
        resizable=True,
    )

    def __init__(self, *args, **kws):
        super(Demo, self).__init__(*args, **kws)
        # Hook up the ranges.
        self.scatter_plot.plot.range2d = self.line_plot.plot.range2d

    def _scatter_plot_default(self):
        return PlotEditor(plot_type="scatter")

    def _line_plot_default(self):
        return PlotEditor(plot_type="line")

demo = Demo()
if __name__ == "__main__":
    demo.configure_traits()