import collections
import decimal
import query, crawl_utils, time, datetime
import loaddb
import sys
import logging
import json

from crawl_utils import clan_link, player_link, linked_text
import crawl_utils
import re

PseudoCol = collections.namedtuple("PseudoCol", ("html_display_name", "numeric_data", "transform_fn"))

BANNER_IMAGES = \
    { 'ashenzari': [ 'banner_ashenzari.png', 'The Explorer' ],
      'cheibriados': [ 'banner_cheibriados.png', 'Slow and Steady' ],
      'dithmenos': [ 'banner_dithmenos.png', 'The Politician' ],
      'fedhas': [ 'banner_fedhas.png', "Nature's Ally" ],
      'gozag': [ 'banner_gozag.png', "Avarice" ],
      'hepliaklqana': [ 'banner_hepliaklqana.png', 'The Inheritor' ],
      'ignis': [ 'banner_ignis.png', 'The Pious' ],
      'jiyva': [ 'banner_jiyva.png', 'Gelatinous Body' ],
      'kikubaaqudgha': [ 'banner_kikubaaqudgha.png', 'Lord of Darkness' ],
      'lugonu': [ 'banner_lugonu.png', 'Spiteful' ],
      'makhleb': [ 'banner_makhleb.png', 'Speed Demon' ],
      'nemelex': [ 'banner_nemelex.png', "Nemelex' Choice" ],
      'okawaru': [ 'banner_okawaru.png', 'The Conqueror' ],
      'qazlal': [ 'banner_qazlal.png', 'The Prophet' ],
      'ru': [ 'banner_ru.png', 'The Ascetic' ],
      'sif': [ 'banner_sif.png', 'The Lorekeeper' ],
      'the_shining_one': [ 'banner_the_shining_one.png', 'Vow of Courage' ],
      'uskayaw': [ 'banner_uskayaw.png', 'Graceful' ],
      'vehumet': [ 'banner_vehumet.png', 'Ruthless Efficiency' ],
      'xom': [ 'banner_xom.png', 'Descent into Madness' ],
      'yredelemnul': [ 'banner_yredelemnul.png', 'The Harvest' ],
      'zin': [ 'banner_zin.png', 'Angel of Justice' ],
      '1top_player': [ 'player.png', 'Top Player'],
      '2top_clan':   [ 'clan.png', 'Top Clan' ],
      'header': ['banner_header.png', '' ],
      'footer': ['banner_footer.png', '' ],
    }

# TODO: this seems redundant with the data in outline.py?
BANNER_TEXT = \
    { 'ashenzari':
        [ 'Enter a branch of the dungeon that contains a rune.',
          'Find 5 distinct runes over the course of the tourney.',
          'Find 17 distinct runes over the course of the tourney.',
        ],
      'cheibriados':
        [ 'Reach experience level 9 in two consecutive games.',
          'Achieve a two-win streak.',
          'Achieve a four-win streak with four distinct species and four distinct backgrounds.',
        ],
      'dithmenos':
        [ 'Steal a combo high score that was previously of at least 1,000 points.',
          'Steal a combo high score for a previously won combo.',
          'Steal a species or background high score that was previously of at least 10,000,000 points.',
        ],
      'fedhas':
        [ 'Enter the Crypt.',
          'Get the golden rune.',
          'Enter Tomb for the first time after picking up the Orb of Zot, and then get the golden rune.',
        ],
      'gozag':
        [ 'Find 1000 gold in a single game.',
          'Find the silver rune.',
          'Find the iron rune before entering Pandemonium or any branch of the dungeon containing any other rune.',
        ],
      'hepliaklqana':
        [ 'Enter the Lair of beasts while worshipping a god from a faded altar, having worshipped no other gods.',
          'Find a rune while worshipping a god from a faded altar, having worshipped no other gods.',
          'Win a game while worshipping a god from a faded altar, having worshipped no other gods.',
        ],
      'ignis':
        [ 'Become the champion of any god.',
          'Become the champion of five different gods over the course of the tournament.',
          'Become the champion of thirteen different gods over the course of the tournament.',
        ],
      'jiyva':
        [ 'Reach experience level 9 with at least 5 distinct species and at least 5 distinct backgrounds.',
          'Get a rune with at least 5 distinct species and at least 5 distinct backgrounds.',
          'Win with at least 5 distinct species and at least 5 distinct backgrounds.',
        ],
      'kikubaaqudgha':
        [ 'Reach the last level of the Orcish Mines without having entered the Lair.',
          'Reach the last level of the Depths without having entered the Lair.',
          'Win a game without having entered the Lair, the Orcish Mines, or the Vaults.',
        ],
      'lugonu':
        [ 'Become the champion of Ru',
          'After becoming the champion of Ru, abandon Ru and become the champion of a different god.',
          'Win a game in which you become the champion of Ru and then abandon Ru before entering any branches other than the Temple and the Lair.',
        ],
      'makhleb':
        [ 'Reach D:15 in 54 minutes as a non-formicid.',
          'Find a rune in 81 minutes.',
          'Win the game in 3 hours.',
        ],
      'nemelex':
        [ "Reach experience level 9 with a Nemelex' choice combo.",
          "Get a rune with a Nemelex' choice combo.",
          "Be one of the first 9 players to win a given Nemelex' choice combo.",
        ],
      'okawaru':
        [ 'Reach experience level 13.',
          'Win a game.',
          'Win a game in under 50000 turns.',
        ],
      'qazlal':
        [ 'Reach the Lair of Beasts with an Invocations title.',
          'Win a game with an Invocations title.',
          'Over the course of the tournament, win with three different Invocations titles.',
        ],
      'ru':
        [ "Reach the Ecumenical Temple without using any potions or scrolls.",
          "Reach the last level of the Lair of Beasts without using any potions or scrolls.",
          "Find a rune (non-slimy, non-abyssal) without using any potions or scrolls.",
        ],
      'sif':
        [ 'Reach the last level of the Lair as a non-formicid without raising any skill to 13.',
          'Win without raising any skill to 20.',
          'Win without raising any skill to 13.',
        ],
      'trog':
        [ 'Enter the Vaults with no runes.',
          'Pick up the silver rune before any other runes.',
          'Pick up the golden rune before any other runes.',
        ],
      'the_shining_one':
        [ 'Kill Sigmund before entering the Depths.',
          'Get four runes before entering the Depths.',
          'Get six runes before entering the Depths.',
        ],
      'uskayaw':
        [ 'Enter the Temple in under 3000 turns.',
          'Enter the third floor of the Elven Halls in under 9000 turns.',
          'Enter the final floor of Gehenna in under 27000 turns.',
        ],
      'vehumet':
        [ 'Reach the last level of the Lair as a non-formicid before reaching experience level 13.',
          'Find a rune before reaching experience level 17.',
          'Win the game before reaching experience level 22.',
        ],
      'xom':
        [ 'Enter the Abyss.',
          'Reach the 10th floor of a ziggurat.',
          'Leave a ziggurat from its lowest floor.',
        ],
      'yredelemnul':
        [ 'Kill 27 distinct uniques over the course of the tournament.',
          'Kill 54 distinct uniques over the course of the tournament.',
          'Kill 81 distinct uniques over the course of the tournament.',
        ],
      'zin':
        [ 'Enter either Pandemonium or any branch of Hell.',
          'Kill at least one unique pan lord and at least one unique hell lord over the course of the tournament.',
          'Kill all four unique pan lords, all four unique hell lords, and the Serpent of Hell (at least once) over the course of the tournament.',
        ],
      '1top_player':
        [ 'Individual with the most tournament points.',
          'Individual with the second-most tournament points.',
          'Individual with the third-most tournament points.',
        ],
      '2top_clan':
        [ 'Clan with the most tournament points.',
          'Clan with the second-most tournament points.',
          'Clan with the third-most tournament points.',
        ],
      'header': [ '' ],
      'footer': [ '' ],
    }

EXT_WIN_COLUMNS = \
    [ ('score', 'Score', True),
      ('race', 'Species'),
      ('class', 'Background'),
      ('god', 'God'),
      ('title', 'Title'),
      ('xl', 'XL'),
      ('turn', 'Turns'),
      ('duration', 'Duration'),
      ('runes', 'Runes'),
      ('end_time', 'Date')
    ]

STOCK_COLUMNS = \
    [ ('player', 'Player'),
      ('score', 'Score', True),
      ('charabbrev', 'Character'),
      ('place', 'Place'),
      ('verb_msg', 'Death'),
      ('turn', 'Turns'),
      ('duration', 'Duration'),
      ('god', 'God'),
      ('runes', 'Runes'),
      ('end_time', 'Time', True)
    ]

STOCK_WIN_COLUMNS = [col for col in STOCK_COLUMNS if col[0] not in ('place', 'verb_msg')]

EXT_COLUMNS = \
    [ ('score', 'Score', True),
      ('race', 'Species'),
      ('class', 'Background'),
      ('god', 'God'),
      ('title', 'Title'),
      ('place', 'Place'),
      ('verb_msg', 'Death'),
      ('xl', 'XL'),
      ('turn', 'Turns'),
      ('duration', 'Duration'),
      ('runes', 'Runes'),
      ('end_time', 'Date')
    ]

WHERE_COLUMNS = \
    [ ('race', 'Species'),
      ('cls', 'Background'),
      ('god', 'God'),
      ('title', 'Title'),
      ('place', 'Place'),
      ('xl', 'XL'),
      ('turn', 'Turns'),
      ('time', 'Time'),
      ('status', 'Status')
    ]

R_STR_DATE = re.compile(r'^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})')

def fixup_column(col, data, game):
  if col.find('time') != -1:
    return pretty_date(data)
  elif col.find('duration') != -1:
    return pretty_dur(data)
  elif col == 'place' and game.get('killertype') == 'winning':
    return ''
  elif col == 'score' and data == '':
    return 0
  return data

def pretty_dur(dur):
  if (not dur) and dur != 0:
    return ""
  try:
    secs = dur % 60
  except:
    print("FAIL on %s" % dur)
    raise
  dur /= 60
  mins = dur % 60
  dur /= 60
  hours = dur
  stime = "%d:%02d:%02d" % (hours, mins, secs)

  return stime

def is_stringlike(s):
  try:
    return isinstance(s, (str, unicode)) #py2
  except:
    return isinstance(s, (str, bytes)) #py3

def pretty_date(date):
  if not date:
    return ''

  if is_stringlike(date):
    m = R_STR_DATE.search(date)
    if not m:
      return date
    return "%s-%s-%s %s:%s:%s" % (m.group(1), m.group(2), m.group(3),
                                  m.group(4), m.group(5), m.group(6))

  return "%04d-%02d-%02d %02d:%02d:%02d" % (date.year, date.month, date.day,
                                            date.hour, date.minute,
                                            date.second)

def pretty_time(time):
  return "%04d-%02d-%02d %02d:%02d:%02d" % (time.tm_year, time.tm_mon,
                                            time.tm_mday,
                                            time.tm_hour, time.tm_min,
                                            time.tm_sec)

def how_old(date, bold_cutoff = 0): #cutoff in hours
  if not date:
    return None,None
  if type(date) == "string":
    delta = datetime.datetime.utcnow() - query.time_from_str(date)
  else:
    delta = datetime.datetime.utcnow() - date
  h = 24*delta.days + delta.seconds / 60 / 60
  m = (delta.seconds / 60) % 60
  s = delta.seconds % 60
  if h < 0:
    return "0:00:00", (0 < bold_cutoff)
  return ("%d:%02d:%02d" % (h, m, s)), (h < bold_cutoff)

def update_time():
  return '''

              Last updated %s UTC.

''' % pretty_time(time.gmtime())

def wrap_tuple(x):
  if isinstance(x, tuple):
    return x
  else:
    return (x,)

def is_player_header(header):
  return header.lower() in ['player', 'captain']

def is_clan_header(header):
  return header.lower() in ['clan', 'team', 'teamname']

def _is_numeric_table_value(value, header=""):
  if is_player_header(header) or is_clan_header(header):
    return False
  if isinstance(value, (int, float, decimal.Decimal)):
    return True
  if is_stringlike(value):
    if value.isdigit():
      return True
    try:
      float(value)
    except ValueError:
      return False
    return True
  return False

# DEPRECATED: Build a wrapper function around _table to replace this
def table_text(headers, data, count=True,
               place_column=-1, stub_text='No data', skip=False, bold=False,
               extra_wide_support=False, caption=None, datatables=False):
  """Create a HTML table of players.

  :param List[str] headers: Column headers
  :param List[List[str]] data: Column data
  :param bool count: Add a "count" column at the start
  :param int place_column: Use this column to determine ranking ties (-1 there are no ties)
  :param str stub_text: Text to show if the table has no data.
  :param bool skip: Use sparse rank numbers (eg two people at rank n means next rank is n+2)
  :param bool bold: Mark winning rows
  :param Optional[str] caption: Table description (for screen readers).

  """
  table_classes = set(("table",
    "table-hover",
    "table-striped",
    "table-dark",
  ))
  if extra_wide_support:
    table_classes.add("table-bordered")
  if datatables:
    table_classes.add("dcss-datatable-wide" if extra_wide_support else "dcss-datatable")
    table_classes.add("nowrap") # keep table rows on one line
  else:
    table_classes.add("table-sm")
    table_classes.add("w-auto")
  out = '''
\n
\n''' % (" ".join(table_classes))

  if caption is not None:
    out += '''
%s
\n''' % caption

  out += '''
\n
'''

  headers = [ wrap_tuple(x) for x in headers ]

  if count:
    out += '''#	'''
  for head in headers:
    out += '''%s	''' % head[0]
  out += "
\n
\n"

  if not data:
    ncols = len(headers) + (1 if count else 0)
    out += '''
%s
''' % (ncols, stub_text)

  # XXX: this is something to do with ranking?
  nplace = 0
  rplace = 0
  last_value = None

  for row in data:
    # TODO: not sure this is necessary with DataTables?
    if bold and row[-1]:
      # Invert colours
      out += '''
'''
    else:
      out += '''
'''

    rplace += 1
    if place_column == -1:
      nplace += 1
    elif last_value != row[place_column]:
      nplace += 1
      if skip:
        nplace = rplace
      last_value = row[place_column]

    if count:
      out += '''%s	''' % (
        "py-1 px-2" if datatables else "",
        nplace,
      )

    for c in range(len(headers)):
      call_classes = set()
      if datatables:
        # More compact display
        call_classes.update(["py-1", "px-2"])
      val = row[c]
      if isinstance(val, str):
        # TODO: is there a better sort value than something like this to use?
        sort_val = "9999999999" if val == "" or val == "-" else val
        val_embedded_str = sort_val.find('"')
        if val_embedded_str > -1:
          # pretty hacky: if there is an embedded '"', we take the contents of
          # that, in order to handle clan links. Better would be to construct the
          # clan link in this function (like player), but doing so is annoyingly
          # complicated. Could also extract the inner text.
          try:
            sort_val = sort_val[val_embedded_str:].split('"')[1]
          except:
            print(sort_val)
            raise
      else:
        sort_val = str(val)
      header = headers[c]

      numeric_col = _is_numeric_table_value(val, header[0])
      pseudo_numeric_col = val == '-'
      if numeric_col:
        val = '{:,}'.format(isinstance(val, str) and int(val) or val)
      if numeric_col or pseudo_numeric_col:
        call_classes.add("text-right")
        call_classes.add("text-monospace")
      if extra_wide_support and is_player_header(header[0]):
        call_classes.add('text-dark')

      if c == place_column:
        out += '''''' % " ".join(call_classes)
      else:
        out += '''''' % (
                  " ".join(call_classes),
                  sort_val)
      val = str(val)
      if is_player_header(header[0]):
        val = linked_text(val, player_link)
      out += val
      if c == place_column:
        out += '	'
      else:
        out += '	'
    out += "
\n"
  out += '
\n
'
  return out

# DEPRECATED: Build a wrapper function around _table to replace this
def games_table(games, first=None, excluding=None, columns=None,
                including=None,
                count=True, win=True,
                place_column=-1, skip=False,
                caption=None):
  """Create a HTML table of games.

  :param List[List[str]] games: Games to list
  :param bool first: ?
  :param Optional[List[str]] excluding: If set, a list of column names to exclude from columns
  :param Optional[List[Union[Tuple[str,str],Tuple[str,str,bool]]]] columns: If set, a list of colum descriptions. The format is (sql column name, html column name[, link col=False]). Defaults to STOCK_WIN_COLUMNS if win is True, otherwise STOCK_COLUMNS.
  :param Optional[List[Tuple[int,Union[Tuple[str,str],Tuple[str,str,bool]]]]] including: If set, a list of (pos, name) tuples of columns to include. pos is the position to insert at.
  :param bool count: Add a count column at the start.
  :param bool win: Select default columns if `columns` is None. See `columns` param for more info.
  :param Optional[str] caption: Table description (for screen readers).
  """
  if columns is None:
    columns = STOCK_WIN_COLUMNS if win else STOCK_COLUMNS

  # Copy columns.
  columns = list(columns)

  if excluding:
    columns = [c for c in columns if c[0] not in excluding]

  if including:
    for pos, col in including:
      columns.insert(pos, col)

  if first and not isinstance(first, tuple):
    first = (first, 1)

  if first and columns[0][0] != first[0]:
    firstc = [ c for c in columns if c[0] == first[0] ]
    columns = [ c for c in columns if c[0] != first[0] ]
    columns.insert( first[1], firstc[0] )

  table_classes = set(("table",
    "table-sm",
    "table-hover",
    "table-striped",
    "table-dark",
    "w-auto",
    ))

  out = '''
\n
\n''' % " ".join(table_classes)

  if caption is not None:
    out += '''
%s
\n''' % caption
  out += '''
\n
'''
  if count:
    out += '''#	'''
  for col in columns:
    out += '''%s	''' % col[1]
  out += "
\n
\n"

  if not games:
    ncols = len(columns) + (1 if count else 0)
    out += '''
No games
''' % ncols

  # XXX: this is something to do with ranking?
  nplace = 0
  rplace = 0
  last_value = None

  for game in games:
    row_class = "table-secondary" if game.get('killertype') == 'winning' else ""

    out += '''
''' % row_class

    rplace += 1
    if place_column == -1:
      nplace += 1
    elif last_value != game.get(columns[place_column][0]):
      nplace += 1
      if skip:
        nplace = rplace
      last_value = game.get(columns[place_column][0])

    if count:
      out += '''%s	''' % nplace

    for i, c in enumerate(columns):
      val = fixup_column(c[0], game.get(c[0]) or '', game)
      numeric_col = _is_numeric_table_value(val, c[0])
      pseudo_numeric_col = val == '-'
      if numeric_col:
        val = '{:,}'.format(isinstance(val, str) and int(val) or val)
      td_class = "text-right text-monospace" if (numeric_col or pseudo_numeric_col) else ""
      if i == place_column:
        out += '''''' % td_class
      else:
        out += '''''' % td_class

      # XXX: this should change
      need_link = len(c) >= 3 and c[2]
      if need_link:
        try:
          out += r'' % crawl_utils.morgue_link(game)
        except:
          sys.stderr.write("Error processing game: " + loaddb.xlog_str(game))
          raise
      elif is_player_header(c[1]):
        val = linked_text(val, player_link)
      out += str(val)
      if need_link:
        out += ''
      if i == place_column:
        out += '	'
      else:
        out += '	'
    out += "
\n"
  out += "
\n
\n"
  return out

def _table(columns, rows, row_classes_fn=None, brief=False):
  # type: (Sequence[ColumnDisplaySpec], Sequence[Sequence[Any]], Optional[Callable[[Any], str]]) -> str
  '''
  Display a HTML table.

  @param columns: Details about columns in display order.
  @param rows: Rows of data in display order.
  @param row_classes_fn: If set, a function which gets passed each row to
                         determine classes for its
.
  '''

  n_columns = len(columns)

  out = ''

  table_classes = set(("table",
    "table-hover",
    "table-striped",
    "table-dark",
    ))
  if brief:
    table_classes.add("dcss-datatable-compact")
  else:
    table_classes.add("dcss-datatable")
    table_classes.add("table-bordered")

  out += '
\n'
  out += '
\n'.format(classes=" ".join(table_classes))

  # Table column headers
  out += '
\n'
  out += '
'
  for column in columns:
    out += '{name}	'.format(name=column.html_display_name)
  out += '
\n'
  out += '
\n'

  for row in rows:
    if len(row) != n_columns:
      raise ValueError("Row length {n_row} != columns length {n_cols}. Row data: {row} Col data: {cols}".format(
        n_row=len(row),
        n_cols=n_columns,
        row=repr(row),
        cols=columns,
      ))

    out += '
'.format(
      row_classes="" if row_classes_fn is None else row_classes_fn(row)
    )

    for column, base_value in zip(columns, row):
      cell_classes = set([
        # compact rows
        "py-1",
        "px-2",
      ])
      if column.numeric_data:
        cell_classes.update(['text-right', 'text-monospace'])

      if callable(column.transform_fn):
        display_value = column.transform_fn(base_value)
      elif column.numeric_data:
        # Basic thousands separators for untransformed numeric data
        display_value = '{:,}'.format(base_value)
      else:
        try:
          display_value = unicode(base_value)
        except:
          display_value = base_value # already unicode in py3

      out += '{value}	'.format(
        cell_classes=" ".join(cell_classes),
        value = display_value,
      )
    out += '
\n'
  out += '
\n'
  out += '
\n'
  return out

def category_table(category, rows, row_classes_fn=None, brief=False):
  """
  Display a HTML table for a given category. We just need to add the
  Rank & Player/Captain columns.
  """
  cols = [col for col in category.columns if (not brief or col.include_in_compact_display)]
  # logging.info(
  #   "category_table %s %s brief:%s base_cols:%s row0:%s",
  #   category.type, category.name, brief, len(cols),
  #   rows[0] if len(rows) else "<empty>")
  if category.type == 'individual':
    cols.insert(0, PseudoCol(
      "Player",
      False,
      lambda player: crawl_utils.linked_text(key=player, link_fn=crawl_utils.player_link),
    ))
  else:
    def _pretty_clan_name(data):
      info = json.loads(data)
      name = info["name"]
      return '{name}'.format(
          url=crawl_utils.clan_link(name, info["captain"]), name=name,
      )
    cols.insert(0, PseudoCol("Team", False, _pretty_clan_name))
  cols.insert(0, PseudoCol("#", True, None))

  return _table(
    columns=cols,
    rows=rows,
    row_classes_fn=row_classes_fn,
    brief=brief,
  )

def full_games_table(games, **pars):
  if not pars.get('columns'):
    if 'win' in pars:
      win = pars['win']
    else:
      win = True
    pars['columns'] = win and EXT_WIN_COLUMNS or EXT_COLUMNS
  return games_table(games, **pars)

def ext_games_table(games, win=True, **pars):
  cols = win and EXT_WIN_COLUMNS or EXT_COLUMNS
  pars.setdefault('including', []).append((1, ('player', 'Player')))
  return games_table(games, columns=cols, count=False, **pars)

def combo_highscorers(c):
  hs = query.get_top_combo_highscorers(c)
  return table_text( [ 'Player', 'Combo scores' ],
                     hs, place_column=1, skip=True )

def deepest_xl1_games(c):
  games = query.get_deepest_xl1_games(c)
  return games_table(games, first = 'place', win=False)

def most_pacific_wins(c):
  games = query.most_pacific_wins(c)
  return games_table(games,
                     columns = STOCK_WIN_COLUMNS + [('kills', 'Kills')])

def hyperlink_games(games, field):
  hyperlinks = [ crawl_utils.morgue_link(g) for g in games ]
  text = [ '%s' % (link, g[field])
           for link, g in zip(hyperlinks, games) ]
  return ", ".join(text)

def youngest_rune_finds(c):
  runes = query.youngest_rune_finds(c)
  runes = [list(r) for r in runes]
  for r in runes:
    r[3] = pretty_date(r[3])
  return table_text([ 'Player', 'Rune', 'XL', 'Time' ], runes)

def most_deaths_to_uniques(c):
  rows = query.most_deaths_to_uniques(c)
  for r in rows:
    r.insert(1, len(r[1]))
    r[2] = ", ".join(r[2])
  return table_text([ 'Player', '#', 'Uniques', 'Time'], rows)

def streak_table(streaks, active=False, place=False):
  # Replace the list of streak games with hyperlinks.
  result = []

  nplace = 0
  rplace = 0
  last_value = None
  for s in streaks:
    games = s['games']
    game_text = hyperlink_games(games, 'charabbrev')
    if active and s['is_active']:
      game_text += ", " + (s['next_char'] or "?")
    rplace += 1
    if last_value != s['length']:
      nplace = rplace
      last_value = s['length']
    row = [nplace, s['player'], s['length'], game_text]
    result.append(row)

  return _table([ PseudoCol('#', True, None),
                  PseudoCol('Player', False, lambda player: crawl_utils.linked_text(key=player, link_fn=crawl_utils.player_link)),
                  PseudoCol('Streak Length', True, None),
                  PseudoCol('Games', False, None), ],
                  result, brief=True)

def best_active_streaks(c):
  return streak_table(query.get_top_active_streaks(c), active=True)

def best_streaks(c):
  streaks = query.get_top_streaks(c)
  return streak_table(streaks, place=True)

def fixup_clan_rows(rows):
  rows = [ list(r) for r in rows ]
  for clan in rows:
    clan[0] = linked_text(clan[1], clan_link, clan[0])
  return rows

def best_clans(c):
  clans = fixup_clan_rows(query.get_top_clan_scores(c))
  return table_text( [ 'Clan', 'Captain', 'Points' ],
                     clans, place_column=2, skip=True )

def clan_unique_kills(c):
  ukills = fixup_clan_rows(query.get_top_clan_unique_kills(c))
  return table_text( [ 'Clan', 'Captain', 'Kills', 'Time' ],
                     ukills)

def clan_combo_highscores(c):
  return table_text( [ 'Clan', 'Captain', 'Scores' ],
                     fixup_clan_rows(query.get_top_clan_combos(c)),
                     place_column=2, skip=True )

def make_milestone_string(w, src, make_links=False):
  ago,new = how_old(w[0])
  if ago == None:
    return None
  if make_links:
    plink = crawl_utils.linked_text(w[1], crawl_utils.player_link)
  else:
    plink = w[1]
  if w[5] == None:
    god_phrase = ''
  else:
    god_phrase = ' of %s' % w[5]
  where_nice = (ago, plink) + w[2:5] + (god_phrase, ) + w[6:9] + (pretty_dur(w[9]),src)
  return ("%s ago: %s the %s (L%d %s%s) %s (%s, turn %d, dur %s, %s)
" % where_nice)

def whereis(c, *players):
  where_data = []
  for p in players:
    for src in ['cao','cbr2','cdo', 'cko', 'cpo', 'cue','cwz','cxc','lld']:
      where = query.whereis_player(c, p, src)
      if not where:
        continue
      mile_string = make_milestone_string(where, src)
      if not mile_string:
        continue
      where_data.append([where[0], mile_string])
  where_data.sort(key=lambda e: e[0], reverse=True)
  where_string = ""
  for w in where_data:
    where_string += w[1]
  return where_string

def whereis_table(c):
  where_data = []
  for w in query.whereis_all_players(c):
    where = w[1]
    if w[0] == 'csz':
      pretty_src = 'cszo'
    elif w[0] == 'cbr':
      pretty_src = 'cbr2'
    else:
      pretty_src = w[0]
    ago,new = how_old(where[0],1)
    if ago == None:
      continue
    if where[5] == None:
      god_phrase = ''
    else:
      god_phrase = ' of %s' % where[5]
    mile_data = [where[1], where[7], where[3], '%s%s' % (where[4], god_phrase), where[2], where[6], '%s ago' % ago, pretty_src.upper()]
    where_data.append([where[7], where[3], where[0], mile_data])
  where_data.sort(key=lambda e: (e[0],e[1],e[2]), reverse=True)

  where_list = []
  for w in where_data:
    where_list.append(w[3])
    if where_list[-1][1] == 0:
      where_list[-1][1] = ''
  return _table([
      PseudoCol("Player", False,
      lambda player: crawl_utils.linked_text(key=player,
          link_fn=crawl_utils.player_link)),
      PseudoCol("Runes", False, None),
      PseudoCol('Level', False, None),
      PseudoCol('Character', False, None),
      PseudoCol('Title', False, None),
      PseudoCol('Location', False, None),
      PseudoCol('Time', False, None),
      PseudoCol('Server', False, None) ], where_list)

def _strip_banner_suffix(banner):
  if ':' in banner:
    return banner[ : banner.index(':')]
  return banner

def banner_suffix(banner):
  if ':' in banner:
    return banner[banner.index(':') + 1 :]
  return ''

def banner_image(banner, prestige, full_name=False):
  p = prestige
  while p > 3:
    p = p/10
  i_string = ''
  for i in range(p):
    i_string = i_string + 'I'
  name_suffix = banner_suffix(banner)
  banner_subkey = _strip_banner_suffix(banner)
  img = BANNER_IMAGES.get(banner) or BANNER_IMAGES.get(banner_subkey)
  banner_text = BANNER_TEXT[banner_subkey][p-1]
  name = ''
  if img and img[1]:
    name = img[1] + " " + i_string
  if full_name and name_suffix:
    name = name + " (" + name_suffix + ")"
  if img and img[1]:
    name = name + ": " + banner_text
  if img and img[0]:
    filename = img[0][:-4]+("%d" % p)+img[0][-4:]
    return (crawl_utils.banner_link(filename), name)
  return img

def banner_img_for(b, nth):
  if nth:
    bid = " id=\"banner-%d\" " % nth
  else:
    bid = ""
  return '''

''' % (b[0], b[1], b[1], bid)

def banner_named(name, prestige):
  img = banner_image(name, prestige)
  if not img:
    return None
  return banner_img_for(img, 0)

def banner_images(banners):
  # First remove duplicates. We assume that higher prestige versions come first.
  seen_banners = set()
  deduped = []
  for b in banners:
    if not _strip_banner_suffix(b[0]) in seen_banners:
      deduped.append(b)
      seen_banners.add(_strip_banner_suffix(b[0]))
  images = [banner_image(x[0],x[1]) for x in deduped]
  images = [i for i in images if i and i[0]]
  return images

def banner_div(all_banners):
  res = ''
  banner_n = 1
  for b in all_banners:
    res += banner_img_for(b, banner_n)
    banner_n += 1
  return res

def _scored_win_text(g, text):
  if g['killertype'] == 'winning':
    text += '*'
  return text

def player_combo_scores(c, player):
  games = query.get_combo_scores(c, player=player)
  games = [ [ crawl_utils.linked_text(g, crawl_utils.morgue_link,
                                      _scored_win_text(g, g['charabbrev'])),
              g['score'] ]
            for g in games ]
  return games

def player_species_scores(c, player):
  games = query.game_hs_species(c, player)

  games = [
    [ crawl_utils.linked_text(g, crawl_utils.morgue_link,
                              _scored_win_text(g, g['charabbrev'][:2])),
      g['score'] ]
    for g in games ]
  return games

def player_class_scores(c, player):
  games = query.game_hs_classes(c, player)
  games = [
    [ crawl_utils.linked_text(g, crawl_utils.morgue_link,
                              _scored_win_text(g, g['charabbrev'][2:])),
      g['score'] ]
    for g in games ]
  return games

def clan_combo_scores(c, captain):
  games = [i[1] for i in query.get_clan_combo_scores(c, captain=captain)]
  games = [ [ crawl_utils.linked_text(g, crawl_utils.morgue_link,
                                      _scored_win_text(g, g['charabbrev'])),
              g['score'] ]
            for g in games ]
  return games

def player_scores_block(c, scores, title):
  asterisk = [ s for s in scores if '*' in s[0] ]
  score_table = (scores
                 and (", ".join([ "%s (%d)" % (s[0], s[1])
                                  for s in scores ]))
                 or "None")
  text = """
%(title)s

                %(score_table)s

         """ % {'title': title, 'score_table': score_table}
  if asterisk:
    text += "
* Winning Game
"
  return text

def english_join(items, final="and"):
  """Join a list of items with an oxford comma and joining word.

  eg [1,2,3] => "1, 2, and 3".
  """
  # type: (Sequence[str], str) -> str
  if not items:
    return ""
  if len(items) == 1:
    return items[0]
  elif len(items) == 2:
    return "%s %s %s" % (items[0], final, items[1])
  else:
    items[-1] = "%s %s" % (final, items[-1])
    return ", ".join(str(item) for item in items)

def category_column_title(cat):
    if cat.proportional:
        return "%s (max %d)" % (cat.name, cat.max)
    else:
        return cat.name


# Create your views here.
from django.http import HttpResponse
from django.shortcuts import render
from django.utils import simplejson as json
from django.db.models import Q

from dasdemo.models import Country

def main(request):
	return render(request, "typeahead/typeahead.html")

def main2(request):
	return render(request, "typeahead/typeahead2.html")

def main3(request):
	return render(request, "typeahead/typeahead3.html")

def api(request):
	qry  =  request.GET.get('q')
	res = {"stat": "error"}
	if qry:
		res = Country.objects.filter(name__istartswith = qry).values('id','name')
		res = list(res)

	return HttpResponse(json.dumps(res), mimetype="application/json")

def api4(request):
	qry  =  request.GET.get('q')
	err = {"stat": "error"}
	data ={}
	suggestions = []

	if qry:
		res = Country.objects.filter(name__istartswith = qry)

		for country in res:
			temp = {}
			temp['id'] = country.id
			temp['name'] = country.name
			temp['img'] = 'http://localhost:8000/static/flags/' + country.short + '.png'
			suggestions.append(temp)

	if suggestions:
		return HttpResponse(json.dumps(suggestions), mimetype = 'application/json')
	else:
		return HttpResponse(json.dumps(err), mimetype = 'application/json')

def api2(request):
	qry  =  request.GET.get('q')
	res = {"stat": "error"}
	data ={}

	if qry:
		res = Country.objects.filter(name__istartswith = qry).values('id','name')
		res = list(res)
		data['suggestions'] = res

	if data:
		return HttpResponse(json.dumps(data), mimetype = 'application/json')
	else:
		return HttpResponse(json.dumps(res), mimetype = 'application/json')

def api3(request):
	qry  =  request.GET.get('query')
	res = {"stat": "error"}
	data ={}
	suggestions = []

	if qry:
		res = Country.objects.filter(name__istartswith = qry)

		for country in res:
			temp = {}
			temp['id'] = country.id
			temp['value'] = country.name #res[i]['name']
			temp['data'] = country.name #res[i]['id']
			suggestions.append(temp)

		data['query'] = qry
		data['suggestions'] = suggestions

	if data:
		return HttpResponse(json.dumps(data), mimetype = 'application/json')
	else:
		return HttpResponse(json.dumps(res), mimetype = 'application/json')

def headings_api(request, qry=""):

	err = {"stat": "error"}
	data ={}
	suggestions = []

	if qry:
		#return HttpResponse("yes")

		res = Country.objects.filter(name__istartswith = qry)

		for country in res:
			temp = {}
			temp['id'] = country.id
			temp['value'] = country.name + ' ' + qry
			temp['tokens'] = [country.name]
			temp['img'] = 'http://localhost:8000/static/flags/' + country.short + '.png'
			suggestions.append(temp)

	if suggestions:
		return HttpResponse(json.dumps(suggestions), mimetype = 'application/json')
	else:
		return HttpResponse(json.dumps(err), mimetype = 'application/json')


import requests
from bs4 import BeautifulSoup
import csv

# 사용자로부터 검색어 입력받기
search_query = input("검색어를 입력하세요: ")

# 검색 페이지 URL 생성
url = f"https://search.shopping.naver.com/search/all?query={search_query}&cat_id=&frm=NVSHATC"

# HTTP 요청 보내기
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
}
response = requests.get(url, headers=headers)

# HTTP 요청이 성공했는지 확인하기
if response.status_code == 200:
    soup = BeautifulSoup(response.text, "html.parser")
    product_items = soup.select(".adProduct_title__amInq")
    product_list = []  # 제품 정보를 담을 리스트

    for idx, product_item in enumerate(product_items, start=1):
        name_element = product_item.select_one('a')
        name = name_element.get('title').strip() if name_element else "No name"
        price_element = product_item.find_next(class_="price_num__2WUXn")
        price = price_element.text.strip() if price_element else "No price"
        detail_element = product_item.find_next(class_="basicList_detail_box__3ta3h")
        detail = detail_element.text.strip() if detail_element else "No detail"
        product_list.append({'번호': idx, '제품명': name, '가격': price, '상세정보': detail})
else:
    print("HTTP 요청 실패")

# CSV 파일로 저장
filename = 'product_list.csv'
fields = ['번호', '제품명', '가격', '상세정보']

with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
    writer = csv.DictWriter(csvfile, fieldnames=fields)
    writer.writeheader()
    writer.writerows(product_list)

print("CSV 파일 저장 완료.")

cart_list = []  # 장바구니 담을 리스트

# 장바구니 CSV 파일로 저장
filename = './cart_list.csv'
fields = ['번호', '제품명', '가격', '상세정보']

with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
    writer = csv.DictWriter(csvfile, fieldnames=fields)
    writer.writeheader()
    writer.writerows(cart_list)

# 장바구니에 추가
cart_list = []

while True:
    product_number = input("장바구니에 추가할 제품 번호를 입력하세요 (종료: q): ")
    if product_number == 'q':
        break

    product_number = int(product_number)
    if product_number < 1 or product_number > len(product_list):
        print("잘못된 제품 번호입니다. 다시 입력해주세요.")
        continue

    selected_product = product_list[product_number - 1]
    cart_list.append(selected_product)
    print(f"제품 '{selected_product['제품명']}'이 장바구니에 추가되었습니다.")


import argparse
import shutil
from pathlib import Path

from . import fetcher, meta
from .slicer import Slicer
from .storage import File, get_files_metadata

def list_files(args: argparse.Namespace):
    if not args.datasets:
        datasets = meta.datasets
    else:
        datasets = args.datasets

    ftp = fetcher.connect()

    total_size = 0
    total_n_files = 0

    for dataset in sorted(datasets):
        dataset_files_list = fetcher.list_dataset_files(ftp, dataset)
        dataset_size = sum(f.size for f in dataset_files_list)
        dataset_n_files = len(dataset_files_list)
        total_size += dataset_size
        total_n_files += dataset_n_files
        if dataset_files_list:
            if "year" in meta.datasets[dataset]["partition"]:
                first = min(dataset_files_list, key=lambda x: x.partition.year)
                first = f"{first.partition.year}"
                last = max(dataset_files_list, key=lambda x: x.partition.year)
                last = f"{last.partition.year}"
            elif "yearmonth" in meta.datasets[dataset]["partition"]:
                first = min(
                    dataset_files_list,
                    key=lambda x: f"{x.partition.year}{x.partition.month:02}",
                )
                first = f"{first.partition.year}-{first.partition.month:02}"
                last = max(
                    dataset_files_list,
                    key=lambda x: f"{x.partition.year}{x.partition.month:02}",
                )
                last = f"{last.partition.year}-{last.partition.month:02}"
        else:
            first = last = "----"
        msg = " ".join(
            [
                f"{dataset: <27}",
                f"{dataset_n_files: >6} files ",
                f"{dataset_size / 2**20: >9.1f} MB",
                f"from {first} to {last}",
            ]
        )
        print(msg)

    print(f"Total size: {total_size / 2**30:.1f} GB")
    print(f"Total files: {total_n_files} files")

    ftp.close()

def fetch_data(args: argparse.Namespace):
    data_dir = args.data_dir
    threads = args.threads
    if not args.datasets:
        datasets = meta.datasets
    else:
        datasets = args.datasets

    slicer = Slicer(
        start_time=args.start,
        end_time=args.end,
        regions=args.regions,
    )

    fetcher.download_data(
        datasets=sorted(datasets),
        destdir=data_dir,
        threads=threads,
        slicer=slicer,
    )

def fetch_docs(args: argparse.Namespace):
    data_dir = args.data_dir
    if args.datasets is None:
        datasets = meta.docs
    else:
        datasets = args.datasets

    ftp = fetcher.connect()
    if datasets:
        datasets_ = set(datasets) & set(meta.docs.keys())
    else:
        datasets_ = meta.docs.keys()
    for dataset in sorted(datasets_):
        for file in fetcher.download_documentation(ftp, dataset, data_dir):
            print(file)
    ftp.close()

def fetch_aux(args: argparse.Namespace):
    data_dir = args.data_dir
    if args.datasets is None:
        datasets = meta.auxiliary_tables
    else:
        datasets = args.datasets

    ftp = fetcher.connect()
    if datasets:
        datasets_ = set(datasets) & set(meta.auxiliary_tables.keys())
    else:
        datasets_ = meta.auxiliary_tables.keys()
    for dataset in sorted(datasets_):
        for file in fetcher.download_auxiliary_tables(ftp, dataset, data_dir):
            print(file)
    ftp.close()

def archive(args: argparse.Namespace):
    data_dir = args.data_dir
    archivedatadir = args.archive_data_dir
    extension = args.extension
    dry_run = args.dry_run

    for datasetdir in data_dir.iterdir():
        print(datasetdir)
        datasetname = datasetdir.name
        for datepartitiondir in datasetdir.iterdir():
            files = get_files_metadata(datepartitiondir, extension=extension)
            for file in files:
                file: File
                if not file.is_most_recent:
                    print(file)
                    if dry_run:
                        continue
                    archivedatasetdir: Path = archivedatadir / datasetname
                    archivedatasetdir.mkdir(parents=True, exist_ok=True)
                    archivefilepath = archivedatasetdir / file.filepath.name
                    shutil.move(file.filepath, archivefilepath)

def get_args():
    parser = argparse.ArgumentParser(
        description="Download raw data files from DATASUS",
    )
    subparsers = parser.add_subparsers(required=True)

    # * list-files ------------------------------------------------------------
    subparser_list_files = subparsers.add_parser("list-files")
    subparser_list_files.add_argument(
        "datasets",
        nargs="*",
        help="Datasets to list",
    )
    subparser_list_files.set_defaults(func=list_files)

    # * fetch data ------------------------------------------------------------
    subparser_fetch = subparsers.add_parser("data")
    subparser_fetch.add_argument(
        "datasets",
        nargs="*",
        help="Datasets to download",
    )
    subparser_fetch.add_argument(
        "--data-dir",
        dest="data_dir",
        type=Path,
        required=True,
        help="Directory to download to",
    )
    subparser_fetch.add_argument(
        "-t",
        "--threads",
        dest="threads",
        type=int,
        default=2,
        help="Number of concurrent fetchers",
    )
    subparser_fetch.add_argument("--start")
    subparser_fetch.add_argument("--end")
    subparser_fetch.add_argument("--regions", nargs="+")
    subparser_fetch.set_defaults(func=fetch_data)

    # * fetch docs ------------------------------------------------------------
    subparser_docs = subparsers.add_parser("docs")
    subparser_docs.add_argument(
        "datasets",
        nargs="*",
        help="Datasets documentation to download",
    )
    subparser_docs.add_argument(
        "--data-dir",
        dest="data_dir",
        type=Path,
        help="Directory to download to",
    )
    subparser_docs.set_defaults(func=fetch_docs)

    # * fetch aux -------------------------------------------------------------
    subparser_aux = subparsers.add_parser("aux")
    subparser_aux.add_argument(
        "datasets",
        nargs="*",
        help="Datasets auxiliary tables to download",
    )
    subparser_aux.add_argument(
        "--data-dir",
        dest="data_dir",
        type=Path,
        help="Directory to download to",
    )
    subparser_aux.set_defaults(func=fetch_aux)

    # * archive ---------------------------------------------------------------
    subparser_archive = subparsers.add_parser("archive")
    subparser_archive.add_argument("--data-dir", type=Path, required=True)
    subparser_archive.add_argument(
        "--archive-data-dir",
        type=Path,
        required=True,
    )
    subparser_archive.add_argument("--extension", required=True)
    subparser_archive.add_argument("--dry-run", action="store_true")
    subparser_archive.set_defaults(func=archive)

    args = parser.parse_args()
    return args

def main():
    args = get_args()
    args.func(args)

if __name__ == "__main__":
    main()


import torch
from torch.autograd import grad, gradcheck

def check_grad(solve_func, inputs, eps, atol, rtol):
    assert gradcheck(solve_func, inputs, eps=eps, atol=atol)

    A_val, b = inputs[0], inputs[1]
    # Check that the gradient works correctly for floating point data
    out = solve_func(*inputs).sum()
    gA, gb = grad(out, (A_val, b))

    A_float = A_val.float()
    b_float = b.float()
    inputs2 = (A_float, b_float) + inputs[2:]
    out_float = solve_func(*inputs2).sum()
    gA_float, gb_float = grad(out_float, (A_float, b_float))

    # This mostly checks that backward() is not accumulating
    # additional floating point errors on top of those expected by
    # converting the input from double to float. Naive float casting
    # in the backward python ops can result in differences in order of magnitude
    # even for well-conditioned systems. These checks cover that case.
    #
    # Do note that, in general, it's possible to construct very
    # ill-conditioned systems where the initial loss or precision is enough
    # to get large gradient errors no matter what we do. This checks
    # are not meant to capture such scenarios.
    torch.testing.assert_close(gA, gA_float.double(), rtol=rtol, atol=atol)
    torch.testing.assert_close(gb, gb_float.double(), rtol=rtol, atol=atol)


import psycopg2
from psycopg2.extensions import AsIs

import argparse
import csv
from configparser import ConfigParser
import datetime
import logging
import os
import requests
import sys
import shutil

logger = None

def configure_logger(args):
    global logger
    if args.verbose:
        level = logging.DEBUG
    else:
        level = logging.INFO
    formatter = "[%(levelname)s] %(asctime)s | %(filename)s:%(lineno)d | %(message)s"
    logger = logging.getLogger(__name__)
    handler = logging.StreamHandler()
    fhandler = logging.FileHandler("run.log")
    handler.setLevel(level)
    fhandler.setLevel(logging.DEBUG)
    handler.setFormatter(logging.Formatter(formatter))
    fhandler.setFormatter(logging.Formatter(formatter))
    logger.setLevel(logging.DEBUG)
    logger.addHandler(handler)
    logger.addHandler(fhandler)

def set_db_config(dbname):
    config = ConfigParser()
    config.read('init.ini')
    config.set("Database","dbname",dbname)
    with open('init.ini',"w") as configfile:
        config.write(configfile)

def parse_args():
    config = ConfigParser()
    config.read("init.ini")
    print(config["Database"]["dbname"])
    todays_date = str(datetime.date.today())

    parser = argparse.ArgumentParser(
        prog="hr.py",
        description="This a program for managing Hr operations",
        epilog="use these subcommands to do specific operations",
    )

    parser.add_argument(
        "-b", "--db", help="Database name (Default : %(default)s)", action="store_true" , default=config["Database"]["dbname"]
    )
    parser.add_argument(
        "-v", "--verbose", help="print out detailed logs", action="store_true"
    )

    subparser = parser.add_subparsers(dest="subcommand", help="subcommand help")

    # database command
    parser_createdb = subparser.add_parser("createdb", help="create a database")
    parser_createdb.add_argument("-b", "--db", help="database name", type=str)

    # loading csv into database
    parser_load = subparser.add_parser(
        "loadcsv", help="insert data from file into employee table in database ",description="Imports list of employees into the database from specified file '-l'"
    )
    parser_load.add_argument("file",help="File name ",type=str,)

    parser_load.add_argument(
        "-d",
        "--address",
        help="Add a new address",
        type=str,
        default="100 Flat Grape Dr.;Fresno;CA;95555;United States of America",
    )

    # commad for creating vcards
    parser_vcard = subparser.add_parser("vcard", help="generate vcards")

    parser_qrcode = subparser.add_parser("qrcode", help="generate qrcode")
    parser_qrcode.add_argument("-s","--size",
        help="Qr dimension ",
        default=300,)

    parser_leave_emp = subparser.add_parser("leave-emp", help="add leaves for employees",description="Adds leaves taken by the employee")

    parser_leave_emp.add_argument("empid", help="Employee id ", type=str)
    parser_leave_emp.add_argument("-d", "--date", help="Date 'YYYY-MM-DD' (Default : %(default)s) ", type=str, default=todays_date)
    parser_leave_emp.add_argument( "-r", "--reason",help=" Reason for leave ",type=str,default="Not mentioned")
    parser_export = subparser.add_parser("export", help="Get the employee detail as csv file",description="Get a detailed summary of all employees in a csv file")

    args = parser.parse_args()
    return args

def parse_data(args):
    filename = args.file
    datalist = []
    try:
        with open(filename, "r") as data:
            detail = csv.reader(data)
            for i in detail:
                datalist.append(i)

        return datalist
    except FileNotFoundError as e:
        logger.error("Error %s", (e,))
        sys.exit(-1)

def fetch_from_db(args, user):
    try:
        connection = psycopg2.connect(f"dbname={args.db} user={user}")
        curs = connection.cursor()
        curs.execute(
            f"""SELECT first_name,last_name,designation,email,phone,company_address
                         FROM
                         employees """
        )
        data = [list(row) for row in curs.fetchall()]
        connection.commit()
        curs.close()
        connection.close()
        logger.info(f"Fetched all data from employees table in {args.db} database")
        return data
    except psycopg2.OperationalError as e:
        logger.error(e)
        sys.exit(-1)

def generate_vcard_content(lname, fname, designation, email, phone, address):
    return f"""BEGIN:VCARD
VERSION:2.1
N:{lname};{fname}
FN:{fname} {lname}
ORG:Authors, Inc.
TITLE:{designation}
TEL;WORK;VOICE:{phone}
ADR;WORK:;;{address}
EMAIL;PREF;INTERNET:{email}
REV:20150922T195243Z
END:VCARD
"""

def generate_qrcode(dimension, i):
    qr_code = requests.get(
        f"https://chart.googleapis.com/chart?cht=qr&chs={dimension}x{dimension}&chl={i}"
    )
    return qr_code

def create_vcards(data):
    if os.path.exists("vcard"):
        shutil.rmtree("vcard")
    os.mkdir("vcards")
    for i in data:
        lname, fname, designation, email, phone, address = i
        vcard = generate_vcard_content(lname, fname, designation, email, phone, address)
        with open(f"vcards/{lname}.vcf", "w") as d:
            d.write(vcard)
        logger.debug("Created vcard for %s ", lname)
    logger.info("Created all vcards")

def create_qrcode_images(data, args):
    if type(args.size) != int or int(args.size) > 500:
        logger.error("size must be an integer between 100 - 500")

    else:
        if os.path.exists("qrcode"):
            shutil.rmtree("vcard")
        os.mkdir("qrcode")
        for i in data:
            with open(f"qrcode/{i[0]}.qr.png", "wb") as Q:
                qr_code = generate_qrcode(args.size, i)
                Q.write(qr_code.content)
            logger.debug("Created QRcode for %s", i[0])
        logger.info("Created QRs for all")

def create_database(args, user):
    try:
        connection = psycopg2.connect(database="postgres", user=user)
        curs = connection.cursor()
        curs.execute("commit")
        curs.execute("create database %s ;", (AsIs(args.db),))
        curs.close()
        connection.close()
        logger.info("Database created")
    except psycopg2.errors.DuplicateDatabase as e:
        logger.error("Error %s", e)

def create_tables(args, user):
    try:
        dbname = args.db
        connnection = psycopg2.connect(f"dbname={dbname} user={user}")
        curs = connnection.cursor()
        query = open("queries.sql", "r")
        curs.execute(query.read())
        connnection.commit()
        curs.close()
        connnection.close()
        logger.info(f"Table employees created in {dbname} database")

    except psycopg2.errors.DuplicateTable as e:
        logger.error("Table already exists: %s", e)
        sys.exit(-1)

def load_csv_into_db(args, data, user):
    try:
        connection = psycopg2.connect(f"dbname={args.db} user={user}")
        curs = connection.cursor()
        fname, lname, designation, email, phone = data

        curs.execute(
            f"""INSERT
                         INTO
                         employees(first_name, last_name, designation, email, phone, company_address)
                          VALUES(%s,%s,%s,%s,%s,%s)""",
            (fname, lname, designation, email, phone, args.address),
        )

        connection.commit()
        curs.close()
        connection.close()
        logger.debug(" Inserted datas of %s into table employees ", fname)

    except psycopg2.OperationalError as e:
        logger.error("Error %s", e)
    except psycopg2.errors.ForeignKeyViolation as e:
        logger.warning("Foreign key violation: %s")

    except psycopg2.Error as e:
        logger.error(f"Error {e}")

def load_leave_employee(args, user):
    try:
        with psycopg2.connect(f"dbname={args.db} user={user}") as connection:
            with connection.cursor() as curs:
                try:
                    curs.execute(
                        "SELECT designation FROM employees WHERE s_no = %s ",
                        (args.empid,),
                    )
                    designation = curs.fetchone()[0]
                except (TypeError, IndexError) as e:
                    logger.error(f"Error while fetching designation: %s", (e,))
                    raise  # for exiting the fun'

                try:
                    curs.execute(
                        "SELECT total_leaves FROM designation WHERE id = %s ",
                        (designation,),
                    )
                    total_leave = curs.fetchone()[0]
                except (TypeError, IndexError) as e:
                    logger.error(f"Error while fetching total_leave: %s", (e,))
                    raise

                try:
                    curs.execute(
                        "SELECT count(leave_date) FROM employee_leave WHERE employee_id = %s ",
                        (args.empid,),
                    )
                    leave_count = curs.fetchone()[0]
                except (TypeError, IndexError) as e:
                    logger.error(f"Error while fetching leave count: %s", (e,))
                    raise

                try:
                    if leave_count < total_leave:
                        curs.execute(
                            "INSERT INTO employee_leave(leave_date, employee_id,reason) VALUES (%s,%s,%s)",
                            (args.date, args.empid, args.reason),
                        )
                        logger.info("Leave added")
                    else:
                        logger.warning("Maximum leave attained")
                except psycopg2.Error as e:
                    logger.error("Error while inserting leave data: %s", e)

    except psycopg2.Error as e:
        logger.error("Error: %s", e)

def join_tables(args, user):
    try:
        connection = psycopg2.connect(f"dbname={args.db} user={user}")
        curs = connection.cursor()
        curs.execute(
            """SELECT
  e.s_no,
  e.first_name,
  e.last_name,
  d.title,
  d.total_leaves,
  e.email,
  e.phone,
  e.company_address,
  COUNT(DISTINCT l.leave_date) AS leave_taken,
  d.total_leaves - COUNT(DISTINCT l.leave_date) AS leave_remaining
FROM employees e
JOIN designation d ON e.designation = d.id
LEFT JOIN employee_leave l ON e.s_no = l.employee_id
GROUP BY e.s_no, e.first_name, e.last_name, d.title, d.total_leaves, e.email, e.phone, e.company_address;
"""
        )
        data = curs.fetchall()
        connection.commit()
        curs.close()
        connection.close()
        logger.info(f"Joined tables and data fetched")
        return data
    except psycopg2.Error as e:
        logger.error("Error %s", e)

def export_employee_details(data):
    with open("employees_summary.csv", "w", newline="") as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(
            [
                "Employee ID",
                "First Name",
                "Last Name",
                "Designation",
                "Total Leaves",
                "Email",
                "Phone",
                "Company Address",
                "Leaves Taken",
                "Leaves Remaining",
            ]
        )

        for row in data:
            writer.writerow(row)
            logger.debug("Writed row on csv file")
    logger.info("Exported File as employee_summary.csv ")

def main():
    args = parse_args()
    user = os.getenv("USERNAME")
    configure_logger(args)

    if args.subcommand == "createdb":
        set_db_config(args.db)
        create_database(args, user)
        create_tables(args, user)

    if args.subcommand == "loadcsv":
        data = parse_data(args)
        for i in range(len(data)):
            load_csv_into_db(args, data[i], user)

    if args.subcommand == "vcard":
        data = fetch_from_db(args, user)
        create_vcards(data)

    if args.subcommand == "qrcode":
        data = fetch_from_db(args, user)
        create_qrcode_images(data, args)

    if args.subcommand == "leavemp":
        load_leave_employee(args, user)

    if args.subcommand == "export":
        data = join_tables(args, user)
        export_employee_details(data)

if __name__ == "__main__":
    main()


import os
import sys
import time
import json
import random
import matplotlib
import numpy as np
import pandas as pd
import tensorflow as tf
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from os import listdir
from keras.preprocessing import sequence
from functools import reduce
from s2vt import Video_Caption_Generator
import bleu as bl

os.environ['CUDA_VISIBLE_DEVICES'] = "0"
#%%

dim_image = 2048#feature dimension
n_frame_step = 50#feature sample
n_video_lstm_step = 50#?????
dim_hidden= 100
n_caption_lstm_step = 20 #?????
schedule_sample_probability = 1
n_epochs = 1000
batch_size = 50
learning_rate = 0.0001
decay_epoch = 100
sample_size = 6513
train_feature_folder = '../MSRVTT/training_data_sequence/'
test_folder_path = '../MSRVTT/testing_data_sequence/'
train_label_path = '../MSRVTT/training_label.json'
test_label_path = '../MSRVTT/testing_label.json'
model_path = './models/pyf_5/'
word_count_threshold = 5
#%%

def preProBuildWordVocab(sentence_iterator, word_count_threshold=5):
    # borrowed this function from NeuralTalk
    print ('preprocessing word counts and creating vocab based on word count threshold %d' % (word_count_threshold))
    word_counts = {}
    nsents = 0
    for sent in sentence_iterator:
        nsents += 1
        for w in sent.lower().split(' '):
            word_counts[w] = word_counts.get(w, 0) + 1
    vocab = [w for w in word_counts if word_counts[w] >= word_count_threshold]

    print ('filtered words from %d to %d' % (len(word_counts), len(vocab)))

    ixtoword = {}
    ixtoword[0] = '<pad>'
    ixtoword[1] = '<bos>'
    ixtoword[2] = '<eos>'
    ixtoword[3] = '<unk>'

    wordtoix = {}
    wordtoix['<pad>'] = 0
    wordtoix['<bos>'] = 1
    wordtoix['<eos>'] = 2
    wordtoix['<unk>'] = 3

    for idx, w in enumerate(vocab):
        wordtoix[w] = idx+4
        ixtoword[idx+4] = w

    word_counts['<pad>'] = nsents
    word_counts['<bos>'] = nsents
    word_counts['<eos>'] = nsents
    word_counts['<unk>'] = nsents

    bias_init_vector = np.array([1.0 * word_counts[ ixtoword[i] ] for i in ixtoword])
    bias_init_vector /= np.sum(bias_init_vector) # normalize to frequencies
    bias_init_vector = np.log(bias_init_vector)
    bias_init_vector -= np.max(bias_init_vector) # shift to nice numeric range

    return wordtoix, ixtoword, bias_init_vector
#%%
path_list = listdir(train_feature_folder)
train_feature_path = [train_feature_folder+path for path in path_list]
#### train_ID_list ####
train_ID_list = [ path[:-4] for path in path_list]

#### train_feature_dict ####
train_feature_dict = {}
for path in train_feature_path:
    feature = np.load(path)
    train_feature_dict[path[:-4].replace(train_feature_folder,"")] = feature

def clean_string(string):
    return string.replace('.', '').replace(',', '').replace('"', '').replace('\n', '').replace('?', '').replace('!', '').replace('\\', '').replace('/', '')

train_label_dict={}
with open(train_label_path) as data_file:
    train_label = json.load(data_file)
captions_corpus = []
for sample in train_label:
    cleaned_captions = [clean_string(sentence) for sentence in sample["caption"]]
    captions_corpus += cleaned_captions
    train_label_dict[sample["id"]] = cleaned_captions
    #print(cleaned_captions)

wordtoix, ixtoword, bias_init_vector = preProBuildWordVocab(captions_corpus, word_count_threshold)
np.save("./wordtoix_pyf", wordtoix)
np.save('./ixtoword_pyf', ixtoword)
np.save("./bias_init_vector_pyf", bias_init_vector)

model = Video_Caption_Generator(
            dim_image=dim_image,
            n_words=len(wordtoix),
            dim_hidden=dim_hidden,
            batch_size=batch_size,
            n_lstm_steps=n_frame_step,
            n_video_lstm_step=n_video_lstm_step,
            n_caption_lstm_step=n_caption_lstm_step,
            bias_init_vector=bias_init_vector,
            schedule_p = schedule_sample_probability)

tf_loss, tf_video, tf_video_mask, tf_caption, tf_caption_mask, tf_probs= model.build_model()

## testing graph
video_tf, video_mask_tf, caption_tf, probs_tf, last_embed_tf = model.build_generator()

test_path = listdir(test_folder_path)
test_features = [ (file[:-4],np.load(test_folder_path + file)) for file in test_path]
test = json.load(open(test_label_path,'r'))
ixtoword_series = pd.Series(np.load('./ixtoword_pyf.npy').tolist())

sess = tf.InteractiveSession()
saver = tf.train.Saver(max_to_keep=100)

train_op = tf.train.AdamOptimizer(0.001).minimize(tf_loss)
tf.global_variables_initializer().run()

loss_fd = open('loss3_pyf.txt', 'w')
bleu_fd = open('bleu3_pyf.txt', 'w')
loss_to_draw = []

X_y_pairs = []
target_texts = []

words_list = []
for ID in train_ID_list:
#     text = np.random.choice(train_label_dict[ID],1)[0]
    print(ID)
    for text in train_label_dict[ID]:
        X_y_pairs.append((train_feature_dict[ID],text))
        words = text.split()
        target_texts.append(words)
        for word in words:
            words_list.append(word)
target_words_set = np.unique(words_list, return_counts=True)[0]
num_decoder_tokens = len(target_words_set)
max_decoder_seq_length = max([len(txt) for txt in target_texts])
print("sample counts", np.shape(X_y_pairs))
print("here")
print("number of decoder tokens", num_decoder_tokens)
print("max length of label", max_decoder_seq_length)

print("tuple first element shape", X_y_pairs[1][0].shape)
print("tuple second element:", X_y_pairs[1][1])

print("tuple second element:", X_y_pairs[1][1])
loss_to_draw_epoch = []
print("Total sample:", sample_size)
for epoch in range(0, n_epochs):
    random.seed(epoch)
    random.shuffle(X_y_pairs)
    X_y_pairs_sub =  random.sample(X_y_pairs,sample_size)
    # modify schedule p
    '''
    if epoch<decay_epoch:
        model.schedule_p = 1
    else:
        #linear
        model.schedule_p = np.max([1-(epoch/decay_epoch-1), 0])
    '''
        #inversesigmoid decay
        #model.schedule_p = decay_epoch/(decay_epoch+np.exp(epoch/decay_epoch))

    for batch_start, batch_end in zip(range(0, sample_size, batch_size), range(batch_size, sample_size, batch_size)):
        start_time = time.time()
        current_batch = X_y_pairs_sub[batch_start:batch_end]
        current_feats = [ row[0] for row in current_batch]
        #current_feats = np.array(current_feats)
        #current_feats = np.reshape(current_feats, (32, 1, 11264))
        current_video_masks = np.zeros((batch_size, n_video_lstm_step))

        current_captions = np.array(["<bos> "+ row[1] for row in current_batch])
        for idx, single_caption in enumerate(current_captions):
            word = single_caption.lower().split(" ")
            if len(word) < n_caption_lstm_step:
                current_captions[idx] = current_captions[idx] + " <eos>"
            else:
                new_word = ""
                for i in range(n_caption_lstm_step-1):
                    new_word = new_word + word[i] + " "
                current_captions[idx] = new_word + "<eos>"
        current_caption_ind = []
        for cap in current_captions:
            current_word_ind = []
            for word in cap.lower().split(' '):
                if word in wordtoix:
                    current_word_ind.append(wordtoix[word])
                else:
                    current_word_ind.append(wordtoix['<unk>'])
            current_caption_ind.append(current_word_ind)

        current_caption_matrix = sequence.pad_sequences(current_caption_ind, padding='post', maxlen=n_caption_lstm_step)
        current_caption_matrix = np.hstack( [current_caption_matrix, np.zeros( [len(current_caption_matrix), 1] ) ] ).astype(int)
        current_caption_masks = np.zeros( (current_caption_matrix.shape[0], current_caption_matrix.shape[1]) )
        nonzeros = np.sum((current_caption_matrix != 0),1) # 算每個row 有幾個字
        for ind, row in enumerate(current_caption_masks):
            row[:nonzeros[ind]] = 1 # 把前幾個有字的在mask上塗成1
        '''
        probs_val = sess.run(tf_probs, feed_dict={
                    tf_video:current_feats,
                    tf_caption: current_caption_matrix
                    })
        '''
        _, loss_val = sess.run(
                        [train_op, tf_loss],
                        feed_dict={
                            tf_video: current_feats,
                            tf_video_mask : current_video_masks,
                            tf_caption: current_caption_matrix,
                            tf_caption_mask: current_caption_masks
                            })
        loss_to_draw_epoch.append(loss_val)
    print (" Epoch: ", epoch, " loss: ", loss_val, ' Elapsed time: ', str((time.time() - start_time)))
    loss_fd.write('epoch ' + str(epoch) + ' loss ' + str(loss_val) + '\n')

    test_sentences = []
    id_list = []

    #validation
    random.shuffle(test_features)
    #test_features_sub =  random.sample(test_features, 50)
    for idx, video_feat in test_features:
        video_feat = video_feat.reshape(1, n_frame_step, dim_image)
        if video_feat.shape[1] == n_frame_step:
            video_mask = np.ones((video_feat.shape[0], video_feat.shape[1]))
        generated_word_index = sess.run(caption_tf, feed_dict={video_tf:video_feat, video_mask_tf:video_mask})
        generated_words = ixtoword_series[generated_word_index]
        generated_sentence = ' '.join(generated_words)
        generated_sentence = generated_sentence.replace('<bos> ', '')
        generated_sentence = generated_sentence.replace(' <eos>', '')
        generated_sentence = generated_sentence.replace('<pad> ', '')
        generated_sentence = generated_sentence.replace(' <pad>', '')
        generated_sentence = generated_sentence.replace(' <unk>', '')
        id_list.append(idx)
        test_sentences.append(generated_sentence)
        if idx in ["video6632","video6551","video6514","video6590","video6587"] and np.mod(epoch, 5) == 0:
            print(idx)
            print(generated_sentence)
    submit = pd.DataFrame(np.array([id_list,test_sentences]).T)
    submit.to_csv("ADL_hw2_pyf.txt",index = False,  header=False)
    result = {}
    with open("ADL_hw2_pyf.txt",'r') as f:
        for line in f:
            line = line.rstrip()
            comma = line.index(',')
            test_id = line[:comma]
            caption = line[comma+1:]
            result[test_id] = caption
    '''
    Calcalate the bleu score
    '''

    #count by the method described in the paper https://aclanthology.info/pdf/P/P02/P02-1040.pdf
    bleu=[]
    for item in test:
        score_per_video = []
        captions = [x.rstrip('.') for x in item['caption']]
        score_per_video.append(bl.BLEU(result[item['id']], captions, 4, True))
        bleu.append(score_per_video[0])
    average = sum(bleu) / len(bleu)

    print("By another method, average bleu score is " + str(average))

    bleu_fd.write('epoch: ' + str(epoch) + ' bleu: ' + str(average) + '\n')
    loss_to_draw.append(np.mean(loss_to_draw_epoch))
    plt_save_dir = "./loss_imgs"
    plt_save_img_name = str(epoch) + '.png'
    plt.plot(range(len(loss_to_draw)), loss_to_draw, color='g')
    plt.grid(True)
    plt.savefig(os.path.join(plt_save_dir, plt_save_img_name))

    if average >  0.23:
        print ("Epoch ", epoch, " is done. Saving the model ...")
        saver.save(sess, os.path.join(model_path, 'model'+str(average)[2:6]), global_step=epoch)


#!/usr/bin/python3
import gi
import trackpoint

gi.require_version("Gtk", "3.0")
from gi.repository import Gtk
tp_values=trackpoint.retrieve_config_settings('CURRENT')
class MyWindow(Gtk.Window):
    def __init__(self) -> None:
        self.rowSkip = 0
        super().__init__(title="Trackpoint Settings")
        self.Box1 = Gtk.VBox(spacing=6)
        self.add(self.Box1)
        self.Grid1 = Gtk.Grid()
        self.Grid1.set_column_homogeneous(True)
        self.Grid1.set_row_baseline_position(1, 1)
        self.Box1.pack_start(self.Grid1, True, True, 0)
        self.ButtonBox = Gtk.Box(spacing=6)
        self.Box1.pack_start(self.ButtonBox, False, False, 0)

        self.tp_values = trackpoint.retrieve_config_settings('CURRENT')

        self.Widgets = []
        for count, item in enumerate(trackpoint.tp_labels):
            if (item[1] == trackpoint.scale):
                WidgetHolder = Gtk.Label.new(item[0])
                self.Widgets.append(WidgetHolder)
                WidgetHolder = Gtk.Scale.new_with_range(0,0,item[2],1)
                WidgetHolder.set_round_digits(1)
                WidgetHolder.set_value(float(tp_values[count]))
            if (item[1] == trackpoint.checkbutton):
                WidgetHolder = Gtk.CheckButton.new_with_label(item[0])
                WidgetHolder.set_halign(Gtk.Align.CENTER)
            self.Widgets.append(WidgetHolder)

        nextColumn= 0
        nextRow = 0
        for count, item in enumerate(self.Widgets):
            if (count + self.rowSkip >= 12):
               nextColumn = 2
               nextRow = 11
            if ((count + self.rowSkip) % 2 == 0):
                if (item.get_name() == "GtkCheckButton"):
                    self.Grid1.attach(item, nextColumn, count-nextRow, 1, 1)
                    self.rowSkip += 1
                else:
                    self.Grid1.attach(item, nextColumn, count-nextRow, 1, 1)
            if ((self.rowSkip + count) % 2 != 0):
                self.Grid1.attach(item, nextColumn+1, count-nextRow-1, 1, 1)

        self.DefaultButton = Gtk.Button(label="Default")
        self.DefaultButton.connect("clicked", self.on_default_clicked)
        self.ButtonBox.pack_start(self.DefaultButton, False, False, 0)

        self.AdvancedButton = Gtk.Button(label="Advanced >>")
        self.AdvancedButton.connect("clicked", self.on_advanced_clicked)
        self.ButtonBox.pack_start(self.AdvancedButton, False, False, 0)

        self.ApplyButton = Gtk.Button(label="Apply")
        self.ApplyButton.connect("clicked", self.on_apply_clicked)
        self.ButtonBox.pack_end(self.ApplyButton, False, False, 0)

    def on_advanced_clicked(self,widget):
        if (widget.get_label() != "Advanced <<"):
            widget.set_label("Advanced <<")
            win.show_all()
            self.resize(700,250)
        else:
            widget.set_label("Advanced >>")
            for count, item in enumerate (self.Widgets):
                if (count > 4):
                    item.hide()
            self.resize(400,100)

    def on_default_clicked(self,widget):
        trackpoint.apply_stored_settings('DEFAULT')
        values=trackpoint.retrieve_config_settings('DEFAULT')

    def on_apply_clicked(self,widget):
        valueCount = 0
        for item in self.Widgets:
            if (item.get_name() == "GtkCheckButton"):
                trackpoint.set_setting(trackpoint.tp_path + trackpoint.tp_labels[valueCount] [0], \
                    int(item.get_active()))
                valueCount += 1
            if (item.get_name() == "GtkScale"):
                trackpoint.set_setting(trackpoint.tp_path + trackpoint.tp_labels[valueCount] [0], \
                    int(item.get_value()))
                valueCount +=1
        trackpoint.store_current_settings()

    def on_scale_draw(self,widget,something):
            widget.set_value(200)

    def initial_show(self):
        win.show_all()
        for count, item in enumerate (self.Widgets):
            if (count > 4):
                item.hide()
        self.resize(200,50)

win = MyWindow()
win.connect("destroy", Gtk.main_quit)
win.initial_show()
Gtk.main()


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

import numpy as np
import pickle
import cv2
from scipy import interpolate, linalg
from reconstruction import epipolar
from tools import util

def sync_iter(fps1, fps2, detect1, detect2, frame1, frame2, maxIter=200, threshold=10, step=10, p_min=0, p_max=6, verbose=False):
    '''
    This function is a modified implementation of the Iterative Algorithm from the following paper

    >>> Albl, Cenek, et al. "On the two-view geometry of unsynchronized cameras." Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2017.

    The function returns both the estimated time shift (beta) and the according ratio of inliers, which can be considered as scoring
    '''

    def solver(s1,s2,ds,d):
        '''
        This function reads data of 9 points and return a list of possible solution for Beta and F
        '''

        # Create design matrices A1, A2
        A1 = np.array([s1[0]*s2[0],s1[0]*s2[1],s1[0], s1[1]*s2[0],s1[1]*s2[1],s1[1], s2[0],s2[1],np.ones(9)]).T
        A2 = np.array([s1[0]*ds[0],s1[0]*ds[1],np.zeros(9), s1[1]*ds[0],s1[1]*ds[1],np.zeros(9), ds[0],ds[1],np.zeros(9)]).T

        # Compute eigenvalue
        w = -linalg.eigvals(A1,A2)
        w[np.iscomplex(w)] = np.inf
        betas = w.real[np.isfinite(w)]

        # compute fundamental matrix F using shifted data
        M = np.empty((0,10))
        for beta in betas:
            x2 = s2 + beta*ds
            # F = ep.compute_fundamental(s1,x2)
            F, mask = cv2.findFundamentalMat(s1[:2].T,x2[:2].T,method=cv2.FM_8POINT)

            if len(np.ravel(F)) == 9:
                M_i = np.append(np.ravel(F),np.array([beta*d]))
                M = np.vstack((M, M_i))

        return M

    def error(M):
        try:
            detect2_temp = np.vstack((detect2[0]-M[-1],detect2[1:]))
            pts1, pts2 = util.match_overlap(detect1, detect2_temp)
            return epipolar.Sampson_error(util.homogeneous(pts1[1:]), util.homogeneous(pts2[1:]), M[:9].reshape((3,3)))
        except:
            return None

    def ransac(d, maxIter, threshold):
        InliersMax = 0
        numSample = min(detect1.shape[1], detect2.shape[1])
        result = np.zeros((10))
        for i in range(maxIter):
            sampleIdx = np.random.choice(np.arange(numSample-2*abs(d)), size=9, replace=False)
            timestamp = detect2[0,sampleIdx]
            s1 = np.asarray(interpolate.splev(timestamp, spline1))
            s2 = np.asarray(interpolate.splev(timestamp, spline2))
            ds = np.asarray(interpolate.splev(timestamp+d, spline2)) - s2
            M = solver(s1,s2,ds,d)
            if len(M) is not 0:
                if len(M.shape)==1:
                    M = np.expand_dims(M,axis=0)
                for Mi in M:
                    err = error(Mi)
                    if err is not None:
                        numInliers = np.sum(err<threshold) / len(err)
                        if numInliers > InliersMax:
                            result = Mi
                            InliersMax = numInliers
        return -result[-1], InliersMax

    # Pre-processing
    alpha = fps1 / fps2
    beta_prior = frame1/alpha - frame2
    detect1_ori, detect2_ori = detect1, detect2
    detect1 = np.vstack((detect1[0]/alpha,detect1[1:]))
    detect2 = np.vstack((detect2[0]+beta_prior,detect2[1:]))

    spline1, _ = interpolate.splprep(detect1[1:], u=detect1[0], s=0, k=3)
    spline2, _ = interpolate.splprep(detect2[1:], u=detect2[0], s=0, k=3)

    # The Iterative Algorithm
    skip, maxInlier, k = 0, 0, 0
    d, p, beta = 2**p_min, p_min, beta_prior

    while k < step:

        # Ransac with d
        beta1, Inlier1 = ransac(d=d, maxIter=maxIter, threshold=threshold)

        # Ransac with -d
        beta2, Inlier2 = ransac(d=-d, maxIter=maxIter, threshold=threshold)

        # Select the better one
        beta_temp = beta1 if Inlier1 >= Inlier2 else beta2
        Inlier = Inlier1 if Inlier1 >= Inlier2 else Inlier2

        if verbose:
            print('d:{}, beta:{:.3f}, maxInlier:{}, Inlier:{}'.format(d, (beta+beta_temp)*alpha, maxInlier, Inlier))

        if skip > p_max:
                break
        elif Inlier < maxInlier:
            if p < p_max:
                p += 1
            else:
                p = 0
            d = 2**p
            skip += 1
        else:
            beta += beta_temp
            maxInlier = Inlier
            detect2 = np.vstack((detect2_ori[0]+beta,detect2_ori[1:]))
            spline2, _ = interpolate.splprep(detect2[1:], u=detect2[0], s=0, k=3)
            skip = 0
            k += 1

    return beta*alpha, maxInlier

def sync_bf(fps1, fps2, detect1, detect2, frame1, frame2, r=10):
    '''
    Brute-force method for temporal synchronization of two series of detections

    r is the half length of the search interval in unit second

    Function returns both the time shift (beta) and the temporal overlap of the two series of detections in unit second
    '''

    def search(beta_list, thres=8):
        maxInlier = 0
        beta_est = 0
        for beta in beta_list:
            detect2_temp = np.vstack((detect2[0]+beta,detect2[1:]))
            pts1, pts2 = util.match_overlap(detect1_temp, detect2_temp)

            F, mask = cv2.findFundamentalMat(pts1[1:].T, pts2[1:].T, method=cv2.FM_RANSAC, ransacReprojThreshold=thres)
            inlier = sum(mask.reshape(-1,)) #/ len(pts1[0])

            if inlier > maxInlier:
                maxInlier = inlier
                beta_est = beta

        return beta_est, maxInlier

    # Pre-processing
    alpha = fps1 / fps2
    detect1_temp = np.vstack((detect1[0]/alpha,detect1[1:]))
    beta_prior = frame1/alpha - frame2

    # Two-stage search
    beta_coarse = np.arange(beta_prior-r*fps2, beta_prior+r*fps2, fps2)
    beta_est, _ = search(beta_coarse)
    beta_fine = np.arange(beta_est-fps2/2, beta_est+fps2/2, fps2/20)
    beta_est, numInlier  = search(beta_fine)

    # Result
    beta = beta_est * alpha
    overlap_second = numInlier/fps1

    return beta, overlap_second

if __name__ == "__main__":

    from reconstruction import common
    import cProfile
    from datetime import datetime

    path = ''
    with open(path, 'rb') as file:
        flight0 = pickle.load(file)

    flight = common.create_scene('')
    flight.cut_detection(second=flight.settings['cut_detection_second'])

    i, j = 0, 1
    fps1, fps2 = flight.cameras[i].fps, flight.cameras[j].fps
    detect1, detect2 = flight.detections[i], flight.detections[j]
    frame1, frame2 = flight.cf[i]-20, flight.cf[j]+30

    start = datetime.now()
    print('Correct beta: {:.3f}\n'.format(flight0.beta[j]))
    cProfile.run("beta, InlierRatio = sync_iter(fps1, fps2, detect1, detect2, frame1, frame2, p_max=5, threshold=10)",sort='cumulative')
    # beta, InlierRatio = sync_iter(fps1, fps2, detect1, detect2, frame1, frame2, p_max=5, threshold=15)
    # beta, overlap = sync_bf(fps1, fps2, detect1, detect2, frame1, frame2)

    print('\nTotal time: {}\n\n\n'.format(datetime.now()-start))
    print('Finished!')


"""
General Game Playing script. Trains a Tangled Program Graph model on multiple games.
Created by: Ryan Amaral
Created on: Feb. 25, 2019 (based on much earlier version)
"""

from __future__ import division
from tpg.tpg_trainer import TpgTrainer
from tpg.tpg_agent import TpgAgent
from optparse import OptionParser
from tpg_helpers import *
import random
import datetime
import multiprocessing as mp
import pickle
import time

"""
Command line arguments.
"""

def separgs(option, opt, value, parser):
    setattr(parser.values, option.dest, value.split(','))

parser = OptionParser()
# number of games to play
parser.add_option('-n', '--ngames', type='int', dest='numGames', default=16)
# number of previous titles to measure fitness on
parser.add_option('-f', '--fitGamesNum', type='int', dest='fitGamesNum', default=-1)
# specify any games
parser.add_option('--games', type='string', action='callback', callback=separgs, dest='games')
# choose games randomly
parser.add_option('-r', '--randg', action='store_true', dest='randomGames', default=False)
# starting tpg population size
parser.add_option('-p', '--pop', type='int', dest='popSize', default=360)
# population size to work up to
parser.add_option('-m', '--popMax', type='int', dest='popSizeMax', default=360)
# concurrent workers for parallization
parser.add_option('-w', '--workers', type='int', dest='workers', default=3)
# method of training
parser.add_option('-t', '--type', type='int', dest='trainType', default=0)
# number of frames to train on
parser.add_option('--trainFrames', type='int', dest='trainFrames', default=18000)
# number of frames to test on
parser.add_option('--testFrames', type='int', dest='testFrames', default=18000)
# number of episodes to train on
parser.add_option('--trainEps', type='int', dest='trainEps', default=3)
# number of episodes to train on
parser.add_option('--testEps', type='int', dest='testEps', default=10)
# every how many generations to do champ eval
parser.add_option('-c', '--champEvalGen', type='int', dest='champEvalGen', default=50)
# file for trainer with pretrained populations
parser.add_option('--popsFile', type='string', dest='popsFile', default=None)

# trainType is 0 for 'all at once', 1 for 'merge', 2 for 'gradual merge'.

(options, args) = parser.parse_args()

"""
General setup for GGP.
"""

# all of the titles we will be general game playing on
# we chose games that we know TPG does at-least OK in alone
allEnvNames = ['Alien-v0','Asteroids-v0','Atlantis-v0','BankHeist-v0',
               'BattleZone-v0','Bowling-v0','Boxing-v0','Centipede-v0',
               'ChopperCommand-v0','DoubleDunk-v0','FishingDerby-v0',
               'Freeway-v0','Frostbite-v0','Gravitar-v0','Hero-v0',
               'IceHockey-v0','Jamesbond-v0','Kangaroo-v0','Krull-v0',
               'KungFuMaster-v0','MsPacman-v0','PrivateEye-v0',
               'RoadRunner-v0','Skiing-v0','Tennis-v0','TimePilot-v0',
               'UpNDown-v0','Venture-v0','WizardOfWor-v0','Zaxxon-v0']

if options.games is not None:
    allEnvNames = sorted(options.games)
else:
    # shuffle if random and take certain number of games
    if options.randomGames:
        random.shuffle(allEnvNames)
    allEnvNames = sorted(allEnvNames[:options.numGames])
    #allEnvNames = ['Assault-v0', 'Boxing-v0'] # hardcode in games to use

print('All Games: ' + str(allEnvNames))

pool = mp.Pool(processes=options.workers, initializer=limit_cpu, maxtasksperchild=1)
man = mp.Manager() # manager for shared memory lists

"""
Create and initialize log files.
"""

timeStamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M")

# game score summary of each game in each generation
logFileGameScoresName = 'ggp-log-gamescores-' + timeStamp + '.txt'
with open(logFileGameScoresName, 'a') as f:
    f.write('tpgGen,hoursElapsed,envName,tpgMin,tpgMax,tpgAvg,eliteSize,eliteUid\n')

# every few generations evaluate top overall few teams on all titles
logFileChampionsName = 'ggp-log-champions-' + timeStamp + '.txt'
with open(logFileChampionsName, 'a') as f:
    f.write('tpgGen,hoursElapsed,teamId')
    for envName in allEnvNames:
        f.write(',score' + envName)
    for envName in allEnvNames:
        f.write(',vis' + envName)
    f.write(',visTotal\n')

# every few generations, save the outcomes of the root teams
logFileOutcomesName = 'ggp-log-outcomes-' + timeStamp + '.txt'
with open(logFileOutcomesName, 'a') as f:
    f.write('tpgGen,hoursElapsed,teamId')
    for envName in allEnvNames:
        f.write(',score' + envName)
    f.write('\n')

# population fitness summary each generation
logFileFitnessName = 'ggp-log-fitness-' + timeStamp + '.txt'
with open(logFileFitnessName, 'a') as f:
    f.write('tpgGen,hoursElapsed,envs,fitMin,fitMax,fitAvg,champSize,champUid,totalTeams,totalRootTeams\n')

# and create file to save TPG
trainerFileName = 'tpg-trainer-' + timeStamp + '.pkl'

"""
Method for training TPG on all games at once. Each individual in the single population
will see all of the games before evolution will occur.
"""
def ggpTrainAllAtOnce():

    # create TPG
    trainer = TpgTrainer(actions=range(18), teamPopSize=options.popSize,
                    rTeamPopSize=options.popSize, maxProgramSize=128)

    envNamesSrt = sorted(list(allEnvNames)) # for reporting envs played

    fitnessEnvs = []

    while True: # train indefinately
        print('TPG Gen: ' + str(trainer.populations[None].curGen))

        if options.fitGamesNum < options.numGames and options.fitGamesNum > 0:
            fitnessEnvs.append(random.choice(list(set(allEnvNames)-set(fitnessEnvs))))
            if len(fitnessEnvs) > options.fitGamesNum:
                fitnessEnvs.pop(0)
        else:
            fitnessEnvs = list(allEnvNames)

        for envName in fitnessEnvs:
            print('Playing Game: ' + envName)

            scoreList = man.list()

            # run all agents on env
            pool.map(runAgent,
                [(agent, envName, scoreList, options.trainEps, options.trainFrames, None)
                    for agent in trainer.getAllAgents(skipTasks=[envName], noRef=True)])

            trainer.applyScores(scoreList)

            # report curEnv results to log
            scoreStats = trainer.getTaskScores(envName)
            bestTeam = trainer.getBestTeams(tasks=[envName])[0]
            with open(logFileGameScoresName, 'a') as f:
                f.write(str(trainer.populations[None].curGen) + ','
                    + str((time.time()-tstart)/3600) + ','
                    + envName + ','
                    + str(scoreStats['min']) + ','
                    + str(scoreStats['max']) + ','
                    + str(scoreStats['average']) +  ','
                    + str(len(bestTeam.getRootTeamGraph()[0])) + ','
                    + str(bestTeam.uid) + '\n')

        # do evolution after all envs played
        trainer.multiEvolve(tasks=[fitnessEnvs]+[[en] for en in fitnessEnvs],
                            weights=[0.5]+[0.5/len(fitnessEnvs) for _ in fitnessEnvs],
                            fitMethod='min', elitistTasks=allEnvNames)

        # report generational fitness results
        bestTeam = trainer.getBestTeams(tasks=fitnessEnvs)[0]
        with open(logFileFitnessName, 'a') as f:
            f.write(str(trainer.populations[None].curGen) + ','
                + str((time.time()-tstart)/3600) + ','
                + '/'.join(sorted(list(fitnessEnvs))) + ','
                + str(trainer.populations[None].scoreStats['min']) + ','
                + str(trainer.populations[None].scoreStats['max']) + ','
                + str(trainer.populations[None].scoreStats['average']) +  ','
                + str(len(bestTeam.getRootTeamGraph()[0])) + ','
                + str(bestTeam.uid) + ','
                + str(len(trainer.populations[None].teams)) + ','
                + str(len(trainer.populations[None].rootTeams)) + '\n')

        # save model after every gen
        with open(trainerFileName,'wb') as f:
            pickle.dump(trainer,f)

        # every 10 generations save all agent outcomes
        if trainer.populations[None].curGen % 10 == 0:
            for rt in trainer.populations[None].rootTeams:
                with open(logFileOutcomesName, 'a') as f:
                    f.write(str(trainer.populations[None].curGen) + ','
                        + str((time.time()-tstart)/3600) + ','
                        + str(rt.uid))
                    for envName in allEnvNames:
                        f.write(',' + str(rt.outcomes.get(envName, '-999999')))
                    f.write('\n')

        # every 50 generations evaluate top agents on all games
        if trainer.populations[None].curGen % options.champEvalGen == 0:
            champEval(envNamesSrt, trainer, logFileChampionsName, pool, man,
                        tstart, frames=options.testFrames, eps=options.testEps)

"""
Method for training TPG on each game separately, in separate populations. All of
the populations will be merged together at once.
"""
def ggpTrainMerge():
    # create TPG from existing file of populations
    trainer = pickle.load(open(options.popsFile, 'rb'))

    envNamesSrt = sorted(list(allEnvNames)) # for reporting envs played

    fitnessEnvs = []

    # merge populations together
    prevName = envNamesSrt[0]
    for i in range(len(envNamesSrt)-1):
        if i+1 < len(envNamesSrt)-1:
            curName = 'tmp'+str(i)
        else:
            curName = None

        trainer.merge2Populations(prevName, envNamesSrt[i+1], curName,
                teamPopSize=options.popSize, rTeamPopSize=options.popSize)
        prevName = curName

    # continue on as regular all at once
    while True: # train indefinately
        print('TPG Gen: ' + str(trainer.populations[None].curGen))

        if options.fitGamesNum < options.numGames and options.fitGamesNum > 0:
            fitnessEnvs.append(random.choice(list(set(allEnvNames)-set(fitnessEnvs))))
            if len(fitnessEnvs) > options.fitGamesNum:
                fitnessEnvs.pop(0)
            else:
                fitnessEnvs = list(allEnvNames)

        for envName in fitnessEnvs:
            print('Playing Game: ' + envName)

            scoreList = man.list()

            # run all agents on env
            pool.map(runAgent,
                [(agent, envName, scoreList, options.trainEps, options.trainFrames, None)
                    for agent in trainer.getAllAgents(skipTasks=[envName], noRef=True)])

            trainer.applyScores(scoreList)

            # report curEnv results to log
            scoreStats = trainer.getTaskScores(envName)
            bestTeam = trainer.getBestTeams(tasks=[envName])[0]
            with open(logFileGameScoresName, 'a') as f:
                f.write(str(trainer.populations[None].curGen) + ','
                    + str((time.time()-tstart)/3600) + ','
                    + envName + ','
                    + str(scoreStats['min']) + ','
                    + str(scoreStats['max']) + ','
                    + str(scoreStats['average']) +  ','
                    + str(len(bestTeam.getRootTeamGraph()[0])) + ','
                    + str(bestTeam.uid) + '\n')

        # do evolution after all envs played
        trainer.multiEvolve(tasks=[fitnessEnvs]+[[en] for en in fitnessEnvs],
                            weights=[0.5]+[0.5/len(fitnessEnvs) for _ in fitnessEnvs],
                            fitMethod='min', elitistTasks=allEnvNames)

        # report generational fitness results
        bestTeam = trainer.getBestTeams(tasks=fitnessEnvs)[0]
        with open(logFileFitnessName, 'a') as f:
            f.write(str(trainer.populations[None].curGen) + ','
                + str((time.time()-tstart)/3600) + ','
                + '/'.join(sorted(list(fitnessEnvs))) + ','
                + str(trainer.populations[None].scoreStats['min']) + ','
                + str(trainer.populations[None].scoreStats['max']) + ','
                + str(trainer.populations[None].scoreStats['average']) +  ','
                + str(len(bestTeam.getRootTeamGraph()[0])) + ','
                + str(bestTeam.uid) + ','
                + str(len(trainer.populations[None].teams)) + ','
                + str(len(trainer.populations[None].rootTeams)) + '\n')

        # save model after every gen
        with open(trainerFileName,'wb') as f:
            pickle.dump(trainer,f)

        # every 50 generations evaluate top agents on all games
        if trainer.populations[None].curGen % options.champEvalGen == 0:
            champEval(envNamesSrt, trainer, logFileChampionsName, pool, man,
                        tstart, frames=options.testFrames, eps=options.testEps)

"""
Method for training TPG on each game separately, in separate populations. The populations
will be merged gradually, meaning that first separate populations will be paired,
then those pairs will be paired, and so on untill in one population.
"""
def ggpTrainGradualMerge(envNames, popSize, popSizeMax, lfGSName, lfCName, lfFName,
        pool, man):
    pass

"""
Run the training unending.
"""
tstart = time.time() # start timing

if options.trainType == 0:
    ggpTrainAllAtOnce()
elif options.trainType == 1:
    ggpTrainMerge()
else:
    ggpTrainGradualMerge()