import pandas as pd
from bs4 import BeautifulSoup
import requests
import re
import os
import calendar
import time
import csv
from fuzzywuzzy import process
from fuzzywuzzy import fuzz

cfbr_folder_path = 'cfbr_scrape\\csv\\'
pos_group_df = pd.read_csv(cfbr_folder_path + 'cfbr_pos_group.csv', header = 0)

header = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1216.0 Safari/537.2'}

remove_char = '([\\t\'\.\,]+)|([^\x00-\x7F]+)| (i+(v*)$|(j|s)r)'

def team_info_cfbr_scrape():
    try:
        team_info_df = pd.read_csv(cfbr_folder_path + 'cfbr_team_info.csv', header = 0)

        modified_time = os.path.getmtime(cfbr_folder_path + 'cfbr_team_info.csv')
        current_time = calendar.timegm(time.gmtime())
        if float(current_time - modified_time)/(60*60*24) > 365:
            pass
        else:
            return team_info_df
    except:
        pass

    team_info_df = pd.DataFrame(columns = ['team_fullname', 'team_schoolname','team_href', 'capacity', 'city'])

    cap_re = re.compile('(?<=cap\. ).*(?=\))')
    city_re = re.compile('(?<=Location:).*(?= )')

    url = 'https://www.sports-reference.com/cfb/schools/'
    req = requests.get(url, headers = header)
    soup = BeautifulSoup(req.content, 'lxml')

    teams = soup.find('table', {'id': 'schools'}).find('tbody').find_all('tr', {'class': None})
    for team in teams:
        max_year = int(team.contents[3].text)
        if max_year > 1980:
            name = team.contents[1].text
            href = team.contents[1].contents[0].get('href')
            print(name,href)

            info_url = 'https://www.sports-reference.com' + href
            info_req = requests.get(info_url, headers = header)
            info_soup = BeautifulSoup(info_req.content, 'lxml')

            school_info = info_soup.find('div', {'id': 'info'}).contents[1].contents[3]
            full_name = school_info.find('h1').contents[1].text
            cap = int(re.sub(',','',cap_re.search(school_info.text).group(0))) if cap_re.search(school_info.text) else None
            city = city_re.search(school_info.text).group(0).strip() if city_re.search(school_info.text) else None

            team_info_df.loc[len(team_info_df),] = [full_name,name,href,cap,city]

    for index, row in team_info_df.iterrows():
        name_split = row['team_schoolname'].split()
        abv = ''
        if len(name_split) > 1:
            for split in name_split:
                abv += split[0]
        abv = abv + ' ' + re.sub(row['team_schoolname'] + ' ','',row['team_fullname'])
        team_info_df.set_value(index,'team_abvname', row['team_fullname'] if (len(name_split) < 2) else abv)

    team_info_df.to_csv(cfbr_folder_path + 'cfbr_team_info.csv', index = False)
    return team_info_df

def roster_scrape(season, team_href, team_name):
    roster_df = pd.DataFrame(columns = ['season', 'team_href', 'team_schoolname', 'player_name', 'player_href', 'class', 'pos'])

    roster_url = 'https://www.sports-reference.com' + team_href + str(season) + '-roster.html'
    for i in range(0,100):
        try:
            roster_req = requests.get(roster_url, headers = header)
            break
        except requests.exceptions.ConnectionError:
            time.sleep(2)
            pass
    roster_soup = BeautifulSoup(roster_req.content, 'lxml')

    try:
        players = roster_soup.find('table', {'id': 'roster'}).find('tbody').find_all('tr', {'class': None})
    except:
        return
    for player in players:
        player_name = player.contents[0].text[:-1] if player.contents[0].text[-1] == '*' else player.contents[0].text.strip()
        if (' ' in player_name):
            player_name = re.sub(remove_char,'',player_name.lower())
        else:
            continue

        try:
            player_href = player.contents[0].contents[0].get('href')
        except:
            player_href = None
        class_ = None if player.contents[1].text == '' else player.contents[1].text
        pos = None if player.contents[2].text == '' else player.contents[2].text

        roster_df.loc[len(roster_df),] = [season, team_href, team_name, player_name, player_href, class_, pos]

    for index2, row2 in roster_df.loc[pd.isnull(roster_df['player_href'])].iterrows():
        match = process.extractOne(row2['player_name'], roster_df.loc[(pd.isnull(roster_df['player_href']) == False),'player_name'])
        if match[1] > 90:
            roster_df.drop(index2, inplace = True)

    roster_df = roster_df.reset_index(drop = True)
    return roster_df

def stats_scrape(roster_df,team_name,skip_stats):
    team_href = roster_df.iloc[0]['team_href']
    season = int(roster_df.iloc[0]['season'])

    stats_url = 'https://www.sports-reference.com' + team_href + str(season) + '.html'
    for i in range(0,100):
        try:
            stats_req = requests.get(stats_url, headers = header)
            break
        except requests.exceptions.ConnectionError:
            time.sleep(2)
            pass
    stats_soup = BeautifulSoup(stats_req.content, 'lxml')

    for table_type in ['passing','rushing_and_receiving','defense_and_fumbles','returns','kicking_and_punting']:
        try:
            table_soup = BeautifulSoup(stats_soup.find('div', {'id': 'all_' + table_type}).contents[-2], 'lxml')
        except:
            continue
        player_rows = table_soup.find('tbody').find_all('tr')
        for player_row in player_rows:
            try:
                player_loc = player_row.contents[1].contents[0].get('href')
                var = 'player_href'
            except:
                player_loc = player_row.contents[1].text
                player_loc = re.sub(remove_char,'',player_loc.lower().strip())
                var = 'player_name'
            if player_loc not in list(roster_df[var]):
                player_href = player_loc if var == 'player_href' else None
                player_name = re.sub(remove_char,'',player_row.contents[1].contents[0].text.lower().strip())\
                    if (var == 'player_href') else player_loc
                if (' ' in player_name):
                    row_list = [season,team_href,team_name,player_name,player_href]
                    roster_df.loc[len(roster_df)] = row_list + [None] * (len(roster_df.columns) - 5)
                else:
                    continue
            index = roster_df.loc[roster_df[var] == player_loc].iloc[0].name
            for stat in player_row.contents[2:]:
                roster_df.set_value(index,stat.get('data-stat'),float(0 if ((stat.text == '') or (skip_stats == True)) else stat.text))

    stats_df = roster_df.drop_duplicates()
    stats_df['dedupe'] = 0
    return stats_df

def active_roster(min_season, current_date):
    try:
        active_roster_df = pd.read_csv(cfbr_folder_path + 'cfbr_active_rosters.csv', header = 0)

        modified_time = os.path.getmtime(cfbr_folder_path + 'cfbr_team_info.csv')
        roster_year = current_date[0] if (current_date[1] > 7) else current_date[0] - 1
        new_roster_time = calendar.timegm(time.strptime(str(roster_year)+str(8),'%Y%m'))
        current_time = calendar.timegm(time.gmtime())

        time_since_modify = current_time - modified_time
        time_since_roster = current_time - new_roster_time
        if time_since_modify > time_since_roster:
            pass
        else:
            return active_roster_df
    except:
        pass

    active_roster_df = pd.DataFrame(columns = ['season','team_href'])

    school_re = re.compile('.*(?=[0-9]{4}\.html)')

    for season in range(min_season, roster_year + 1):
        url = 'https://www.sports-reference.com/cfb/years/' + str(season) + '-standings.html'
        req = requests.get(url, headers = header)
        soup = BeautifulSoup(req.content, 'lxml').find('table', {'id': 'standings'})

        teams = soup.find('tbody').find_all('tr')
        for team in teams:
            try:
                href = school_re.search(team.contents[1].contents[0].get('href')).group(0)
                active_roster_df.loc[len(active_roster_df)] = [season,href]
            except:
                continue

    active_roster_df.to_csv(cfbr_folder_path + 'cfbr_active_rosters.csv', index = False)
    return active_roster_df

def href_dedupe_process(seasons,team_href,match_list):
    temp_df = pd.DataFrame(columns = ['player_href','player_name','season','team_href','class','class_ind','pos_group_list','phys'])
    for href in match_list:
        if 'custom_href' in href:
            continue
        player_url = 'https://www.sports-reference.com' + href
        for i in range(0,100):
            try:
                player_req = requests.get(player_url, headers = header)
                break
            except requests.exceptions.ConnectionError:
                time.sleep(2)
                pass
        player_soup = BeautifulSoup(player_req.content, 'lxml')

        name = player_soup.find('div', {'id': 'info'}).contents[1].contents[1].contents[1].text.strip().lower()
        name = re.sub(remove_char,'',name)
        try:
            pos_list = re.sub(': ','',
                              player_soup.find('div', {'id': 'info'}).contents[1].contents[1].contents[4].contents[2].strip()).split('/')
            pos_group_list = []
            for pos in pos_list:
                if pos in list(pos_group_df['pos']):
                    pos_group_list.append(pos_group_df.loc[pos_group_df['pos'] == pos,'pos_group'].iloc[0])
                else:
                    pos_group_list.append('unknwn')
        except:
            pos_group_list = ['unknwn']
        try:
            phys = True if player_soup.find('div', {'id': 'info'}).contents[1].contents[1].contents[6].text.strip() != '' else False
        except:
            phys = False

        player_tables = player_soup.find_all('tbody')
        for table in player_tables:
            player_rows = table.find_all('tr')
            for player_row in player_rows:
                season = int(player_row.contents[0].text) if player_row.contents[0].text[0] != '*'\
                            else int(player_row.contents[0].text[1:])
                new_team_href = re.sub('[0-9]{4}\.html','',player_row.contents[1].contents[0].get('href'))
                class_ = None if player_row.contents[3].text == '' else player_row.contents[3].text
                class_ind = 0 if class_ == None else 1
                temp_df.loc[len(temp_df)] = [href,name,season,new_team_href,class_,class_ind,pos_group_list,phys]

    rem_href = []
    for href in match_list:
        prev_list = []
        prev_class = None
        for index, row in temp_df.loc[temp_df['player_href'] == href].iterrows():
            if row['class'] in prev_list:
                if row['class'] == prev_class:
                    continue
                else:
                    rem_href.append(href)
            else:
                prev_list.append(row['class'])
                prev_class = row['class']
    temp_df = temp_df.loc[~temp_df['player_href'].isin(rem_href)]

    for season in list(temp_df['season'].drop_duplicates()):
        temp_rows = temp_df.loc[temp_df['season'] == season]
        if len(temp_rows) > 2:
            raise ValueError('too many players in one season')
        elif len(temp_rows) == 2:
            pos_list1, pos_list2 = temp_rows['pos_group_list'].iloc[0], temp_rows['pos_group_list'].iloc[1]
            class1, class2 = temp_rows['class'].iloc[0], temp_rows['class'].iloc[1]
            if ((class1 != class2) & (None not in [class1,class2]))\
                | ((len(list(set(pos_list1) & set(pos_list2))) == 0) & ('unknwn' not in (pos_list1 + pos_list2))):
                raise ValueError('likely not same player')
            else:
                break

    if (len(temp_df['player_href'].drop_duplicates()) == 0):
        winner = match_list[-1]
        losers = list(set(match_list) - set([winner]))
        return winner, losers
    elif len(temp_df['player_href'].drop_duplicates()) == 1:
        winner = temp_df['player_href'].iloc[0]
        losers = list(set(match_list) - set([winner]))
        return winner, losers
    elif len(temp_df['player_href'].drop_duplicates()) > 2:
        if len(temp_df['player_name'].drop_duplicates()) == 2:
            names_hrefs = temp_df[['player_href','player_name']].drop_duplicates()
            names_hrefs = names_hrefs.groupby('player_name').agg(lambda x: x.nunique())
            dedup_href_name = names_hrefs.loc[names_hrefs['player_href'] == 2].index[0]

            temp_df2 = temp_df.loc[temp_df['player_name'] == dedup_href_name]
            href1 = temp_df2['player_href'].drop_duplicates().iloc[0]
            href1_df = temp_df2.loc[temp_df2['player_href'] == href1]
            href2 = temp_df2['player_href'].drop_duplicates().iloc[1]
            href2_df = temp_df2.loc[temp_df2['player_href'] == href2]

            if len(href1_df) > len(href2_df):
                winner = href1
                losers = list(set(match_list) - set([winner]))
            elif len(href1_df) < len(href2_df):
                winner = href2
                losers = list(set(match_list) - set([winner]))
            else:
                raise ValueError('too many player_hrefs2')
            return winner, losers
        elif len(temp_df['player_name'].drop_duplicates()) == 1:
            hrefs_w_class = temp_df.groupby('player_href').agg({"class_ind": lambda x: x.sum()})
            class_max = hrefs_w_class['class_ind'].max()

            if len(hrefs_w_class.loc[hrefs_w_class['class_ind'] == class_max]) == 1:
                winner = hrefs_w_class.loc[hrefs_w_class['class_ind'] == class_max].index[0]
                losers = list(set(match_list) - set([winner]))
                return winner, losers
            elif len(hrefs_w_class.loc[hrefs_w_class['class_ind'] == class_max]) == 2:
                href1 = hrefs_w_class.loc[hrefs_w_class['class_ind'] == class_max].index[0]
                href1_df = temp_df.loc[temp_df['player_href'] == href1]
                href1_name = href1_df['player_name'].iloc[0]

                href2 = hrefs_w_class.loc[hrefs_w_class['class_ind'] == class_max].index[1]
                href2_df = temp_df.loc[temp_df['player_href'] == href2]
                href2_name = href2_df['player_name'].iloc[0]
                pass
            else:
                raise ValueError('too many player_hrefs3')
        else:
            raise ValueError('too many player_hrefs')
    else:
        href1 = temp_df['player_href'].drop_duplicates().iloc[0]
        href1_df = temp_df.loc[temp_df['player_href'] == href1]
        href1_name = href1_df['player_name'].iloc[0]

        href2 = temp_df['player_href'].drop_duplicates().iloc[1]
        href2_df = temp_df.loc[temp_df['player_href'] == href2]
        href2_name = href2_df['player_name'].iloc[0]

    href1_pos_set = set(pos for sublist in href1_df['pos_group_list'] for pos in sublist)
    href2_pos_set = set(pos for sublist in href2_df['pos_group_list'] for pos in sublist)
    if (len(list(set(href1_pos_set - set(['unknwn'])) & set(href2_pos_set - set(['unknwn'])))) == 0)\
        & (href1_pos_set != set(['unknwn'])) & (href2_pos_set != set(['unknwn'])):
        if href1_name == href2_name:
            raise ValueError('same names and mismatch pos_group')
        else:
            raise ValueError('mismatching names and pos_group')
    else:
        if (len(href1_df) == len(href2_df)) | (team_href not in list(href1_df['team_href']))\
                | (team_href not in list(href2_df['team_href'])):
            href1_df = href1_df.loc[href1_df['team_href'] == team_href]
            href2_df = href2_df.loc[href2_df['team_href'] == team_href]
        if len(href1_df) == len(href2_df):
            href1_df = href1_df.loc[pd.isnull(href1_df['class']) == False]
            href2_df = href2_df.loc[pd.isnull(href2_df['class']) == False]

        if len(href1_df) > len(href2_df):
            winner = href1
            losers = list(set(match_list) - set([href1]))
        elif len(href1_df) < len(href2_df):
            winner = href2
            losers = list(set(match_list) - set([href2]))
        elif (True in list(href1_df['phys'])) and (False in list(href2_df['phys'])):
            winner = href1
            losers = list(set(match_list) - set([href1]))
        elif (True in list(href2_df['phys'])) and (False in list(href1_df['phys'])):
            winner = href2
            losers = list(set(match_list) - set([href2]))
        else:
            winner = href2
            losers = list(set(match_list) - set([href2]))
        return winner, losers

def player_stats_deduped(stats_df):
    try:
        matches = pd.read_csv(cfbr_folder_path + 'cfbr_player_matches.csv', header = None)
        skip_list = [set(re.sub('\[|\]|\'|,','',x).split()) for x in matches.loc[pd.isnull(matches[3]) == False,0].head()]
    except:
        skip_list = []

    for min_season in range(stats_df['season'].min(),stats_df['season'].max() - 3):
        seasons = range(min_season,min_season + 5)
        print(seasons)
        for team_href in list(stats_df.loc[stats_df['season'].isin(seasons) & (stats_df['dedupe'] == 0),'team_href'].drop_duplicates()):
            print('\t' + team_href)
            players_df = stats_df.loc[stats_df['season'].isin(seasons) & (stats_df['team_href'] == team_href),
                                      ['player_name','player_href']].drop_duplicates()
            for index, row in players_df.iterrows():
                name_list = [row['player_name']]
                fuzzymatches = process.extract(row['player_name'],
                                   list(players_df.loc[players_df['player_name'] != row['player_name'],'player_name'].drop_duplicates()),
                                   limit = 2,scorer = fuzz.token_set_ratio)
                for fuzzymatch in fuzzymatches:
                    if (fuzzymatch[1] >= 87):
                        name_list += [fuzzymatch[0]]
                match_list = list(players_df.loc[players_df['player_name'].isin(name_list),'player_href'].drop_duplicates())
                if (len(match_list) >= 2) and (set(match_list) not in skip_list):
                    try:
                        winner,losers = href_dedupe_process(seasons,team_href,match_list)
                        error_msg = None
                    except Exception as e:
                        error_msg = str(e)
                        skip_list.append(set(match_list))

                    match = 'likely' if error_msg == None else 'unsure'
                    with open(cfbr_folder_path + 'cfbr_player_matches.csv', 'ab') as matchcsv:
                        matchwriter = csv.writer(matchcsv, delimiter = ',', quotechar = '"', quoting = csv.QUOTE_MINIMAL)
                        matchwriter.writerow([match_list,match,error_msg])

                    if error_msg != None:
                        continue

                    stats_df.loc[(stats_df['season'].isin(seasons) & (stats_df['team_href'] == team_href))
                            & (stats_df['player_href'].isin(losers) | stats_df['player_name'].isin(name_list)),'player_href'] = winner
                    players_df.loc[players_df['player_href'].isin(losers),'player_href'] = winner
                elif (len(match_list) == 1) and (set(match_list) not in skip_list) and (name_list > 1):
                    stats_df.loc[stats_df['season'].isin(seasons) & (stats_df['team_href'] == team_href)
                            & stats_df['player_name'].isin(name_list),'player_href'] = match_list[0]
                    players_df.loc[players_df['player_name'].isin(name_list),'player_href'] = match_list[0]
            stats_df.loc[stats_df['season'].isin(seasons) & (stats_df['dedupe'] == 0) & (stats_df['team_href'] == team_href),'dedupe'] = 1
    return stats_df

def player_stats_scrape(min_season, current_date):
    try:
        player_stats_df = pd.read_csv(cfbr_folder_path + 'cfbr_player_stats.csv', header = 0)
    except:
        player_stats_df = pd.DataFrame(columns = ['season', 'team_schoolname'])

    team_info_df = team_info_cfbr_scrape()
    active_rosters = active_roster(min_season, current_date)

    season = None
    for index, row in active_rosters.sort_values(['season','team_href']).iterrows():
        if int(row['season']) != season:
            season = int(row['season'])
            print(season)
        team_schoolname = team_info_df.loc[team_info_df['team_href'] == row['team_href'],'team_schoolname'].iloc[0]
        if len(player_stats_df.loc[(player_stats_df['team_schoolname'] == team_schoolname) & (player_stats_df['season'] == season),]) < 1:
            print('\t' + team_schoolname)
            roster_df = roster_scrape(season, row['team_href'],team_schoolname)
            try:
                roster_df.iloc[0]
            except:
                continue

            skip_stats = (season >= (current_date[0] - (1 if current_date[1] == 1 else 0)))
            stats_df = stats_scrape(roster_df,team_schoolname,skip_stats)

            if len(player_stats_df) == 0:
                player_stats_df = stats_df
            else:
                player_stats_df = pd.concat([stats_df,player_stats_df])

    player_stats_df = pd.merge(player_stats_df, pos_group_df, how = 'left', on = 'pos')
    player_stats_df['pos_group'] = player_stats_df['pos_group'].fillna('unknwn')

    player_stats_df.loc[:,player_stats_df.columns[7:-2]] = player_stats_df.loc[:,player_stats_df.columns[7:-2]].fillna(0)
    player_stats_df = player_stats_df.drop_duplicates().reset_index(drop = True)
    i, prev_team, prev_name = 1, None, None
    for index, row in player_stats_df.loc[pd.isnull(player_stats_df['player_href'])].sort_values(['player_name','team_href']).iterrows():
        if (row['team_href'] == prev_team) & (row['player_name'] == prev_name):
            pass
        elif (row['player_name'] == prev_name):
            i += 1
            prev_team = row['team_href']
        else:
            i, prev_team, prev_name = 1, row['team_href'], row['player_name']
        player_stats_df.set_value(index,'player_href','custom_href/' + re.sub(' ','-',row['player_name']) + '-' + str(i))
    player_stats_df.to_csv(cfbr_folder_path + 'cfbr_player_stats.csv', index = False)

    dedupe_in = pd.read_csv(cfbr_folder_path + 'cfbr_player_stats.csv', header = 0)
    if len(dedupe_in.loc[dedupe_in['dedupe'] == 0]) > 0:
        dedupe_out = player_stats_deduped(dedupe_in).drop_duplicates().reset_index(drop = True)
        dedupe_out.to_csv(cfbr_folder_path + 'cfbr_player_stats_dedupe.csv', index = False)

        player_stats_df['dedupe'] = 1
        player_stats_df.to_csv(cfbr_folder_path + 'cfbr_player_stats.csv', index = False)
        return dedupe_out
    else:
        return pd.read_csv(cfbr_folder_path + 'cfbr_player_stats_dedupe.csv', header = 0)


# --------------------------------------------
# Plots for cross validation
# Mark van der Plaat
# September 2020 -- Update: November 2021
# --------------------------------------------

# --------------------------------------------
# Import Packages
# --------------------------------------------

# Data manipulation
import pandas as pd
import numpy as np

# Plotting
import matplotlib.pyplot as plt
import seaborn as sns

sns.set(style='whitegrid', font_scale=2)

# Set WD
import os
os.chdir(r'D:\RUG\PhD\Materials_papers\01-Note_on_securitization')

#--------------------------------------------
# Load data
#--------------------------------------------

# Get file names from the directory
lst_directory = os.listdir('Results/Cross_validation')
lst_fitfiles = [file for file in lst_directory if '_fit_' in file]
lst_r2files = [file for file in lst_directory if '_r2_' in file]

# Load data and concat
def readFiles(files):
    data_lst = list()

    for file in files:
        # Load file
        data = pd.read_csv('Results/Cross_validation/' + file, index_col = 0).T

        # Append to list
        data_lst.append(data)

    # Make df and reset index
    dataframe = pd.concat(data_lst)
    dataframe.reset_index(drop = True, inplace = True)

    return dataframe

df_fit = readFiles(lst_fitfiles)
df_r2 = readFiles(lst_r2files)

#--------------------------------------------
# Plot fit indices
#--------------------------------------------

# Plot SRMR (absolute fit)
fig, ax = plt.subplots(figsize=(8,8))
ax.set(ylabel='Value', xlabel = 'SRMR')
sns.violinplot(y = 'srmr',
               data = df_fit,
               inner = None,
               alpha=0.1)
sns.boxplot(y = 'srmr',
            data = df_fit,
            boxprops={'zorder': 2}).set(ylabel = '')
plt.tight_layout()

fig.savefig('Results/Cross_validation/Box_absolute_fit.png')

# Plot RMSEA and RNI (parsimonious fit)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16,8))
ax1.set(ylabel='Value', xlabel = 'RMSEA')
ax2.set(ylabel='', xlabel = 'RNI')
sns.violinplot(ax=ax1,
               y = 'rmsea.robust',
               data = df_fit,
               inner = None,
               alpha=0.1)
sns.boxplot(ax=ax1,
            y = 'rmsea.robust',
            data = df_fit,
            boxprops={'zorder': 2}).set(ylabel = '')
sns.violinplot(ax=ax2,
               y = 'rni.robust',
               data = df_fit,
               inner = None,
               alpha=0.1)
sns.boxplot(ax=ax2,
            y = 'rni.robust',
            data = df_fit,
            boxprops={'zorder': 2}).set(ylabel = '')
plt.tight_layout()
fig.savefig('Results/Cross_validation/Box_parsimonious_fit.png')

# Plot CFI and TLI (comparative and relative fit)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16,8))
ax1.set(ylabel='Value', xlabel = 'CFI')
ax2.set(ylabel='', xlabel = 'TLI')
sns.violinplot(ax=ax1,
               y = 'cfi.robust',
               data = df_fit,
               inner = None,
               alpha=0.1)
sns.boxplot(ax=ax1,
            y = 'cfi.robust',
            data = df_fit,
            boxprops={'zorder': 2}).set(ylabel = '')
sns.violinplot(ax=ax2,
               y = 'tli.robust',
               data = df_fit,
               inner = None,
               alpha=0.1)
sns.boxplot(ax=ax2,
            y = 'tli.robust',
            data = df_fit,
            boxprops={'zorder': 2}).set(ylabel = '')
plt.tight_layout()
fig.savefig('Results/Cross_validation/Box_comparative_fit.png')

#--------------------------------------------
# Plot communalities
#--------------------------------------------

# Set labels
lst_vars = df_r2.columns.tolist()
lst_labels = ['Small Bus. Obl. Transf.','Sec. Residential Loans',
              'Sec. Other Assets','Sec. Residential Mortgages',
              'TA Sec. Vehicles' ,
              'Sec. Income', 'Servicing Fees',
              'CDSs Purchased','TA ABCP Conduits',
              'Un. Com. Own ABCP Conduits',
              'Credit Exp. Own ABCP Conduits','Un. Com. Other ABCP Conduits']

# Plot
fig = plt.figure(figsize=(24,24))
for i in range(len(lst_vars)):
    ax = fig.add_subplot(4,3,i+1)
    ax.set(ylabel='', xlabel=lst_labels[i])
    sns.violinplot(ax=ax,
                   y=lst_vars[i],
                   data=df_r2,
                   inner=None,
                   alpha=0.1)
    sns.boxplot(ax=ax,
                y=lst_vars[i],
                data=df_r2,
                boxprops={'zorder': 2}).set(ylabel='')
plt.tight_layout()
fig.savefig('Results/Cross_validation/Box_communalities.png')


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

ticker = ['AAPL', 'AVB', 'AMD', 'CCC', 'BA', 'BLL', 'C', 'INTC', 'GPS', 'MMM', 'T', 'TJX']
STK = len(ticker)

print('Reading data... \n')
price = pd.read_csv('price.csv').values
T = price.shape[0] - 1
R = np.ones((T, STK))  # data of returns
for idx in range(STK):
    R[:, idx] = (price[1:, idx] - price[:T, idx])/price[:T, idx]
R = R - 0.001  # excess return
prob = np.zeros(T)  # probability that model t is true
prob[:2] = 0.5
mu = np.ones((T, STK))/STK  # model t's mean
oldmu = np.ones(STK)/STK
cov = np.ones((T, STK, STK))  # model t's covariance
oldcov = np.zeros((T, STK, STK))
for i in range(T):
    cov[i, :, :] = np.eye(STK)
kappa = np.ones(T)
delta = np.ones(T)
L = np.zeros(T)
phi = np.zeros((T, STK))

print('Caculating the BA strategy... \n')
for t in range(3, T):
    # new model t is born
    mu[t, :] = np.sum(R[:t, :]) / (t - 1) / STK
    cov[t, :, :] = np.sum(np.power(R[:t, :] - np.sum(R[:t, :], axis=0), 2)) / (t - 2) / STK * np.eye(STK)
    # probability updated
    oldprob = np.copy(prob)
    for q in range(t):
        oldprob[q] = oldprob[q] / (t + 1 - q)
    prob[:t] = np.cumsum(oldprob[:t])
    prob[t] = prob[t - 1]

    # prob[:t+1] = 1 / (t + 1)

    # update mean and cov when new return on time t observed
    for idx in range(t + 1):
        oldmu = np.copy(mu[idx, :])
        mu[idx, :] = (kappa[idx] * oldmu + R[t, :]) / (kappa[idx] + 1)
        oldcov[idx, :, :] = np.copy(cov[idx, :, :])
        cov[idx, :, :] = (delta[idx] * (kappa[idx] + 1) * cov[idx, :, :] +
                          kappa[idx] * np.outer(R[t, :] - oldmu, R[t, :] - oldmu)) / (delta[idx] + 1) / (kappa[idx] + 1)
        delta[idx] = delta[idx] + 1
        kappa[idx] = kappa[idx] + 1

    # update probability when new return on t observed
    for idx in range(t + 1):
        detold = np.power(
             np.linalg.det((delta[idx] - 1) * oldcov[idx, :, :]) / np.linalg.det(delta[idx] * cov[idx, :, :]),
             (delta[idx] + STK) / 2)
        detnew = np.power(np.linalg.det(delta[idx] * cov[idx, :, :]), 1 / 2)
        # gammafrac = spy.gamma((delta[idx] + STK + 1) / 2) / spy.gamma((delta[idx] + STK) / 2 + (1-STK) / 2)
        kappafrac = np.power(1 - 1 / kappa[idx], STK / 2)
        L[idx] = detold / detnew * (delta[idx] + 10 + 1) / 2 * (delta[idx] + 8 + 1) / 2 * (delta[idx] + 6 + 1) / 2 * (
                 delta[idx] + 4 + 1) / 2 * (delta[idx] + 2 + 1) / 2 * (delta[idx] + 1) / 2 * kappafrac

    totalprob = np.dot(L, prob)
    prob = np.multiply(L, prob) / totalprob

    # calculate strategy
    MU = np.sum(np.multiply(mu[:t + 1, :], np.tile(prob[:t + 1], (STK, 1)).transpose()), axis=0)
    SIGMA = np.zeros((STK, STK))
    for idx in range(t + 1):
        SIGMA = SIGMA + (1 + kappa[idx]) / kappa[idx] * cov[idx, :, :] + np.outer(mu[idx, :], mu[idx, :]) * prob[idx]
    SIGMA = SIGMA - np.outer(MU, MU)
    if np.isnan(np.linalg.cond(SIGMA, 2)):
        print('Condition number for time {0} is NaN\n'.format(t))
    phi[t, :] = np.dot(np.linalg.inv(SIGMA), MU)

# backtest of the strategy, starting from Week 701 to Week 800

print('Backtesting... \n')
start = 700
ter = 800
profit = np.ones(ter-start)
weekrtn = np.zeros(ter-start-1)
for t in range(start, ter-1):
    profit[t-start+1] = profit[t-start] * (R[t+1, :].dot(1000*phi[t, :]) + 1.001)
    weekrtn[t-start] = R[t+1, :].dot(1000*phi[t, :])
print('Sharpe ratio of BA is %f\n' % (np.mean(weekrtn)/np.std(weekrtn)*np.sqrt(52)))
print('Annualized return is %f\n' % (np.power(profit[ter-start-1], 52/(ter-start))-1))
# rebalanced strategy
RBprofit = np.ones(ter-start)
RBweekrtn = np.zeros(ter-start+1)
for t in range(start, ter-1):
    RBprofit[t-start+1] = RBprofit[t-start] * (R[t+1, :].dot(np.ones(12))/13 + 1.001)
    RBweekrtn[t-start] = R[t+1, :].dot(np.ones(12)/13)
print('Sharpe ratio of Rebalanced strategy is %f\n' % (np.mean(RBweekrtn)/np.std(RBweekrtn)*np.sqrt(52)))
print('Annualized return is %f\n' % (np.power(RBprofit[ter-start-1], 52/(ter-start))-1))
# best stock
print('Final wealth of holding each stock is {0} \n'.format(price[ter, :]/price[start+1, :]))

# plot the comparison figure
plt.plot(profit, label='BA')
plt.plot(RBprofit, label='RB')
plt.plot(price[start+1:ter+1, 0]/price[start+1, 0], label='AAPL')
plt.plot(price[start+1:ter+1, 5]/price[start+1, 5], label='BLL')
plt.legend(loc='best')
plt.title('Comparison')
plt.savefig('compare.png', dpi=300)
#  plt.show()

# save figure showing the money invested in stock
plt.figure(2, figsize=(12, 9), dpi=80)
for idx in range(12):
    ax = plt.subplot(4, 3, idx+1)
    ax.set_title(ticker[idx])
    ax.plot(phi[start:ter, idx]*1000)
plt.tight_layout()
plt.savefig('strategy.png', dpi=300)


from django import forms
from django.forms import SelectDateWidget
from .models import PotentialStu, CounsellingInfo

class PotentialForm(forms.ModelForm):

    class Meta:
        model = PotentialStu
        fields = ('name', 'agent', 'dob', 'gender', 'phone', 'email', 'parent_phone',
                  'parent_email', 'city', 'state', 'school', 'grade', 'gpa', 'toefl')
        widgets = {
            'dob': SelectDateWidget(years=range(1990, 2016)),
        }

class CounsellingForm(forms.ModelForm):

    class Meta:
        model = CounsellingInfo
        fields = ('semester', 'apply_grade', 'school1', 'school2', 'school3',
                  'note', 'counsellor', 'date', 'status', 'fee', 'pay_date',)
        widgets = {
            'date': SelectDateWidget(years=range(2013, 2020)),
            'pay_date': SelectDateWidget(years=range(2013, 2020)),
        }

class ConfirmForm(forms.Form):
    CONFIRM_CHOICES = (
        ('yes', 'Yes'),
        ('no', 'No'),
    )

    confirm = forms.ChoiceField(choices=CONFIRM_CHOICES, widget=forms.RadioSelect())


"""Functions to build data pipelines for the Kaggle NLP with Disaster Tweets challenge."""

import csv
from typing import Callable, Iterable, Optional
import torch
import torchtext.data
from torchdata.datapipes.map import MapDataPipe, SequenceWrapper
from torchdata.datapipes.iter import IterDataPipe

def load_csv_dataset(filepath: str) -> list[tuple[int, str, Optional[int]]]:
    """Load a dataset file for the Kaggle NLP with Disaster Tweets challenge.
    Result will be a list of tuples, each representing a sample and containing the sample id (int),
    tweet text (str) and label (int or None).
    A 1 (0) label indicates that the tweet is (not) about a disaster event.
    When loading the challenge test set, the labels will be None.
    """
    with open(filepath, mode='r', newline='', encoding="utf8") as file:
        return _parse_csv(file)

def _parse_csv(f: Iterable[str]):
    data = []
    reader = csv.DictReader(f)
    for row in reader:
        # row is a dict with keys id, keyword, location, text, target (not for test dataset)
        text = row["text"]

        label=None
        if "target" in row:
            label = int(row["target"])

        sample_id = int(row["id"])

        data.append((sample_id, text, label))

    return data

def tokenize_dataset(data: list[tuple[int, str, Optional[int]]], tokenize: Callable) -> list[tuple[int, list[int], Optional[int]]]:
    """Tokenize the text of each sample of a dataset.
    Since the dataset for the Kaggle challenge is small, we can just tokenize all
    sentences once at the beginning.
    """
    return [(id, tokenize(text), label) for id, text, label in data]

class BasicTokenizer:
    """BasicTokenizer loads its token dictionary from a file and uses the basic_english tokenizer
    from the torchtext.data package. This will lowercase the text and split it into words.
    Note however that e.g. fastText word embeddings are case-sensitive. There are separate
    embeddings for "and" and "And", so some information will be lost.
    """
    def __init__(self, token_dict_file: str) -> None:
        self._token_dict = torch.load(token_dict_file)
        self._tokenizer = torchtext.data.get_tokenizer("basic_english")

    def tokenize(self, text: str | list[str], unknown_index=1) -> list[int] | list[list[int]]:
        """Split text into tokens and map them to token indices.
        Supports single elements (str) or batches (list[str]).
        """
        if isinstance(text, list):
            return [self._tokenize_single(x, unknown_index=unknown_index) for x in text]
        else:
            return self._tokenize_single(text, unknown_index=unknown_index)

    def _tokenize_single(self, text: str, unknown_index=1) -> list[int]:
        return [self._token_dict.get(y, unknown_index) for y in self._tokenizer(text)]

def build_pipeline(data, transform: Optional[Callable], batch_size: Optional[int], shuffle=True) -> IterDataPipe | MapDataPipe:
    """Returns a pipeline that applies the given transform function to each element/batch.
    If batch_size is None, no batching is performed.
    """
    pipeline = SequenceWrapper(data)
    if shuffle:
        pipeline = pipeline.shuffle()
    if batch_size is not None:
        pipeline = pipeline.batch(batch_size)
    if transform is not None:
        pipeline = pipeline.map(transform)
    return pipeline

def default_transform(samples, seq_len=None, padding_index=0, train=True):
    """Transform the given sample or batch of samples to tensors.
    It is assumed that text is already tokenized. The list of token indices is padded to
    the given length (max length of the batch if seq_len is None).
    If train is true returns tokens and labels, otherwise sample ids and tokens.
    """
    if isinstance(samples, list):
        sample_id, tokens, label = [x[0] for x in samples], [x[1] for x in samples], [x[2] for x in samples]
        if seq_len is None:
            seq_len = max(len(x) for x in tokens)
        tokens = [to_length(x, seq_len, padding_index) for x in tokens]
    else:
        sample_id, tokens, label = samples
        if seq_len is not None:
            tokens = to_length(tokens, seq_len, padding_index)

    if train:
        return torch.tensor(tokens), torch.tensor(label)
    else:
        return sample_id, torch.tensor(tokens)

def to_length(seq: list[int], length: int, padding_index) -> list[int]:
    """Pad or truncate sequence to given length."""
    if len(seq) > length:
        return seq[:length]
    if len(seq) < length:
        seq.extend([padding_index for i in range(length-len(seq))])
    return seq


import os
from django.urls.conf import path
from files.models import FilesAdmin
from django.shortcuts import redirect, render
from django.conf import settings
from django.http import Http404,HttpResponse, response
from .form import FilesAdminForm

# Create your views here.

def home(request):
    context={'file' : FilesAdmin.objects.all()}
    return render(request,'files/index.html',context)

def uploadfile(request):
    forms = FilesAdminForm()
    if request.method == 'POST':
        forms = FilesAdminForm(request.POST,request.FILES)
        if forms.is_valid():
            forms.save()
            context = {'Upload':"Success"}
            return render(request,'files/upload.html',context)
        else:
            form = FilesAdminForm()
    return render(request,'files/upload.html',context = {'form':forms})

def download(request,path):
    file_path=os.path.join(settings.MEDIA_ROOT,path)
    if os.path.exists(file_path):
        with open(file_path,'rb')as fh:
            response = HttpResponse(fh.read(),content_type="application/adminupload")
            response['Content-Disposition']='inline;filename='+os.path.basename(file_path)
            return response

        raise Http404


from django.contrib import messages
from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required

from .forms import ProfileForm

@login_required
def view(request):
    if request.method == 'POST':
        form = ProfileForm(request.POST, instance=request.user.profile)

        if form.is_valid():
            form.save()

            messages.success(request, "Your settings were saved successfully.")

            return redirect('profile:view')

    else:
        form = ProfileForm(instance=request.user.profile)

    return render(request, 'profile/view.html', {
        'form': form,
    })


import pandas as pd
from mlxtend.preprocessing import TransactionEncoder
from mlxtend.frequent_patterns import apriori
from mlxtend.frequent_patterns import association_rules

df = pd.read_csv("./bank-data.csv")
del df["id"]
df["income"] = pd.cut(df["income"],10)
dataset = []
for index, row in df.iterrows():
    row = [col+"="+str(row[col]) for col in list(df)]
    dataset.append(row)

te = TransactionEncoder()
te_ary = te.fit(dataset).transform(dataset)
df = pd.DataFrame(te_ary, columns=te.columns_)
frequent_itemsets = apriori(df, min_support=0.3, use_colnames=True)
print("Frequent Itemset:\n")
print(frequent_itemsets)

rules_confidence = association_rules(frequent_itemsets, metric="confidence", min_threshold=0.7)
print("\nConfidence 0.7:\n")
print(rules_confidence)

rules_support = association_rules(frequent_itemsets, metric="support", min_threshold=0.4)
print("\nSupport 0.4:\n")
print(rules_support)

rules_lift = association_rules(frequent_itemsets, metric="lift", min_threshold=1)
print("\nLift 1:\n")
print(rules_lift)


import uuid
import boto3
import  pandas as pd

def transform_to_parquet(file_name,source_folder, bucket_name, target_folder):
    print(f"Creating json reader for {file_name}")
    json_reader = pd.read_json(
        f's3://{bucket_name}/{source_folder}/{file_name}',
        lines=True,
        orient='records',
        chunksize=10000
    )
    year = file_name.split("-")[0]
    month = file_name.split("-")[1]
    dayofmonth = file_name.split("-")[2]
    hour = file_name.split("-")[3].split(".")[0]
    print("Read each chunks of maximum 10000 records")
    for idx, df in enumerate(json_reader):
        print(f"processing chunk # {idx} which has # {df.shape[0]} records")
        target_file_name = f'part-{year}-{month}-{dayofmonth}-{hour}-{uuid.uuid1()}.snappy.parquet'
        df.drop(columns=['payload']). \
            to_parquet(
            f's3://{bucket_name}/{target_folder}/year={year}/month={month}/dayofmonth={dayofmonth}/{target_file_name}',
            index=False
        )

    return  {
        "last_run_src_file_name":file_name,
        "last_run_tgt_file_pattern":f's3://{bucket_name}/{target_folder}/year={year}/month={month}/dayofmonth={dayofmonth}/{hour}'
    }


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Minimap and miniasm in Python.

Requirements
------------
For this program the minimum required is Python version 3.6.

No additional requirements needed.

README
------
This program consists of two subprograms, minimap, and miniasm.

You can run one or the other using the first argument, followed by the other
arguments.

For example:
python3 hielkewalinga_4374561_minimap-miniasm.py minimap query.fq target.fq > output.paf

For more details, you can use the --help flag to one of the subprograms:
python3 hielkewalinga_4374561_minimap-miniasm.py miniasm --help

The main program (without supplying a subprogram) accepts a --help as well.

NB.

There is also a third subprogram, which is the 'test' subprogram. This will
run the unittest included in this program. These unittest reflect the tests
given on the project page of this project.
https://abeellab.github.io/cs4255/minimap-doc.html

There is also a fourth subprogram 'mytest', used for debugging purposes.
"""

import argparse
import heapq
import sys
import unittest
from argparse import Namespace
from collections import defaultdict, deque
from functools import partial
from itertools import chain, groupby, islice, repeat, tee
from operator import itemgetter
from typing import (
    IO,
    Callable,
    DefaultDict,
    Dict,
    Iterable,
    Iterator,
    List,
    Optional,
    Set,
    Tuple,
)

###########
# MINIMAP #
###########

# Type aliases minimap

# A set with tuples representing the hash value, the position, and the strand.
Minimizers = Set[Tuple[int, int, bool]]

# A function that takes a sequence and returns the minimizers.
Minimizer_Sketch = Callable[[str], Minimizers]

# A function that takes a sequence and returns a hash integer.
Seq_Hash = Callable[[str], int]

# A dictionary that maps the hash to a list of tuples with
# target sequence id, position, and strand.
Target_Index = Dict[int, List[Tuple[str, int, bool]]]

# A dictionary that maps the sequence name to the length.
Seq_Lens = Dict[str, int]

# For PAF we will use fields 1-11.
# So not using the quality one, and optional additional fields.
Strand_Range = Tuple[str, int, int, int]
PAF = Tuple[Strand_Range, str, Strand_Range, Tuple[int, int]]

def reverse_complement(
    s: str, complement: Dict[int, Optional[int]] = str.maketrans("ACGT", "TGCA")
) -> str:
    """
    Create the reverse complement of sequence
    """
    return s.translate(complement)[::-1]

def sliding_windowing(s: str, k: int) -> Iterator[str]:
    """
    Creates a sliding window of k-mers over a sequence (s).

    Parameters
    ----------
    s : str
        The sequence.
    k : int
        The size of the k-mer.

    Returns
    -------
    it : Iterator[str]
        The k-mers yield from this iterator.
    """
    if len(s) < k:
        return iter(())
    # To create this k-mer iterator, we create k letter iterators which
    # start all one letter further than the previous ones.
    # Then, get one letter from all these letter iterators and concatenate them.
    texts = []
    for ind, t in enumerate(tee(s, k)):
        for _ in repeat(None, ind):
            next(t)
        texts.append(t)
    return map("".join, zip(*texts))

def hash_seq(
    s: str, base_values: Dict[str, int] = {"A": 0, "C": 1, "G": 2, "T": 3}
) -> int:
    """
    Hash a sequence (or k-mer).

    Using Phi(A)=0, Phi(C)=1, Phi(G)=2, and Phi(T)=3

    Calculate with s = a_1 ... a_k:
    Phi(s) = Phi(a_1) x 4 ^ (k-1) + Phi(a_2) x 4 ^ (k-2) ... + Phi(a_k)

    Parameters
    ----------
    s : str
        s is the sequence (or k-mer) that needs to be hashed
    base_values : Dict[str, int]
        This is a dict that maps the base pair to the value used in the hash.
        Default is {"A": 0, "C": 1, "G": 2, "T": 3}

    Returns
    -------
    x : int
        Returns the integer hashed
    """
    k = len(s)
    return sum(base_values[b] * 4 ** (k - i - 1) for i, b in enumerate(s))

def invertable_hash(x: int, p: int = 64) -> int:
    """
    Invertible integer hash function.

    Invert a hash integer.

    Algorithm 2.

    A problem with :func:`hash_seq` is that the uninformative poly-A sequence
    will get the hash value of 0, which is the minimum. To prevent this,
    the hash function is altered using this function. Invertibility of the
    hash function is not a requirement, but prevent hash collisions.

    Parameters
    ----------
    x : int
        The hash integer.
    p : int
        The amount of bits of the hash.

    Returns
    -------
    x : int
        Return the new hash integer.

    Notes
    -----
    The paper mentions that the hash can be a 2 * k bit hash. However
    the tests included of in the project assume p=64, so I went with that instead.
    However, p=63 would make a bit more sense, as that is the maximum size of
    Python integer (So, one bit smaller than a C integer size). Luckily this
    isn't a big problem as Python will increase the memory size of an integer
    automatically if it will overflow.
    """
    m = 2 ** p - 1
    x = (~x + (x << 21)) & m
    x = x ^ (x >> 24)
    x = (x + (x << 3) + (x << 8)) & m
    x = x ^ (x >> 14)
    x = (x + (x << 2) + (x << 4)) & m
    x = x ^ (x >> 28)
    x = (x + (x << 31)) & m
    return x

def compute_minimizers(s: str, w: int, k: int, hash_func: Seq_Hash) -> Minimizers:
    """
    Compute the (w, k)-minimizers.

    A minimizer is the smallest k-mer in a window of w k-mer.

    Algorithm 1.

    Parameters
    ----------
    s : str
        The input sequence. For the size of the sequence it should be
        |s| => w + k - 1
    w : int
        The amount of k-mers to use in a window in which to find the smallest k-mer.
    k : int
        The size of the k-mer.

    Returns
    -------
    M : Minimizers
        The minimizers are a set with a tuple with the numbers m, i, r
        Where m is the hash integer,
        i is the position where the k-mer starts. (Also start position on reverse)
        r is the strand

    Notes
    -----
    NB. In the test set the starting coords of the minimizers is
    reversed if the minimizer is found on the reverse strand.
    This is however different from the given pseudocode which does not perform
    this reversal. The reason is explained here.

    Algo 4 will also not work if you perform the reversal.
    The reason for this is that the clustering is performed
    on i + i_t instead of i - i_t for opposite strands.

    Different strands use a different invariant to find minimizers
    with the same maplen, as shown below:
    The invariants:
    Same strand: i - i_t = qlen - maplen
    Opposite strand: i + i_t = qlen + tlen - maplen

    Of course this wasn't needed when we would use reversed coords for different
    strands, but the convenience comes in the final end of the implementation.

    Here we made a distinction between a qeury-target mapping on the same strands
    and a query-target mapping on different strands. What is convenient is that
    a query-target mapping is the same as a query(reversed)-target(reversed) mapping,
    and that a query(reversed)-target mapping is the same as a query-target(reversed)
    mapping. However, since we did not reverse any coords we can just output a
    query-target mapping for a query(reversed)-target(reversed) mapping (same strand),
    and a query-target(reversed) mapping for a query(reversed)-target mapping.
    Conveniently we only reverse coords for the target strand, when the mapping
    is not on the same strand.
    """
    kmers = sliding_windowing(s, k)
    w_window = deque(islice(kmers, w))
    M: Minimizers = set()

    i = 0  # We use 0-indexing.
    while True:

        # Find minimum k-mer
        m = sys.maxsize  # 2 ** 63 - 1 on 64 bit platforms.
        for j, kmer in enumerate(w_window):
            u, v = hash_func(kmer), hash_func(reverse_complement(kmer))
            if u != v:  # If reverse complement the same sequence, do not use.
                m = min(m, u, v)

        # Add minimizer
        for j, kmer in enumerate(w_window):
            u, v = hash_func(kmer), hash_func(reverse_complement(kmer))
            if u == m or v == m:
                # NB. We do not reverse the coord here, but use different invariants
                # for the opposite strands. See Algo 4: compute_minimizers
                M.add((m, i + j, v < u))

        # Move w window one k-mer further. (If not more k-mers, we are done.)
        new_kmer = next(kmers, None)
        if not new_kmer:
            break
        w_window.popleft()
        w_window.append(new_kmer)
        i += 1

    return M

def get_func_minimizer_sketch(
    w: int, k: int, hash_func: Seq_Hash = lambda s: invertable_hash(hash_seq(s))
) -> Minimizer_Sketch:
    """
    A function to set all values to the :func:`compute_minimizers` function.

    Paramaters
    ----------
    w : int
        Amount of k-mers in window.
    k : int
        Length of the k-mer.
    hash_func : Seq_Hash
        A function to map a sequence (str) to a hash integer.
        Defaults to the invertible hash from the minimap/miniasm paper.

    Returns
    -------
    minimizer_sketch : Minimizer_Sketch
        A function that takes a sequence and returns the minimizers

    Raises
    ------
    Exception
        If k * 2 is larger than the bit of the hash, do not proceed.

        As integers can be arbitrarily large in Python, changing the hardcoded
        hash size (p) can be used to run with a 'k' larger than 32.

        You can probably run with a larger k anyway, as hash collisions aren't
        that bad. During clustering of minimizer hits this is dealt with as
        minimizer hits aren't guaranteed to be correct.
    """
    if k >= 32:
        raise Exception("k is too big")
    return partial(compute_minimizers, w=w, k=k, hash_func=hash_func,)

def index_targets(
    target_seqs: Iterable[Tuple[str, str]], minimizer_sketch: Minimizer_Sketch
) -> Tuple[Target_Index, Seq_Lens]:
    """
    Algorithm 3.
    """
    H_list = []
    seq_lens: Seq_Lens = dict()
    for name, seq in target_seqs:

        seq_lens[name] = len(seq)

        M = minimizer_sketch(seq)
        for h, i, r in M:
            H_list.append((h, (name, i, r)))

    H_list.sort(key=itemgetter(0))

    H_dict = {
        h: list(map(itemgetter(1), vals))
        for h, vals in groupby(H_list, key=itemgetter(0))
    }

    return H_dict, seq_lens

def map_query(
    qseq: str, qname: str, seq_lens: Seq_Lens, H: Target_Index, args: Namespace,
) -> Iterable[PAF]:
    """
    Takes a query and maps it to the target indexers.

    Algorithm 4.

    Returns
    -------
    paf_it : Iterable[PAF]
        PAF tuple, this is a structured PAF line.

    Notes
    -----
    PAF file : A PAF file is used to describe mapings. It is a tab seperated file
    with te following fields:

    1	string	Query sequence name
    2	int	Query sequence length
    3	int	Query start coordinate (0-based)
    4	int	Query end coordinate (0-based)
    5	char	‘+’ if query/target on the same strand; ‘-’ if opposite
    6	string	Target sequence name
    7	int	Target sequence length
    8	int	Target start coordinate on the original strand
    9	int	Target end coordinate on the original strand
    10	int	Number of matching bases in the mapping
    11	int	Number bases, including gaps, in the mapping
    12	int	Mapping quality (0-255 with 255 for missing)
    """
    M = args.minimizer_sketch(qseq)
    A = []

    # Collect minimizers hit
    for h, i, r in M:
        for t, i_t, r_t in H[h]:
            if t == qname:
                continue  # Do not map the same read to the same read.
            # NB. For opposite strand the coord is not reversed,
            # therefore use the invariants for the third element as such:
            # Same strand: i - i_t = qlen - maplen
            # Opposite strand: i + i_t = qlen + tlen - maplen
            # The third element can now be used to group minimizers that create
            # the same kind of mapping (i.e. that have the same maplen).
            if r == r_t:
                A.append((t, 0, i - i_t, i_t))
            else:
                A.append((t, 1, i + i_t, i_t))

    # Python automatically sorts tuples with radix sort if no key provided.
    # This is what we want here.
    A.sort()

    b = 0
    for e in range(len(A)):
        if (
            e + 1 == len(A)
            or A[e + 1][0] != A[e][0]
            or A[e + 1][1] != A[e][1]
            # Here we perform single linkage if the link < args.epsilon
            or A[e + 1][2] - A[e][2] >= args.epsilon
        ):
            # lh3/minimap also merges minimizers
            # if there is a -m=50% overlap between the two, I do not.

            potential_overlap = A[b:e]
            b = e + 1

            orientation = "-" if potential_overlap[0][1] == 1 else "+"

            # Find the maximum colinear subset by finding the longest increasing
            # subset of the target positions.
            # Notice that for the invariants above, the only factor that can
            # change slightly is the map length.
            # Find the longest increasing subset will make sure that the small
            # variations in the map length does not cause a target coord to
            # be shifted that far that it starts mixing with another match.

            if orientation == "-":
                # Since we use coords that have not been reversed for opposite
                # mappings we have to find the longest decreasing subset here.
                potential_overlap.reverse()
                indices = longest_decreasing_subset(
                    map(itemgetter(3), potential_overlap)
                )
            else:
                indices = longest_increasing_subset(
                    map(itemgetter(3), potential_overlap)
                )

            # We have now a (potential) cluster that represents a mapping.
            # Now use this cluster to create paf or reject the cluster.

            # NB. lh3/minimap breaks this chain if
            # there is a gap longer than -g=10000, I do not.

            if len(indices) <= args.min_subset:
                # Not enough minimizers in this mapping, we do not use this cluster.
                continue

            bh = potential_overlap[indices[0]]  # First minimizer for query
            eh = potential_overlap[indices[-1]]  # Last minimizer for query

            # Does not check for overlapping minimizers
            mapped_bp = len(indices) * args.k

            qlen = len(qseq)
            tname = potential_overlap[0][0]
            tlen = seq_lens[tname]

            # Undo the transformation, and find the exact coords of the ranges.
            if orientation == "+":  # (+) same strand
                tstart, tend = bh[3], eh[3] + args.k
                qstart, qend = bh[2] + bh[3], eh[2] + eh[3] + args.k
            else:  # Opposite strand (-)
                # Note: here we finally reverse the coordinates of the target strand.
                # The only place where we do that is here!
                # It could be that in fact query was reversed here, but still
                # that doesn't matter as query(reversed)-target is equivalent
                # to query-target(reversed), so we pick the latter always,
                # and do not have to bother with reversing query, as we never
                # reversed before.
                tstart, tend = tlen - bh[3] - args.k, tlen - eh[3]
                qstart, qend = bh[2] - bh[3], eh[2] - eh[3] + args.k

            maplen = max(qend - qstart, tend - tstart)
            if maplen < args.min_overlap:
                # Mapping not large enough. Do not use.
                continue

            query = qname, qlen, qstart, qend
            target = tname, tlen, tstart, tend
            mapping_info = mapped_bp, maplen

            paf = query, orientation, target, mapping_info

            yield paf

def longest_decreasing_subset(seq: Iterable[int]) -> List[int]:
    seq = list(seq)
    seq.reverse()
    indices = longest_increasing_subset(seq)
    indices_reversed = [len(seq) - ind - 1 for ind in indices]
    indices_reversed.reverse()
    return indices_reversed

def longest_increasing_subset(seq: Iterable[int]) -> List[int]:
    """
    Find the longest increasing subsequence.

    Source: https://stackoverflow.com/questions/3992697/longest-increasing-subsequence

    Returns
    -------
    indices : List[int]
        Returns the indices of the longest increasing subsequence.
    """
    seq = list(seq)
    if not seq:
        return seq

    P: List[Optional[int]] = [None]
    M = [0]

    for i in range(1, len(seq)):

        # Binary search
        lo = 0
        up = len(M)

        if seq[M[up - 1]] < seq[i]:
            j = up
            M.append(i)
        else:
            while up - lo > 1:
                mid = (lo + up) // 2
                if seq[M[mid - 1]] < seq[i]:
                    lo = mid
                else:
                    up = mid
            j = lo

            if seq[i] < seq[M[j]]:
                M[j] = i

        # Update P
        P.append(None if j == 0 else M[j - 1])

    # Trace back using the predecessor array (P).
    def trace(i):
        if i is not None:
            yield from trace(P[i])
            yield i

    indices = list(trace(M[-1]))
    return indices

def read_fastx(file: IO) -> Iterator[Tuple[str, str]]:
    """
    Currently, this function just assumes for fasta that the sequences and the
    fasta header are on alternating lines.

    Parameters
    file : IO
        Name of the input file.
    format : str
        The format ("fasta" or "fastq")

    Returns
    -------
    it : Iterator[Tuple[str, str]]
        Returns an iterator that yields tuples with the sequence name and the sequence.
    """
    first_line = next(file)
    if first_line[0] == ">":
        group_size = 2
    elif first_line[0] == "@":
        group_size = 4
    else:
        raise Exception(f"Something is wrong with the file {file.name}")

    ind_line = enumerate(chain([first_line], file))
    for ind, line in ind_line:
        if ind % group_size == 0:
            seq_name = line.strip()[1:]
            _, seq = next(ind_line)
            # We will always use upper case sequences
            yield seq_name, seq.strip().upper()

def minimap(target_file: IO, query_file: IO, output_file: IO, args: Namespace):
    """
    This is the minimap function
    """
    if not args.w:
        args.w = round(2 * args.k / 3)  # Default is 2 / 3 of k.

    target_seqs = read_fastx(target_file)
    query_seqs = read_fastx(query_file)

    args.minimizer_sketch = get_func_minimizer_sketch(args.w, args.k)

    H, seq_lens = index_targets(target_seqs, args.minimizer_sketch)

    for qname, qseq in query_seqs:
        for t in map_query(qseq, qname, seq_lens, H, args):
            # We don't calculate quality, so we just print 255 there.
            print(*chain(*t), 255, sep="\t", file=output_file)

###########
# MINIASM #
###########

# A Mapping is described by two strings: The names of the query and the target,
# and one more string to describe if they match on the same strand or opposite
# (Just like in PAF.)
Mappings = Dict[Tuple[str, str, str], Tuple[int, int, int, int]]

# The all mappings dict is used to trim ranges that don't have enough mapped coverage.
All_Mappings = Dict[str, List[Tuple[int, int, int, int, str, str]]]

# In the genome graph each strand is described with the name and a bool
# to indicate strand side. If True, it is the same strand,
# if False, it is the opposite.
# (This is equivalent to {True: '+', '-': False} compared with PAF.)
# The other integer indicates the lenght of the mapping.
# The key value pairs indicate from-to mapping.
Vertex = Tuple[str, bool]
Genome_Graph = DefaultDict[Vertex, List[Tuple[Vertex, int]]]

def read_paf_file(paf_file: IO) -> Iterator[PAF]:
    """
    Just read the file and yields each line as a paf tuple.

    Notes
    -----
    PAF file : A PAF file is used to describe mapings. It is a tab seperated file
    with te following fields:

    1	string	Query sequence name
    2	int	Query sequence length
    3	int	Query start coordinate (0-based)
    4	int	Query end coordinate (0-based)
    5	char	‘+’ if query/target on the same strand; ‘-’ if opposite
    6	string	Target sequence name
    7	int	Target sequence length
    8	int	Target start coordinate on the original strand
    9	int	Target end coordinate on the original strand
    10	int	Number of matching bases in the mapping
    11	int	Number bases, including gaps, in the mapping
    12	int	Mapping quality (0-255 with 255 for missing)
    1	string	Query sequence name
    2	int	Query sequence length
    3	int	Query start coordinate (0-based)
    4	int	Query end coordinate (0-based)
    5	char	‘+’ if query/target on the same strand; ‘-’ if opposite
    6	string	Target sequence name
    7	int	Target sequence length
    8	int	Target start coordinate on the original strand
    9	int	Target end coordinate on the original strand
    10	int	Number of matching bases in the mapping
    11	int	Number bases, including gaps, in the mapping
    12	int	Mapping quality (0-255 with 255 for missing)

    Parameters
    ----------
    paf_file : IO
        IO readable object with the paf file.

    Returns
    -------
    paf : tuple
        Returns the PAF tuple, but without quality and additional fields.
    """
    for paf_line in paf_file:
        p = paf_line.strip().split("\t")
        # Convert the rigth fields to integer, and
        # discard mapping quality, and additional fields after that.
        query = p[0], int(p[1]), int(p[2]), int(p[3])
        orientation = str(p[4])
        target = p[5], int(p[6]), int(p[7]), int(p[8])
        mapping_info = int(p[9]), int(p[10])
        paf = query, orientation, target, mapping_info
        yield paf

def clean_small_overlaps(
    pafs: Iterable[PAF], min_overlap_size: int, min_matching_bp: int
) -> Iterator[PAF]:
    """
    Step 2.1

    Remove all pafs that have an overlap too small and/or a matching region too small.

    Returns
    -------
    pafs : Iterator[PAF]
        An iterable with pafs.
    """
    return filter(
        lambda p: p[3][0] > min_matching_bp and p[3][1] > min_overlap_size, pafs
    )

def filter_overlaps_and_create_seq_lens(
    pafs: Iterable[PAF], min_coverage: int
) -> Tuple[Mappings, Seq_Lens]:
    """
    Step 2.1

    This function gathers all pafs and only returns those overlaps that
    have the minimum coverage. It will also trim the mapping outside the
    region with not enough coverage.

    This function will also make the sequence lengths dictionary.

    Returns
    ------
    overhangs : Tuple[dict, dict]
        The first value in the tuple is the overhangs, which is a dictionary
        with as a key the query->target mapping with strand char
        and as a value a list with tuples
        with qstart, qend, tstart, tend.

        The second tuple is the sequence dictionary which maps the sequence name to
        its length.
    """
    mappings: All_Mappings = defaultdict(list)
    seq_lens: Seq_Lens = dict()

    for query, orientation, target, mapping_info in pafs:
        mappings[query[0]].append(
            (query[2], query[3], target[2], target[3], target[0], orientation)
        )
        seq_lens[query[0]] = query[1]

    # trim_mapping function will trim the mapping based on the list of mappings
    # and the min_coverage.

    trimmed_mappings: Mappings = {
        (qname, trimmed_mapping[4], trimmed_mapping[5]): trimmed_mapping[:4]
        for qname, mappings_list in mappings.items()
        for trimmed_mapping in trim_mappings(mappings_list, min_coverage)
    }
    return trimmed_mappings, seq_lens

def trim_mappings(
    mappings_list: List[Tuple[int, int, int, int, str, str]], min_coverage: int
) -> Iterator[Tuple[int, int, int, int, str, str]]:
    """
    Trim the mappings where the min_coverage is not achieved.

    This function will calculate the mapping for each strand (query and target)
    separately. (They can differ slightly.)

    Returns
    -------
    trimmed_mapping_list : List[Tuple[int, int, int, int, str, str]]
        The mapping with query start and end, and target start and end.

    If there will be no mapping left it is not included in the list again.

    Notes
    -----
    For this algorithm there is still an unresolved edge case. That is, when a
    mapping can be found in two different trimmed ranges, it is unclear if
    this mapping should be cut in two. Here we just keep the first half.
    """
    trimmed_ranges_it = find_minimum_coverage_range(
        ((m[0], m[1]) for m in mappings_list), min_coverage
    )
    return trim_mappings_with_ranges(mappings_list, trimmed_ranges_it)

def trim_mappings_with_ranges(
    mappings_list: List[Tuple[int, int, int, int, str, str]],
    trimmed_ranges_it: Iterator[Tuple[int, int]],
) -> Iterator[Tuple[int, int, int, int, str, str]]:

    trim_start, trim_end = next(trimmed_ranges_it)
    for mapping in sorted(mappings_list, key=itemgetter(0)):
        qstart, qend, tstart, tend, tname, ori = mapping

        # Move to a new trimmed range if needed
        while qstart > trim_end:
            trim_range = next(trimmed_ranges_it, None)
            if not trim_range:
                return
            trim_start, trim_end = trim_range

        qtrim, etrim = 0, 0

        if trim_start > qstart:  # Not inside trimmed range -> trim
            strim = trim_start - qstart
            qstart = trim_start

        if trim_end < qend:  # Not inside trimmed_range
            etrim = qend - trim_end
            qend = trim_end

        if not qtrim and not etrim:  # No trimming, prevent tuple reconstruction.
            yield mapping
        else:
            # We trim the target range as well with the same parameters.
            # There can however be an edge case where the target range is
            # slightly different and give an invalid range, then we just
            # skip it completely
            tstart = tstart + strim
            tend = tend - etrim
            if tstart < tend and tstart > 0 and tend > 0:
                yield qstart, qend, tstart, tend, tname, ori

def find_minimum_coverage_range(
    coords: Iterable[Tuple[int, int]], min_coverage: int
) -> Iterator[Tuple[int, int]]:
    """
    Trim the coordinates for minimum coverage.

    We do this by looping over the collection in sorted order of the first
    coordinate. We keep a priority queue with the end coordinates of the
    ranges. Each time we process a new range, we remove all the end coordinates
    that are before the beginning of the current range. Now the size of the
    priority queue is equal to the coverage.

    Returns
    -------
    trimmed_mappings_it : Iterator[Tuple[int, int]]
        Returns the trimmed ranges in sorted order as an iterator.
    """
    # Use heapq as a min heap to use as a priority qeueu
    end_coords: List[int] = []
    trimmed_range_start: List[int] = []
    trimmed_range_end: List[int] = []
    in_range = False
    for b, e in sorted(coords, key=itemgetter(0)):

        # Remove all end_coords that are behind the begin of the current range.
        while end_coords and end_coords[0] < b:
            last_end = heapq.heappop(end_coords)

            # If this removing, reduces the coverage below min_coverage and we were
            # in_range, add the last removed end as the end of the trimmed_range.
            if len(end_coords) < min_coverage:
                if in_range:
                    trimmed_range_end.append(last_end)
                    in_range = False

        # Now add the new end coord to the end_coords
        heapq.heappush(end_coords, e)

        # If this new range bring the coverage on or over the min_coverage, and we
        # were not in range, start a new range with the beginning of the current range.
        if len(end_coords) >= min_coverage:
            if not in_range:
                trimmed_range_start.append(b)
                in_range = True

    # Now return all the trimmed ranges with the minimum coverage
    return zip(trimmed_range_start, trimmed_range_end)

def create_genome_graph(
    mappings: Mappings,
    seq_lens: Seq_Lens,
    max_overhang: int,
    max_overhang_ratio: float,
) -> Genome_Graph:
    """
    Step 2.2; Algorithm 5

    This function finds the overlaps with the correct characteristics.

    With all the overlaps with the correct characteristics a new dictionary
    is created which represents the genome graph.

    Parameters
    ----------
    mappings : Mappings
        A dictionary with all the mappings.
    seq_lens : Seq_Lens
        A dictionary that maps the sequence name to the lenght of the sequence.
    max_overhang : int
        The maximum size the overhangs of the mappings can be to still be a
        viable mapping for the genome graph.
    max_overhang_ratio : float
        The maximum ratio between the overhang and the mapping length for
        the mapping to still be a viable candidate for the genome graph.

    Returns
    -------
    genome_graph : Genome_Graph
        Returns a dictionary that represents the genome graph.
    """
    genome_graph: Genome_Graph = defaultdict(list)
    for (query, target, ori), (qstart, qend, tstart, tend) in mappings.items():
        # We only add one edge per mapping
        # as we assume the PAF file is already
        # symmetric. (I.e. it contains all-vs-all mappings where all reads can be
        # found in both the target role as well as the query role.)

        qlen = seq_lens[query]
        tlen = seq_lens[target]

        # First we change the situation to the 'normal' one, where we look
        # to the same site of both strands.
        if ori == "-":
            tstart, tend = tlen - tend, tlen - tstart

        overhang = min(qstart, tstart) + min(qlen - qend, tlen - tend)
        maplen = max(qend - qstart, tend - tstart)

        if overhang > min(max_overhang, max_overhang_ratio * maplen):
            # Internal match
            continue
        elif qstart <= tstart and qlen - qend <= tlen - tend:
            # Query contained
            continue
        elif qstart >= tstart and qlen - qend >= tlen - tend:
            # Target contained
            continue
        elif qstart > tstart:
            # Query to target
            target_ori = ori != "-"  # False if on opposite strands.
            genome_graph[query, True].append(((target, target_ori), maplen))
        else:
            # Target to query
            # For the symmetry we only add query to target mappings.
            # For this target to query mapping, we just swap the strand side
            # and add query complement strand to target complement strand.
            # (NB. The target to query mapping will be added with another
            # mapping that swaps this roles. Provided the PAF is symmetrical.)

            # To change target-to-query mapping, to query-to-target mapping,
            # we swap strands. If mapping is on opposite strand, target is
            # on the same strand again.
            target_ori = ori == "-"
            genome_graph[query, False].append(((target, target_ori), maplen))

    return genome_graph

# Graph cleaning functions.

def remove_transitive_edges(
    genome_graph: Genome_Graph, vertices: List[Vertex], seq_lens: Seq_Lens, fuzz: int
) -> None:
    """
    Myers 2005

    This function goes through the genome graph and removes transitive edges.
    These are edges that have a mapping that is not really needed as other
    edges already show that the nodes relate.
    """

    def edge_len(edge_info: Tuple[Vertex, int]) -> int:
        vertex, maplen = edge_info
        return seq_lens[vertex[0]] - maplen

    for val in genome_graph.values():
        # Sort all edges so that the longest
        val.sort(key=edge_len)

    # This dict has the following convention: {0: vacant, 1: inplay, 2: eliminated}
    vertices_dict: Dict[Vertex, int] = dict.fromkeys(vertices, 0)

    for v in vertices:
        for w, _ in genome_graph[v]:
            vertices_dict[w] = 1

        longest = edge_len(genome_graph[v][-1]) + fuzz

        for w, maplen_w in genome_graph[v]:
            if vertices_dict[w] != 1:
                continue
            v2w_len = edge_len((w, maplen_w))
            edge_it = iter(genome_graph[w])

            # Check the smallest edge of w
            x, maplen = next(edge_it)
            if vertices_dict[x] == 1:
                vertices_dict[x] == 2  # eliminated

            for x, maplen_x in edge_it:

                # Check all edges smaller than fuzz.
                if edge_len((x, maplen_x)) < fuzz and vertices_dict[x] == 1:
                    vertices_dict[x] == 2  # eliminated
                # With fuzz=10, this might seem very small,
                # but the coverage trimming might
                # reduce a mapping below what you could get from minimap

                # When edge too long, we don't have to check anymore
                if v2w_len + edge_len((x, maplen_x)) > longest:
                    break

                if vertices_dict[x] == 1:
                    vertices_dict[x] = 2  # eliminated

        new_edges: List[Tuple[Vertex, int]] = []
        for w, maplen in genome_graph[v]:
            if vertices_dict[w] != 2:
                new_edges.append((w, maplen))
            vertices_dict[w] = 0

        if new_edges:
            genome_graph[v] = new_edges
        else:
            del genome_graph[v]

def remove_small_tip(genome_graph: Genome_Graph, min_size_tip: int, v: Vertex) -> None:
    """
    Checks if v is the end of a small tip (a small tip is defined if it connects
    to something biggers within min_size_tip nodes).
    """
    # We find the tip of edges,
    # by inspecting if the symmetry vertex does not contain edges.
    if genome_graph.get((v[0], not v[1])):
        return

    # This is a tip, see if it is small.

    # Follow incoming edges and see if we hit a junction within min_size_tip.
    tip_consists: List[Vertex] = []
    for _ in repeat(None, min_size_tip):
        # We check if there are more than one edges in the vertex and its complement.
        # If there is more than one in one of them, this is a junction,
        # not part of the tip.
        if len(genome_graph[v]) > 1 or len(genome_graph[v[0], not v[1]]) > 1:
            # Junction: remove tip and return
            for w in tip_consists:
                del genome_graph[w]
                del genome_graph[w[0], not v[1]]
            return
        tip_consists.append(v)
        v = genome_graph[v][0][0]

    # Moved over edges and no junction, so this is a long tip.
    return

def pop_bubble(
    genome_graph: Genome_Graph, seq_lens: Seq_Lens, probe_distance: int, start: Vertex,
) -> None:
    """
    Popping small bubbles from the genome_graph.

    Algorithm 6.

    A bubble is defined as the isolated part of the genome from node A to node B
    where there are multiple paths from A to B. The probe_distance here defines
    how big a bubble can be.

    We remove all of the bubble accept a chosen path in the bubble. This path
    is chosen by greedily pick the largest edge when we traverse the bubble.
    """
    if len(genome_graph[start]) < 2:  # Cannot be the source of a bubble.
        return

    # Keep track of a path through the bubble, and all other vertices
    bubble_path = {start}
    path_tip = start
    other_vertices = set()

    # Keeping track of things to find the bubble.
    unvisited_incoming = dict()
    distances = dict.fromkeys(genome_graph.keys(), sys.maxsize)
    distances[start] = 0
    S = [start]
    p = 0  # Number of visited vertices not yet in S.

    while S:
        v = S.pop()
        first = True
        for w, maplen in sorted(genome_graph[v], key=itemgetter(1), reverse=True):
            if path_tip == v and first:
                # We keep track of a path by checking if the current vertex
                # is in the last in the path and appending the first next vertex.
                # The sorting puts the longest map length as the preferred path.
                bubble_path.add(w)
                path_tip = w
                first = False
            elif w not in bubble_path:
                other_vertices.add(w)

            if w == start:  # A circle is not a bubble.
                return

            new_dist = distances[v] + seq_lens[v[0]] - maplen
            if new_dist > probe_distance:  # Too far away
                return

            if distances[w] == sys.maxsize:  # Not visited
                # We make use of the symmetry to find the incoming edges.
                unvisited_incoming[w] = len(genome_graph[w[0], not [1]])
                p += 1
            elif new_dist < distances[w]:
                distances[w] = new_dist

            unvisited_incoming[w] -= 1
            if unvisited_incoming[w] == 0:  # We have visisted all incoming edges.
                if genome_graph[w]:  # Not a tip
                    S.append(w)
                p -= 1

        if len(S) == 1 and p == 0:  # This is the sink.
            # Now remove all vertices that are part of the bubble, but not the path.
            # As a bonus we remove the other side as well.
            for v in other_vertices:
                del genome_graph[v]
                del genome_graph[v[0], not v[1]]
            return

def create_unitigs(
    genome_graph: Genome_Graph, vertices: List[Vertex]
) -> Dict[str, List[Tuple[Vertex, int]]]:
    """
    Goes over the graph and when a path does not branches it merges the nodes of
    that path. These mergings are saved in a dict and returned. This dict can
    later be used to create the sequences that form the unitigs.
    """

    def utg_to_name(utg_nr: int) -> str:
        return f"{utg_nr:06}"

    unitig_to_reads: Dict[str, List[Tuple[Vertex, int]]] = dict()
    utg_nr = 1
    converted_start: Dict[str, str] = dict()  # dict with start tip to converted utg

    for start in vertices:
        # First see if the start of the unitig goes one way.
        if len(genome_graph[start]) != 1:
            continue

        # The start of a unitig must be a junction or a tip.
        other_start = start[0], not start[1]  # Using symmetry vertex again
        in_edges = len(genome_graph[other_start])
        if in_edges == 1:
            continue

        # Continue till junction or tip
        utg_path = [(start, 0)]
        v = start
        while len(genome_graph[v]) == 1:
            del genome_graph[v]
            v_info = genome_graph[v][0]
            v = v_info[0]
            utg_path.append(v_info)

        end = v
        utg_name = converted_start[end[0]]
        if utg_name:
            ori = False
        else:
            utg_name = utg_to_name(utg_nr)
            utg_nr += 1
            ori = True

        unitig_to_reads[utg_name] = utg_path

        incoming_edges = [(w[0], not w[1]) for w, _ in genome_graph[other_start]]
        for w in incoming_edges:
            genome_graph[w] = [
                ((utg_name, ori) if x[0] == start else x, maplen)
                for x, maplen in genome_graph[w]
            ]

        genome_graph[utg_name, ori] = genome_graph[end]
        del genome_graph[end]

    return unitig_to_reads

# Print to file functions.

def gfa(
    genome_graph: Genome_Graph,
    unitig_to_reads: Dict[str, List[Tuple[Vertex, int]]],
    read_seqs: Dict[str, str],
) -> Iterator[Tuple[str, ...]]:
    """
    This function takes the genome_graph that is completely cleaned and
    with the reads, a dict to outlines how the unitigs are formed it will
    output the genome assembly lines of the GFA file.
    """

    def get_ori(ori: bool) -> str:
        return "+" if ori else "-"

    for name, ori in genome_graph.keys():
        seq = read_seqs[name]
        if seq:
            yield "S", name, seq
            continue

        # It's a utg
        utg_name = name
        utg_path = iter(unitig_to_reads[utg_name])
        name, ori = next(utg_path)[0]
        new_seq = read_seqs[name] if ori else reverse_complement(read_seqs[name])
        for (name, ori), maplen in utg_path:
            add_seq = read_seqs[name] if ori else reverse_complement(read_seqs[name])
            new_seq += add_seq[maplen:]
        yield "S", utg_name, new_seq

    for v, edges in genome_graph.items():
        for (w, maplen) in edges:
            yield "L", v[0], get_ori(v[1]), w[0], get_ori(w[1]), f"{maplen}M"

def print_gfa_file(
    genome_graph: Genome_Graph,
    unitig_to_reads: Dict[str, List[Tuple[Vertex, int]]],
    read_seqs: Dict[str, str],
    out: IO,
):
    """
    Prints the GFA lines to the output file.
    """
    for line in gfa(genome_graph, unitig_to_reads, read_seqs):
        print(*line, sep="\t", file=out)

def miniasm(paf_file: IO, reads_file: IO, out: IO, args: Namespace):
    """
    This is the miniasm function.
    """
    # First read the PAF file and filter the pafs (2.1).
    pafs = read_paf_file(paf_file)

    pafs = clean_small_overlaps(pafs, args.min_overlap_size, args.min_matching_bp)
    mappings, seq_lens = filter_overlaps_and_create_seq_lens(pafs, args.min_coverage)

    # Create graph from pafs (2.2).
    genome_graph = create_genome_graph(
        mappings, seq_lens, args.max_overhang, args.max_overhang_ratio
    )

    # Graph cleaning (2.3).

    # Looping the keys-view, while modifying dict leads to bizarre behavior.
    # Thus we make a copy of the vertices for every cleaning operation.

    vertices = list(genome_graph.keys())
    remove_transitive_edges(genome_graph, vertices, seq_lens, args.fuzz)

    vertices = list(genome_graph.keys())
    for v in vertices:
        remove_small_tip(genome_graph, args.min_size_tip, v)

    vertices = list(genome_graph.keys())
    for start in vertices:
        pop_bubble(genome_graph, seq_lens, args.probe_distance, start)

    vertices = list(genome_graph.keys())
    unitig_to_reads = create_unitigs(genome_graph, vertices)

    # Convert to gfa format

    read_seqs: Dict[str, str] = dict(read_fastx(reads_file))

    print_gfa_file(genome_graph, unitig_to_reads, read_seqs, out)

#########
# Tests #
#########

def mytest():
    pass

class TestSuite(unittest.TestCase):
    def test_hash(self):
        seq = "TTA"
        h = hash_seq(seq)
        self.assertEqual(h, 60)

    def test_invertible_hash(self):
        new_h = invertable_hash(60)
        self.assertEqual(new_h, 9473421487448830983)

    def test_find_minimizers(self):
        """
        Note: As can be seen, here the coords are not the same as given in the
        project page. The reason is that we only apply reversal of the coords
        in the very end at the paf outputting. This is explained more in-depth
        in the notes of the :func:`compute_minimizers` function and all the
        details of reversal or dealt with in the :func:`map_query` function.
        """
        seq = "GATTACAACT"
        minimizer_sketch = get_func_minimizer_sketch(w=4, k=3)
        self.assertEqual(
            minimizer_sketch(seq),
            {(1396078460937419741, 1, True), (2859083004982788208, 4, True)},
        )

    def test_longest_increasing(self):
        alist = [3, 1, 2, 5, 4, 8, 6]
        indices = [1, 2, 4, 6]
        self.assertEqual(longest_increasing_subset(alist), indices)

if __name__ == "__main__":

    parser = argparse.ArgumentParser(
        prog="MINIMAP-MINIASM",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description="This program contains a minimap and a miniasm implementation."
        " Run one of them by providing the first word the same as that program."
        " Those subprograms contain a --help as well.",
    )

    # The '-' for files types indicates stdin or stdout.
    parser.add_argument("--out", type=argparse.FileType("w"), default="-")

    subparsers = parser.add_subparsers(help="MINIMAP-MINIASM")

    # minimap argument parsing
    minimap_argparser = subparsers.add_parser(
        "minimap",
        help="Map sequences.",
        description="This is the minimap program."
        " Arguments have the same flags as the minimap program,"
        " but also have a longer name option. Defaults are equal as well."
        " Not all options are included."
        " The -L flag can only be found in the original minimap program,"
        " but not in the new minimap2 program.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    minimap_argparser.add_argument("target", type=argparse.FileType("r"))
    minimap_argparser.add_argument("query", type=argparse.FileType("r"))

    indexing = minimap_argparser.add_argument_group("Indexing options")
    indexing.add_argument("-k", help="Minimizer length.", default=15, type=int)
    indexing.add_argument(
        "-w",
        help="Minimizer window size (default is 2/3 of k).",
        type=int,
        default=argparse.SUPPRESS,
    )

    mapping = minimap_argparser.add_argument_group("Mapping options")
    mapping.add_argument(
        "-n",
        "--min-subset",
        help="Minimum minimizers in mappings",
        default=4,
        type=int,
    )
    mapping.add_argument(
        "-r",
        "--epsilon",
        help="Maximum gap size between minimizers. The bandwidth used in the"
        " single linkage clustering.",
        default=500,
        type=int,
    )
    mapping.add_argument(
        "-L",
        "--min-overlap",
        help="Minimum overlap of a mapping",
        default=100,
        type=int,
    )

    # miniasm argument parsing
    miniasm_argparser = subparsers.add_parser(
        "miniasm",
        help="Assembly from PAF file.",
        description="This is the miniasm program."
        " Arguments have the same flags as the minimap program,"
        " but also have a longer name option. Defaults are equal as well."
        " Not all options are included."
        " The -h option for miniasm has been changed to the -H option"
        " as it conflicts with the -h, --help option.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    miniasm_argparser.add_argument("paf", type=argparse.FileType("r"))

    preselection = miniasm_argparser.add_argument_group("Preselection options")
    preselection.add_argument(
        "-m",
        "--min-matching-bp",
        default=100,
        type=int,
        help="Drop mappings having less mapped bp than min-matching-bp.",
    )
    preselection.add_argument(
        "-s",
        "--min-overlap-size",
        default=1000,
        type=int,
        help="Drop mappings that are shorter than min-overlap-size.",
    )

    overlapping = miniasm_argparser.add_argument_group("Overlapping options")
    overlapping.add_argument(
        "-H", "--max-overhang", help="Maximum overhang length", default=1000, type=int
    )
    overlapping.add_argument(
        "-I",
        "--max-overhang-ratio",
        help="Maximum overhang ratio.",
        default=0.8,
        type=float,
    )

    graph = miniasm_argparser.add_argument_group("Graph layout options")
    graph.add_argument(
        "-n", "--min-coverage", help="Minimum coverage of overlap", default=3, type=int
    )
    graph.add_argument(
        "-e", "--min-size-tip", help="The minimum size of a tip", default=4, type=int
    )
    graph.add_argument(
        "-f",
        "--reads",
        type=argparse.FileType("r"),
        required=True,
        help="This is the read file. It is for now required.",
    )
    graph.add_argument(
        "-d",
        "--probe-distance",
        help="Maximum probe distance for bubble popping.",
        default=50000,
        type=int,
    )

    subparsers.add_parser("test", help="Run the unittests.")
    subparsers.add_parser("mytest", help="Run some debugging function.")

    subprograms = {"minimap", "miniasm", "test", "mytest", "--help", "-h"}

    if len(sys.argv) < 1:
        print("Debug mode for Python REPL.")
    elif sys.argv[1] not in subprograms:
        print(
            "You did not select a correct sub program"
            f" from the subprograms: {subprograms}"
        )
        sys.exit(1)
    elif sys.argv[1] == "test":
        unittest.main(argv=["ignore-this"])
    elif sys.argv[1] == "mytest":
        mytest()
    else:
        args = parser.parse_args()
        if sys.argv[1] == "minimap":
            minimap(args.target, args.query, args.out, args)
        elif sys.argv[1] == "miniasm":
            args.fuzz = 10  # Used by the removal of transitive edges
            miniasm(args.paf, args.reads, args.out, args)


# File for the variance threshold feature selection algorithm
from sklearn.feature_selection import VarianceThreshold

# The function which will be called
def get_features(raw_data, raw_data_ids):
    '''
    Perform feature selection using variance threshold
    Removes features without any variance
    '''

    # Returns columns that meet the amount of variance
    sel = VarianceThreshold()
    sel.fit(raw_data)

     # Set aside correct columns
    return_columns = []
    index = 0
    for feature in sel.get_support():
        if feature:
            return_columns.append(index)
        index += 1

    # return
    return return_columns


import argparse
from datetime import datetime as dt
import json
from typing import Dict

"""
Potential additions:
    Profile specific clients
    Check for burst traffic (time-series analysis)
    Check for number of different paths requested (may be able to detect scanning activity)
"""

def parse_timestamp(ts_string: str, ts_format: str):
    """
    Parse a given string with the given datetime format and return the corresponding Unix epoch timestamp.
    :param ts_string: string containing a datetime object of some kind
    :param ts_format: format string presumably matching the format of ts_string
    :return: number of seconds since epoch
    """
    return dt.strptime(ts_string, ts_format).timestamp()

def profile_apache_logs(input_file: str) -> Dict:
    """
    Create a log profile from a parsed Apache access log file.
    :param input_file: Apache access log file (JSON format)
    :return: dictionary with profiled statistics
    """
    apache_ts_format = '%d/%b/%Y:%H:%M:%S %z'
    with open(input_file, 'r') as f:
        logs = json.load(f)
    baseline = {'requests': {'all': {'total': 0}}, 'methods_breakdown': {}}
    first_datetime = None
    last_datetime = None
    for log in logs:
        if not first_datetime:
            first_datetime = log['datetime']
        baseline['requests']['all']['total'] += 1
        method = log['request_method'].lower()
        if method not in baseline['methods_breakdown']:
            baseline['methods_breakdown'][method] = 0
        baseline['methods_breakdown'][method] += 1
        last_datetime = log['datetime']

    earliest_timestamp = parse_timestamp(first_datetime, apache_ts_format)
    latest_timestamp = parse_timestamp(last_datetime, apache_ts_format)
    baseline['first_timestamp'] = earliest_timestamp
    baseline['last_timestamp'] = latest_timestamp
    baseline['seconds_elapsed'] = latest_timestamp - earliest_timestamp
    baseline['minutes_elapsed'] = baseline['seconds_elapsed'] / 60
    baseline['requests']['all']['avg_per_sec'] = baseline['requests']['all']['total'] / baseline['seconds_elapsed']
    baseline['requests']['all']['avg_per_min'] = baseline['requests']['all']['total'] / baseline['minutes_elapsed']

    for k, v in baseline['methods_breakdown'].items():
        baseline['requests'][k] = {'total': v}
        baseline['requests'][k]['avg_per_sec'] = v / baseline['seconds_elapsed']
        baseline['requests'][k]['avg_per_min'] = v / baseline['minutes_elapsed']

    del baseline['methods_breakdown']
    return baseline

def main():
    parser = argparse.ArgumentParser(
        description='This application calculates a log profile from a provided parsed log file.')
    parser.add_argument('-i', '--input', required=True, help='Parsed log file (JSON format) to read from')
    parser.add_argument('-l', '--log-format', required=True, choices=['apache'], help='Type of log to parse')
    parser.add_argument('-o', '--output', help='Output file to write to')
    args = parser.parse_args()

    input_file = args.input
    log_format = args.log_format
    output = args.output

    profile_function = globals()['profile_{}_logs'.format(log_format)]
    profile = profile_function(input_file)

    if output:
        with open(output, 'w') as of:
            json.dump(profile, of, indent=2)
    else:
        print(json.dumps(profile, indent=2))

if __name__ == '__main__':
    main()