from math import log, pi

from fluids.constants import g
from fluids.friction import friction_factor

__all__ = ['Lehrer', 'Stein_Schmidt']

def Lehrer(m, Dtank, Djacket, H, Dinlet, rho, Cp, k, mu, muw=None,
           isobaric_expansion=None, dT=None, inlettype='tangential',
           inletlocation='auto'):
    r'''Calculates average heat transfer coefficient for a jacket around a
    vessel according to [1]_ as described in [2]_.

    .. math::
        Nu_{S,L} = \left[\frac{0.03Re_S^{0.75}Pr}{1 + \frac{1.74(Pr-1)}
        {Re_S^{0.125}}}\right]\left(\frac{\mu}{\mu_w}\right)^{0.14}

    .. math::
        d_g = \left(\frac{8}{3}\right)^{0.5}\delta

    .. math::
        v_h = (v_Sv_{inlet})^{0.5} + v_A

    .. math::
        v_{inlet} = \frac{Q}{\frac{\pi}{4}d_{inlet}^2}

    .. math::
        v_s = \frac{Q}{\frac{\pi}{4}(D_{jacket}^2 - D_{tank}^2)}

    For Radial inlets:

    .. math::
        v_A = 0.5(2g H \beta\delta \Delta T)^{0.5}

    For Tangential inlets:

    .. math::
        v_A = 0

    Parameters
    ----------
    m : float
        Mass flow rate of fluid, [kg/s]
    Dtank : float
        Outer diameter of tank or vessel surrounded by jacket, [m]
    Djacket : float
        Inner diameter of jacket surrounding a vessel or tank, [m]
    H : float
        Height of the vessel or tank, [m]
    Dinlet : float
        Inner diameter of inlet into the jacket, [m]
    rho : float
        Density of the fluid at Tm [kg/m^3]
    Cp : float
        Heat capacity of fluid at Tm [J/kg/K]
    k : float
        Thermal conductivity of fluid at Tm [W/m/K]
    mu : float
        Viscosity of fluid at Tm [Pa*s]
    muw : float, optional
        Viscosity of fluid at Tw [Pa*s]
    isobaric_expansion : float, optional
        Constant pressure expansivity of a fluid, [m^3/mol/K]
    dT : float, optional
        Temperature difference of fluid in jacket, [K]
    inlettype : str, optional
        Either 'tangential' or 'radial'
    inletlocation : str, optional
        Either 'top' or 'bottom' or 'auto'

    Returns
    -------
    h : float
        Average heat transfer coefficient inside the jacket [W/m^2/K]

    Notes
    -----
    If the fluid is heated and enters from the bottom, natural convection
    assists the heat transfer and the Grashof term is added; if it were to enter
    from the top, it would be subtracted. The situation is reversed if entry
    is from the top.

    Examples
    --------
    Example as in [2]_, matches completely.

    >>> Lehrer(m=2.5, Dtank=0.6, Djacket=0.65, H=0.6, Dinlet=0.025, dT=20.,
    ... rho=995.7, Cp=4178.1, k=0.615, mu=798E-6, muw=355E-6)
    2922.128124761829

    Examples similar to in [2]_ but covering the other case:

    >>> Lehrer(m=2.5, Dtank=0.6, Djacket=0.65, H=0.6, Dinlet=0.025, dT=20.,
    ... rho=995.7, Cp=4178.1, k=0.615, mu=798E-6, muw=355E-6,
    ... inlettype='radial', isobaric_expansion=0.000303)
    3269.4389632666557

    References
    ----------
    .. [1] Lehrer, Isaac H. "Jacket-Side Nusselt Number." Industrial &
       Engineering Chemistry Process Design and Development 9, no. 4
       (October 1, 1970): 553-58. doi:10.1021/i260036a010.
    .. [2] Gesellschaft, V. D. I., ed. VDI Heat Atlas. 2nd edition.
       Berlin; New York:: Springer, 2010.
    '''
    delta = (Djacket-Dtank)/2.
    Q = m/rho
    Pr = Cp*mu/k
    vs = Q/H/delta
    vo = Q/(pi/4*Dinlet**2)
    if dT is not None and isobaric_expansion is not None and inlettype == 'radial' and inletlocation is not None:
        if dT > 0: # Heating jacket fluid
            if inletlocation == 'auto' or inletlocation == 'bottom':
                va = 0.5*(2*g*H*isobaric_expansion*abs(dT))**0.5
            else:
                va = -0.5*(2*g*H*isobaric_expansion*abs(dT))**0.5
        else: # cooling fluid
            if inletlocation == 'auto' or inletlocation == 'top':
                va = 0.5*(2*g*H*isobaric_expansion*abs(dT))**0.5
            else:
                va = -0.5*(2*g*H*isobaric_expansion*abs(dT))**0.5
    else:
        va = 0
    vh = (vs*vo)**0.5 + va
    dg = (8/3.)**0.5*delta
    Res = vh*dg*rho/mu
    if muw is not None:
        NuSL = (0.03*Res**0.75*Pr)/(1 + 1.74*(Pr-1)/Res**0.125)*(mu/muw)**0.14
    else:
        NuSL = (0.03*Res**0.75*Pr)/(1 + 1.74*(Pr-1)/Res**0.125)
    return NuSL*k/dg

def Stein_Schmidt(m, Dtank, Djacket, H, Dinlet,
                  rho, Cp, k, mu, muw=None, rhow=None,
                  inlettype='tangential', inletlocation='auto', roughness=0.0):
    r'''Calculates average heat transfer coefficient for a jacket around a
    vessel according to [1]_ as described in [2]_.

    .. math::
        l_{ch} = \left[\left(\frac{\pi}{2}\right)^2 D_{tank}^2+H^2\right]^{0.5}

    .. math::
        d_{ch} = 2\delta

    .. math::
        Re_j = \frac{v_{ch}d_{ch}\rho}{\mu}

    .. math::
        Gr_J = \frac{g\rho(\rho-\rho_w)d_{ch}^3}{\mu^2}

    .. math::
        Re_{J,eq} = \left[Re_J^2\pm \left(\frac{|Gr_J|\frac{H}{d_{ch}}}{50}
        \right)\right]^{0.5}

    .. math::
        Nu_J = (Nu_A^3 + Nu_B^3 + Nu_C^3 + Nu_D^3)^{1/3}\left(\frac{\mu}
        {\mu_w}\right)^{0.14}

    .. math::
        Nu_J = \frac{h d_{ch}}{k}

    .. math::
        Nu_A = 3.66

    .. math::
        Nu_B = 1.62 Pr^{1/3}Re_{J,eq}^{1/3}\left(\frac{d_{ch}}{l_{ch}}
        \right)^{1/3}

    .. math::
        Nu_C = 0.664Pr^{1/3}(Re_{J,eq}\frac{d_{ch}}{l_{ch}})^{0.5}

    .. math::
        \text{if } Re_{J,eq} < 2300: Nu_D = 0

    .. math::
        Nu_D = 0.0115Pr^{1/3}Re_{J,eq}^{0.9}\left(1 - \left(\frac{2300}
        {Re_{J,eq}}\right)^{2.5}\right)\left(1 + \left(\frac{d_{ch}}{l_{ch}}
        \right)^{2/3}\right)

    For Radial inlets:

    .. math::
        v_{ch} = v_{Mit}\left(\frac{\ln\frac{b_{Mit}}{b_{Ein}}}{1 -
        \frac{b_{Ein}}{b_{Mit}}}\right)

    .. math::
        b_{Ein} = \frac{\pi}{8}\frac{D_{inlet}^2}{\delta}

    .. math::
        b_{Mit} = \frac{\pi}{2}D_{tank}\sqrt{1 + \frac{\pi^2}{4}\frac
        {D_{tank}^2}{H^2}}

    .. math::
        v_{Mit} = \frac{Q}{2\delta b_{Mit}}

    For Tangential inlets:

    .. math::
        v_{ch} = (v_x^2 + v_z^2)^{0.5}

    .. math::
        v_x = v_{inlet}\left(\frac{\ln[1 + \frac{f_d D_{tank}H}{D_{inlet}^2}
        \frac{v_x(0)}{v_{inlet}}]}{\frac{f_d D_{tank}H}{D_{inlet}^2}}\right)

    .. math::
        v_x(0) = K_3 + (K_3^2 + K_4)^{0.5}

    .. math::
        K_3 = \frac{v_{inlet}}{4} -\frac{D_{inlet}^2v_{inlet}}{4f_d D_{tank}H}

    .. math::
        K_4 = \frac{D_{inlet}^2v_{inlet}^2}{2f_d D_{tank} H}

    .. math::
        v_z = \frac{Q}{\pi D_{tank}\delta}

    .. math::
        v_{inlet} = \frac{Q}{\frac{\pi}{4}D_{inlet}^2}

    Parameters
    ----------
    m : float
        Mass flow rate of fluid, [kg/m^3]
    Dtank : float
        Outer diameter of tank or vessel surrounded by jacket, [m]
    Djacket : float
        Inner diameter of jacket surrounding a vessel or tank, [m]
    H : float
        Height of the vessel or tank, [m]
    Dinlet : float
        Inner diameter of inlet into the jacket, [m]
    rho : float
        Density of the fluid at Tm [kg/m^3]
    Cp : float
        Heat capacity of fluid at Tm [J/kg/K]
    k : float
        Thermal conductivity of fluid at Tm [W/m/K]
    mu : float
        Viscosity of fluid at Tm [Pa*s]
    muw : float, optional
        Viscosity of fluid at Tw [Pa*s]
    rhow : float, optional
        Density of the fluid at Tw [kg/m^3]
    inlettype : str, optional
        Either 'tangential' or 'radial'
    inletlocation : str, optional
        Either 'top' or 'bottom' or 'auto'
    roughness : float, optional
        Roughness of the tank walls [m]

    Returns
    -------
    h : float
        Average  transfer coefficient inside the jacket [W/m^2/K]

    Notes
    -----
    [1]_ is in German and has not been reviewed. Multiple other formulations
    are considered in [1]_.

    If the fluid is heated and enters from the bottom, natural convection
    assists the heat transfer and the Grashof term is added; if it were to enter
    from the top, it would be subtracted. The situation is reversed if entry
    is from the top.

    Examples
    --------
    Example as in [2]_, matches in all but friction factor:

    >>> Stein_Schmidt(m=2.5, Dtank=0.6, Djacket=0.65, H=0.6, Dinlet=0.025,
    ... rho=995.7, Cp=4178.1, k=0.615, mu=798E-6, muw=355E-6, rhow=971.8)
    5695.2041698088615

    References
    ----------
    .. [1] Stein, Prof Dr-Ing Werner Alexander, and Dipl-Ing (FH) Wolfgang
       Schmidt. "Wärmeübergang auf der Wärmeträgerseite eines Rührbehälters mit
       einem einfachen Mantel." Forschung im Ingenieurwesen 59, no. 5
       (May 1993): 73-90. doi:10.1007/BF02561203.
    .. [2] Gesellschaft, V. D. I., ed. VDI Heat Atlas. 2nd edition.
       Berlin; New York:: Springer, 2010.
    '''
    delta = (Djacket-Dtank)/2.
    Q = m/rho
    Pr = Cp*mu/k
    lch = (pi**2/4*Dtank**2 + H**2)**0.5
    dch = 2*delta
    if inlettype == 'radial':
        bEin = pi/8*Dinlet**2/delta
        bMit = pi/2*Dtank*(1 + pi**2/4*Dtank**2/H**2)**0.5
        vMit = Q/(2*delta*bMit)
        vch = vMit*log(bMit/bEin)/(1 - bEin/bMit)
        ReJ = vch*dch*rho/mu
    elif inlettype == 'tangential':
        f = friction_factor(1E5, roughness/dch)
        for run in range(5):
            vinlet = Q/(pi/4*Dinlet**2)
            vz = Q/(pi*Dtank*delta)
            K4 = Dinlet**2*vinlet**2/(2*f*Dtank*H)
            K3 = vinlet/4. - Dinlet**2*vinlet/(4*f*Dtank*H)
            vx0 = K3 + (K3**2 + K4)**0.5
            vx = vinlet*log(1 + f*Dtank*H/Dinlet**2*vx0/vinlet)/(f*Dtank*H/Dinlet**2)
            vch = (vx**2 + vz**2)**0.5
            ReJ = vch*dch*rho/mu
            f = friction_factor(ReJ, roughness/dch)
    if inletlocation and rhow:
        GrJ = g*rho*(rho-rhow)*dch**3/mu**2
        if rhow < rho: # Heating jacket fluid
            if inletlocation == 'auto' or inletlocation == 'bottom':
                ReJeq = (ReJ**2 + GrJ*H/dch/50.)**0.5
            else:
                ReJeq = (ReJ**2 - GrJ*H/dch/50.)**0.5
        else: # Cooling jacket fluid
            if inletlocation == 'auto' or inletlocation == 'top':
                ReJeq = (ReJ**2 + GrJ*H/dch/50.)**0.5
            else:
                ReJeq = (ReJ**2 - GrJ*H/dch/50.)**0.5
    else:
        ReJeq = (ReJ**2)**0.5
    NuA = 3.66
    NuB = 1.62*Pr**(1/3.)*ReJeq**(1/3.)*(dch/lch)**(1/3.)
    NuC = 0.664*Pr**(1/3.)*(ReJeq*dch/lch)**0.5
    if ReJeq < 2300:
        NuD = 0
    else:
        NuD = 0.0115*Pr**(1/3.)*ReJeq**0.9*(1 - (2300./ReJeq)**2.5)*(1 + (dch/lch)**(2/3.))
    if muw:
        NuJ = (NuA**3 + NuB**3 + NuC**3 + NuD**3)**(1/3.)*(mu/muw)**0.14
    else:
        NuJ = (NuA**3 + NuB**3 + NuC**3 + NuD**3)**(1/3.)
    return NuJ*k/dch


import random
from typing import List

import numpy as np
import torch
import torch.nn.functional as F
from torch import nn

from environments import MVCEnvironment
from graph_utils import draw_graph, get_adjacency_matrix

class Struc2Vec(nn.Module):

    def __init__(self, p: int = 64):
        super(Struc2Vec, self).__init__()
        self.theta1 = torch.Tensor(np.random.randn(*(1, p)) * 0.01)
        self.theta1.requires_grad = True
        self.theta2 = torch.Tensor(np.random.randn(*(p, p)) * 0.01)
        self.theta2.requires_grad = True
        self.theta3 = torch.Tensor(np.random.randn(*(p, p)) * 0.01)
        self.theta3.requires_grad = True
        self.theta4 = torch.Tensor(np.random.randn(*(1, 1, p)) * 0.01)
        self.theta4.requires_grad = True

    def forward(self, x: torch.Tensor, mu: torch.Tensor, w: torch.Tensor, adj_mat: torch.Tensor):
        t1 = self.theta1.mul(x)  # (N, p)

        t2 = adj_mat.mm(mu)
        t2 = t2.mm(self.theta2)  # (N, p)

        t3 = self.theta4.mul(w.unsqueeze(2))
        t3 = F.relu(t3)
        t3 = t3.sum(dim=1)
        t3 = t3.mm(self.theta3)  # (N, p)
        return F.relu(torch.add(t1, t2.add(t3)))

    @staticmethod
    def s2x(s: List[int], n_nodes: int):
        """One hot encode the state [1, 4, 5] to the Tensor x"""
        eye = np.eye(n_nodes)
        x = eye[s, :].sum(axis=0).reshape((-1, 1))
        return torch.Tensor(x)

class ReplayMemory(object):

    def __init__(self, capacity):
        self.capacity = capacity
        self.memory = []
        self.position = 0

    def push(self, transition):
        if len(self.memory) < self.capacity:
            self.memory.append(None)
        self.memory[self.position] = transition
        self.position = (self.position + 1) % self.capacity

    def sample(self, batch_size):
        if batch_size >= len(self.memory):
            return random.sample(self.memory, len(self.memory))
        return random.sample(self.memory, batch_size)

    def __len__(self):
        return len(self.memory)

class DQN(nn.Module):

    def __init__(self, emb_dim=64):
        super().__init__()
        self.theta5 = torch.Tensor(np.random.randn(*(2 * emb_dim, 1))*0.01)
        self.theta5.requires_grad = True
        self.theta6 = torch.Tensor(np.random.randn(*(emb_dim, emb_dim)) * 0.01)
        self.theta6.requires_grad = True
        self.theta7 = torch.Tensor(np.random.randn(*(emb_dim, emb_dim)) * 0.01)
        self.theta7.requires_grad = True

    def forward(self, mu: torch.Tensor):
        t1 = mu.sum(dim=(0,)).repeat((mu.shape[0], 1))
        t1 = t1.mm(self.theta6)  # (N, emb_dim)

        t2 = mu.mm(self.theta7)  # (N, emb_dim)

        t3 = torch.cat((t1, t2), dim=1)
        t3 = F.relu(t3)
        t3 = t3.mm(self.theta5)  # (N, 1)
        return t3

class Test:

    def __init__(self, n_graphs: int = 100, n_nodes_max: int = 200):
        np.random.seed(1234)

        self.graphs = []

        for i in range(n_graphs):
            n_nodes = random.randint(10, n_nodes_max)
            graph = draw_graph(n_nodes=n_nodes)
            self.graphs.append(graph)

    def evaluate_policy(self, agent):
        cumulated_reward = 0

        for graph in self.graphs:
            env = MVCEnvironment(name='test', graph=graph)
            adj_mat = get_adjacency_matrix(graph=graph)
            weights = adj_mat

            while not env.is_terminated():
                best_action = agent.pick_action(s=env.s, v_bar=env.v_bar, w=weights, adj_mat=adj_mat, train=False)
                _ = env.make_transition(best_action)

            cumulated_reward += env.get_cumulated_reward()

        return cumulated_reward


import pandas as pd, json, pickle as pkl, numpy as np, pickle
from sklearn.preprocessing import MultiLabelBinarizer

with open("all_opportunities_df.pkl", 'rb') as infile:
    df = pkl.load(infile)

# Prefix Ages and manipulate and unify the ages group and fix an age type (9 to 6)
for i, el in enumerate(df.ages,1):
    for age in range(len(el)):
        #print('age', el[age])
        if el[age] != '':
            el[age] = "Ages_" + el[age]

print('result: ',df.ages)

# Unify REACH
df.reach = df.reach.apply(lambda x: 'Missouri' if x == ['Missouri', 'Illinois'] else x)
df.reach = df.reach.apply(lambda x: np.nan if x == [] else x)
df.reach = df.reach.apply(lambda x: str(x).strip("[]''"))
df.reach = df.reach.apply(lambda x: 'None' if x == '' else x)
df.reach = df.reach.apply(lambda x: np.nan if x == 'None' else x)

print(df.reach)
print(df.attention)
df.attention = df.attention.apply(lambda x: ['None'] if x == [] or x == None else x)
for i, row in enumerate(df.attention, 1):
    for element in range(len(row)):
        if row[element] == 'Gifted and talented students' or row[element] == 'Gifted and Talented':
            row[element] = "Att_" + "GTS"
        else:
            if row[element] =='Students with disabilities' or row[element] == 'Students with disabilites':
                row[element] = "Att_" + "SwDis"
            else:
                if row[element] == 'Students at risk from dropping out of school':
                    row[element] = "Att_" + "SRDoS"
                else:
                    row[element] = "Att_" + row[element]
print(df.attention)

# LANGUAGE unifying
df.language = df.language.apply(lambda x: ['None'] if x == [] or x == None else x)
for i, row in enumerate(df.language,1):
    for el in range(len(row)):
        row[el] = "L_" + row[el]

print(df.areaOfInterest)
# Replacing/abbreviating strings in ATTENTION
for i,row in enumerate(df.areaOfInterest,1):
    for el in range(len(row)):
        if (row[el] == 'Earth and Environmental Science' or row[el] == 'earth and environmental science' or row[el] == 'earth and Environmental Science'):
            row[el] = "AoI_EES"
        else:
            if (row[el] == 'Computer Science' or row[el] =='computer science'):
                row[el] = "AoI_CS"
            else:
                if (row[el] == 'Coding/Programming'):
                    row[el] = "AoI_" + 'Coding/Programming'
                else:
                    if (row[el] == 'General Science'):
                        row[el] = "AoI_GS"
                    else:
                        if (row[el] == 'Architecture and Design'):
                            row[el] = "AoI_AaD"
                        else:
                            if (row[el] == 'Space Science/Aerospace/Rocketry'):
                                row[el] = "AoI_SSAR"
                            else:
                                row[el] = "AoI_" + row[el]
print(df.areaOfInterest)

# TYPEOFOPPORTUNITY stripping the list brackets
df.typeOfOpportunity = df.typeOfOpportunity.apply(lambda x: str(x).strip("[]'"))

# run the OHE on AGES, ATTENTION, LANGUAGE, AREAOFINTEREST
mlb = MultiLabelBinarizer()
variables = [df.ages,df.attention, df.language, df.areaOfInterest]
for el in variables:
     df = df.join(pd.DataFrame(mlb.fit_transform(el),
                          columns=mlb.classes_,
                          index=df.index))

#Varibales Checking
for n,el in enumerate(df.reach,1):
    print(n, el)
df.reach[df.reach.isna()]
df.attention[df.attention.isna()]

for n, el in enumerate(df.attention,1):
    print(n, el)
for n, el in enumerate(df.language,1):
    print(n, el)
for n, el in enumerate(df.areaOfInterest,1):
    print(n, el)
for n, el in enumerate(df.areaOfInterest,1):
    print(n, el)

# Check are there any locationCity's format is not correct
for i in range(1, len(df)):
    if df.locationCity[i][0].isdigit():
        print(df.locationCity[i])
        location = df.locationCity[i].split(",")
        print(location)
        print(len(location[1]))
        df.locationCity[i] = location[1][1:len(location[1])]
        print(df.locationCity[i])

#Store the data
with open("cleaned_data.pkl", "wb") as outfile:
    pickle.dump(df, outfile)

#df.to_csv("data/all_opportunities_df_cleaned.csv")
print(df)

import os

if os.path.isfile("all_opportunities_df_cleaned.csv") == True:
    old_dataframe = pd.read_csv("all_opportunities_df_cleaned.csv")
    old_dataframe = old_dataframe.drop(['Unnamed: 0'], axis=1)
    result = pd.concat([old_dataframe, df], ignore_index=True, sort=False)
    print(result)
    result.drop(columns=result.columns[-1], axis=1, inplace=True)
    print(result)
    result.to_csv("all_opportunities_df_cleaned.csv")
else:
    df.to_csv("all_opportunities_df_cleaned.csv")
#print(old)
#print(list(set(old)- set(new)))


from __future__ import print_function
import pickle
import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
import logging

# If modifying these scopes, delete the file token.pickle.
SCOPES = ['https://www.googleapis.com/auth/gmail.readonly']

class GmailAPI():
    def __init__(self):
        """Shows basic usage of the Gmail API.
        Lists the user's Gmail labels.
        """
        creds = None
        # The file token.pickle stores the user's access and refresh tokens, and is
        # created automatically when the authorization flow completes for the first
        # time.
        if os.path.exists('token.pickle'):
            with open('token.pickle', 'rb') as token:
                creds = pickle.load(token)
        # If there are no (valid) credentials available, let the user log in.
        if not creds or not creds.valid:
            if creds and creds.expired and creds.refresh_token:
                logging.info("Refreshing OAuth Token, Check browser...")
                creds.refresh(Request())
            else:
                flow = InstalledAppFlow.from_client_secrets_file(
                    'credentials.json', SCOPES)
                creds = flow.run_local_server(port=0)
            # Save the credentials for the next run
            with open('token.pickle', 'wb') as token:
                pickle.dump(creds, token)

        self.service = build('gmail', 'v1', credentials=creds)

    def get_email_list(self, options, pageToken=None):
        logging.info("Getting emails, Options: %s", options)

        response = self.service.users().messages().list(**options).execute()
        emails = response['messages']

        if response.get('nextPageToken'):
            options['pageToken'] = response['nextPageToken']
            next_page_response = self.get_email_list(options)
            emails.extend(next_page_response)

        return emails

    def get_emails(self, email_ids):
        logging.info("Downloading emails: %s", len(email_ids))
        response_array = []
        exception_array = []
        global total_downloaded
        total_downloaded = 0

        def callback_callable(request_idx, response, exception):
            global total_downloaded
            if exception:
                exception_array.append(request_idx) if request_idx not in exception_array else None
            else:
                total_downloaded += 1
                print("\rEmail Downloaded:", total_downloaded + 1, end="")
                response_array.append(response)

        batch = self.service.new_batch_http_request(callback=callback_callable)

        [batch.add(self.service.users().messages().get(
            userId='me', id=e_id), request_id=e_id) for e_id in email_ids]

        batch.execute()

        if exception_array:
            logging.warning("Exception Handling for: %s", exception_array)
            response_array.extend(self.get_emails(exception_array))

        return response_array


"""
Module: df_to_txt
This module provides a class for writing pandas DataFrame to a text file with custom headers.
"""

import os

import pandas as pd

from .log import logger
from .units import units
from .utils import convert_bytes_to_gb

class DataFrameToFile:
    """
    A class used to write a pandas DataFrame to a text file with a custom header.
    """

    def __init__(self, df: pd.DataFrame):
        """
        Parameters
        ----------
        df : pd.DataFrame
            The pandas DataFrame to be written to a text file.
        """
        self.df = df
        self.units = units
        self.include_weights = True
        self.include_energy = True

    def exclude_weights(self):
        self.include_weights = False
        return self

    def exclude_energy(self):
        self.include_energy = False
        return self

    def write_to_file(self, file_path):
        # Select columns to write based on include_weights and include_energy
        columns_to_write = self.df.columns.tolist()
        if not self.include_weights:
            columns_to_write.remove("weights")
        if not self.include_energy:
            columns_to_write.remove("energy_mev")

        # Write header to file
        with open(file_path, "w", encoding="utf-8") as f:
            header = ", ".join(
                f"{column} ({self.units[column]})" for column in columns_to_write
            )
            f.write(header + "\n")

        # Append DataFrame to file
        logger.info("Writing dataframe to file. This may take a while...\n")
        self.df.to_csv(
            file_path,
            columns=columns_to_write,
            index=False,
            header=False,
            sep=",",
            float_format="%.7e",
            mode="a",
        )
        logger.info("Wrote %s\n", file_path)

        # Compute and log the file size
        file_size_bytes = os.path.getsize(file_path)
        file_size_gb = convert_bytes_to_gb(file_size_bytes)
        logger.info("Final file size: %.2f GB", file_size_gb)


# -*- coding: utf-8 -*-
"""
Created on Wed Apr  8 19:57:00 2020

@author: xs27
"""

# import ECE552_Project
# import previous stamps
import numpy as np
import scipy.linalg as linalg
import matplotlib.pyplot as plt

#####################
# stamp for DC source
#####################

def VS_stamp(G,s,n1,n2,b12,v):
    # b12: branch number
    # n1,n2: node number
    G[b12,n1] += 1
    G[b12,n2] -= 1
    G[n1,b12] += 1
    G[n2,b12] -= 1
    s[b12] += v
    return G, s

#######################
# stamp for MOSFET
#######################

def NMOS_stamp(G,x,s,k,Vg,Vs,Vd,Vth):
    # construct the stamp of a NMOS
    # Input:
    #     Vg:  node number for gate
    #     Vd:  node number for drain
    #     Vs:  node number for source
    #     k:   coefficient
    #     Vth: threshold voltage
    #     G:   NA matrix
    #     x:   current solution
    #     s:   RHS of the NA equation
    # Output:
    #     G,s: updated G & s
    # check region of operation
    # and construct NA stamp
    # notation from (6.3.23)
    Vov = x[Vg]-x[Vs]-Vth
    Vds = x[Vd]-x[Vs]
    Vgs = x[Vg]-x[Vs]
    if Vgs <= Vth: # cut-off
        a11 = 0
        a12 = 0 #1e-5# large resistor
        b1 = 0
    elif Vds < Vov: # linear
        a11 = k*Vds
        a12 = k*(Vov-Vds)
        b1 = k*(Vov*Vds-0.5*Vds*Vds)-a11*Vgs-a12*Vds
    else: # saturation
        a11 = 0.5*k*(2*Vov*(1+lamda*(Vds-Vov)-lamda*Vov*Vov))
        a12 = 0.5*k*lamda*Vov*Vov
        b1 = 0.5*k*Vov*Vov*(1+lamda*(Vds-Vov))-a11*Vgs-a12*Vds
    # update on the matrix
    a12 += 1e-9 # large resistor
    G[Vd,Vg] += a11
    G[Vs,Vg] -= a11
    G[Vd,Vd] += a12
    G[Vs,Vd] -= a12
    G[Vd,Vs] -= a11+a12
    G[Vs,Vs] += a11+a12
    s[Vd] -= b1
    s[Vs] += b1
    return G,s

def PMOS_stamp(G,x,s,k,Vg,Vs,Vd,Vth):
    # construct the stamp of a PMOS
    # Input:
    #     Vg:  node number for gate
    #     Vd:  node number for drain
    #     Vs:  node number for source
    #     k:   coefficient
    #     Vth: threshold voltage
    #     G:   NA matrix
    #     x:   current solution
    #     s:   RHS of the NA equation
    # Output:
    #     G,s: updated G & s
    # check region of operation
    # and construct NA stamp
    # notation from (6.3.23)
    Vov = x[Vs]-x[Vg]-Vth
    Vsd = x[Vs]-x[Vd]
    Vsg = x[Vs]-x[Vg]
    if Vsg <= Vth: # cut-off
        a11 = 0
        a12 = 0#1e-5 # large resistor
        b1 = 0
    elif Vsd < Vov: # linear
        a11 = k*Vsd
        a12 = k*(Vov-Vsd)
        b1 = k*(Vov*Vsd-0.5*Vsd**2)-a11*Vsg-a12*Vsd
    else: # saturation
        a11 = 0.5*k*(2*Vov*(1+lamda*(Vsd-Vov)-lamda*Vov**2))
        a12 = 0.5*k*lamda*Vov**2
        b1 = 0.5*k*Vov**2*(1+lamda*(Vsd-Vov))-a11*Vsg-a12*Vsd
    # update on the matrix
    a12 += 1e-9 # large resistor
    G[Vd,Vg] += a11
    G[Vs,Vg] -= a11
    G[Vd,Vd] += a12
    G[Vs,Vd] -= a12
    G[Vd,Vs] -= a11+a12
    G[Vs,Vs] += a11+a12
    s[Vd] += b1
    s[Vs] -= b1
    return G,s

# find number of nodes and extra banches in MNA
def node_and_branch_number(total_list):
    # return the largest number of node in the netlist
    # starting from 0 (index for python)
    # and determine extra lines for the netlist(inductor current etc)
    node_max = 0
    extra_branch = 0
    for line in total_list:
        component = line.split()
        # find the largest node
        if int(component[1]) > node_max:
            node_max = int(component[1])
        if int(component[2]) > node_max:
            node_max = int(component[2])
        # find the number of extra branches for MNA
        if 'V' == component[0][0]:
            extra_branch += 1
    return node_max, extra_branch

# read the netlist, and stamp the matrix G(real) and C(imaginary)
def read_netlist(netlist,total_list,node_max,G,source):
    for line in total_list:
        component = line.split()
        if 'V' == component[0][0]:
            G,source = VS_stamp(G,source,int(component[1])-1,int(component[2])-1,\
                                node_max,float(component[3]))
            node_max += 1
    return G,source

def update_G(G,source,x):
# go through the netlist and update the G & source matrices
    G[0:5,0:5] = 0
    source[0:5] = 0
    for line in total_list:
        component = line.split()
        if 'NMOS' == component[0][0:4]:
            G, source = NMOS_stamp(G,x,source,kn,int(component[1])-1,int(component[2])-1,int(component[3])-1,Vth)
        elif 'PMOS' == component[0][0:4]:
            G, source = PMOS_stamp(G,x,source,kp,int(component[1])-1,int(component[2])-1,int(component[3])-1,Vth)
    A[0:node_max-1,0:node_max-1] = G[0:node_max-1,0:node_max-1]
    A[0:node_max-1,node_max-1:node_max+extra_branch-1] = G[0:node_max-1,node_max:node_max+extra_branch]
    A[node_max-1:node_max+extra_branch-1,0:node_max-1] = G[node_max:node_max+extra_branch,0:node_max-1]
    A[node_max-1:node_max+extra_branch-1,node_max-1:node_max+extra_branch-1] = G[node_max:node_max+extra_branch,node_max:node_max+extra_branch]
    s[0:node_max-1] = source[0:node_max-1]
    s[node_max-1:node_max + extra_branch-1] = source[node_max:node_max + extra_branch]
    return A,s
# G[:,node_max-1:node_max+extra_branch-1] = G[:,node_max:node_max+extra_branch]
# G[node_max-1:node_max+extra_branch-1,:] = G[node_max:node_max+extra_branch,:]

################
#   Constants  #
################

kn = 1e-4
kp = 5e-5
lamda = 0.02
Vth = 1
x = np.ndarray(8,dtype = float)
x[:] = 0
x_old = np.ndarray(8,dtype = float)
x_old[:] = 0

###########################
###########################
#  Start of the program   #
###########################
###########################

netlist = open('netlist_MOS.txt','r')
total_list = netlist.readlines()
# node = np.ndarray((len(total_list),2),dtype = int)

# find the number of nodes and determine the number of extra current branches
node_max, extra_branch = node_and_branch_number(total_list)

# construct the NA matrix: G (real) and source (real)
G = np.ndarray((node_max+extra_branch,node_max+extra_branch), dtype = float)
G[:,:] = 0
A = np.ndarray((node_max+extra_branch-1,node_max+extra_branch-1), dtype = float)
source = np.ndarray(node_max+extra_branch,dtype = float)
source[:] = 0
s = np.ndarray(node_max+extra_branch-1,dtype = float)

G, source = read_netlist(netlist,total_list,node_max,G,source)

# prepare for iteration updates
def iteration_solve(G,source,x,x_old,epsilon):
    cnt = 0
    while (max(abs(x_old-x))>epsilon) and (cnt<50):
        x_old = x
        A, s = update_G(G,source,x)
        lu, piv = linalg.lu_factor(A)
        x = linalg.lu_solve((lu,piv),s)
        cnt += 1
    return x,cnt

# Q1
# x, cnt = iteration_solve(G, source, x, x_old, 1e-9)

N = 501
solution = np.ndarray((N,8,3),dtype = float)

VA = np.linspace(0,5,N)
Vout = np.ndarray(N+1,dtype = float)

VB = np.array([2, 3.5, 5])

for jj in range(3):
    source[7] = VB[jj]
    for ii in range(N):
        x[4] = VA[ii]
        source[6] = VA[ii]
        x_old[:] = 0
        x,cnt = iteration_solve(G,source,x,x_old,1e-9)
        solution[ii,:,jj] = x

########################
# Plotting results
########################
Vout1 = solution[1:N+1,2,0]
Vout2 = solution[1:N+1,2,1]
Vout3 = solution[1:N+1,2,2]
# Vout = Vout.reshape(501)
plt.plot(VA[1:N+1], Vout1, label='VB = 2V', color = 'r')
plt.plot(VA[1:N+1], Vout2, label='VB = 3.5V', color = 'g')
plt.plot(VA[1:N+1], Vout3, label='VB = 5V', color = 'b')
plt.scatter(3,Vout1[int(3*(N+1)/5)])
plt.text(3.1,Vout1[int(3*(N+1)/5)],'%.4f V'%Vout1[int(3*(N+1)/5)],ha="left")
plt.xlabel('Input Voltage')
plt.ylabel('Output Voltage')
plt.legend()
plt.show()

############################
############################
#    End of ECE552 pj2     #
############################
############################


import pandas as pd
import numpy as np
from sklearn.compose import make_column_transformer
from sklearn.pipeline import make_pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.model_selection import RandomizedSearchCV

# Acknowledgement:
# The format of the function documentations are derived from the sample project provided.
# Proper citation to the sample projct is included in `notebooks/fraud_detection.ipynb`.
# https://github.com/ttimbers/demo-tests-ds-analysis-python/blob/main/src/count_classes.py
# Proper citation to the sample projct is included in `UGFraud`
# https://github.com/safe-graph/UGFraud/blob/master/UGFraud/Detector/SVD.py

def run_model_with_random_search(
    X_train, y_train, numerical_features, credit_feature, categorical_features,
    drop_features, model_class, param_dist, n_iter=5, cv=2, scoring="f1", need_class_weights=True):
    """
    Run a machine learning model with hyperparameter tuning using RandomizedSearchCV.

    Parameters:
    - X_train: Training input data.
    - y_train: True labels for training data.
    - numerical_features: List of numerical feature names.
    - credit_feature: Credit feature name.
    - categorical_features: List of categorical feature names.
    - drop_features: List of features to drop.
    - model_class: Machine learning model class (e.g., LogisticRegression, RandomForestClassifier).
    - param_dist: Parameter distribution for RandomizedSearchCV.
    - n_iter: Number of iterations for RandomizedSearchCV.
    - cv: Number of cross-validation folds.
    - scoring: Scoring metric for RandomizedSearchCV.

    Returns:
    - RandomizedSearchCV object

    Example:
    ```
    param_dist_logreg = {"logisticregression__C": 10.0 ** np.arange(-2, 3),
                        "logisticregression__solver": ["newton-cholesky", "lbfgs"]}
    logreg_search = run_model_with_random_search(
    X_train, y_train, numerical_features, credit_feature, categorical_features, drop_features,
    LogisticRegression, param_dist_logreg)
    logreg_search.fit(X_train, y_train)
    ```
    """
    # Input validation
    if not isinstance(X_train, pd.DataFrame):
        raise ValueError("X_train must be a DataFrame.")
    if not isinstance(y_train, pd.Series):
        raise ValueError("y_train must be a Series.")
    if X_train.shape[0] != y_train.shape[0]:
        raise ValueError("Number of rows in X_train and y_train must match.")

    # Check for duplicate feature names
    all_features = set(numerical_features + credit_feature + categorical_features + drop_features)
    if len(all_features) != len(numerical_features) + len(credit_feature) + len(categorical_features) + len(drop_features):
        raise ValueError("Duplicate feature names found.")

    # Ensure all features are non-empty strings
    if not all(isinstance(feature, str) and feature for feature in all_features):
        raise ValueError("All feature names must be non-empty strings.")

    # Check for overlapping features between numerical and categorical features
    if any(feature in numerical_features for feature in categorical_features):
        raise ValueError("Numerical and categorical features must not overlap.")

    # Check if numerical and credit features have any common elements
    if any(feature in numerical_features for feature in credit_feature):
        raise ValueError("Numerical and credit features must not have common elements.")

    ct = make_column_transformer(
        (
        make_pipeline(SimpleImputer(strategy="median"), StandardScaler()),
        numerical_features,
        ),
        (
        make_pipeline(SimpleImputer(strategy="mean"), StandardScaler()),
        credit_feature,
        ),
        (OneHotEncoder(drop="if_binary", handle_unknown="ignore"), categorical_features),
        ("drop", drop_features)
    )
    if need_class_weights:
        pipe = make_pipeline(ct, model_class(class_weight="balanced"))
    else:
        pipe = make_pipeline(ct, model_class())
    random_search = RandomizedSearchCV(
        pipe, param_dist, n_jobs=-1, n_iter=n_iter, cv=cv, scoring=scoring, return_train_score=True
    )

    return random_search


from django.db import models

class Trainer(models.Model):
    base_user = models.OneToOneField(
        to='users.User',
        on_delete=models.CASCADE,
        related_name='trainer',
        verbose_name='id пользователя'
    )

    class Meta:
        verbose_name = 'Тренер'
        verbose_name_plural = 'Тренер'
        ordering = ('base_user',)

    def __str__(self):
        return self.base_user


import argparse
from ReadStream import StreamReader

""" USAGE:

    python download_data.py -o <output_data_directory> -run <last_4_digits_of_run>
    -n <number_of_files_to_read> -format <upper or lower>

    The output data directory should be ../data/ unless you want to save data somewhere else (not recommended).
    The ../data/ folder will be created while running this script the first time, and it has already been added to .gitignore!

    Example: to download the old data RUN001231

    python download_data.py -run 1231 -format upper

    Example: to download a small portion of RUN001231 into the ../data/ folder simply type

    python download_data.py -run 1231 -n 20 -format upper

    IF data is NEW data then set -format lower

    Example: to download the new data Run000054

    python download_data.py -run 0054 -format lower

    IF data is the NEWEST data then set -format lower and -capital lower

    Example: to download the new data run000057

    python download_data.py -run 0054 -format lower -capital lower

    If you change the output directory please make sure to add the new folder to .gitignore

"""

def argParser():
    """manages command line arguments"""

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-o", "--output", type=str, default="../data/", help="output directory"
    )
    parser.add_argument(
        "-format", "--format", type=str, default="upper", help="CVeneto container format"
    )
    parser.add_argument(
        "-capital", "--capital", type=str, default="upper", help="first letter capital or not"
    )
    parser.add_argument(
        "-run", "--run", type=str, default="0054", help="run number"
    )
    parser.add_argument(
        "-n", "--nfiles", type=int, default=-1, help="number of files"
    )

    return parser.parse_args()

def main(args):
    """
        MAIN FUNCTION gets data from CloudVeneto and save them into a .txt file

        args: the command line arguments
    """

    # store the command line arguments
    RUNNUMBER = args.run
    OUT_PATH  = args.output
    N_FILES   = args.nfiles
    FORMAT    = args.format
    CAPITAL   = args.capital

    # create an instance of StreamReader to manage data I/O from CloudVeneto
    reader = StreamReader(RUNNUMBER, OUT_PATH, N_FILES, FORMAT, CAPITAL)

    # get data from CloudVeneto
    reader.readStream()
    # save data into a .txt file
    reader.saveData()

    print("\n\nExiting...\n\n")

if __name__ == "__main__":
    args = argParser()
    main(args)


#!/usr/bin/env python3

import numpy as np
import matplotlib.pyplot as plt
plt.style.use( 'publication.sty' )

from UtilitiesModule import GetFileArray, GetData

#### ========== User Input ==========

rootDirectory_NR \
  = '/home/kkadoogan/Work/Codes/thornado/SandBox/AMReX/Applications/\
StandingAccretionShock_NonRelativistic/'
rootDirectory_GR \
  = '/home/kkadoogan/Work/Codes/thornado/SandBox/AMReX/Applications/\
StandingAccretionShock_Relativistic/'

#IDs = np.array( [ '1D_M2.8_Mdot0.3_Rs060_nX380', \
#                  '1D_M2.8_Mdot0.3_Rs075_nX380', \
#                  '1D_M2.8_Mdot0.3_Rs090_nX380', \
#                  '1D_M2.8_Mdot0.3_Rs120_nX380', \
#                  '1D_M2.8_Mdot0.3_Rs150_nX380', \
#                  '1D_M2.8_Mdot0.3_Rs180_nX380' ], str )

IDs = np.array( [ '1D_M3.0_Mdot0.3_Rs020_nX400' ], str )

R_PNS = 10.0
#Rsh = np.array( [ 60.0, 75.0, 90.0, 120.0, 150.0, 180.0 ], np.float64 )
Rsh = np.array( [ 20.0 ], np.float64 )

nRows = 2
nCols = 1

yLabels = np.array( [ r'$p/\left(\rho\,c^{2}\right)$', \
                      r'$v/c$' ], str )

saveFigAs = 'fig.CompareNRvsGR_IC_Fluid.png'

verbose = False

#### ====== End of User Input =======

### Plotting

fig = plt.figure( figsize = (12,9) )
fig.suptitle( 'M3.0, Mdot0.3, Rpns010', fontsize = 15, y = 0.92 )

ax0 = fig.add_subplot( nRows, nCols, 1 )
ax1 = fig.add_subplot( nRows, nCols, 2 )

ax0.set_ylabel( yLabels[0] )
ax1.set_ylabel( yLabels[1] )

plotFileDirectory_NR = rootDirectory_NR
plotFileDirectory_GR = rootDirectory_GR

c = [ 'red',  \
      'orange',  \
      'yellow',  \
      'green',  \
      'blue',  \
      'magenta' ]

y0Min = +np.inf
y0Max = -np.inf
y1Min = +np.inf
y1Max = -np.inf
for i in range( IDs.shape[0] ):

    plotFileBaseName_NR = 'NR' + IDs[i] + '.plt'
    plotFileBaseName_GR = 'GR' + IDs[i] + '.plt'

    plotFileArray_NR \
      = GetFileArray( plotFileDirectory_NR, plotFileBaseName_NR )
    plotFileArray_GR \
      = GetFileArray( plotFileDirectory_GR, plotFileBaseName_GR )

    plotFile_NR = plotFileDirectory_NR + plotFileArray_NR[-1]
    plotFile_GR = plotFileDirectory_GR + plotFileArray_GR[-1]

    time, rho_NR, dataUnits, X1, X2, X3, dX1, dX2, dX3, nX \
      = GetData( plotFile_NR, 'PF_D', verbose = verbose )
    time, rho_GR, dataUnits, X1, X2, X3, dX1, dX2, dX3, nX \
      = GetData( plotFile_GR, 'PF_D', verbose = verbose )
    time, p_NR, dataUnits, X1, X2, X3, dX1, dX2, dX3, nX \
      = GetData( plotFile_NR, 'AF_P', verbose = verbose )
    time, p_GR, dataUnits, X1, X2, X3, dX1, dX2, dX3, nX \
      = GetData( plotFile_GR, 'AF_P', verbose = verbose )
    time, v_NR, dataUnits, X1, X2, X3, dX1, dX2, dX3, nX \
      = GetData( plotFile_NR, 'PF_V1', verbose = verbose )
    time, v_GR, dataUnits, X1, X2, X3, dX1, dX2, dX3, nX \
      = GetData( plotFile_GR, 'PF_V1', verbose = verbose )

    rho_NR *= (2.99792458e10)**2
    rho_GR *= (2.99792458e10)**2

    v_NR /= 2.99792458e5
    v_GR /= 2.99792458e5

    lab = IDs[i][16:21]

    eta = np.copy( ( X1 - R_PNS ) / ( Rsh[i] - R_PNS ) )
    #eta = np.copy( X1 )

    ind = np.where( eta <= 1.0 )[0]
    eta = np.copy( eta[ind] )

    y0_NR = np.copy( p_NR[ind,0,0] / rho_NR[ind,0,0] )
    y0_GR = np.copy( p_GR[ind,0,0] / rho_GR[ind,0,0] )
    y1_NR = np.copy( v_NR[ind,0,0] )
    y1_GR = np.copy( v_GR[ind,0,0] )

    ax0.plot( eta, y0_NR, color = c[i], ls = '-', \
              label = r'\texttt{{NR_'+'{:}}}'.format( lab ) )
    ax0.plot( eta, y0_GR, color = c[i], ls = '--', \
              label = r'\texttt{{GR_'+'{:}}}'.format( lab ) )

    ax1.plot( eta, y1_NR, color = c[i], ls = '-' )
    ax1.plot( eta, y1_GR, color = c[i], ls = '--' )

#ax0.set_xlim( R_PNS, 5.0e1 )
#ax1.set_xlim( R_PNS, 5.0e1 )

#ax0.set_xlabel( r'Radial Coordinate $\left[\mathrm{km}\right]$' )
#ax1.set_xlabel( r'Radial Coordinate $\left[\mathrm{km}\right]$' )
ax0.set_xlabel \
  ( r'$\left(r-R_{\textsc{pns}}\right)/\left(R_{s}-R_{\textsc{pns}}\right)$' )
ax1.set_xlabel \
  ( r'$\left(r-R_{\textsc{pns}}\right)\left(R_{s}-R_{\textsc{pns}}\right)$' )

ax0.tick_params( axis = 'x', colors = 'k', labelsize = 15 )
ax0.tick_params( axis = 'y', colors = 'k', labelsize = 15 )
ax1.tick_params( axis = 'x', colors = 'k', labelsize = 15 )
ax1.tick_params( axis = 'y', colors = 'k', labelsize = 15 )

ax0.legend( loc = 1, prop = { 'size' : 13 } )

plt.subplots_adjust( hspace = 0.3 )
plt.savefig( saveFigAs, dpi = 300 )
#plt.show()
plt.close()

import os
os.system( 'rm -rf __pycache__ ' )


from fastapi import APIRouter, Body, Depends, Path

from api.database.config import DBConnectionHandler

from . import domain, models

router = APIRouter()

@router.post("/", response_model=models.Condominium)
async def insert_condominium(
    payload: models.Condominium = Body(...), context: DBConnectionHandler = Depends()
):
    return await domain.InsertCondominiumUseCase(context, payload).execute()

@router.post("/{condominium_id}", response_model=models.BaseUser, status_code=202)
async def create_user(
    condominium_id: int = Path(...),
    payload: models.CreateUser = Body(...),
    context: DBConnectionHandler = Depends(),
):
    return await domain.CreateUserUseCase(condominium_id, context, payload).execute()


#!/usr/bin/env python
# coding: utf-8

import re
import argparse
import pickle
import json
import os
from datetime import datetime, timedelta

import boto3
import numpy as np
import pandas as pd
from  tqdm import tqdm
import editdistance
from sklearn.metrics.pairwise import cosine_similarity

from autolocal.aws import aws_config
from autolocal.documentdb import S3DocumentManager

from autolocal import AUTOLOCAL_HOME

def single_vector_per_doc(vectors):
    # vectors is a list of np arrays where:
    # dims: (LAYERS(3), TOKENS(varies), DIMENSIONS(1024))
    """
    https://towardsdatascience.com/elmo-helps-to-further-improve-your-word-embeddings-c6ed2c9df95f
    In the ELMo paper, there are 3 layers of word embedding,
    layer zero is the character-based context independent layer,
    followed by two Bi-LSTM layers. The authors have empirically
    shown that the word vectors generated from the first Bi-LSTM
    layer can better capture the syntax, and the second layer can
    capture the semantics better.
    """
    vectors = np.concatenate([v[2] for v in vectors], 0)
    return vectors

def read_metadata(args):
    # start_date = [int(d) for d in args.start_date.split("-")]
    # end_date = [int(d) for d in args.end_date.split("-")]
    # for year in range(start_date[0], end_date[0]):
    #     for month in range(start_date[1], end_date[1]):
    #         for day in range(start_date[2], end_date[2]):
    table = boto3.resource(
        'dynamodb',
        region_name=aws_config.region_name,
        ).Table(aws_config.db_document_table_name)
    s3_client = boto3.client(
        's3',
        region_name=aws_config.region_name
        )

    response = table.scan()
    data = response['Items']

    while 'LastEvaluatedKey' in response:
        response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
        data.extend(response['Items'])

    metadata = pd.DataFrame(data)
    metadata["date"] = [datetime.strptime(d, '%Y-%m-%d') for d in metadata["date"]]
    return metadata

def sentence_split(s):
    sentences = re.split('[.\n!?"\f]', s)
    return [s for s in sentences if len(s.strip())>0]

def tokenize(s):
    tokens = re.findall(r'\w+', s)
    return tokens

def read_queries():
    table = boto3.resource(
        'dynamodb',
        region_name=aws_config.region_name,
        ).Table(aws_config.db_query_table_name)
    queries = table.scan()["Items"]
    return queries

def parse_dates(start_date, end_date):
    starting_dates_for_filtering = {
        'upcoming_only': datetime.now(),
        'upcoming': datetime.now(),
        'this_week': datetime.now() - timedelta(weeks=1),
        'this_year': datetime.now() - timedelta(days=365),
        'this_month': datetime.now() - timedelta(weeks=5),
        'past_six_months':datetime.now() - timedelta(days=183),
        'last_six_months':datetime.now() - timedelta(days=183),
        'past': None,
        'all': None
    }
    if start_date:
        start_date = datetime.strptime(start_date, '%Y-%m-%d')
    else:
        start_date = starting_dates_for_filtering['upcoming']
    if end_date:
        end_date = datetime.strptime(end_date, '%Y-%m-%d')
    if start_date and end_date:
      assert(start_date <= end_date)
    return start_date, end_date

def find_relevant_doc_ids(queries, metadata, start_date=None, end_date=None, agenda_only=False):

    cities = set()

    # filter metadata to only those files that match the query municipality and time_window
    for query in queries:
        cities.update(query["municipalities"])

    potential_documents = metadata

    print("start", start_date)
    print("end", end_date)
    potential_documents = potential_documents[potential_documents["date"] >= start_date]
    if end_date:
        potential_documents = potential_documents[potential_documents["date"] <= end_date]

    potential_documents = potential_documents[[(c in cities) for c in potential_documents["city"]]]

    if agenda_only:
        potential_documents = potential_documents[potential_documents["doc_type"]=="Agenda"]

    return list(potential_documents['doc_id'])

def read_docs(doc_ids, document_manager):
  log_every = 100

  documents = {}
  n_docs_total = len(doc_ids)

  i = 0
  n_docs_read = 0
  for doc_id in doc_ids:
    doc = document_manager.get_doc_by_id(doc_id)
    try:
      doc_string = document_manager.get_doc_text(doc)
    except Exception as e:
      print("could not read document {}".format(doc_id))
      print(e)
      print("adding document")
      document_manager.add_doc(doc)
      print("document added")
      try:
        print("reading document {}".format(doc_id))
        doc_string = document_manager.get_doc_text(doc)
        print("document read")
      except:
        doc_string = None
    if doc_string:
      vector_data = document_manager.get_doc_vectors(doc)
      if vector_data:
        documents[doc_id] = {
          "original_text": doc_string,
          "sentences": vector_data["sentences"],
          "vectors": vector_data["vectors"]
        }
    n_docs_read+=1
    if n_docs_read % log_every == 0:
        print("{} of {} documents read".format(n_docs_read, n_docs_total))

  return documents

def select_relevant_docs(municipalities, all_docs, metadata, start_date=None, end_date=None):
    # filter metadata to only those files that match the query municipality and time_window
    potential_documents = metadata
    start_date, end_date = parse_dates(start_date, end_date)
    if start_date:
        potential_documents = potential_documents[potential_documents["date"] >= start_date]
    if end_date:
        potential_documents = potential_documents[potential_documents["date"] >= end_date]
    potential_documents = potential_documents[[(c in municipalities) for c in potential_documents["city"]]]
    # filter all docs to only filenames in subset of metadata
    doc_ids = list(potential_documents['doc_id'])
    urls = list(potential_documents['url'])
    docs_to_return = []
    for i in range(len(doc_ids)):
        f = doc_ids[i]
        u = urls[i]
        if f in all_docs:
            docs_to_return.append({**all_docs[f], 'doc_id':f, 'url':u})
        else:
            print("file was not collected in 'all_docs': {}".format(f))
    # return [{**all_docs[f], 'filename':f, 'url':"example.com"} for f in potential_documents['local_path_txt'] if f in all_docs]
    return docs_to_return

def segment_docs(relevant_docs):
    min_section_length = 50 # tokens
    # TODO: this cuts of end of doc

    sections = []
    for doc in relevant_docs:
        original_text = doc["original_text"]
        pages = original_text.split('\f')
        page_numbers = []
        for p, page in enumerate(pages):
            page_sentences = sentence_split(page)
            # for each sentence, what page was it on?
            for sentence in page_sentences:
                sentence_tokens = tokenize(sentence)
                page_numbers.append(p+1)
        doc_sentences = doc["sentences"]
        doc_vectors = doc["vectors"]
        section = []
        section_tokens = 0
        if (len(doc_sentences) == len(doc_vectors)):
            for i in range(len(doc_sentences)):
                sentence = doc_sentences[i]
                page = page_numbers[i]
                sentence_vectors = doc_vectors[i]
                sentence_tokens = tokenize(sentence)
                section.append({
                    "sentence": sentence,
                    "page": page,
                    "sentence_vectors": sentence_vectors,
                    "sentence_tokens": sentence_tokens
                })
                section_tokens += len(sentence_tokens)
                if section_tokens >= min_section_length:
                    section_text = ". ".join([s["sentence"].strip() for s in section])
                    sections.append({
                        "sentences": section,
                        "section_text": section_text,
                        "doc_id": doc["doc_id"],
                        "url": doc["url"]
                    })
                    section = []
                    section_tokens = 0
    return sections

def set_casing(x, casing="lower_non_acronyms"):
    if casing=="cased":
        return x
    elif casing=="lower":
        return x.lower()
    elif casing=="lower_non_acronyms":
        return (x if x.isupper() else x.lower())
    else:
        raise Exception

# TODO: use vectors to find closes words to keywords
def score_doc_sections(doc_sections, orig_keywords, vectorizer):
    orig_keywords = [k.strip() for k in orig_keywords]
    keywords = []
    for k in orig_keywords:
        words = k.split(" ")
        for word in words:
            keywords.append(word)
    vector_data = vectorizer.vectorize(" ".join(keywords))
    keyword_vectors = single_vector_per_doc(vector_data["vectors"])
    doc_sections_scores = []
    for s, section in enumerate(doc_sections):
        section_vectors = single_vector_per_doc([s["sentence_vectors"] for s in section["sentences"]])
        section_text = section['section_text']
        no_keywords_found = True
        for k in orig_keywords:
            if bool(re.search("([^\w]|^)" + k + "([^\w]|$)", section_text)):
            # if k in section_text:
            #     TODO: consider casing
            # TODO: lemmatize!
                no_keywords_found = False
        for k in orig_keywords:
            if k.islower():
                if bool(re.search("([^\w]|^)" + k + "([^\w]|$)", section_text.lower())):
                    no_keywords_found = False
        if no_keywords_found:
            score = 0
        elif section_vectors.shape[0]>0:
            similarities = cosine_similarity(section_vectors, keyword_vectors)
            # if threshold_similarity > -1:
            #     similarities = similarities*(similarities>threshold_similarity)
            keyword_similarities = np.mean(similarities, axis=0)
            # score = np.sum(keyword_similarities*keyword_weights)
            score = np.mean(keyword_similarities)
        else:
            score = 0
        doc_sections_scores.append(score)

    return doc_sections_scores

def text_is_too_similar(a, b):
    # if there are only 2 edits to get from one text to the other, it's not good
    return editdistance.eval(a, b) < 50

# make sure we're not giving similar text among the top k (e.g. shows up on both minutes and agenda)
def check_repeated_text(top_k, section_text):
    for old in top_k:
        if text_is_too_similar(old[1]["section_text"], section_text):
            return True
    return False

def select_top_k(doc_sections, doc_sections_scores, k):
    sorted_sections = sorted(zip(doc_sections_scores, doc_sections), key=lambda pair: pair[0], reverse=True)
    top_k = []
    text_returned = []
    for x in sorted_sections:
        score = x[0]
        if score > 0:
            doc_id = x[1]["doc_id"]
            starting_page = x[1]["sentences"][0]["page"]
            ending_page = x[1]["sentences"][-1]["page"]
            section_text = x[1]["section_text"]
            if check_repeated_text(top_k, section_text):
                print("this excpert has already been returned")
            else:
                top_k.append(x)
            if len(top_k) >= k:
                break
    return top_k

def update_with_top_k(results, top_k_sections, query):
    for section in top_k_sections:
        x = section[1]
        x.update(query)
        results.append(x)
    return results

def write_results(results, query_id, batch):

    results_to_return = []
    for result in results:
        new_result = {
            "start_page": result["sentences"][0]["page"],
            'doc_id': result['doc_id'],
            'section_text': result['section_text']
        }
        results_to_return.append(new_result)

    print({
            'id': "{}_{}".format(query_id, datetime.now()),
            'email_address': result['email_address'],
            'query_id': result['id'],
            'recommendations': results_to_return,
            'keywords': result['keywords'],
            'municipalities': result['municipalities']
        })

    batch.put_item(
       Item={
            'id': "{}_{}".format(query_id, datetime.now()),
            'email_address': result['email_address'],
            'query_id': result['id'],
            'recommendations': results_to_return,
            'keywords': result['keywords'],
            'municipalities': result['municipalities'],
            'time': "{}".format(datetime.now())
        }
    )

def run_queries(document_manager, input_args):
    k = input_args.k
    start_date, end_date = parse_dates(input_args.start_date, input_args.end_date)
    print("reading queries")
    queries = read_queries()
    desired_subscription_status = input_args.query_type
    queries = [q for q in queries if ('subscription_status' in q and q['subscription_status'] == desired_subscription_status)]
    if input_args.filter:
        queries = [q for q in queries if ('email_address' == input_args.filter)]
    print("reading metadata")
    metadata = read_metadata(input_args)
    print("finding relevant doc_ids")
    relevant_doc_ids = find_relevant_doc_ids(queries, metadata, start_date = start_date, end_date = end_date, agenda_only=input_args.agenda_only)
    print("reading {} relevant documents".format(len(relevant_doc_ids)))
    # (not actually *all*, but all the ones we care about for queries)
    all_docs = read_docs(relevant_doc_ids, document_manager)
    print("read {} relevant documents".format(len(all_docs)))

    for q, orig_query in enumerate(queries):
        query = orig_query
        results = []
        print("running query {} of {}".format(q, len(queries)))
        query_id = query["id"]
        if input_args.emails != "production":
            query["email_address"] = input_args.emails
        email_address = query["email_address"]
        print("email address: {}".format(email_address))
        keywords = query["keywords"]
        print("keywords: {}".format(keywords))
        municipalities = query["municipalities"]
        print("municipalities: {}".format(municipalities))
        relevant_docs = select_relevant_docs(municipalities, all_docs, metadata, input_args.start_date, input_args.end_date)
        print("{} relevant documents identified for this query".format(len(relevant_docs)))
        print("segmenting documents")
        doc_sections = segment_docs(relevant_docs)
        print("{} document sections to choose from".format(len(doc_sections)))
        print("scoring documents")
        doc_sections_scores = score_doc_sections(
            doc_sections,
            keywords,
            document_manager.vectorizer
        )
        top_k_sections = select_top_k(doc_sections, doc_sections_scores, k)
        results = update_with_top_k(results, top_k_sections, query)
        #print(results)
        if len(results) == 0:
            print("no results found")
        else:
            print("sending email")
            # update dynamo db table
            table = boto3.resource(
                'dynamodb',
                region_name=aws_config.region_name,
                ).Table(aws_config.db_recommendation_table_name)
            with table.batch_writer() as batch:
                write_results(results, query_id, batch)

        # DEBUG

    # print("sending emails")
    # this will be handled by dynamodb/lambda function
    # send_emails(results, args)
    print("finished")

if __name__=='__main__':

    parser = argparse.ArgumentParser(description='Section documents and rank by relevance to queries')

    # parser.add_argument('--email', type=str, required=True,
    #     help='Who should I send emails to?\nUse `--email P` to send to the actual addresses in the queries.')

    parser.add_argument('--start_date', type=str, default=None,
        # TODO implement this
        # help="".join([
        #     'Documents have dates associated with them. ',
        #     'What is the earliest date of the documents that we should return?\n',
        #     'Format is YYYY-MM-DD, e.g. 2019-12-4 for December 4, 2019.\n',
        #     'Default is to include Minutes from meetings that took place this ',
        #     'past week and Agendas for upcoming meetings'
        # ]))
        help="".join([
            'Documents have dates associated with them. ',
            'What is the earliest date of the documents that we should return?\n',
            'Format is YYYY-MM-DD, e.g. 2019-12-4 for December 4, 2019.\n',
            'Default is to include only upcoming meetings'
        ]))

    parser.add_argument('--end_date', type=str, default=None,
        help="".join([
            'Documents have dates associated with them. ',
            'What is the latest date of the documents that we should return?\n',
            'Format is YYYY-MM-DD, e.g. 2019-12-4 for December 4, 2019.\n',
            'Default is to include *all* documents after the start date.'
        ]))

    parser.add_argument('--k', type=int, default=5,
        help="".join([
            'We will return the top k results. Default is k=5.'
        ]))

    parser.add_argument('--agenda_only', action='store_true',
        help="".join([
            "If this flag is included, return only agenda documents, no minutes."
        ]))

    parser.add_argument('--query_type', type=str, default='testing',
        help="".join([
            "Queries have a type, e.g. 'subscribed'. By default, run only queries labeled 'testing'."
        ]))

    parser.add_argument('--emails', type=str, default=None,
        help="".join([
            'Send all results to this one email address. If the flag is set to "production", send the actual emails to the actual users'
        ]))

    parser.add_argument('--filter', type=str, default=None,
        help="".join([
            'Filter queries to only those with this email address set. If empty, run all queries of this type.'
        ]))

    args = parser.parse_args()
    print(args)

    document_manager = S3DocumentManager()

    run_queries(
        document_manager=document_manager,
        input_args=args
    )


import datetime

from flask import Flask, render_template, request

app = Flask(__name__)

@app.route("/", methods=["GET"])
# GETメソッドでのアクセスは以下が実行される．
def cookie():
    count = request.cookies.get("count")
    if count is None:
        # クッキー変数"count"が存在しない場合は，countに"1"を代入する．
        count = id
    id = "あなたはこのサイトに{}回アクセスしました．".format(count)
    max_age = 60 * 60 * 24 * 120
    # クッキーの賞味期限を120日にする
    response = make_response(render_template("a4-1.html", title="クッキーの利用", id=id, max_age=max_age))

    # クッキー変数"count"にcountに1加えたものを代入する．
    return response

@app.route("/", methods=["POST"])
# POSTメソッドでのアクセスは以下が実行される．
def output():
    id = request.form["id"]
    # フォームからnameデータを取得する．
    try:
        if id == "1":
                name="桜祭り"
                time="2022/3/20"
                place="1号公園"
                color="black"
        elif id == "2":
                name="夏祭り"
                time="2023/8/5"
                place="2号公園"
                color="red"
        elif id == "3":
                name="冬祭り"
                time="2023/12/30"
                place="3号公園"
                color="red"

        else:
                message="idが正しくありません"
                color = "black"

        message = "{}は{}に{}で開催".format(name, time, place)
    except Exception:
        message = "idが正しくありません"
        color = "black"
    return render_template("a4-2out.html", title="フォームの利用", message=message, color = color)

if __name__ == "__main__":
    app.debug = True
    app.run(host="localhost")


# https://data.oecd.org/price/inflation-cpi.htm
# https://data.oecd.org/interest/long-term-interest-rates.htm#indicator-chart

from re import A
import pandas as pd
import os
import tqdm
from pathlib import Path as pl

from data_prep.sbf120.stock import stock_processing
from data_prep.sbf120.main_data_prep import read_files

def load_macroeconomy_data(base_path):

    macro_data = {}

    macro_data["INFLATION"] = pd.read_csv(base_path / pl("data/external/inflation_rates_since_1980.csv"), sep=",")
    macro_data["LT_RATES"] = pd.read_csv(base_path / pl("data/external/long_term_rates_since_1980.csv"), sep=",")
    macro_data["ST_RATES"] = pd.read_csv(base_path / pl("data/external/short_term_rates_since_1980.csv"), sep=",")
    macro_data["PPI_RATES"] = pd.read_csv(base_path / pl("data/external/producer_price_index_since_1980.csv"), sep=",")
    macro_data["UNEMPLOYMENT_RATES"] = pd.read_csv(base_path / pl("data/external/unemployment_rate_since_1980.csv"), sep=",")

    #commodities
    macro_data["GOLD"] = pd.read_csv(base_path / pl("data/extracted_data/commodities/gold.csv"), sep=",")
    macro_data["SILVER"] = pd.read_csv(base_path / pl("data/extracted_data/commodities/silver.csv"), sep=",")
    macro_data["s&p500"] = pd.read_csv(base_path / pl("data/extracted_data/commodities/s&p500.csv"), sep=",")
    macro_data["WHEAT"] = pd.read_csv(base_path / pl("data/extracted_data/commodities/wheat.csv"), sep=",")
    macro_data["BRENT"] = pd.read_csv(base_path / pl("data/extracted_data/commodities/brent.csv"), sep=",")
    macro_data["GAS"] = pd.read_csv(base_path / pl("data/extracted_data/commodities/gas.csv"), sep=",")
    macro_data["SUGAR"] = pd.read_csv(base_path / pl("data/extracted_data/commodities/sugar.csv"), sep=",")
    macro_data["COPPER"] = pd.read_csv(base_path / pl("data/extracted_data/commodities/copper.csv"), sep=",")
    macro_data["COFFEE"] = pd.read_csv(base_path / pl("data/extracted_data/commodities/coffee.csv"), sep=",")
    macro_data["CACAO"] = pd.read_csv(base_path / pl("data/extracted_data/commodities/cacao.csv"), sep=",")
    macro_data["COTTON"] = pd.read_csv(base_path / pl("data/extracted_data/commodities/cotton.csv"), sep=",")
    macro_data["VIX"] = pd.read_csv(base_path / pl("data/extracted_data/commodities/vix.csv"), sep=",")

    for indic in macro_data.keys():
        if len(set(["INDICATOR", "SUBJECT", "FREQUENCY", "Flag Codes", "MEASURE"]).intersection(macro_data[indic].columns)) == 5:
            macro_data[indic] = macro_data[indic].drop(["INDICATOR", "SUBJECT", "FREQUENCY", "Flag Codes", "MEASURE"], axis=1)
            macro_data[indic]["TIME"] = pd.to_datetime(macro_data[indic]["TIME"], format="%Y-%m").dt.to_period("M")

    return macro_data

def leverage_oecd_data(base_path, data, df):

    macro_data = load_macroeconomy_data(base_path)

    mapping_location = {"IT": "ITA", "GB": "GBR", "DE": "DEU",
                        "BE": "BEL", "CH" : "CHN",
                        "US": "USA", "FR" : "FRA",
                        "CHF" : "CHE", "ES" : "ESP",
                        "LU" : "LUX", "PT" : "ESP", "PL" : "POL",
                        "IE" : "IRL", "TW" : "CHN", "AT" : "AUT",
                        "KS" : "KOR", "NO" : "NOR", "FI" : "FIN",
                        "DK" : "DNK", "NL" : "NLD", "AUS" : "DEU",
                        "JPY": "CHN",
                        "SE" : "SWE"}

    file_path = base_path / pl("data/extracted_data")
    results_analysis = {}
    inflation = macro_data["INFLATION"]
    rates= macro_data["LT_RATES"]

    for company in tqdm.tqdm(data["REUTERS_CODE"].unique()):

        results_analysis[company] = {}
        params = {"specific" : "",
                  "company" : company,
                  "base_path" : file_path}

        try:
            inputs = read_files(params)
            _, stock = stock_processing(inputs)
            stock["DATE"] = stock["DATE"].dt.to_period("M")
            agg_stock = stock[["DATE", "CLOSE", "VOLUME"]].groupby("DATE").mean().reset_index()

            country = data.loc[data["REUTERS_CODE"] == company, "Country"].values[0]
            infl = inflation.loc[inflation["LOCATION"] == mapping_location[country]]
            rte = rates.loc[rates["LOCATION"] == mapping_location[country]]

            comparison_df= pd.merge(agg_stock, infl, left_on="DATE", right_on="TIME", how="inner")
            comparison_df= pd.merge(comparison_df, rte, left_on="DATE", right_on="TIME", how="left", suffixes=("_INFLATION", "_RATE"))

            comparison_df = comparison_df[["DATE", "CLOSE", "VOLUME", "Value_INFLATION", "Value_RATE"]]
            comparison_df = comparison_df.loc[~comparison_df["CLOSE"].isnull()]
            results_analysis[company] = {"INFLATION" : comparison_df[["CLOSE", "Value_INFLATION"]].corr().iloc[0,1],
                                        "RATE" : comparison_df[["CLOSE", "Value_RATE"]].corr().iloc[0,1]}

        except Exception:
            pass

    results = pd.DataFrame(results_analysis).sort_index().T

    # results = pd.merge(results, data[["REUTERS_CODE", "SECTOR", "SUB INDUSTRY", "NAME"]], left_index=True, right_on="REUTERS_CODE", how="left")
    # agg = results[["SUB INDUSTRY", "INFLATION", "RATE"]].groupby("SUB INDUSTRY").mean().sort_values("INFLATION")

    df = pd.merge(df, results, left_index=True, right_index=True, how="left", validate="1:1")

    return df


import pandas as pd
import os
from datetime import datetime
from module import get_start_endtime, get_exechost

def main():
        '''
        From darshan log, jobid, starttime, endtime are gathered with get_start_endtime() from module.py.
        '''
        print('please input darshan log file name')
        filename = input() # Manual input the name we want to analyze
        jobid, start, end = get_start_endtime(filename)

        '''
        Get nodelist used by the corresponding jobid from the scheduled log of the date obtained above.
        get_start_endtime() from  module.py
        '''
        jobid, hostlist = get_exechost(start[:8], jobid) #return jobid, nodelist

        '''
        # Open csv of the date and make newdf between start time-10 and end time+10.
        # groupby('{}').mean().format(feature)
        # The average of counts per feature in the time period is obtained.
        # save one row in (csv/DB)
        # index = jobid, nodename / column = each feature
        '''
        for i in range(len(hostlist)): #for all host list for one jobid
                bound = datetime.timedelta(seconds=10)
                path1 = str(os.getcwd())+'/res/{}/{}data.csv'.format(hostlist[i],start[2:8])
                df1 = pd.read_csv(path1,'/','')
                df1['timestamp'] = pd.to_datetime(df1['timestamp'], infer_datetime_format=True)

                if start[8]==end[8]:
                        mask = (df1['timestamp'] > start - bound) & (df1['timestamp'] < end + bound)
                        selected_df = df1.loc[mask]
                else:
                        path2 = str(os.getcwd())+'/res/{}/{}data.csv'.format(hostlist[i],end[2:8])
                        df2 = pd.read_csv(path2)
                        df2['timestamp'] = pd.to_datetime(df1['timestamp'], infer_datetime_format=True)
                        mask1 = (df1['timestamp'] > start - bound)
                        mask2 = (df2['timestamp'] < end + bound)
                        selected_df = pd.concat(df1.loc[mask1],df2.loc[mask2])

                # get mean, std of each feature per one node that is used. save each row in csv
                print( hostlist[i],':\n', selected_df.describe() )

if __name__== '__main__':
    main()


#!/usr/bin/python
# coding=UTF-8
'''
@Author: recar
@Date: 2019-09-27 11:37:22
@LastEditTime: 2019-09-27 15:51:48
'''

from datetime import datetime
from threading import Thread
import redis
import time
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

redis_server = "127.0.0.1"
redis_port = 6379

class Scheduler(object):
    # 基于redis实现一个定时调度功能

    def  __init__(self, schedule=False):
        self.is_schedule = schedule
        self.conn = self.get_connection()
        self.zset_delay_key = "zset_delay:"
        self.zset_schedule_key = "zset_schedule:"
        if self.is_schedule:
            self.schedule()

    def get_connection(self):
        connect = redis.Redis(host=redis_server, port=redis_port,db=0)
        return connect

    def add_task(self, task_id, delay):
        # 添加时间间隔
        self.conn.zadd(self.zset_delay_key, {task_id: delay})
        # 添加调度任务 这个会要先生成一个立即调度的
        self.conn.zadd(self.zset_schedule_key, {task_id: time.time()})

    def schedule(self):
        while True:
            next = self.conn.zrange(self.zset_schedule_key, 0, 0, withscores=True)
            now = time.time()
            # print(next)
            if not next or next[0][1] > now:
                # print('next: {0} -> now: {1}'.format(next[0][1], now))
                time.sleep(0.5)
                continue
            task_id = next[0][0]
            # 获取时间间隔
            delay = self.conn.zscore(self.zset_delay_key, task_id)
            # 如果更新了时间间隔 为0 那么都删除掉
            if delay <=0:
                self.conn.zrem(self.zset_delay_key, task_id)
                self.conn.zrem(self.zset_schedule_key, task_id)
                continue
            print('exec task: {0} time: {1}'.format(task_id, now))
            # 否则更新调度任务
            self.conn.zrem(self.zset_schedule_key, task_id)
            self.conn.zadd(self.zset_schedule_key, {task_id: now+delay})

def main():
    try:
        # 启动一个定时调度监控
        def statr_schedule():
            schedule = Scheduler(schedule=True)
        t = Thread(target=statr_schedule, args=())
        t.start()
        print('start schedule')
        schedule = Scheduler()
        schedule.add_task(1, 5)
        print('add task: {0}'.format(1))
        schedule.add_task(2, 10)
        print('add task: {}'.format(2))
    except Exception as e:
        print(e)
        exit(1)

if __name__ == "__main__":
    main()


import random
import re
import time
import pymongo
import requests
from urllib.parse import quote
from multiprocessing import Pool
from config import *

'''
爬取淘宝1688商品信息，运行前请先打开代理
'''

client = pymongo.MongoClient(MONGO_URI)
db = client[MONGO_DB]
item_id = set()

def get_proxy():
    '''
    获取代理
    '''
    return requests.get("http://127.0.0.1:5010/get/").text

def get_one_page(page):
    '''
    得到单页商品信息
    '''
    try:
        print('正在爬取第%d页' % page)
        # for syncreq in range(1, MAX_SYNCREQ + 1):
        proxy = get_proxy()
        url = 'https://data.p4psearch.1688.com/data/ajax/get_premium_offer_list.json'
        data = {
            'beginpage': str(page),  # 页数
            'syncreq': '1',  # 同步请求参数  str(syncreq)
            'keywords': KEYWORD  # 搜索关键词
        }
        headers = {
            'User_Agent': random.choice(USER_AGENT),
            'Referer': 'https://p4psearch.1688.com/p4p114/p4psearch/offer.htm?keywords=' + quote(
                KEYWORD) + '&sortType=&descendOrder=&province=&city=&priceStart=&priceEnd=&dis=&provinceValue=%E6%89%80%E5%9C%A8%E5%9C%B0%E5%8C%BA',
            'Cookie': COOKIE,
        }
        proxies = {"http": "http://{}".format(proxy)}
        response = requests.get(url=url, headers=headers, params=data, proxies=proxies, timeout=5)
        time.sleep(1)
        if response.status_code == 200:
            data = response.json()
            get_info(data=data)
    except Exception:
        print('出现异常，重新爬取第%d页' % page)
        return get_one_page(page)

def get_info(data):
    '''
    得到并保存商品信息
    '''
    try:
        print('正在获取商品信息..')
        items = data['data']['content']['offerResult']
        # item为每一个商品
        for item in items:
            param1 = item['attr']['company']
            company = param1['name']
            type = param1['bizTypeName']
            city = param1['city']
            province = param1['province']
            param2 = item['attr']['tradePrice']['offerPrice']
            originalValue = param2['originalValue']['integer'] + param2['originalValue']['decimals'] / 10
            quantityPrices = param2['value']['integer'] + param2['value']['decimals'] / 10
            param3 = item['attr']['tradeQuantity']
            sales = param3['number']
            saleType = param3['sortType']
            detailUrl = item['eurl']
            imgUrl = item['imgUrl']
            originaltitle = item['title']
            title = re.sub('<.*>', '', originaltitle)
            result = {
                '标题': title,
                '原价': originalValue,
                '最低批发价': quantityPrices,
                '销售量': sales,
                '销售形式': saleType,
                '详细链接': detailUrl,
                '图片链接': imgUrl,
                '公司': company,
                '公司类型': type,
                '城市': city,
                '省份': province,
            }
            save_to_mongo(result=result)
    except TypeError:
        pass

def save_to_mongo(result):
    '''
    信息保存至mongoDB
    '''
    # 去除标题相同的商品
    if result['标题'] in item_id:
        print('发现重复item，丢弃--->%s<---' % result['标题'])
    else:
        item_id.add(result['标题'])
        if db[MONGO_TABLE].insert(result):
            print('成功保存至mongoDB', result['标题'])

def main(page):
    get_one_page(page)
    print('有效商品数为%d'%len(item_id))  # 打印有效商品数量

if __name__ == '__main__':
    page = [x for x in range(1, MAX_PAGE + 1)]
    # 多进程
    pool = Pool()
    pool.map(main, page)

# todo 待解决问题：
'''
关于遍历syncreq获取单页更多信息，每页的第一个请求正常，
后续获得商品重复，怀疑与请求头里面的一些参数有关，若无
法分析参数规律，该爬虫只可扩大MAX_PAGE获得更多采集量。
或者换用selenium的方式来爬取。
2019.1.10
'''


import os.path
from itertools import groupby

import pros.common.ui as ui
import pros.conductor as c
from pros.cli.common import *
from pros.conductor.templates import ExternalTemplate
from pros.ga.analytics import analytics

@pros_root
def conductor_cli():
    pass

@conductor_cli.group(cls=PROSGroup, aliases=['cond', 'c', 'conduct'], short_help='Perform project management for PROS')
@default_options
def conductor():
    """
    Conductor is PROS's project management facility. It is responsible for obtaining
    templates for which to create projects from.

    Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more
    """
    pass

@conductor.command(aliases=['download'], short_help='Fetch/Download a remote template',
                   context_settings={'ignore_unknown_options': True})
@template_query(required=True)
@default_options
def fetch(query: c.BaseTemplate):
    """
    Fetch/download a template from a depot.

    Only a template spec is required. A template spec is the name and version
    of the template formatted as name@version (libblrs@1.0.0). Semantic version
    ranges are accepted (e.g., libblrs@^1.0.0). The version parameter is also
    optional (e.g., libblrs)

    Additional parameters are available according to the depot.

    Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more
    """
    analytics.send("fetch-template")
    template_file = None
    if os.path.exists(query.identifier):
        template_file = query.identifier
    elif os.path.exists(query.name) and query.version is None:
        template_file = query.name
    elif query.metadata.get('origin', None) == 'local':
        if 'location' not in query.metadata:
            logger(__name__).error('--location option is required for the local depot. Specify --location <file>')
            logger(__name__).debug(f'Query options provided: {query.metadata}')
            return -1
        template_file = query.metadata['location']

    if template_file and (os.path.splitext(template_file)[1] in ['.zip'] or
                          os.path.exists(os.path.join(template_file, 'template.pros'))):
        template = ExternalTemplate(template_file)
        query.metadata['location'] = template_file
        depot = c.LocalDepot()
        logger(__name__).debug(f'Template file found: {template_file}')
    else:
        if template_file:
            logger(__name__).debug(f'Template file exists but is not a valid template: {template_file}')
        template = c.Conductor().resolve_template(query, allow_offline=False)
        logger(__name__).debug(f'Template from resolved query: {template}')
        if template is None:
            logger(__name__).error(f'There are no templates matching {query}!')
            return -1
        depot = c.Conductor().get_depot(template.metadata['origin'])
        logger(__name__).debug(f'Found depot: {depot}')
    # query.metadata contain all of the extra args that also go to the depot. There's no way for us to determine
    # whether the arguments are for the template or for the depot, so they share them
    logger(__name__).debug(f'Additional depot and template args: {query.metadata}')
    c.Conductor().fetch_template(depot, template, **query.metadata)

@conductor.command(context_settings={'ignore_unknown_options': True})
@click.option('--upgrade/--no-upgrade', 'upgrade_ok', default=True, help='Allow upgrading templates in a project')

@click.option('--install/--no-install', 'install_ok', default=True, help='Allow installing templates in a project')
@click.option('--download/--no-download', 'download_ok', default=True,
              help='Allow downloading templates or only allow local templates')
@click.option('--upgrade-user-files/--no-upgrade-user-files', 'force_user', default=False,
              help='Replace all user files in a template')
@click.option('--force', 'force_system', default=False, is_flag=True,
              help="Force all system files to be inserted into the project")
@click.option('--force-apply', 'force_apply', default=False, is_flag=True,
              help="Force apply the template, disregarding if the template is already installed.")
@click.option('--remove-empty-dirs/--no-remove-empty-dirs', 'remove_empty_directories', is_flag=True, default=True,
              help='Remove empty directories when removing files')
@click.option('--beta', is_flag=True, default=False, show_default=True,
              help='Allow applying beta templates')
@project_option()
@template_query(required=True)
@default_options
def apply(project: c.Project, query: c.BaseTemplate, **kwargs):
    """
    Upgrade or install a template to a PROS project

    Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more
    """
    analytics.send("apply-template")
    return c.Conductor().apply_template(project, identifier=query, **kwargs)

@conductor.command(aliases=['i', 'in'], context_settings={'ignore_unknown_options': True})
@click.option('--upgrade/--no-upgrade', 'upgrade_ok', default=False)
@click.option('--download/--no-download', 'download_ok', default=True)
@click.option('--force-user', 'force_user', default=False, is_flag=True,
              help='Replace all user files in a template')
@click.option('--force-system', '-f', 'force_system', default=False, is_flag=True,
              help="Force all system files to be inserted into the project")
@click.option('--force-apply', 'force_apply', default=False, is_flag=True,
              help="Force apply the template, disregarding if the template is already installed.")
@click.option('--remove-empty-dirs/--no-remove-empty-dirs', 'remove_empty_directories', is_flag=True, default=True,
              help='Remove empty directories when removing files')
@click.option('--beta', is_flag=True, default=False, show_default=True,
              help='Allow applying beta templates')
@project_option()
@template_query(required=True)
@default_options
@click.pass_context
def install(ctx: click.Context, **kwargs):
    """
    Install a library into a PROS project

    Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more
    """
    analytics.send("install-template")
    return ctx.invoke(apply, install_ok=True, **kwargs)

@conductor.command(context_settings={'ignore_unknown_options': True}, aliases=['u'])
@click.option('--install/--no-install', 'install_ok', default=False)
@click.option('--download/--no-download', 'download_ok', default=True)
@click.option('--force-user', 'force_user', default=False, is_flag=True,
              help='Replace all user files in a template')
@click.option('--force-system', '-f', 'force_system', default=False, is_flag=True,
              help="Force all system files to be inserted into the project")
@click.option('--force-apply', 'force_apply', default=False, is_flag=True,
              help="Force apply the template, disregarding if the template is already installed.")
@click.option('--remove-empty-dirs/--no-remove-empty-dirs', 'remove_empty_directories', is_flag=True, default=True,
              help='Remove empty directories when removing files')
@click.option('--beta', is_flag=True, default=False, show_default=True,
              help='Allow upgrading to beta templates')
@project_option()
@template_query(required=False)
@default_options
@click.pass_context
def upgrade(ctx: click.Context, project: c.Project, query: c.BaseTemplate, **kwargs):
    """
    Upgrade a PROS project or one of its libraries

    Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more
    """
    analytics.send("upgrade-project")
    if not query.name:
        for template in project.templates.keys():
            click.secho(f'Upgrading {template}', color='yellow')
            q = c.BaseTemplate.create_query(name=template, target=project.target,
                                            supported_kernels=project.templates['kernel'].version)
            ctx.invoke(apply, upgrade_ok=True, project=project, query=q, **kwargs)
    else:
        ctx.invoke(apply, project=project, query=query, upgrade_ok=True, **kwargs)

@conductor.command('uninstall')
@click.option('--remove-user', is_flag=True, default=False, help='Also remove user files')
@click.option('--remove-empty-dirs/--no-remove-empty-dirs', 'remove_empty_directories', is_flag=True, default=True,
              help='Remove empty directories when removing files')
@click.option('--no-make-clean', is_flag=True, default=True, help='Do not run make clean after removing')
@project_option()
@template_query()
@default_options
def uninstall_template(project: c.Project, query: c.BaseTemplate, remove_user: bool,
                       remove_empty_directories: bool = False, no_make_clean: bool = False):
    """
    Uninstall a template from a PROS project

    Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more
    """
    analytics.send("uninstall-template")
    c.Conductor().remove_template(project, query, remove_user=remove_user,
                                  remove_empty_directories=remove_empty_directories)
    if no_make_clean:
        with ui.Notification():
            project.compile(["clean"])

@conductor.command('new-project', aliases=['new', 'create-project'])
@click.argument('path', type=click.Path())
@click.argument('target', default=c.Conductor().default_target, type=click.Choice(['v5', 'cortex']))
@click.argument('version', default='latest')
@click.option('--force-user', 'force_user', default=False, is_flag=True,
              help='Replace all user files in a template')
@click.option('--force-system', '-f', 'force_system', default=False, is_flag=True,
              help="Force all system files to be inserted into the project")
@click.option('--force-refresh', is_flag=True, default=False, show_default=True,
              help='Force update all remote depots, ignoring automatic update checks')
@click.option('--no-default-libs', 'no_default_libs', default=False, is_flag=True,
              help='Do not install any default libraries after creating the project.')
@click.option('--compile-after', is_flag=True, default=True, show_default=True,
              help='Compile the project after creation')
@click.option('--build-cache', is_flag=True, default=None, show_default=False,
              help='Build compile commands cache after creation. Overrides --compile-after if both are specified.')
@click.option('--beta', is_flag=True, default=False, show_default=True,
              help='Create a project with a beta template')
@click.pass_context
@default_options
def new_project(ctx: click.Context, path: str, target: str, version: str,
                force_user: bool = False, force_system: bool = False,
                no_default_libs: bool = False, compile_after: bool = True, build_cache: bool = None, **kwargs):
    """
    Create a new PROS project

    Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more
    """
    analytics.send("new-project")
    if version.lower() == 'latest' or not version:
        version = '>0'
    if not force_system and c.Project.find_project(path) is not None:
        logger(__name__).error('A project already exists in this location at ' + c.Project.find_project(path) +
                               '! Delete it first. Are you creating a project in an existing one?', extra={'sentry': False})
        ctx.exit(-1)
    try:
        _conductor = c.Conductor()
        if target is None:
            target = _conductor.default_target
        project = _conductor.new_project(path, target=target, version=version,
                                         force_user=force_user, force_system=force_system,
                                         no_default_libs=no_default_libs, **kwargs)
        ui.echo('New PROS Project was created:', output_machine=False)
        ctx.invoke(info_project, project=project)

        if compile_after or build_cache:
            with ui.Notification():
                ui.echo('Building project...')
                ctx.exit(project.compile([], scan_build=build_cache))

    except Exception as e:
        pros.common.logger(__name__).exception(e)
        ctx.exit(-1)

@conductor.command('query-templates',
                   aliases=['search-templates', 'ls-templates', 'lstemplates', 'querytemplates', 'searchtemplates'],
                   context_settings={'ignore_unknown_options': True})
@click.option('--allow-offline/--no-offline', 'allow_offline', default=True, show_default=True,
              help='(Dis)allow offline templates in the listing')
@click.option('--allow-online/--no-online', 'allow_online', default=True, show_default=True,
              help='(Dis)allow online templates in the listing')
@click.option('--force-refresh', is_flag=True, default=False, show_default=True,
              help='Force update all remote depots, ignoring automatic update checks')
@click.option('--limit', type=int, default=15,
              help='The maximum number of displayed results for each library')
@click.option('--beta', is_flag=True, default=False, show_default=True,
              help='View beta templates in the listing')
@template_query(required=False)
@click.pass_context
@default_options
def query_templates(ctx, query: c.BaseTemplate, allow_offline: bool, allow_online: bool, force_refresh: bool,
                    limit: int, beta: bool):
    """
    Query local and remote templates based on a spec

    Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more
    """
    analytics.send("query-templates")
    if limit < 0:
        limit = 15
    templates = c.Conductor().resolve_templates(query, allow_offline=allow_offline, allow_online=allow_online,
                                                force_refresh=force_refresh, beta=beta)
    if beta:
        templates += c.Conductor().resolve_templates(query, allow_offline=allow_offline, allow_online=allow_online,
                                                force_refresh=force_refresh, beta=False)

    render_templates = {}
    for template in templates:
        key = (template.identifier, template.origin)
        if key in render_templates:
            if isinstance(template, c.LocalTemplate):
                render_templates[key]['local'] = True
        else:
            render_templates[key] = {
                'name': template.name,
                'version': template.version,
                'location': template.origin,
                'target': template.target,
                'local': isinstance(template, c.LocalTemplate)
            }
    import semantic_version as semver
    render_templates = sorted(render_templates.values(), key=lambda k: (k['name'], semver.Version(k['version']), k['local']), reverse=True)

    # Impose the output limit for each library's templates
    output_templates = []
    for _, g in groupby(render_templates, key=lambda t: t['name'] + t['target']):
        output_templates += list(g)[:limit]
    ui.finalize('template-query', output_templates)

@conductor.command('info-project')
@click.option('--ls-upgrades/--no-ls-upgrades', 'ls_upgrades', default=False)
@project_option()
@default_options
def info_project(project: c.Project, ls_upgrades):
    """
    Display information about a PROS project

    Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more
    """
    analytics.send("info-project")
    from pros.conductor.project import ProjectReport
    report = ProjectReport(project)
    _conductor = c.Conductor()
    if ls_upgrades:
        for template in report.project['templates']:
            import semantic_version as semver
            templates = _conductor.resolve_templates(c.BaseTemplate.create_query(name=template["name"],
                                                                                version=f'>{template["version"]}',
                                                                                target=project.target))
            template["upgrades"] = sorted({t.version for t in templates}, key=lambda v: semver.Version(v), reverse=True)

    ui.finalize('project-report', report)

@conductor.command('add-depot')
@click.argument('name')
@click.argument('url')
@default_options
def add_depot(name: str, url: str):
    """
    Add a depot

    Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more
    """
    _conductor = c.Conductor()
    _conductor.add_depot(name, url)

    ui.echo(f"Added depot {name} from {url}")

@conductor.command('remove-depot')
@click.argument('name')
@default_options
def remove_depot(name: str):
    """
    Remove a depot

    Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more
    """
    _conductor = c.Conductor()
    _conductor.remove_depot(name)

    ui.echo(f"Removed depot {name}")

@conductor.command('query-depots')
@click.option('--url', is_flag=True)
@default_options
def query_depots(url: bool):
    """
    Gets all the stored depots

    Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more
    """
    _conductor = c.Conductor()
    ui.echo(f"Available Depots{' (Add --url for the url)' if not url else ''}:\n")
    ui.echo('\n'.join(_conductor.query_depots(url))+"\n")


import requests
import re
import xlwt,xlrd
import os

def get_num(url):
    wb=requests.get(url,headers=headers)
    wb.encoding=wb.apparent_encoding
    wbdata=wb.text.split('<')
    for i in range(len(wbdata)):
        if wbdata[i].startswith('em id='):
            aa=wbdata[i].split('>')
            num=aa[1]
            if '万' in num:
                num1=float(num.replace('万',''))*10000
            elif '亿' in num:
                num1=float(num.replace('亿',''))*100000000
            return(num1)
            # print(num)

def get_title(url):
    wb = requests.get(url, headers=headers)
    wb.encoding = wb.apparent_encoding
    wbdata = wb.text
    title_str = re.findall('name="title" content=.*', wbdata)
    if title_str:
        title=title_str[0].split(' ')
        title1=title[1].replace('content="','')
        # print(title[1])
        return(title1)

if __name__ == "__main__":
    cookie='pgv_pvi=9294307328; ptui_loginuin=598564346; RK=LfjkfTO2Y9; ptcz=ff0c0ddbe7b003a1891204d1d575e3ad1627ce75bd6e4f87ef013448667a359a; pgv_pvid=9222305988; tvfe_boss_uuid=c7151d80a1172cf9; video_guid=66066166ce686bb7; video_platform=2; pgv_info=ssid=s5413384000'
    headers={
        'User_Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
        'Cookie':cookie
    }
    url='https://v.qq.com/channel/net_tv'
    wb=requests.get(url,headers=headers)
    # print(wbdata)
    wb.encoding=wb.apparent_encoding
    wbdata=wb.text
    url_list=re.findall('https:.*html',wbdata)
    title_list=[]
    num_list=[]
    for i in range(len(url_list)):
        title=get_title(url_list[i])
        if title not in title_list:
            title_list.append(title)
            # print(title_list)
            num=get_num(url_list[i])
            num_list.append(num)
    os.chdir('C:/Users/Administrator/Desktop')
    wb=xlwt.Workbook()
    ws=wb.add_sheet('腾讯视频观看量')
    for i in range(len(num_list)):
        ws.write(i,1,num_list[i])
        ws.write(i,0,title_list[i])
        wb.save('test.xls')


from flask import Flask, request
from logging.config import dictConfig
from flask_sqlalchemy.record_queries import get_recorded_queries

from config import Config

dictConfig(
    {
        "version": 1,
        "formatters": {
            "default": {
                "format": "[%(asctime)s] [%(levelname)s | %(module)s] %(message)s",
                "datefmt": "%B %d, %Y %H:%M:%S %Z",
            },
        },
        "handlers": {
            "console": {
                "class": "logging.StreamHandler",
                "formatter": "default",
            },
            "file": {
                "class": "logging.FileHandler",
                "filename": "todo.log",
                "formatter": "default",
            },
        },
        "root": {"level": "DEBUG", "handlers": ["file"]},
    }
)

myapp = Flask(__name__)
myapp.config.from_object(Config)

from models import db
from schemas import ma

db.init_app(myapp)
ma.init_app(myapp)

from create.routes import create_bp
from retreive.routes import retreive_bp
from update.routes import update_bp
from delete.routes import delete_bp

myapp.register_blueprint(create_bp)
myapp.register_blueprint(retreive_bp)
myapp.register_blueprint(update_bp)
myapp.register_blueprint(delete_bp)

@myapp.route('/')
def hellofunc():

    myapp.logger.info(">>REQ>> A user visited the home page.")
    return "Hello there!"

@myapp.after_request
def logAfterRequest(response):

    queries = list(get_recorded_queries())
    query_str = ''
    total_duration = 0.0
    for q in queries:
        total_duration += q.duration
        stmt = str(q.statement % q.parameters).replace('\n', '\n       ')
        query_str += 'Query: {0}\nDuration: {1}ms\n\n'.format(stmt, round(q.duration * 1000, 2))

    myapp.logger.info('=' * 80)
    myapp.logger.info(' SQL Queries - {0} Queries Executed in {1}ms'.format(len(queries), round(total_duration * 1000, 2)))
    myapp.logger.info('=' * 80)
    myapp.logger.info(query_str.rstrip('\n'))
    myapp.logger.info('=' * 80)

    myapp.logger.info(
        ">>RESP>> path: %s | method: %s | status: %s | size: %s",
        request.path,
        request.method,
        response.status,
        response.content_length,
    )

    return response

if __name__ == "__main__":
    #with myapp.app_context():
        #db.create_all()

    myapp.run(debug=True)