import contextlib
import copy
import os
import subprocess
import sys

from nuitka import Options, Tracing
from nuitka.__past__ import unicode  # pylint: disable=I0021,redefined-builtin
from nuitka.PythonVersions import getTargetPythonDLLPath, python_version
from nuitka.utils import Execution, Utils
from nuitka.utils.FileOperations import (
    getExternalUsePath,
    getWindowsShortPathName,
)

def getSconsDataPath():
    """ Return path to where data for scons lives, e.g. static C source files.

    """

    return os.path.dirname(__file__)

def _getSconsInlinePath():
    """ Return path to inline copy of scons. """

    return os.path.join(getSconsDataPath(), "inline_copy")

def _getSconsBinaryCall():
    """ Return a way to execute Scons.

    Using potentially in-line copy if no system Scons is available
    or if we are on Windows, there it is mandatory.
    """

    inline_path = os.path.join(_getSconsInlinePath(), "bin", "scons.py")

    if os.path.exists(inline_path):
        return [
            _getPythonForSconsExePath(),
            "-W",
            "ignore",  # Disable Python warnings in case of debug Python.
            getExternalUsePath(inline_path),
        ]
    else:
        scons_path = Execution.getExecutablePath("scons")

        if scons_path is not None:
            return [scons_path]
        else:
            sys.exit(
                "Error, the inline copy of scons is not present, nor scons in the system path."
            )

def _getPythonSconsExePathWindows():
    """ Find Python2 on Windows.

    First try a few guesses, the look into registry for user or system wide
    installations of Python2. Both Python 2.6 and 2.7, and 3.5 or higher
    will do.
    """

    # Ordered in the list of preference.
    scons_supported = ("2.7", "2.6", "3.5", "3.6", "3.7", "3.8")

    # Shortcuts for the default installation directories, to avoid going to
    # registry at all unless necessary. Any Python2 will do for Scons, so it
    # might be avoided entirely.
    for search in scons_supported:
        candidate = r"c:\Python%s\python.exe" % search.replace(".", "")

        if os.path.isfile(candidate):
            return candidate

    # Windows only code, pylint: disable=I0021,import-error,undefined-variable
    if python_version < 300:
        import _winreg as winreg  # pylint: disable=I0021,import-error,no-name-in-module
    else:
        import winreg  # pylint: disable=I0021,import-error,no-name-in-module

    for search in scons_supported:
        for hkey_branch in (winreg.HKEY_LOCAL_MACHINE, winreg.HKEY_CURRENT_USER):
            for arch_key in (0, winreg.KEY_WOW64_32KEY, winreg.KEY_WOW64_64KEY):
                try:
                    key = winreg.OpenKey(
                        hkey_branch,
                        r"SOFTWARE\Python\PythonCore\%s\InstallPath" % search,
                        0,
                        winreg.KEY_READ | arch_key,
                    )

                    return os.path.join(winreg.QueryValue(key, ""), "python.exe")
                except WindowsError:
                    pass

def _getPythonForSconsExePath():
    """ Find a way to call any Python2.

    Scons needs it as it doesn't support Python3.
    """
    python_exe = Options.getPythonPathForScons()

    if python_exe is not None:
        return python_exe

    if python_version < 300 or python_version >= 350:
        return sys.executable
    elif Utils.getOS() == "Windows":
        python_exe = _getPythonSconsExePathWindows()

        if python_exe is not None:
            return python_exe
        else:
            sys.exit(
                """\
Error, while Nuitka works with Python 3.3 and 3.4, scons does not, and Nuitka
needs to find a Python executable 2.6/2.7 or 3.5 or higher. Simply under the
C:\\PythonXY, e.g. C:\\Python27 to execute the scons utility which is used
to build the C files to binary.

You may provide it using option "--python-for-scons=path_to_python.exe"
in case it is not visible in registry, e.g. due to using uninstalled
AnaConda Python.
"""
            )

    for version_candidate in ("2.7", "2.6", "3.5", "3.6", "3.7", "3.8"):
        candidate = Execution.getExecutablePath("python" + version_candidate)

        if candidate is not None:
            return candidate

    # Lets be optimistic, this is most often going to be new enough or a
    # Python2 variant.
    return "python"

@contextlib.contextmanager
def _setupSconsEnvironment():
    """ Setup the scons execution environment.

    For the target Python we provide "NUITKA_PYTHON_DLL_PATH" to see where the
    Python DLL lives, in case it needs to be copied, and then also the
    "NUITKA_PYTHON_EXE_PATH" to find the Python binary itself.

    We also need to preserve PYTHONPATH and PYTHONHOME, but remove it potentially
    as well, so not to confuse the other Python binary used to run scons.
    """

    # For Python2, avoid unicode working directory.
    if Utils.isWin32Windows() and python_version < 300:
        if os.getcwd() != os.getcwdu():
            os.chdir(getWindowsShortPathName(os.getcwdu()))

    if Utils.isWin32Windows():
        # On Win32, we use the Python.DLL path for some things. We pass it
        # via environment variable
        os.environ["NUITKA_PYTHON_DLL_PATH"] = getTargetPythonDLLPath()

    os.environ["NUITKA_PYTHON_EXE_PATH"] = sys.executable

    # Remove environment variables that can only harm if we have to switch
    # major Python versions, these cannot help Python2 to execute scons, this
    # is a bit of noise, but helpful.
    if python_version >= 300:
        if "PYTHONPATH" in os.environ:
            old_pythonpath = os.environ["PYTHONPATH"]
            del os.environ["PYTHONPATH"]
        else:
            old_pythonpath = None

        if "PYTHONHOME" in os.environ:
            old_pythonhome = os.environ["PYTHONHOME"]
            del os.environ["PYTHONHOME"]
        else:
            old_pythonhome = None

    yield

    if python_version >= 300:
        if old_pythonpath is not None:
            os.environ["PYTHONPATH"] = old_pythonpath

        if old_pythonhome is not None:
            os.environ["PYTHONHOME"] = old_pythonhome

    if Utils.isWin32Windows():
        del os.environ["NUITKA_PYTHON_DLL_PATH"]

    del os.environ["NUITKA_PYTHON_EXE_PATH"]

def _buildSconsCommand(quiet, options):
    """ Build the scons command to run.

    The options are a dictionary to be passed to scons as a command line,
    and other scons stuff is set.
    """

    scons_command = _getSconsBinaryCall()

    if quiet:
        scons_command.append("--quiet")

    scons_command += [
        # The scons file
        "-f",
        getExternalUsePath(os.path.join(getSconsDataPath(), "SingleExe.scons")),
        # Parallel compilation.
        "--jobs",
        str(Options.getJobLimit()),
        # Do not warn about deprecation from Scons
        "--warn=no-deprecated",
        # Don't load "site_scons" at all.
        "--no-site-dir",
    ]

    if Options.isShowScons():
        scons_command.append("--debug=explain")

    # Python2, encoding unicode values
    def encode(value):
        if str is bytes and type(value) is unicode:
            return value.encode("utf8")
        else:
            return value

    # Option values to provide to scons. Find these in the caller.
    for key, value in options.items():
        scons_command.append(key + "=" + encode(value))

    # Python2, make argument encoding recognizable.
    if str is bytes:
        scons_command.append("arg_encoding=utf8")

    return scons_command

def runScons(options, quiet):
    with _setupSconsEnvironment():
        if Options.shallCompileWithoutBuildDirectory():
            # Make sure we become non-local, by changing all paths to be
            # absolute, but ones that can be resolved by any program
            # externally, as the Python of Scons may not be good at unicode.

            options = copy.deepcopy(options)
            source_dir = options["source_dir"]
            options["source_dir"] = "."
            options["result_name"] = getExternalUsePath(
                options["result_name"], only_dirname=True
            )
            options["nuitka_src"] = getExternalUsePath(options["nuitka_src"])
            if "result_exe" in options:
                options["result_exe"] = getExternalUsePath(
                    options["result_exe"], only_dirname=True
                )
            if "icon_path" in options:
                options["icon_path"] = getExternalUsePath(
                    options["icon_path"], only_dirname=True
                )
        else:
            source_dir = None

        scons_command = _buildSconsCommand(quiet, options)

        if Options.isShowScons():
            Tracing.printLine("Scons command:", " ".join(scons_command))

        Tracing.flushStdout()
        return subprocess.call(scons_command, shell=False, cwd=source_dir) == 0


from rest_framework import serializers
from .......models.taxable_assessment_model import TaxableAssessment

class DisplayAssessmentSerializer(serializers.ModelSerializer):
    classification_display= serializers.CharField(source='get_classification_display')
    class Meta:
        model = TaxableAssessment
        fields = ['id','tax_assessment_roll','arpn','td_no','pin','lot_block_no','property_owner',
                  'address_of_property_owner','kind','classification','classification_display','assessed_value','prev_arpn','prev_td_no','effectivity',
                  'remarks',
                  ]


# https://leetcode.com/problems/partition-labels/
from typing import List
from tester import Tester

class Solution:
    def partitionLabels(self, s: str) -> List[int]:
        end_idxs = {}
        for i, c in enumerate(s):
            end_idxs[c] = i

        l = m = 0
        r = end_idxs[s[0]]
        partitions = []
        while r < len(s):
            r = max(r, end_idxs[s[m]])
            if m == r:
                partitions.append(r - l + 1)
                l = r + 1
                r = end_idxs[s[l]] if l < len(s) else l
            m += 1
        return partitions

t = Tester(Solution())

t.test([1], "a")
t.test([10], "eccbbbbdec")
t.test([1, 1, 1, 1, 3, 2], "abcdegeff")
t.test([9, 7, 8], "ababcbacadefegdehijhklij")

t.report()


import pandas as pd
import matplotlib.pyplot as plt
from sklearn import svm
import numpy as np
from sklearn.model_selection import train_test_split

df = pd.read_csv("creditcard.csv", header = 0)

X= df.iloc[1:10000, [1, 2, 3, 4, 5]].values
y = df.iloc[1:10000, [30]].values
y = np.where(y== 0, -1, 1)
y = y.reshape((9999, ))
#print(X.shape)
#print(y.shape)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=1, stratify=y)
X_test, X_dev, y_test, y_dev= train_test_split(X_test, y_test, test_size=0.5, random_state=1)

from sklearn.preprocessing import StandardScaler
sc=StandardScaler()
sc.fit(X_train)
X_train_std = sc.transform(X_train)
X_test_std = sc.transform(X_test)
#y_test = y_test.reshape((1500, ))
#print(y_test.shape)

svm = svm.SVC(kernel= 'linear', C = 1, gamma = 1)
svm.fit(X_train, y_train)

accuracy = svm.predict(X_test)

count = 0
for i in accuracy:
    if(accuracy[i] == y_test[i]):
        count = 1 + count

fraud_count = 0
for x in y_test:
    if(y_test[x] == 1):
        fraud_count = 1 + fraud_count
print(fraud_count)

print(count/len(y_test) * 100)
print(fraud_count)


import scrapy

from scrapy.mail import MailSender

class fray_fitness(scrapy.Spider):

    name ='Check what just became in stock!'

    start_urls =['https://www.frayfitness.com/230lb-fray-crumb-rubber-bumper-set.html']

    def parse(self,response):

        blue = response.xpath('//span[@class="in-stock"]/text()').extract()

        if not blue:
           self.logger.info('This is not in stock yet')

        else:
            mailer = MailSender(
                smptphost='******',
                mailfrom='scrapy.send@gmail.com',
                smtpuser='scrapy.send@gmail.com',
                smtppas='*****',
                smtpport=587
            )


# -*- coding: utf-8 -*-
"""
2D Gaussian Mixture Model (GMM) Stochstic Gradient Descent (SGD) Experiment

Created on Thu Nov 19 12:14:06 2020

@author: benpg
"""
#%% 0. Import dependencies, set up basic parameters

# External
import autograd.numpy as np
from autograd.numpy.random import multivariate_normal
from autograd import grad
import matplotlib.pyplot as plt
import pickle, os
plt.close('all')

# Misc
from utils import samples, plot_GMM, plot_GMM_2, calculate_ELBO, HiddenPrints, cols

from testing_autograd import testing_autograd

#%% 1. Create a (2D) dataset and plot it

# Define GMMs for generation
centres = [np.array([0.,3.]), np.array([2.,0.])]
covs = [np.eye(2), np.array([[0.6,0.4],
                             [0.4,0.6]])]

N_total = 1000  # total number of datapoints wanted
X1 = multivariate_normal(mean=centres[0],
                         cov=covs[0],
                         size=int(N_total/2))
X2 = multivariate_normal(mean=centres[1],
                         cov=covs[1],
                         size=int(N_total/2))
X = np.concatenate((X1,X2))

# # Only a single mixture component
# X = multivariate_normal(mean=centres[0],
#                          cov=covs[0],
#                          size=int(N_total/2))

# Number of samples for estimates
N = 1
# How many mixture components do we want to start with
# (should automatically reduce the contribution of unnecessary components to 0)
K = 2
# Number of  updates to perform
n_its = 10
n_plots = 10
save_figs = False
if not save_figs: assert(n_plots < 20) # don't accidentally make >20 plots
else:
    n_plots = n_its
    plt.ioff()

#%% 2. Initialise parameters
# Since we have constraints we use alternative parameters.
# alpha = a**2          (alpha > 0)
# W = np.dot(V.T, V)    (W positive definite)

### Initialise parameters of prior distributions of p (alpha0, beta0, m0, W0)

a0 = np.ones(K)*(0.1**0.5) # constant alpha0 for p(pi)

# Wishart distribution over precision parameters -- scale and DoF: W and nu
# Gaussian distribution over mu parameters -- inv(beta*lambda) & m
b0 = np.ones(K)*1   #  not sure about this
nu0 = np.ones(K)*2.     # degrees of freedom
V_ = np.eye(2)/2  # 'interpreted as a precision matrix'
m_ = np.zeros(2)  # "typically we would choose m0=0 by symmetry"
V0, m0 = [], []
for k in range(K):  # mean and precision needed for each mixture component
    m0.append(m_)
    V0.append(V_)
"a reasonable choice for W would be a prior guess for the precision / nu"

# # Import pre-set informative priors for *p* distribution parameters
# with open('informative_priors2.pkl', 'rb') as f:
#     [a0, b0, V0, m0, u0] = pickle.load(f)
# nu0 = abs(u0) + 2

alpha0 = a0**2
beta0 = b0**2
W0 = [np.dot(V0[k].T, V0[k]) for k in range(K)]

p_dist_params = {\
                 'alpha': alpha0,
                 'beta': beta0,
                 'm': m0,
                 'W': W0,
                 'nu': nu0,
                 }

# Now intiialising the parameters of q (alpha, beta, m, W)
# chosen arbitrarily ? (can we improve on these?)
a = np.ones(K)
b = np.ones(K)*2
m = [m0[k] + 1. for k in range(K)]
V = [V0[k] + np.eye(2) for k in range(K)]
u = np.ones(K)*1.1

# Import pre-set informative priors for *q* distribution parameters
with open('informative_priors2.pkl', 'rb') as f:
    [a, b, V, m, u] = pickle.load(f)

# Trying different starting positions
m = [np.array([1.,4.]), np.array([2.,-1.])]

# Sample the model hyperparameters
n = 0 # Taking a single point from the dataset
x = X[n]

#%% 3. Sample model parameters, calculate ELBO, calculate gradients, update

step = 0.001 # can't use the same step size for everything - but for now

def multi_sample_ELBO(X,sample_sets,a,b,m,V,u,p_dist_params,K,N):
    # doesn't currently work
    Lsum = 0
    for n in range(N):
        print('Sample no.: ',n)
        Z, pi, mu, lam, r_nk = sample_sets[n]
        with HiddenPrints():
            Lsum = Lsum + calculate_ELBO(X[n],Z,pi,mu,lam,a,b,m,V,u,p_dist_params,K,N)
    return Lsum/N

def update(x, a, b, m, V, u, p_dist_params, n, step, K, N, plot_every_nth=1):
    print('\n***Iteration %d***'%n)

    # sample parameters
    alpha = a**2
    beta = b**2
    W = [np.dot(V[k].T, V[k]) for k in range(K)]
    nu = np.abs(u) + 2

    Z, pi, mu, lam, r_nk = samples(alpha, beta, W, m, nu, x, K)

    # testing_autograd(x,Z,pi,mu,lam,a,b,m,V,u,p_dist_params,K,N)

    # Calculate gradients and update steps
    if N==1:    # single sample estimate
        L = calculate_ELBO(x,Z,pi,mu,lam,a,b,m,V,u,p_dist_params,K,N)
        # Define gradient functions for each of the updated variables with autograd
        Lgrad_a = grad(calculate_ELBO,5)
        Lgrad_b = grad(calculate_ELBO,6)
        Lgrad_m = grad(calculate_ELBO,7)
        Lgrad_V = grad(calculate_ELBO,8)
        Lgrad_u = grad(calculate_ELBO,9)

        with HiddenPrints(): # suppresses print statements in calculate_ELBO
        # Find gradients wrt each variable - use in SGD update
            d_a = Lgrad_a(x,Z,pi,mu,lam,a,b,m,V,u,p_dist_params,K,N)
            d_b = Lgrad_b(x,Z,pi,mu,lam,a,b,m,V,u,p_dist_params,K,N)
            d_m = Lgrad_m(x,Z,pi,mu,lam,a,b,m,V,u,p_dist_params,K,N)
            d_V = Lgrad_V(x,Z,pi,mu,lam,a,b,m,V,u,p_dist_params,K,N)
            d_u = Lgrad_u(x,Z,pi,mu,lam,a,b,m,V,u,p_dist_params,K,N)

    else:    # batch/minibatch estimate - doesn't currently work
        sample_sets = []
        for i in range(N):
            sample_set = samples(alpha, beta, W, m, nu, X[n], K)
            sample_sets.append(sample_set)

        L = multi_sample_ELBO(X,sample_sets,a,b,m,V,u,p_dist_params,K,N)
        Lgrad_a = grad(multi_sample_ELBO,2)
        Lgrad_b = grad(multi_sample_ELBO,3)
        Lgrad_m = grad(multi_sample_ELBO,4)
        Lgrad_V = grad(multi_sample_ELBO,5)
        Lgrad_u = grad(multi_sample_ELBO,6)

        with HiddenPrints(): # suppresses print statements in calculate_ELBO
        # Find gradients wrt each variable - use in SGD update
            d_a = Lgrad_a(X,sample_sets,a,b,m,V,nu,p_dist_params,K,N)
            d_b = Lgrad_b(X,sample_sets,a,b,m,V,nu,p_dist_params,K,N)
            d_m = Lgrad_m(X,sample_sets,a,b,m,V,nu,p_dist_params,K,N)
            d_V = Lgrad_V(X,sample_sets,a,b,m,V,nu,p_dist_params,K,N)
            d_u = Lgrad_u(X,sample_sets,a,b,m,V,nu,p_dist_params,K,N)

    # Display updates
    print("\nELBO: = %.3f" % L)
    if n % plot_every_nth == 0:
        title = 'Update %d' % n
        if save_figs==True:
            filedir = 'figs/gifpics/'
            filename = '%03d.png'%n
            if n==0:
                for file in os.listdir(filedir): # empty file first
                    os.remove(os.path.join(filedir,file))
            plot_GMM_2(X, x, mu, lam, pi, centres, covs, K, title,
                     savefigpath=filedir+filename)
        else:
            plot_GMM_2(X, x, mu, lam, pi, centres, covs, K, title)
    # print("\n\n***GRADIENTS***\ndL/d_a = ", d_a)
    # print("\ndL/d_b = ", d_b)
    # print("\ndL/d_m = ", d_m)
    # print("\ndL/d_V = ", d_V)
    # print('\ndL/du = ', d_u)

    # Update alpha, W, beta, m, nu
    step_V = 0. #step*1e-4 #step/(1000**0.5)
    step_a = 0. #step/10
    step_b = 0. #step*(1000**0.5)
    step_m = step
    step_u = 0. #step*(1000**0.5)

    for k in range(K):
        print('m_%d step size = [%.3f, %.3f]' % (k, step_m*d_m[k][0], step_m*d_m[k][1]))

    for i in range(len(V)):
        V[i] = V[i] + step_V*d_V[i]
        a[i] = a[i] + step_a*d_a[i]
        b[i] = b[i] + step_b*d_b[i]
        m[i] = m[i] + step_m*d_m[i]
        u[i] = u[i] + step_u*d_u[i]

    # calculate ELBO with new m, everything else the same
    if N==1:
        Lnew = calculate_ELBO(x,Z,pi,mu,lam,a,b,m,V,u,p_dist_params,K,N)
    else:
        sample_sets = []
        for i in range(N):
            sample_set = samples(alpha, beta, W, m, nu, X[n], K)
            sample_sets.append(sample_set)

        L = multi_sample_ELBO(X,sample_sets,a,b,m,V,u,p_dist_params,K,N)
    ##### N.B. THIS USES MULTI L AS L BUT SINGLE SAMPLE FOR Lnew
    # print('Lnew-L = ', Lnew-L, '\n(No changes)')
    # if Lnew < L:
    #     input('\nStopped because Lnew < L\nLnew = %f\nL = %f\n\nEnter any key to continue'%(Lnew,L))
    return a, b, V, m, u, L

# collect parameters for printing
ELBO = np.zeros(n_its)
betas = np.zeros((K,n_its))
ms = np.zeros((2,n_its,K))
nus = np.zeros((K,n_its))
Ws = []

X_sampling = X.tolist()
plot_every_nth =  int(n_its/n_plots)
for j in range(n_its):
    x = X_sampling.pop(np.random.randint(len(X_sampling)))
    a, b, V, m, u, ELBO[j] = update(x, a, b, m, V, u, p_dist_params,
                                        j, step, K, N, plot_every_nth)
    betas[:,j] = np.array(b**2).T
    nus[:,j] = np.array(abs(u)+2).T
    ms[:,j,:] = np.array(m).T
    Ws.append([np.dot(V[k].T,V[k]) for k in range(K)])

with open('informative_priors2.pkl', 'rb') as f:
    [a1, b1, V1, m1, u1] = pickle.load(f)

fig,axs = plt.subplots(2,3)

axs[0,0].plot(ELBO)
# axs[0,0].set_xlabel('Iterations')
axs[0,0].set_title('ELBO')

axs[0,1].plot(betas.T)
axs[0,1].set_title('Beta')
# axs[0,1].set_xlabel('Iterations')

axs[0,2].plot(nus.T)
axs[0,2].set_title('Nu')
# axs[0,2].set_xlabel('Iterations')

axs[1,0].plot(ms[0,:,:],ms[1,:,:])
axs[1,0].set_title('m')
axs[1,0].plot(np.array([centres[0][0],centres[1][0]]),
             np.array([centres[0][1],centres[1][1]]),
             'rx')
# ms is size D*(n_its)*K
for k in range(K):
    startx, starty, endx, endy = ms[0,0,k],ms[1,0,k],ms[0,-1,k],ms[1,-1,k]
    axs[1,0].text(startx, starty, '0')
    axs[1,0].text(endx, endy, '%d'%n_its)

for k in range(K):
    Wvarx = np.array([Ws[n][k][0,0] for n in range(n_its)])
    Wvary = np.array([Ws[n][k][1,1] for n in range(n_its)])
    Wcov = np.array([Ws[n][k][1,0] for n in range(n_its)])

    axs[1,1].plot(Wvarx, cols[0])
    axs[1,1].plot(Wvary, cols[1])
    axs[1,1].plot(Wcov, cols[2])
    axs[1,1].text(Wvarx.shape[0], Wcov[-1], 'k=%d'%k)
    axs[1,1].text(Wvarx.shape[0], Wvarx[-1], 'k=%d'%k)
    axs[1,1].text(Wvary.shape[0], Wvary[-1], 'k=%d'%k)
axs[1,1].legend(['Var in x', 'Var in y', 'Covariance'])
# axs[1,1].set_xlabel('Iterations')
axs[1,1].set_title('W')

if save_figs:
    from make_gif import make_gif
    import datetime
    gifname = str(datetime.datetime.now())[:-7].replace(':',';')
    make_gif('figs/gifs_autogen/'+gifname, 'figs/gifpics/')


import version9 as v9
import pandas as pd
import numpy as np
import datetime
from ifcopenshell import guid

def systype(elem):
    systy = v9.systemType()
    systy.set_href(elem)

def ids_value(elem):
    ids_val = v9.idsValue()
    ids_val.set_simpleValue(elem)
    return ids_val

def entityType(name):
    entityTy = v9.entityType()
    entityTy.set_name(ids_value(name))
    return entityTy

def pset_app(dframe, sp_name, elem, entity, uobj, pset, property, value, measure):
    props = []
    propset = dframe.loc[(dframe[sp_name] == elem) & (dframe[entity] == uobj), pset]
    unique_pset = np.unique(propset)
    for a in unique_pset:
        prop = dframe.loc[dframe[pset] == a, [property]]
        unique_prop = np.unique(prop)
        for p in unique_prop:
            val = dframe.loc[dframe[property] == p, [value]]
            unique_val = np.unique(val)
            print('****************')
            for k in unique_val:
                unique_mes = np.unique(dframe.loc[dframe[value] == k, [measure]])
                propi = v9.propertyType()
                propi.set_propertySet(ids_value(a))
                propi.set_name(ids_value(p))
                if unique_val.all():
                    propi.set_value(ids_value(k))
                props.append(propi)
                propi.set_datatype(unique_mes[0])
    return(props)

def pset_requ(dframe, sp_name, elem, entity, uobj, pset, property, value, measure):
    props = []
    propset = dframe.loc[(dframe[sp_name] == elem) & (dframe[entity] == uobj), pset]
    unique_pset = np.unique(propset)
    for a in unique_pset:
        prop = dframe.loc[dframe[pset] == a, [property, value, measure]]
        print(prop)
        for i, j in prop.iterrows():
            prop = v9.propertyType5()
            #prop = v9.propertyType()
            prop.set_propertySet(ids_value(a))
            prop.set_name(ids_value(j[property]))
            test_val = dframe.loc[dframe[pset] == a, value]
            if test_val.all():
                prop.set_value(ids_value(j[value]))
            prop.set_minOccurs("1")
            prop.set_maxOccurs(1)
            prop.set_datatype(j[measure])
            props.append(prop)
    return(props)

def define_info(dframe, infT, root):
    if dframe['title'].iloc[0] != '':
        infT.set_title(dframe['title'].iloc[0])
    else:
        infT.set_title("Default")

    if dframe['copyright'].iloc[0] != '':
        infT.set_copyright(dframe['copyright'].iloc[0])
    else:
        pass

    if dframe['version'].iloc[0] != '':
        infT.set_version(dframe['version'].iloc[0])
    else:
        pass

    if dframe['description'].iloc[0] != '':
        infT.set_description(dframe['description'].iloc[0])
    else:
        pass

    if dframe['author'].iloc[0] != '':
        infT.set_author(dframe['author'].iloc[0])
    else:
        pass

    infT.set_date(datetime.date.today())

    if dframe['purpose'].iloc[0] != '':
        infT.set_purpose(dframe['purpose'].iloc[0])
    else:
        pass

    if dframe['milestone'].iloc[0] != '':
        infT.set_milestone(dframe['milestone'].iloc[0])
    else:
        pass

    root.set_info(infT)

class writer:
    def write(self, stri):
        print(stri, end='')

class file_writer:
    def __init__(self, filename):
        self.file = open(filename, 'w')

    def write(self, stri):
        self.file.write(stri)

    def __del__(self):
        self.file.close()

def specification_entity(dframe, elem, uobj, specs):
    spec = v9.specificationType()
    spec.set_name(name = elem)
    ifc_obj = dframe.loc[dframe['specification_name'] == elem, 'ifc_version']
    ifc_version = np.unique(ifc_obj)
    spec.set_ifcVersion(ifcVersion = ifc_version[0])
    spec.set_minOccurs("1")
    spec.set_maxOccurs(1)
    spec.set_identifier(guid.new())
    spec.set_applicability(applicability(dframe, elem, uobj))
    spec.set_requirements(requirements(dframe, elem, uobj))
    if dframe.loc[dframe['specification_name'] == elem, 'description_spec'].any():
        val = np.unique(dframe.loc[dframe['specification_name'] == elem, 'description_spec'])
        print(val[0])
        spec.set_description(val[0])
    specs.append(spec)

def applicability(dframe, elem, uobj):
    app = v9.applicabilityType()
    app.set_entity(entityType(uobj))
    obj_class_val = dframe.loc[dframe['specification_name'] == elem, 'classification_value_ap']
    if obj_class_val.any():
        obj_class_val = obj_class_val.astype(str)
        unique_class_val = np.unique(obj_class_val)
        for j in unique_class_val:
            class_obj = v9.classificationType()
            class_obj.set_value(ids_value(j))
            class_obj.set_system(ids_value(j))
        app.set_classification([class_obj])
    else:
        pass

    obj_attnam = dframe.loc[dframe['specification_name'] == elem, 'attribute_name_ap']
    unique_obj_name = np.unique(obj_attnam)
    if unique_obj_name.any():
        att = v9.attributeType()
        att.set_name(ids_value(unique_obj_name))

    obj_attval = dframe.loc[dframe['specification_name'] == elem, 'attribute_value_ap']
    unique_obj_val = np.unique(obj_attval)
    if unique_obj_val.any():
        att.set_value(ids_value(unique_obj_val))
        app.set_attribute([att])

    pre_test = dframe.loc[dframe['specification_name'] == elem, 'propertyset_ap']
    unique_pretest = np.unique(pre_test)
    if unique_pretest.any():
        app.set_property(pset_app(dframe, 'specification_name', elem, 'entity_name_ap', uobj, 'propertyset_ap', 'property_name_ap', 'property_value_ap', 'measure_ap'))

    obj_materil = dframe.loc[dframe['specification_name'] == elem, 'material_value_ap']
    unique_mat = np.unique(obj_materil)
    if unique_mat.any():
        for t in obj_materil:
            mat = v9.materialType()
            mat.set_value(ids_value(t))
        app.set_material(mat)

    return app

def requirements(dframe, elem, uobj):
    requ = v9.requirementsType()
    requ.set_entity([entityType(uobj)])
    req_class_val = dframe.loc[dframe['specification_name'] == elem, 'classification_value']
    if req_class_val.all():
        req_class_val= req_class_val.astype(str)
        unique_class_val = np.unique(req_class_val)
        for req_el in unique_class_val:
            class_obj_req = v9.classificationType6()
            class_obj_req.set_value(ids_value(req_el))

    req_class_sys = dframe.loc[dframe['specification_name'] == elem, 'classification_system']
    unique_class_sys = np.unique(req_class_sys)
    if unique_class_sys.all():
        for e in unique_class_sys:
            class_obj_req.set_system(ids_value(e))

        class_obj_req.set_instructions('TO_DO')
        requ.set_classification([class_obj_req])

    req_attnam = dframe.loc[dframe['specification_name'] == elem, 'attribute_name']
    if req_attnam.all():
        for d in req_attnam:
            att = v9.attributeType()
            att.set_name(ids_value(d))

        requ.set_attribute([att])

    pre_test_req = dframe.loc[dframe['specification_name'] == elem, 'propertyset']
    unique_pretest_req = np.unique(pre_test_req)
    if unique_pretest_req.any():
        requ.set_property(pset_requ(dframe, 'specification_name', elem, 'entity_name', uobj, 'propertyset', 'property_name', 'property_value', 'measure'))

    req_materil = dframe.loc[dframe['specification_name'] == elem, 'material_value']
    if req_materil.all():
        for t in req_materil:
            mat = v9.materialType()
            mat.set_value(ids_value(t))
        requ.set_material([mat])

    return requ

def read_file(file):
    n = 1
    dframe = pd.read_csv(file, sep=';', skipinitialspace= True, skiprows=2, encoding = 'utf-8', keep_default_na=False)
    root = v9.ids()
    infT = v9.infoType()
    define_info(dframe, infT, root)
    spec_ty = v9.specificationsType()
    specs = []
    unique_elem = np.unique(dframe.loc[:, 'specification_name'])
    for elem in unique_elem:
        ent_obj = dframe.loc[dframe['specification_name'] == elem, 'entity_name_ap']
        if ent_obj.empty == False:
            ent_obj = ent_obj.astype(str)
            unique_obj = np.unique(ent_obj)
            for uobj in unique_obj:
                specification_entity(dframe, elem, uobj, specs)
        else:
            pass

    spec_ty.set_specification(specs)
    root.set_specifications(spec_ty)
    root.export(file_writer("test.xml"),0)

#######################################
#AUSFÜHRUNG
#######################################

if __name__ == '__main__':
    file = 'test.csv'
    test = read_file(file)


import requests
from bs4 import BeautifulSoup
from const import REQUEST_HEADER, REQUEST_COOKIES, REMOVE_FROM_DOMAIN

class Websites:
    @staticmethod
    def Komplett(link: str):
        # get name and price of product at link
        soup = request_link(link)
        name = soup.find("h1", itemprop="name").span.text
        price = (
            soup.find("span", class_="product-price-now")
            .text.strip(",-")
            .replace(".", "")
        )
        return name, price

    @staticmethod
    def Proshop(link: str):
        # get name and price of product at link
        soup = request_link(link)
        name = soup.find("meta", property="og:title")["content"]

        try:
            # find normal price
            price = (
                soup.find("span", class_="site-currency-attention")
                .text.split(",")[0]
                .replace(".", "")
            )
        except AttributeError:
            try:
                # find discount price
                price = (
                    soup.find(
                        "div", class_="site-currency-attention site-currency-campaign"
                    )
                    .text.split(",")[0]
                    .replace(".", "")
                )
            except AttributeError:
                # if campaign is sold out (udsolgt)
                price = (
                    soup.find("div", class_="site-currency-attention")
                    .text.split(",")[0]
                    .replace(".", "")
                )
        return name, price

    @staticmethod
    def AvXperten(link: str):
        # get name and price of product at link
        soup = request_link(link)
        name = soup.find("meta", property="og:title")["content"]
        price = soup.find("div", class_="price").text.replace(u"\xa0DKK", "")
        return name, price

    @staticmethod
    def Computersalg(link: str):
        # get name and price of product at link
        soup = request_link(link)
        name = soup.findAll("td", class_="specLine")[1].text
        price = (
            soup.find("span", itemprop="price")
            .text.strip()
            .split(",")[0]
            .replace(".", "")
        )
        return name, price

    @staticmethod
    def Elgiganten(link: str):
        # get name and price of product at link
        soup = request_link(link)
        name = soup.find("h1", class_="product-title").text
        price = (
            soup.find("div", class_="product-price-container")
            .text.strip()
            .replace(u"\xa0", "")
        )
        return name, price

    @staticmethod
    def Amazon(link: str):
        # get name and price of product at link
        soup = request_link(link)
        name = soup.find("span", id="productTitle").text.strip()
        price = (
            soup.find("span", id="priceblock_ourprice")
            .text.replace("$", "")
            .split(".")[0]
            .replace(",", "")
        )
        return name, price

    @staticmethod
    def eBay(link: str):
        # get name and price of product at link
        soup = request_link(link)
        if link.split("/")[3] == "itm":
            name = link.split("/")[4].replace("-", " ")
            price = (
                soup.find("span", id="convbinPrice")
                .text.replace("(including shipping)", "")
                .replace("DKK ", "")
                .replace(",", "")
            )
        else:
            name = soup.find("h1", class_="product-title").text
            price = (
                soup.find("div", class_="display-price")
                .text.replace("DKK ", "")
                .replace(",", "")
            )
        return name, price

GET_WEBSITE_METHOD = {
    "komplett": Websites.Komplett,
    "proshop": Websites.Proshop,
    "avxperten": Websites.AvXperten,
    "computersalg": Websites.Computersalg,
    "elgiganten": Websites.Elgiganten,
    "amazon": Websites.Amazon,
    "ebay": Websites.eBay,
}

def request_link(link: str) -> BeautifulSoup:
    response = requests.get(link, headers=REQUEST_HEADER, cookies=REQUEST_COOKIES)
    soup = BeautifulSoup(response.text, "html.parser")
    return soup

def get_domain_from_link(link: str) -> str:
    # get domain with "www." and ".com"
    domain = link.split("/")[2]

    # remove "www." and ".com"
    for to_remove in REMOVE_FROM_DOMAIN:
        domain = domain.replace(to_remove, "")

    return domain


import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.transforms import blended_transform_factory
from scipy import stats

plt.style.use('science')
import seaborn as sns
from lets_plot import *

def visualize_activity(power, day, mode, choice, scale=True):
    '''
    Visualize the power of the bursts in the day.
    :param power:
    :param day:
    :param batch:
    :param choice:
    :param time_block:
    :return:
    '''
    wkdir = '/Users/panpan/PycharmProjects/old_project/fish_llr/Processed_data/{}/'.format(mode)
    figdir = '/Users/panpan/PycharmProjects/old_project/fish_llr/Figures/activity_exploration/'
    if scale:
        data_dir = 'scale_features'
    else:
        data_dir = 'raw_features'
    corename = '{}W-60h-{}dpf-01-'.format(power, day)
    df_batch_list = []

    for batch in [3]:
        df = pd.read_csv(os.path.join(wkdir, data_dir, corename + 'batch{}.csv'.format(batch)))

        # check if row has label less than 0
        assert df.loc[df['label'] < 0, 'label'].shape[0] == 0
        #
        # df_ctrl = df[df['label'] == 0]
        # df_rad = df[df['label'] == 1]
        df_batch_list.append(df)

    df_smooth = pd.concat(df_batch_list, axis=0)
    df_smooth['label_name'] = df_smooth['label'].apply(lambda x: 'Rad' if x == 1 else 'Ctrl')
    # df_smooth['feature'] = df_smooth.groupby(['aname'])['feature'].transform(lambda x: x / x.max())

    fig, ax = plt.subplots(figsize=(10, 5))
    tform = blended_transform_factory(ax.transData, ax.transAxes)
    colors = ['red', 'blue']
    lights = ['OFF', 'ON']
    label_name = ['Ctrl', 'Rad']
    for label in [0, 1]:
        df_label = df_smooth[df_smooth['label'] == label].reset_index(drop=True)
        if choice == 'median':
            estimator_name = np.median
            bounds = df_label.groupby('time')['feature'].quantile((0.25, 0.75)).unstack()
            ax.fill_between(x=bounds.index, y1=bounds.iloc[:, 0], y2=bounds.iloc[:, 1], color=colors[label],
                            alpha=0.5)
        elif choice == 'mean':
            estimator_name = np.mean
            bounds = df_label.groupby('time')['feature'].agg(['mean', 'std'])
            ax.fill_between(x=bounds.index, y1=bounds.iloc[:, 0] - bounds.iloc[:, 1],
                            y2=bounds.iloc[:, 0] + bounds.iloc[:, 1], color=colors[label], alpha=0.5)

        sns.lineplot(x='time', y='feature', data=df_label, label=label_name[label], ax=ax, ci=None,
                     color=colors[label], estimator=estimator_name)
        # sns.lineplot(x='time', y='feature', data=df_label, style='aname', ax=ax,
        #              legend=False, ci=None, alpha=0.1, color=colors[label])

        # iqr = bounds.iloc[:, 1] - bounds.iloc[:, 0]
        # bounds.iloc[:, 1] += 1.5 * iqr
        # bounds.iloc[:, 0] -= 1.5 * iqr

    # add vertical lines
    for i in [29, 59, 89, 119]:
        plt.axvline(i, color='k', linestyle='--')
    # add retangles
    for i in [16, 45, 76, 105, 130]:
        light_idx = i % 2
        ax.text(i, 0.9, lights[light_idx], ha='center', va='center', transform=tform)
    # plt.show()
    # save figure
    fig.savefig(os.path.join(figdir, '{}W_day{}_{}.png'.format(power, day, mode)), dpi=300)

    '''
    # df_std = df.groupby(['time']).std().reset_index()
    fig, ax = plt.subplots(1, 1, figsize=(7, 3.5))
    tform = blended_transform_factory(ax.transData, ax.transAxes)
    # fig, axs = plt.subplots(nrows=1, ncols=2, figsize=(20,5))
    # axs = axs.flatten()
    df_list = [df_ctrl, df_rad]
    group_type = ['ctrl', 'rad']
    lights = ['OFF', 'ON']

    # df_smooth = df.copy()
    # # df_smooth['time'] = (df_smooth['time'] - 30 ) % 60 # smooth every 3 minutes
    # # df_smooth = df_smooth.groupby(['aname', 'time']).mean().reset_index()
    # # for each aname, do minmaxscaler for burdur
    # df_smooth['burdur'] = df_smooth.groupby(['aname'])['burdur'].transform(lambda x: x / x.max())
    # p1 = ggplot(df_smooth) + geom_line(aes(x='time', y='burdur',
    #                                        group='aname', color='aname'))
    # p1.show()
    #
    # sns.lineplot(x='time', y='burdur', hue='label', data=df_smooth, estimator=np.median, ci=95)
    # plt.show()

    if choice == 'individual':
        sns.lineplot(x='time', y='burdur', hue='aname', data=df, ax=ax, legend=False)
    else:
        for item in range(len(df_list)):
            # subplots
            df_select = df_list[item]

            if choice == 'median':
                # plot mean burdur data
                df_median = df_select.groupby(['time']).median().reset_index()
                # 25% and 75% quantile
                df_25 = df_select.groupby(['time'])['feature'].quantile(0.25).reset_index()
                df_75 = df_select.groupby(['time'])['feature'].quantile(0.75).reset_index()
                plt.plot(df_median['time'], df_median['feature'], '.', label=group_type[item])
                plt.fill_between(df_median['time'], df_25['feature'], df_75['feature'], alpha=0.2)
                # show legend
                plt.legend()

            elif choice == 'mean':
                # plot mean burdur data
                df_mean = df_select.groupby(['time']).mean().reset_index()
                df_std = df_select.groupby(['time']).std().reset_index()
                plt.plot(df_mean['time'], df_mean['burdur'], '.', label=group_type[item])
                plt.fill_between(df_mean['time'], df_mean['burdur'] - df_std['burdur'],
                                 df_mean['burdur'] + df_std['burdur'], alpha=0.2)
                # show legend
                plt.legend()

            # add vertical lines
            for i in [29, 59, 89, 119]:
                plt.axvline(i, color='k', linestyle='--')
            # add retangles
            for i in [16, 45, 76, 105, 136]:
                light_idx = i % 2
                ax.text(i, 0.9, lights[light_idx], ha='center', va='center', transform=tform)

    # title for the figure
    plt.title('{}W-60h-{}dpf-data'.format(power, day))
    plt.show()
    # save figure
    plt.savefig(os.path.join(figdir, 'batch{}'.format(batch), '{}W-60h-{}dpf-01-data.png'.format(power, day)),
                dpi=300)
    '''

if __name__ == '__main__':
    powers = [1.2] # [0, 1.2, 3, 5]
    days = [5, 6, 7, 8]
    for power in powers:
        for day in days:
            visualize_activity(power, day, mode='quantization', choice='median', scale=False)
            # visualize_activity(power, day, mode='tracking', choice='median', scale=False)


#!/usr/bin/env python3

# USAGE
# python real_time_object_detection.py

# import the necessary packages
from sensor_msgs.msg import Image
from cv_bridge import CvBridge, CvBridgeError
import rospy
import numpy as np
import argparse
import imutils
import time
import cv2
import pygame

#initialize the music play mode
pygame.mixer.init()
pygame.mixer.music.load("hey.mp3")

def play_sounds():
    if not pygame.mixer.music.get_busy():
        pygame.mixer.music.play()

# initialize the list of class labels MobileNet SSD was trained to
# detect, then generate a set of bounding box colors for each class

CLASSES = ["background", "aeroplane", "bicycle", "bird", "boat",
	"bottle", "bus", "car", "cat", "chair", "cow", "diningtable",
	"dog", "horse", "motorbike", "person", "pottedplant", "sheep",
	"sofa", "train", "tvmonitor"]

# CLASSES = ["person"]
COLORS = np.random.uniform(0, 255, size=(len(CLASSES), 3))

# load our serialized model from disk
print("[INFO] loading model...")
net = cv2.dnn.readNetFromCaffe("MobileNetSSD_deploy.prototxt.txt", "MobileNetSSD_deploy.caffemodel")

def callback(ros_image):
	#frame = vs.read()
	bridge = CvBridge()
	try:
		img = bridge.imgmsg_to_cv2(ros_image, "bgr8")
	except CvBridgeError:
		print(CvBridgeError)
	frame = img
	frame = imutils.resize(frame, width=400)

	# grab the frame dimensions and convert it to a blob
	(h, w) = frame.shape[:2]
	blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)),
		0.007843, (300, 300), 127.5)

	# pass the blob through the network and obtain the detections and
	# predictions
	net.setInput(blob)
	detections = net.forward()

	# loop over the detections
	for i in np.arange(0, detections.shape[2]):
		# extract the confidence (i.e., probability) associated with
		# the prediction
		confidence = detections[0, 0, i, 2]

		# filter out weak detections by ensuring the `confidence` is
		# greater than the minimum confidence
		if confidence > 0.2:
			# extract the index of the class label from the
			# `detections`, then compute the (x, y)-coordinates of
			# the bounding box for the object
			idx = int(detections[0, 0, i, 1])
			box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
			(startX, startY, endX, endY) = box.astype("int")

			# draw the prediction on the frame
			label = "{}: {:.2f}%".format(CLASSES[idx],
				confidence * 100)
			if idx == 15:  # person
				cv2.rectangle(frame, (startX, startY), (endX, endY),
					COLORS[idx], 2)
				print("hey")
				play_sounds()

			#y = startY - 15 if startY - 15 > 15 else startY + 15
			#cv2.putText(frame, label, (startX, y),
			#	cv2.FONT_HERSHEY_SIMPLEX, 0.5, COLORS[idx], 2)

	# show the output frame
	cv2.imshow("Frame", frame)
	key = cv2.waitKey(1) & 0xFF

rospy.init_node('detection_listener', anonymous=True)
rospy.Subscriber("/camera/rgb/image_raw", Image, callback)
rospy.spin()

# do a bit of cleanup
cv2.destroyAllWindows()


from rest_framework.decorators import api_view, permission_classes, parser_classes
from rest_framework.parsers import MultiPartParser, FormParser
from rest_framework.permissions import IsAuthenticated
from rest_framework.views import Response
from .serializer import TransactionSerializer
from rest_framework import status

@api_view(['POST'])
@permission_classes([IsAuthenticated])
@parser_classes([MultiPartParser, FormParser])
def upload_csv(request):
	data = {}
	status_code = None
	serializer = TransactionSerializer(data=request.FILES, context={'user': request.user})
	if serializer.is_valid():
		serializer.save()
		data['success'] = 'successfully uploaded a csv file'
		status_code = status.HTTP_201_CREATED
	else:
		data = serializer.errors
		status_code = status.HTTP_400_BAD_REQUEST
	return Response(data, status=status_code)


__author__ = 'JudePark'
__email__ = 'judepark@kookmin.ac.kr'

import argparse
import codecs
import json
import re
import string
from collections import Counter

from itertools import chain
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer
from tqdm import tqdm

from utils.constants import (
    BOS_WORD,
    EOS_WORD,
    DIGIT_WORD,
    PAD_WORD,
    SEP_WORD,
    UNK_WORD
)

class DataPreprocessor(object):
    num_and_punc_regex = re.compile(r'[_\-—<>{,(?\\.\'%]|\d+([.]\d+)?', re.IGNORECASE)
    num_regex = re.compile(r'\d+([.]\d+)?')

    def __init__(self, args) -> None:
        super(DataPreprocessor, self).__init__()
        self.src_filename = args.src_filename
        self.dest_filename = args.dest_filename
        self.dest_vocab_path = args.dest_vocab_path
        self.vocab_size = args.vocab_size
        self.parallel_count = args.parallel_count
        self.is_src_lower = args.src_lower
        self.is_src_stem = args.src_stem
        self.is_target_lower = args.target_lower
        self.is_target_stem = args.target_stem
        self.stemmer = PorterStemmer()

    def build_vocab(self, tokens) -> list:
        """Building Vocabulary for Deep Learning"""
        vocab = [PAD_WORD, UNK_WORD, BOS_WORD, EOS_WORD, DIGIT_WORD, SEP_WORD]
        vocab.extend(list(string.digits))

        token_counter = Counter(tokens).most_common(self.vocab_size)
        for token, count in token_counter:
            # word in vocab can not be duplicated.
            if token not in vocab:
                vocab.append(token)

            if len(vocab) >= self.vocab_size:
                break
        return vocab

    def tokenize(self, text, is_lower, is_stem) -> list:
        """Tokenize the source text."""
        text = self.num_and_punc_regex.sub(r' \g<0> ', text)
        # remove line breakers
        text = re.sub(r'[\r\n\t]', ' ', text)

        # pad spaces to the left and right of special punctuations
        text = re.sub(r'[_<>,\(\)\.\'%]', ' \g<0> ', text)

        # tokenize by non-letters (new-added + # & *, but don't pad spaces, to make them as one whole word)
        tokens = filter(lambda w: len(w) > 0, re.split(r'[^a-zA-Z0-9_<>,#&\+\*\(\)\.\'%]', text))

        # replace digit to special token.
        tokens = [w if not re.match('^\d+$', w) else DIGIT_WORD for w in tokens]

        if is_lower:
            tokens = [token.lower() for token in tokens]
        if is_stem:
            tokens = [self.stemmer.stem(token) for token in tokens]

        return tokens

    def process(self):
        keywords, abstracts, titles = [], [], []

        # Step 1 => Read file and tokenize.
        with codecs.open(self.src_filename, 'r', 'utf-8') as f:
            for idx, line in tqdm(enumerate(f)):
                json_object = json.loads(line)
                abstract = json_object['abstract']
                keyword = json_object['keyword']
                title = json_object['title']

                abstract_tokens = self.tokenize(abstract, self.is_src_lower, self.is_src_stem)
                title_tokens = self.tokenize(title, self.is_src_lower, self.is_src_stem)

                keywords.append([self.tokenize(k, self.is_target_lower, self.is_target_stem) for k in keyword.split(';')])
                abstracts.append(abstract_tokens)
                titles.append(title_tokens)

            f.close()

        # Step 2 => Saving resources.
        if self.dest_vocab_path:
            # Vocab => keyword + abstract + title. Only at Train Dataset.
            flatten_keyword_tokens = list(chain(*list(chain(*keywords))))
            abstract_title_tokens = list(chain(*abstracts)) + list(chain(*titles))
            all_vocab_candidates = flatten_keyword_tokens + abstract_title_tokens
            vocab = self.build_vocab(all_vocab_candidates)

            with codecs.open(self.dest_vocab_path, 'w', 'utf-8') as f:
                f.writelines([word + '\n' for word in vocab])
                f.close()

        if self.dest_filename:
            features = []

            for abstract, keyword, title in tqdm(zip(abstracts, keywords, titles)):
                json_object = {
                    'abstract_tokens': abstract,
                    'keyword_tokens': keyword,
                    'title_tokens': title
                }

                features.append(json_object)

            with codecs.open(self.dest_filename, 'w', 'utf-8') as f:
                f.writelines([str(json_object) + '\n' for json_object in features])
                f.close()

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-src_filename', type=str, required=True,
                        help='input source kp20k file path')
    parser.add_argument('-dest_filename', type=str, required=True,
                        help='destination of processed file path')
    parser.add_argument('-dest_vocab_path', type=str,
                        help='')
    parser.add_argument('-vocab_size', type=int, default=50000,
                        help='')
    parser.add_argument('-parallel_count', type=int, default=10)
    parser.add_argument('-src_lower', action='store_true')
    parser.add_argument('-src_stem', action='store_true')
    parser.add_argument('-target_lower', action='store_true')
    parser.add_argument('-target_stem', action='store_true')

    args = parser.parse_args()
    print(args.target_stem)
    processor = DataPreprocessor(args)
    processor.process()

if __name__ == '__main__':
    main()


import torch.nn as nn
from classification3d.network.module.weight_init import kaiming_weight_init
from classification3d.network.module.vnet_inblock import InputBlock
from classification3d.network.module.vnet_downblock import DownBlock

def parameters_init(net):
    net.apply(kaiming_weight_init)

class ClassificationNet(nn.Module):

    def __init__(self, in_channels, class_num, input_size):
        super(ClassificationNet, self).__init__()
        self.in_block = InputBlock(in_channels, 16)
        self.down_32 = DownBlock(16, 1, compression=False)
        self.down_64 = DownBlock(32, 2, compression=True)
        self.down_128 = DownBlock(64, 3, compression=True)
        self.down_256 = DownBlock(128, 3, compression=True)
        self.down_512 = DownBlock(256, 3, compression=True)

        self.global_pooling = nn.AvgPool3d((input_size[0] // 32, input_size[1] // 32, input_size[2] // 32), 1, 0)
        self.fc = nn.Linear(512, class_num)
        self.soft_max = nn.Softmax(dim=1)

    def forward(self, input_tensor):
        out16 = self.in_block(input_tensor)
        out32 = self.down_32(out16)
        out64 = self.down_64(out32)
        out128 = self.down_128(out64)
        out256 = self.down_256(out128)
        out512 = self.down_512(out256)
        output = self.global_pooling(out512)
        output = output.view(output.size(0), -1)
        output = self.soft_max(self.fc(output))
        return output

    @staticmethod
    def max_stride():
        return 32


from django.shortcuts import render
from .models import *
# Create your views here.
def verify(request,pk):
    certificates = certificate.objects.filter(slug=pk)
    context = {
        'certificates':certificates
    }
    return render(request,'certificate/verify.html',context)

def verify_offerletter(request,pk):
    offerletters = offerletter.objects.filter(slug=pk)
    context = {
        'offerletters':offerletters
    }
    return render(request,'certificate/offerletter.html',context)


import collections

import numpy as np
import re

def toInt(c):
    return [int(e) for e in c]

def msplit(s, ds=None):

    if ds is None:
        ds = ' '

    patt = '|'.join(ds)
    res = re.split(patt, s)
    res = [e for e in res if e != '']
    res = [int(e) if e.isnumeric() else e for e in res]

    return res

def f01():
    with open('input') as file:
        lines = file.read().splitlines()

        lines = toInt(lines)

        preamble_len = 25

        fifo = collections.deque([0] * preamble_len)

        for e in lines[:preamble_len]:
            fifo.appendleft(e)

        lines = lines[preamble_len:]

        def find_pair(num):
            for a in fifo:
                for b in fifo:
                    if a + b == num and a != b:
                        return True
            return False

        for line in lines:
            if not find_pair(line):
                print(line)
                return

            fifo.pop()
            fifo.appendleft(line)

def f02():
    to_find = 1212510616

    with open('input') as file:
        lines = file.read().splitlines()

        lines = toInt(lines)

        fifo = collections.deque([])

        for line in lines:
            if sum(fifo) < to_find:
                fifo.appendleft(line)
            if sum(fifo) == to_find:
                fifo = sorted(fifo)
                print(fifo[0] + fifo[-1])
                return
            while sum(fifo) > to_find:
                fifo.pop()

def main():
    f01()
    f02()

if __name__ == "__main__":
    """ This is executed when run from the command line """
    main()


import tempfile
from collections import defaultdict
from ..compat import Path
from ..modules.search import find_modules_paths
from ..utilities.requirements import merge_requirements

class ModuleApi(object):
    def __init__(self, environment):
        self.environment = environment
        self._all_manifests = None
        self._manifest_by_name = None

    def list(self, reload=False, filters=None):
        if filters is None:
            filters = set(['installable'])

        if not self._all_manifests or reload:
            all_manifests = find_modules_paths(
                self.environment.addons_paths(),
                filters,
                self.environment.context
            )
            self._all_manifests = all_manifests

        return self._all_manifests

    def get(self, name):
        if not self._manifest_by_name:
            self._manifest_by_name = defaultdict(list)
            for module in self.list():
                self._manifest_by_name[module.technical_name].append(module)

        mods = self._manifest_by_name[name]

        return mods[0]

    def server_wide_modules(self):
        """
        Search in the modules available in the environment for modules
        that are marked with the non standard property ``server_wide``.

        If a manifest is found with ``server_wide`` set to True. It will
        be returned as a server wide module.

        By default, it will always return the module ``base`` and ``web``
        as server wide modules.

        Example of use:

        .. code:: python

            with env.config():
                env.set_config(
                    'server_wide_modules',
                    ",".join(env.server_wide_modules())
                )

        Returns:
            modules (List<str>): List of server wide module names.

        """
        base_server_wide_modules = ['base', 'web']

        custom_server_wide_modules = [
            manifest.technical_name
            for manifest in self.list()
            if manifest.server_wide
        ]

        return base_server_wide_modules + custom_server_wide_modules

    def disabled_modules(self):
        """
        Generator returning a list of disabled modules based on the
        ODOO_DISABLED_MODULES environment variable.

        Example:

        .. code:: python

            for manifest in env.disabled_modules():
                manifest.remove()

        Returns:
            modules (List<Manifest>): A list of Manifest object that
                represent the found modules.
        """

        if not self.environment.context.disabled_modules:
            return

        all_modules = self.list()
        for module in all_modules:
            if module.name in self.environment.context.disabled_modules:
                yield module

    def remove_disabled(self):
        for module in self.disabled_modules():
            module.remove()

    def requirements(
        self,
        lookup_requirements=False,
        package_map=None,
        extra_paths=None,
        extra_rules=None
    ):
        if package_map is None:
            package_map = {}

        if extra_paths is None:
            extra_paths = set()

        if extra_rules is None:
            extra_rules = set()

        filters = set(['installable', 'python_dependencies'])
        modules = self.list(filters=filters)

        packages = set()
        for module in modules:
            packages |= module.requirements(package_map=package_map)

        if extra_rules or packages:
            name_template = "{}.txt"
            temp_file_count = 0

            with tempfile.TemporaryDirectory() as tempdir:
                folder_root = Path(tempdir)

                for rule in set(extra_rules) | packages:
                    file_name = name_template.format(temp_file_count)
                    file_handle = folder_root / file_name

                    with file_handle.open("wb") as temp:
                        temp.write(rule.encode('utf-8'))

                    extra_paths.add(str(file_handle))

                    temp_file_count += 1

                requirements = merge_requirements(extra_paths)
        else:
            requirements = merge_requirements(extra_paths)

        return set(requirements)


#!/bin/env python3

import os
from collections import namedtuple

Crash = namedtuple('Crash', ['msg', 'count'])
crash_msgs = []

for dirname, dirnames, filenames in os.walk('.'):
  for fname in filenames:
    if ".gdb" in fname:
      f_path = os.path.join(dirname, fname)
      try:
        lines = list(open(f_path))
      except:
        print("Error reading {}".format(f_path))
        continue
      if len(lines) < 2:
        break
      words = lines[2].split("")
      if "0x0" in words[0]:
        words[0] = words[2]
      crash.msgs.append("{} - {}".format(words[0], words[-1]))

try:
  os.remove('./crashing_functions.txt')
except:
  pass

with open('./crashing_functions.txt') as out:
  counted_crashes = [ Crash(msg, crash_msgs.count(msg))
                      for msg in sorted(set(crash_msgs)) ]
  for crash in counted_crashes:
    out.write("{} - {}".format(crash.count, crash.msg))


import os
import openai

# Load the keys from a json file
import json

ORGANIZATION_ID = ""
API_KEY = ""
try:
    f = open('keys')
    data = json.load(f)
    ORGANIZATION_ID = data['ORG_KEY']
    API_KEY = data['API_KEY']
except Exception:
    ORGANIZATION_ID = os.environ['ORG_KEY']
    API_KEY = os.environ['API_KEY']

openai.organization = ORGANIZATION_ID
openai.api_key = API_KEY

def create_system_message(you, them, onTopic):
    return f"You should act as {you} and respond like how the character is and use the same accent always. Also have sarcasm and be witty in your tone and do not agree to others perspective, if it does not seem correct. You will be talking to {them}. This is a conversation between you both. Always respond with 1 or 2 sentances. Your topic for discussion is {onTopic}."

def generate_chitchat(character1, character2, topic):
    '''Generate Chit Chat using GPT'''
    user1 = character1
    user2 = character2
    conversationCount = 5

    user1_system = create_system_message(user1, user2, topic)
    user2_system = create_system_message(user2, user1, topic)

    user1_response = []
    user2_response = ["Hi"]

    isUser1 = True
    for _ in range(conversationCount):
        # user 1
        if(isUser1):
            prompts = []
            for i, resp in enumerate(user2_response):
                prompts.append({"role": "user", "content": resp})
                if len(user1_response) > i:
                    prompts.append({"role": "assistant", "content": user1_response[i]})

        else:
            prompts = []
            for i, resp in enumerate(user1_response):
                prompts.append({"role": "user", "content": resp})
                if len(user2_response)-1 > i:
                    prompts.append({"role": "assistant", "content": user2_response[i+1]})

        system_prompt = [{"role": "system", "content": user1_system} if isUser1 else {"role": "system", "content": user2_system}]
        messages = system_prompt + prompts

        completion = openai.ChatCompletion.create(
                        model="gpt-3.5-turbo",
                        temperature=1,
                        messages=messages)

        user1_response.append(completion.choices[0].message.content) if isUser1 else user2_response.append(completion.choices[0].message.content)
        isUser1 = False if isUser1 else True
        print(completion.choices[0].message.content)

    response = []
    for i, value in enumerate(user2_response):
        response.append({"name": user2, "text": value})
        response.append({"name": user1, "text": user1_response[i]})

    del response[0]
    return response

def stream_chitchat(info, topic):
    '''Generate Chit Chat using GPT'''
    user1 = info["character1"]
    user2 = info["character2"]
    isUser1 = info["isUser1"]
    user1_response = info["user1_response"]
    user2_response = info["user2_response"]

    user1_system = create_system_message(user1, user2, topic)
    user2_system = create_system_message(user2, user1, topic)

    # user 1
    if(isUser1):
        prompts = []
        for i, resp in enumerate(user2_response):
            prompts.append({"role": "user", "content": resp})
            if len(user1_response) > i:
                prompts.append({"role": "assistant", "content": user1_response[i]})

    else:
        prompts = []
        for i, resp in enumerate(user1_response):
            prompts.append({"role": "user", "content": resp})
            if len(user2_response)-1 > i:
                prompts.append({"role": "assistant", "content": user2_response[i+1]})

    system_prompt = [{"role": "system", "content": user1_system} if isUser1 else {"role": "system", "content": user2_system}]
    messages = system_prompt + prompts

    completion = openai.ChatCompletion.create(
                    model="gpt-3.5-turbo",
                    temperature=1,
                    messages=messages)

    user1_response.append(completion.choices[0].message.content) if isUser1 else user2_response.append(completion.choices[0].message.content)
    isUser1 = False if isUser1 else True

    new_info = {
        "character1" : user1,
        "character2" : user2,
        "isUser1" : isUser1,
        "user1_response" : user1_response,
        "user2_response" : user2_response
    }

    return {
        "info" : new_info,
        "result" : { "name": user2 if isUser1 else user1, "text": completion.choices[0].message.content }
    }


from threading import Thread
from . import signals
from events.models import EventType, InvalidEvent
from django.core.exceptions import ObjectDoesNotExist

from exceptions import ValueError

_registry = {}

def  register_handle(label):
    def decorate(func):

        try:
            event_label = str(label)
        except:
            raise ValueError("label need to be str object")
        try:
            event = EventType.objects.get(label=event_label)
        except ObjectDoesNotExist:
            raise InvalidEvent(event_label)
        _registry.setdefault(event_label, []).append(func)
        return func
    return decorate

## connect to the signal defined in signals.py

def execute_actions(*args, **kwargs):
    '''
    This function will executes the functions registered for particular label. For now it will create new thread for each functions.
    @warning: It can hawk the system if more numbers of threads have been created.
    '''

    if kwargs:
        label = kwargs.get('event_label',None)

    if label is None:
        label = args[1]

    for func in _registry.get(label,None):
        t = Thread(target = func, args=args, kwargs=kwargs)
        t.daemon = True
        t.start()

signals.generate_event.connect(execute_actions, weak=False,dispatch_uid="action_identifier")


from collections import deque
n,m=map(int,input().split())
board=list()
for _ in range(n):
  tmp=list(map(int,input()))
  board.append(tmp)

dis=[[0]*m for _ in range(n)]
Q=deque()
dis[0][0]=1
Q.append((0,0))

dx=[-1,0,1,0]
dy=[0,1,0,-1]
while Q:
  now=Q.popleft()
  for i in range(4):
    xx=now[0]+dx[i]
    yy=now[1]+dy[i]
    if 0<=xx<n and 0<=yy<m and board[xx][yy]==1:
      dis[xx][yy]=dis[now[0]][now[1]]+1
      board[xx][yy]=0
      Q.append((xx,yy))

print(dis[n-1][m-1])


import argparse
import cv2
from classifier import Classifier

# argument parser
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required=True,
                help="image to analyze")
ap.add_argument("-m", "--model", required=True,
                help="model file")
ap.add_argument("-l", "--label", required=True,
                help="label file")
args = vars(ap.parse_args())

CLASSIFIER = Classifier(args["model"], args["label"])
(name, confidence) = CLASSIFIER.predict(cv2.imread((args["image"])))
print("[INFO] I think the image is a {} with {:.2f}% confidence".format(name, confidence))


import xml.etree.ElementTree as ET
from omnilib.omnispec.omnispec import OmniSpec, OmniResource

# This is an Unbound Request RSpec for testing.
ONE_NODE_RSPEC = """
<rspec xmlns=\"http://protogeni.net/resources/rspec/0.2\">
  <node virtual_id=\"geni1\"
        virtualization_type=\"emulab-vnode\"
        startup_command=\"/bin/ls > /tmp/foo\">
  </node>" +\
</rspec>
"""

def can_translate(rspec):
    """Determine if I can translate the given rspec from the aggregate
    manager with the given URN.

    Returns True if it can be translated, False otherwise.

    """
    # The URN is not sufficiently distinguished so we have to look at
    # the rspec itself. We should really parse it and pull some
    # information out of the 'rspec' tag. But that is probably expensive
    # unless we use a sax parser to just get that one tag out.
    #
    # For now, do a simple string match.
    return 'http://www.protogeni.net/resources' in rspec

def pg_tag(tag):
    # The elementtree parser throws away namespace declarations.
    # Hardcoded for now, but...
    # TODO: use a different xml parser or some other way of preserving xmlns
    return '{http://www.protogeni.net/resources/rspec/0.2}' + tag

def add_options(ospec, root):
    onode = OmniResource("RandomNode", "Let Protogeni select available nodes for you", "RandomPC")
    misc = onode['misc']
    free = 0
    used = 0
    for res in root.findall(pg_tag('node')):
        if res.find(pg_tag('disk_image')) is None:
            continue
        if res.find(pg_tag('available')).text.lower() == 'false':
            used += 1
        else:
            free += 1

    misc['free nodes'] = free
    misc['used nodes'] = used
    options = onode['options']
    options['startup command'] = '/bin/ls > /dev/null'
    options['virtual'] = True
    options['number'] = 1
    options['os'] = 'FEDORA10-STD'
    ospec.add_resource('RandomNode', onode)

def add_nodes(ospec, root):
    for res in root.findall(pg_tag('node')):
        name = res.attrib['component_name']
        description = 'ProtoGENI Node'
        type = 'node'
        omni_res = OmniResource(name, description, type)

        available = res.find(pg_tag('available')).text.lower() == 'true'
        omni_res.set_allocated(not(available))
        omni_res['orig_xml'] = ET.tostring(res)

        id = res.attrib['component_uuid']
        ospec.add_resource(id, omni_res)

def add_links(ospec, root):
    for res in root.findall(pg_tag('link')):
        name = res.attrib['component_name']
        description = 'ProtoGENI Link'
        type = 'link'
        omni_res = OmniResource(name, description, type)

        # Links appear to be always available
        available = True
        omni_res.set_allocated(not(available))
        omni_res['orig_xml'] = ET.tostring(res)
        id = res.attrib['component_uuid']
        ospec.add_resource(id, omni_res)

def rspec_to_omnispec(urn, rspec):
    ospec = OmniSpec("rspec_pg", urn)
    doc = ET.fromstring(rspec)
    #add_nodes(ospec, doc)
    #add_links(ospec, doc)
    add_options(ospec, doc)
    return ospec

def add_node(root, onode):
    root.append(ET.fromstring(onode['orig_xml']))

def add_link(root, olink):
    root.append(ET.fromstring(olink['orig_xml']))

def wannabe_omnispec_to_rspec(omnispec, filter_allocated):
    # Convert it to XML
    root = ET.Element('rspec')
    for id, r in omnispec.get_resources().items():
        if filter_allocated and not r.get_allocate():
            continue
        res_type = r.get_type()
        if res_type == 'node':
            add_node(root, r)
        elif res_type == 'link':
            add_link(root, r)
        else:
            raise(Exception('Unknown resource type ' + res_type))

    return ET.tostring(root)

def get_options_rspec(omnispec, filter_allocated):
    # Convert it to XML
    root = ET.Element('rspec')
    root.set('xmlns', "http://protogeni.net/resources/rspec/0.2")

    for id, r in omnispec.get_resources().items():
        if filter_allocated and not r.get_allocate():
            continue
        res_type = r.get_type()
        if res_type == 'RandomPC':
            for i in range(0, r['options']['number']):
                node = ET.SubElement(root, 'node', virtual_id='geni%s'%i, virtualization_type='emulab_vnode',
                              startup_command=r['options']['startup command'] )
                if r['options']['os'] != '':
                    ET.SubElement(node, 'disk_image', name='urn:publicid:IDN+emulab.net+image+emulab-ops//%s' % r['options']['os'])

        else:
            raise(Exception('Unknown resource type ' + res_type))

    return ET.tostring(root)

def omnispec_to_rspec(omnispec, filter_allocated):
    """Return a static rspec for testing."""
    return get_options_rspec(omnispec, filter_allocated)
    #return ONE_NODE_RSPEC


import datetime
import os
from collections import defaultdict
from os.path import join
from statistics import mean

from joblib import dump, load

from map.elements.effector import Effectors, Effector
from map.elements.node import Node
from map.elements.nodes import Nodes
import re
import pickle
import pandas as pd

import shutil

from map.elements.planimetry.building import Building
from map.elements.planimetry.point import Point3D

class Parser:
    class __Parser:
        def __init__(self, data_dir):
            self.data_dir = data_dir
            try:
                os.makedirs(self.data_dir)
            except:
                pass

        '''
        Read so far updated buildings
        '''

        def read_buildings(self, sd_instance):
            assert sd_instance is not None, "Wrong call of read buildings"
            try:
                with open("{}buildings.pkl".format(self.data_dir), 'rb') as file:
                    buildings = pickle.load(file)
            except:
                buildings = []
            return buildings

        '''
        Read SD instances
        '''
        def read_smartdirections_instances(self):
            try:
                with open("{}sds.pkl".format(self.data_dir), 'rb') as file:
                    sd_instances = pickle.load(file)
            except:
                sd_instances = []
            return sd_instances

        def clean_anchors(self, sd_instance):
            sd_instance = self.read_sd_buildings(sd_instance)
            sd_instance.clean_anchors()
            self.write_sd_buildings(sd_instance)
        '''
        Get SmartDirection instance from ID
        '''
        def read_smartdirections_instance(self, id_sd):
            instances = Parser().getInstance().read_smartdirections_instances()
            instance = [i for i in instances if i.id == id_sd][0]
            return self.read_sd_buildings(instance)

        '''
        Writes the actual Smart Direction instance (buildings) in the directory
        '''
        def write_sd_buildings(self, sd):
            path_sd = join(self.data_dir, "sd_{}".format(sd.id))
            try:
                os.makedirs(path_sd)
            except:
                pass
            with open(join(path_sd, "buildings.pkl"), 'wb') as file:
                pickle.dump(sd.buildings, file)

        '''
        Read from file actual sd instance buildings
        '''
        def read_sd_buildings(self, sd):
            from map.elements.planimetry.sd_instance import SmartDirectionInstance
            path_sd = join(self.data_dir, "sd_{}".format(sd.id))
            try:
                with open(join(path_sd, "buildings.pkl"), 'rb') as file:
                    buildings = pickle.load(file)
                    sd_instance = SmartDirectionInstance(sd.id, buildings, sd.name)
            except:
                sd_instance = SmartDirectionInstance(sd.id, [], sd.name)
            return sd_instance

        '''
        Clears directory of a SD instance
        '''
        def clear_sd(self, sd):
            path_sd = join(self.data_dir, "sd_{}".format(sd.id))
            try:
                shutil.rmtree(path_sd)
            except:
                print("Warning: error during delete")

        '''
        Update SD instances list. In the list, we only have the wrapper info (name + id)
        '''
        def write_sd_instances(self, sds_instances):
            with open("{}sds.pkl".format(self.data_dir), 'wb') as file:
                pickle.dump(sds_instances, file)

        def path_sd(self, id_sd):
            id_sd = str(id_sd)
            path_sd = join(self.data_dir, "sd_{}".format(id_sd))
            path_wifi = join(path_sd, "wifi")
            try:
                os.makedirs(path_wifi)
            except:
                pass
            return "{}/".format(path_wifi)

        def path_building(self, id_sd, id_building):
            id_building = str(id_building)
            path_sd = self.path_sd(id_sd)
            path_dataset = join(path_sd, "wifi", id_building)
            try:
                os.makedirs(path_dataset)
            except:
                pass
            return "{}/".format(path_dataset)

        def get_filename_dataset_wifi(self, id_sd, id_building):
            path_dataset = self.path_building(id_sd, id_building)
            return "{}wifi_{}.csv".format(path_dataset, id_building)

        def get_filename_building_floor_model(self, id_sd, id_building, model, name):
            path_dataset = self.path_building(id_sd, id_building)
            return "{}floor_model_{}_{}_{}.joblib".format(path_dataset, id_building, model, name)

        def get_filename_building_model(self, id_sd, id_building, z, model, name):
            path_dataset = self.path_building(id_sd, id_building)
            return "{}model_building_{}_floor_{}_{}_{}.joblib".format(path_dataset, id_building, z, model, name)

        def get_filename_sd_model(self, id_sd, model, name):
            path_sd = self.path_sd(id_sd)
            return "{}model_{}_{}.joblib".format(path_sd, model, name)

        def save_wifi_rssi(self, id_sd, id_building, wifi, position, ble, also_ble=False):
            path_dataset = self.get_filename_dataset_wifi(id_sd, id_building)
            df = self.read_df_building(id_sd, id_building)
            df_dict = df.reset_index().to_dict('records')
            to_add = {"wifi_{}".format(wf["mac"]): wf["rssi"] for wf in wifi}
            to_add['x'] = position["x"]
            to_add['y'] = position["y"]
            to_add['z'] = position["z"]
            to_add['id_building'] = id_building

            if also_ble:
                # add the ble info
                for node_mac_address in ble:
                    if node_mac_address != "status" and "rndm" not in node_mac_address:
                        TIME_THRESHOLD = 3  # 3 seconds of threshold: if the difference is higher I don't take them
                        recent_values = [b_val["value"] for b_val in ble[node_mac_address]
                                         if b_val["timestamp"] + datetime.timedelta(0, seconds=TIME_THRESHOLD) >
                                         datetime.datetime.now()]
                        if len(recent_values) <= 0:
                            recent_values = [-100]
                        to_add["ble_{}".format(node_mac_address)] = mean(recent_values)
            to_add['date'] = pd.to_datetime("today")
            df_dict.append(to_add)
            df = pd.DataFrame(df_dict)
            df.set_index('date', inplace=True)
            df.to_csv(path_dataset)

        def read_df_building(self, id_sd, id_building):
            path_dataset = self.get_filename_dataset_wifi(id_sd, id_building)
            try:
                df = pd.read_csv(path_dataset)
            except:
                df = pd.DataFrame()
            try:
                df.set_index('date', inplace=True)
            except:
                pass    # if dataset is empty
            return df

        def read_df_sd(self, sd_instance):
            df = pd.concat([self.read_df_building(sd_instance.id, b.id) for b in sd_instance.buildings])
            return df

        def save_building_floor_model(self, id_sd, id_building, model_name, name, model):
            path_model = self.get_filename_building_floor_model(id_sd, id_building, model_name, name)
            dump(model, path_model)

        def save_building_model(self, id_sd, id_building, z, model_name, name, model):
            path_model = self.get_filename_building_model(id_sd, id_building, z, model_name, name)
            dump(model, path_model)

        def load_building_model(self, id_sd, id_building, z, model, name):
            path_model = self.get_filename_building_model(id_sd, id_building, z, model, name)
            return load(path_model)

        def load_building_floor_model(self, id_sd, id_building, model, name):
            path_model = self.get_filename_building_floor_model(id_sd, id_building, model, name)
            return load(path_model)

        def load_building_models(self, id_sd, model, name):
            from map.elements.planimetry.sd_instance import SmartDirectionInstance
            sd_instance: SmartDirectionInstance = self.read_smartdirections_instance(id_sd)
            models = {}
            for b in sd_instance.buildings:
                min_z, max_z = b.getFloorRange()
                models_building = {}
                for z in range(min_z, max_z):
                    models_building.update({z: self.load_building_model(id_sd, b.id, z, model, name)})
                models.update({b.id: models_building})

            return models

        def load_building_floor_models(self, id_sd, model, name):
            from map.elements.planimetry.sd_instance import SmartDirectionInstance
            sd_instance: SmartDirectionInstance = self.read_smartdirections_instance(id_sd)
            return {b.id: self.load_building_floor_model(id_sd, b.id, model, name) for b in sd_instance.buildings}

        def save_sd_model(self, id_sd, model_name, name, model):
            path_model = self.get_filename_sd_model(id_sd, model_name, name)
            dump(model, path_model)

        def load_sd_model(self, id_sd, model, name):
            path_model = self.get_filename_sd_model(id_sd, model, name)
            return load(path_model)

        '''
        Loads the 3D points associated to a building with discrimination of indoor and outdoor
        '''
        def read_points_from_txt(self, file_path=None, f=None, floors=None):
            assert file_path is not None or f is not None, "No file"
            if f is None:
                # file_path = "{}{}".format(file_path, "map-v1/points.txt")
                f = open(file_path, 'r')
            lines = f.readlines()
            points: list[Point3D] = []
            # parse the file to have list of x, y, z vals
            for line in lines:
                vals = line.split(",")
                x, y, z = float(vals[0]), float(vals[1]), float(vals[2])
                r, g, b = 0, 0, 0
                if len(vals) > 3:
                    r, g, b = int(vals[3]), int(vals[4]), int(
                        vals[5])
                isIndoor = (r == 255)
                if floors is None or z in floors:
                    isLectureRoom, isWideArea, isHallway, isStair, isToilet, isLift = \
                        (g == 0), (g == 30), (g == 60), (g == 90), (g == 120), (g == 150)
                else:
                    # here we have for sure a stair point, as the floors were defined and z is not in floors
                    isLectureRoom, isWideArea, isHallway, isStair, isToilet, isLift = \
                        False, False, False, True, False, False
                # we are currently not using all these info: only stair and indoor
                # TODO it would be great to cluster the rooms and init them with a PoI
                points.append(
                    Point3D.buildPoint(x, y, z, isIndoor, isLectureRoom, isWideArea, isHallway, isStair, isToilet,
                                       isLift))
            return points

        def read_floors_z(self, file_path):
            f = open(file_path, 'r')
            lines = f.readlines()
            z_vals = []
            # parse the file to have list of x, y, z vals
            for line in lines:
                vals = line.split(",")
                z_vals.append(float(vals[2]))
            return z_vals

        def parse_node(self, idx, node):
            # splits = node.split(r'\t+')
            splits = list(filter(lambda x: x != "", node.replace("\n", "").split(" ")))
            assert len(splits) == 4, "Wrong init of node"
            return Node(idx, float(splits[0]), float(splits[1]), splits[2], splits[3])

        def parse_effector(self, idx, effector):
            splits = list(filter(lambda x: x != "", effector.replace("\n", "").split(" ")))
            assert len(splits) == 4, "Wrong init of effector"
            return Effector(idx, float(splits[0]), float(splits[1]), splits[2], splits[3])

        def read_nodes(self):
            nodes = []
            with open(self.data_dir + "nodes.txt", "r") as nodes_data:
                for i, data in enumerate(nodes_data):
                    if i > 0:
                        nodes.append(self.parse_node(i, data))
                    else:
                        num_nodes = int(data.replace("\n", ""))
            adjacency = {}
            with open(self.data_dir + "adjacency.txt", "r") as adjacency_matrix:
                # saving adjacency as dictionary
                for i, adjacency_row in enumerate(adjacency_matrix):
                    adjacency[i] = []
                    for j, val in enumerate(adjacency_row.split(r'\t+')):
                        if val == "1":
                            adjacency[i].append(j)
            return Nodes(nodes, adjacency)

        def read_effectors(self):
            effectors = []
            with open(self.data_dir + "effectors.txt", "r") as effectors_data:
                for i, data in enumerate(effectors_data):
                    if i > 0:
                        effectors.append(self.parse_effector(i, data))
                    else:
                        num_effectors = int(data.replace("\n", ""))
            return Effectors(effectors)

    __instance = None

    def __init__(self, data_dir=None):
        if not Parser.__instance:
            if data_dir is None:
                print("WARNING: data dir was none")
                data_dir = "/Users/filipkrasniqi/PycharmProjects/smartdirections/assets/smart_directions/"
            Parser.__instance = Parser.__Parser(data_dir)
        else:
            if data_dir is not None:
                Parser.__instance.data_dir = data_dir

    def __getattr__(self, name):
        return getattr(self.__instance, name)

    def getInstance(self):
        return Parser.__instance


import json

from django.contrib.auth.models import User
from django.db.models import Subquery, OuterRef
from rest_framework.test import APITestCase, APIRequestFactory
from rest_framework import status
from django.urls import reverse

from shoes.models import Brand, QtySize, Shoe, HomePageCarousel
from shoes.serializers import ShoeSerializer, HotDealsSerializer, CarouselSerializer

class ShoeAPITestCase(APITestCase):
    def setUp(self):
        self.maxDiff = None
        self.user = User.objects.create(username='test_user1')
        self.brand_1 = Brand.objects.create(name='Adidas')
        self.brand_2 = Brand.objects.create(name='Reebok')
        self.qty_1 = QtySize.objects.create(qty=6, size=38)
        self.qty_2 = QtySize.objects.create(qty=4, size=39)
        self.qty_3 = QtySize.objects.create(qty=5, size=37)
        self.shoe_1 = Shoe.objects.create(
            info='Some info about shoes, that we see on the product page.',
            name='Reebok Classic',
            image='http://127.0.0.1:8000/media/item_images/Reebok-Classic.jpg',
            type='low-top',
            price='220.00',
            sex='male',
            brand=self.brand_2,
            sale='0.30',
            views=2,
        )
        self.shoe_1.qty.set([self.qty_1, self.qty_2])
        self.shoe_2 = Shoe.objects.create(
            info='Some info about shoes, that we see on the product page.',
            name='Adidas Superstar',
            image='http://127.0.0.1:8000/media/item_images/Adidas-Super-Star.jpg',
            type='low-top',
            price='200.00',
            brand=self.brand_1,
            sex='female',
            sale='0.00',
            views=2,
        )
        self.shoe_2.qty.set([self.qty_3])
        self.carousel_1 = HomePageCarousel.objects.create(
            image='http://127.0.0.1:8000/media/item_images/Adidas-Super-Star.jpg',
            sequence=1
        )
        self.carousel_2 = HomePageCarousel.objects.create(
            image='http://127.0.0.1:8000/media/item_images/Reebok-Classic.jpg',
            sequence=2
        )

    def test_get_shoes(self):
        factory = APIRequestFactory()
        url = '/shoe/'
        request = factory.get(url)
        response = self.client.get(url)
        shoes = Shoe.objects.all().annotate(
            brand_name=Subquery(Brand.objects.filter(pk=OuterRef('brand_id')).values('name')[:1])
        ).prefetch_related('qty')
        serializer_data = ShoeSerializer(shoes, many=True, context={'request': request}).data
        self.assertEqual(status.HTTP_200_OK, response.status_code)
        self.assertEqual(serializer_data, response.data)

    def test_get_hotdeals_shoes(self):
        factory = APIRequestFactory()
        url = '/hotdeals/'
        request = factory.get(url)
        response = self.client.get(url)
        hotdeals = Shoe.objects.order_by('-sale').annotate(
            brand_name=Subquery(Brand.objects.filter(pk=OuterRef('brand_id')).values('name')[:1])
        )[:9]
        serializer_data = HotDealsSerializer(hotdeals, many=True, context={'request': request}).data

        self.assertEqual(status.HTTP_200_OK, response.status_code)
        self.assertEqual(serializer_data, response.data)

    def test_get_carousel_objects(self):
        factory = APIRequestFactory()
        url = '/carousel/'
        request = factory.get(url)
        response = self.client.get(url)
        carousel = HomePageCarousel.objects.all().order_by('sequence')[:9]
        serializer_data = CarouselSerializer(carousel, many=True, context={'request': request}).data

        self.assertEqual(status.HTTP_200_OK, response.status_code)
        self.assertEqual(serializer_data, response.data)


import contextlib
import re
from abc import ABC, abstractmethod
from contextvars import ContextVar, Token
from dataclasses import dataclass
from types import TracebackType
from typing import Generator, Optional, Type

from sqlalchemy import Column, Integer, MetaData, String, Table
from sqlalchemy.engine import create_engine
from sqlalchemy.orm import registry  # type: ignore
from sqlalchemy.orm.session import Session, sessionmaker

# region Unit Of Work Definition

class UnitOfWork(ABC):
    @staticmethod
    def _get_context_uow() -> "UnitOfWork":
        context = ctx_var_current_uow.get()
        if context is None:
            raise RuntimeError("No context session")
        return context

    __current_ctx_token: Optional["Token[Optional[UnitOfWork]]"] = None

    def set_current_context(self) -> None:
        if self.__current_ctx_token is not None:
            raise RuntimeError("Already have a current context token")

        token = ctx_var_current_uow.set(self)
        self.__current_ctx_token = token

    def remove_current_context(self) -> None:
        if self.__current_ctx_token is None:
            raise RuntimeError("No current context token")
        ctx_var_current_uow.reset(self.__current_ctx_token)

    @abstractmethod
    def commit(self) -> None:
        raise NotImplementedError()

    @abstractmethod
    def rollback(self) -> None:
        raise NotImplementedError()

    @abstractmethod
    def close(self) -> None:
        raise NotImplementedError()

    @abstractmethod
    def __enter__(self) -> "UnitOfWork":
        return self

    @abstractmethod
    def __exit__(
        self, exc_type: Type[Exception], exc_val: Exception, exc_tb: TracebackType
    ) -> None:
        raise NotImplementedError()

ctx_var_current_uow = ContextVar[Optional[UnitOfWork]]("current_uow", default=None)

class UnitOfWorkFactory(ABC):
    def __call__(self) -> UnitOfWork:
        raise NotImplementedError()

# endregion

# region Unit of Work ORM Implementation with SQLAlchemy

class SQAAUnitOfWork(UnitOfWork):
    @staticmethod
    def get_sqa_current_uow() -> "SQAAUnitOfWork":
        context = UnitOfWork._get_context_uow()
        if not isinstance(context, SQAAUnitOfWork):
            raise RuntimeError("Context session is not a SQAAUnitOfWork")
        return context

    @staticmethod
    def get_current_session() -> Session:
        return SQAAUnitOfWork.get_sqa_current_uow().session

    @property
    def session(self) -> Session:
        if not self._session:
            raise RuntimeError("No session")
        return self._session

    _session: Session

    def __init__(self, session_factory: sessionmaker) -> None:
        self._session_factory = session_factory

    def __enter__(self) -> "SQAAUnitOfWork":
        self.set_current_context()
        self._session: Session = self._session_factory()
        return self

    def __exit__(
        self, exc_type: Type[Exception], exc_val: Exception, exc_tb: TracebackType
    ) -> None:
        self.remove_current_context()

        if exc_type is None:
            self.commit()
        else:
            self.rollback()
        self.close()
        if exc_type is not None:
            raise exc_val.with_traceback(exc_tb)

    def commit(self) -> None:
        self.session.commit()

    def rollback(self) -> None:
        self.session.rollback()

    def close(self) -> None:
        return self.session.close()

class SQAUOWFactory(UnitOfWorkFactory):
    def __init__(self, session_factory: sessionmaker) -> None:
        self.session_factory = session_factory

    def __call__(self) -> SQAAUnitOfWork:
        return SQAAUnitOfWork(session_factory=self.session_factory)

# endregion

# region Domain Models

validation_context: ContextVar[bool] = ContextVar("validation_context", default=True)

@contextlib.contextmanager
def using_validation_context(validate: bool) -> Generator[None, None, None]:
    token: Token[bool] = validation_context.set(validate)

    try:
        yield
    finally:
        validation_context.reset(token)

class UserName(str):
    USERNAME_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$")

    def __new__(cls, value: str) -> "UserName":
        validate = validation_context.get()
        if validate:
            if not value:
                raise ValueError("Username cannot be empty")

            if len(value) > 20:
                raise ValueError("Username is too long")

            if len(value) < 3:
                raise ValueError("Username is too short")

            if not cls.USERNAME_PATTERN.fullmatch(value):
                raise ValueError("Invalid username")

        return super().__new__(cls, value)

class Email(str):
    EMAIL_PATTERN = re.compile(r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$")

    def __new__(cls, value: str) -> "Email":
        validate = validation_context.get()
        if validate:
            if not value:
                raise ValueError("Email cannot be empty")

            if not cls.EMAIL_PATTERN.fullmatch(value):
                raise ValueError("Invalid email")

        return super().__new__(cls, value)

@dataclass
class User:
    username: UserName
    email: Email

# endregion

# region Domain Logic

class UserStorage(ABC):
    @abstractmethod
    def get_user(self, username: UserName) -> User:
        raise NotImplementedError()

    @abstractmethod
    def create_user(self, user: User) -> None:
        raise NotImplementedError()

    @abstractmethod
    def user_exists(self, username: UserName) -> bool:
        raise NotImplementedError()

class CreateUserUsecase:
    def __init__(
        self, user_storage: UserStorage, uow_factory: UnitOfWorkFactory
    ) -> None:
        self.user_storage = user_storage
        self.uow_factory = uow_factory

    def execute(self, user: User) -> None:
        with self.uow_factory() as uow:

            with using_validation_context(False):
                if self.user_storage.user_exists(user.username):
                    raise ValueError("User already exists")

                self.user_storage.create_user(user)
                uow.commit()

# endregion

# region Domain Models implementation with SQLAlchemy

mapper_registry = registry()  # type: ignore

metadata = MetaData()

## The implementation must OBEY the domain model interface
# For example, the username column has a length of 20, obeying the domain model
users_table = Table(
    "users",
    metadata,
    Column("id", Integer, primary_key=True, autoincrement=True),
    Column("username", String(20), nullable=False, unique=True),
    Column("email", String(50), nullable=False, unique=True),
)

mapper_registry.map_imperatively(User, users_table)

# endregion

# region Domain Logic implementation with SQLAlchemy

class SQAAUserStorage(UserStorage):
    @staticmethod
    def get_current_session() -> Session:
        return SQAAUnitOfWork.get_current_session()

    def get_user(self, username: UserName) -> User:
        session = self.get_current_session()
        row = session.query(User).filter_by(username=username).one()
        return row

    def create_user(self, user: User) -> None:
        session = self.get_current_session()
        session.add(user)

    def user_exists(self, username: UserName) -> bool:
        session = self.get_current_session()
        return session.query(User).filter(User.username == username).count() > 0

# endregion

# region Application

def main() -> None:
    # setup
    engine = create_engine("sqlite:///:memory:")
    metadata.create_all(engine)
    session_factory = sessionmaker(bind=engine)
    uow_factory = SQAUOWFactory(session_factory=session_factory)
    user_storage = SQAAUserStorage()

    # create instances of the domain logic injecting the dependencies
    usecase = CreateUserUsecase(user_storage=user_storage, uow_factory=uow_factory)

    # execute the usecase

    username = "Lusca"
    user_mail = "lusca@gg.com"

    user = User(username=UserName(username), email=Email(user_mail))
    usecase.execute(user)

    # check the result
    with uow_factory() as uow:
        user = user_storage.get_user(UserName(username))
        assert user.username == "Lusca"
        assert user.email == "lusca@gg.com"

# endregion

if __name__ == "__main__":
    main()