import numpy as np
from config_tva import hdfe_dir
import warnings
import sys
import pandas as pd
from scipy.optimize import minimize
from vam import VAM
sys.path += [hdfe_dir, '/Users/lizs/Documents/bfgs']
from multicollinearity import find_collinear_cols
from bfgs import bfgs_outer

class MLE(VAM):
    def __init__(self, data: pd.DataFrame, outcomes: list, teacher: str, covariates: list,
                 class_level_var: str, categorical_controls=None,
                 verbose=False, bias_correction=True):
        """

        :param data:
        :param outcomes:
        :param teacher:
        :param covariates:
        :param class_level_var:
        :param categorical_controls:
        :param verbose:
        :param bias_correction:
        """
        VAM.__init__(self, data, outcomes, teacher, covariates, class_level_var,
                     categorical_controls, verbose)
        self.get_bias_correction = bias_correction

        # Make Groupby objects
        self.n_classes = len(self.group_student_to_class.first_occurrences)
        self.n_students = len(self.y)

        self.y_tilde = self.group_student_to_class.apply(lambda vec: vec - np.mean(vec, 0), self.y, shape=self.y.shape)

        self.x_tilde = self.group_student_to_class.apply(lambda arr: arr - np.mean(arr, 0), self.x,
                                                         shape=self.x.shape, order='F')

        self.x_bar = self.group_student_to_class.apply(lambda arr: np.mean(arr, 0), self.x, broadcast=False,
                                                       shape=(self.group_student_to_class.n_keys, self.x.shape[1]))

        del self.x
        self.y_bar = self.group_student_to_class.apply(lambda arr: np.mean(arr, 0), self.y, broadcast=False,
                                                       shape=(self.n_classes, len(outcomes)))

        del self.y
        assert self.group_class_to_teacher.already_sorted

        # Make sure x bar bar is not rank deficient
        x_bar_bar = self.group_class_to_teacher.apply(lambda arr: np.mean(arr, 0), self.x_bar,
                                                      shape=(self.n_teachers, self.x_bar.shape[1]),
                                                      broadcast=False)
        x_bar_bar = np.hstack((np.ones((self.n_teachers, 1)), x_bar_bar))
        collin_x_bar_bar, not_collin_x_bar_bar = find_collinear_cols(x_bar_bar, tol=.001,
                                                                     verbose=True)
        if len(collin_x_bar_bar) > 0:
            not_collin_x_bar_bar = np.array(not_collin_x_bar_bar)
            if self.verbose:
                print('Found', len(collin_x_bar_bar), 'collinear columns in x bar bar')
            self.x_tilde = self.x_tilde[:, not_collin_x_bar_bar[1:] - 1]
            self.x_bar = self.x_bar[:, not_collin_x_bar_bar[1:] - 1]
        self.xx_tilde = self.x_tilde.T.dot(self.x_tilde)
        self.xy_tilde = self.x_tilde.T.dot(self.y_tilde)
        assert self.xy_tilde.shape[1] == len(outcomes)
        self.n_students = self.x_tilde.shape[0]
        assert self.n_students > self.n_classes
        assert self.n_classes > self.n_teachers

        self.beta = np.zeros((self.x_bar.shape[1], len(outcomes)))
        self.lambda_ = self.beta.copy()
        self.alpha = np.zeros(len(outcomes))
        self.h = np.full((self.group_student_to_class.n_keys, len(outcomes)), np.nan)
        del self.group_student_to_class
        self.h_sum = np.full((self.n_teachers, len(outcomes)), np.nan)
        self.h_sum_long = self.h.copy()
        self.x_bar_tilde = np.full((self.n_classes, self.x_bar.shape[1], len(outcomes)), np.nan)
        self.y_bar_tilde = self.h.copy()
        self.x_bar_bar = np.full((self.n_teachers, self.x_bar.shape[1], len(outcomes)), np.nan)
        self.y_bar_bar = np.full((self.n_teachers, len(outcomes)), np.nan)
        # A bunch of (n_outcomes,) shaped arrays
        self.bias_correction = np.full((len(outcomes),), np.nan)
        self.total_var = np.full((len(outcomes),), np.nan)
        self.predictable_var = np.full((len(outcomes),), np.nan)
        self.total_var = np.full((len(outcomes),), np.nan)
        self.hessian = None
        self.var_teacher_effect_variance = None

    def set_params(self, old_estimator: VAM):
        params = {'sigma_mu_squared', 'sigma_theta_squared',
                  'sigma_epsilon_squared', 'alpha', 'beta', 'lambda'}
        for k in params:
            attr = getattr(old_estimator, k)
            setattr(self, k, attr)
        return self

    def _get_ll_grad(self, sigma_mu_squared: np.ndarray = None,
                     sigma_theta_squared: np.ndarray =None,
                     sigma_epsilon_squared: np.ndarray=None, beta=None, lambda_=None,
                     alpha: np.ndarray=None, get_grad=False, variances_only=False):

        if beta is None:
            beta = self.beta
        else:
            assert beta.shape == self.beta.shape
        if lambda_ is None:
            lambda_ = self.lambda_
        else:
            assert lambda_.shape == self.lambda_.shape
        if alpha is None:
            alpha = self.alpha
        else:
            assert alpha.shape == self.alpha.shape

        change_variances = sigma_mu_squared is not None or sigma_theta_squared is not None \
            or sigma_epsilon_squared is not None
        if sigma_mu_squared is None:
            sigma_mu_squared = self.sigma_mu_squared
        assert np.all(sigma_mu_squared > 0)

        if sigma_theta_squared is None:
            sigma_theta_squared = self.sigma_theta_squared
        assert (sigma_theta_squared >= 0).all()

        if sigma_epsilon_squared is None:
            sigma_epsilon_squared = self.sigma_epsilon_squared
        assert (sigma_epsilon_squared > 0).all()

        if change_variances:
            # 41.6% of the time in this function
            h, h_sum, x_bar_tilde, y_bar_tilde, x_bar_bar, y_bar_bar, \
                _, _ = self._setup_means(sigma_theta_squared, sigma_epsilon_squared)

            assert x_bar_tilde.shape == (self.n_classes, self.x_bar.shape[1], len(self.outcomes))
        else:
            h = self.h
            h_sum = self.h_sum
            x_bar_tilde = self.x_bar_tilde
            assert x_bar_tilde.shape == (self.n_classes, self.x_bar.shape[1], len(self.outcomes))
            y_bar_tilde = self.y_bar_tilde
            x_bar_bar = self.x_bar_bar
            y_bar_bar = self.y_bar_bar

        assert isinstance(x_bar_tilde, np.ndarray)
        assert x_bar_tilde.ndim == 3
        assert isinstance(x_bar_bar, np.ndarray)

        assert isinstance(h_sum, np.ndarray)
        assert np.all(h_sum > 0)

        # Done with setup
        one_over_h_sum = 1 / h_sum
        assert one_over_h_sum.shape == h_sum.shape
        assert one_over_h_sum.shape == (self.n_teachers, len(self.outcomes))

        assert x_bar_tilde.ndim == 3
        ll = (self.n_classes - self.n_students) * np.log(sigma_epsilon_squared)
        ll += np.sum(np.log(h), 0) - np.sum(np.log(h_sum), 0)
        ll -= np.sum(np.log(sigma_mu_squared + one_over_h_sum), 0)
        if beta is self.beta:
            tilde_err_2 = self.tilde_err_2
        else:
            tilde_err = self.y_tilde - self.x_tilde.dot(beta)
            tilde_err_2 = np.sum(tilde_err**2, 0)
            del tilde_err
        ll -= tilde_err_2 / sigma_epsilon_squared

        # See if this works better stored differently
        beta = np.array(beta, order='F')
        lambda_ = np.array(lambda_, order='F')
        bar_bar_err = y_bar_bar - alpha - \
            np.einsum('i...,i...', np.swapaxes(x_bar_bar, 0, 1), (beta + lambda_)[:, None, :])

        s_mu_sq_plus_over_h = sigma_mu_squared + one_over_h_sum
        ll -= np.sum(bar_bar_err**2 / s_mu_sq_plus_over_h, 0)

        # Slowish
        within_teacher_err = y_bar_tilde - np.einsum('i...,i...', np.swapaxes(x_bar_tilde, 0, 1), beta[:, None, :])
        assert within_teacher_err.shape == h.shape
        ll -= np.sum(h * within_teacher_err**2, 0)
        assert isinstance(ll, np.ndarray)

        assert ll.shape == (len(self.outcomes),)
        ll /= -2
        assert np.all(np.isfinite(ll))
        if not get_grad:
            return ll

        n_params = 4 + 2 * len(self.beta)
        gradient = np.full((3 if variances_only else n_params, len(self.outcomes)), np.nan)
        # Gradient for log sigma mu squared
        assert s_mu_sq_plus_over_h.shape == (self.group_class_to_teacher.n_keys, len(self.outcomes))
        grad_s_mu = np.sum((bar_bar_err / s_mu_sq_plus_over_h)**2, 0) - np.sum(1 / s_mu_sq_plus_over_h, 0)
        # grad_s_mu *= -sigma_mu_squared / 2
        grad_s_mu /= -2
        assert grad_s_mu.shape == (len(self.outcomes),)

        gradient[0, :] = grad_s_mu

        # Get gradient for log sigma theta squared
        first = 1 / h - within_teacher_err**2
        assert first.shape == h.shape
        second = -one_over_h_sum + one_over_h_sum**2 / s_mu_sq_plus_over_h \
                 - (one_over_h_sum / s_mu_sq_plus_over_h)**2 * bar_bar_err**2
        assert second.shape == one_over_h_sum.shape

        d_h_d_theta = -h**2
        assert d_h_d_theta.shape == h.shape
        d_h_sum = self.group_class_to_teacher.apply(lambda arr: np.sum(arr, 0), d_h_d_theta,
                                                    broadcast=False,
                                                    shape=(self.n_teachers, len(self.outcomes)))
        assert d_h_sum.shape == (self.n_teachers, len(self.outcomes))

        grad_s_theta = np.sum(d_h_d_theta * first, 0) + np.sum(d_h_sum * second, 0)
        gradient[1, :] = grad_s_theta / -2

        # Get gradient for log sigma epsilon squared
        d_h_d_eps = -h**2 / self.n_students_per_class[:, None]
        assert d_h_d_eps.shape == h.shape
        d_e_sum = self.group_class_to_teacher.apply(lambda arr: np.sum(arr, 0), d_h_d_eps,
                                                    broadcast=False,
                                                    shape=(self.n_teachers, len(self.outcomes)))
        assert d_e_sum.shape == (self.n_teachers, len(self.outcomes))

        grad_s_eps = (self.n_classes - self.n_students) / sigma_epsilon_squared + np.sum(d_h_d_eps * first, 0)
        grad_s_eps += np.sum(d_e_sum * second, 0)
        grad_s_eps += tilde_err_2 / sigma_epsilon_squared**2

        gradient[2, :] = -grad_s_eps / 2

        if variances_only:
            return ll, gradient

        # Gradient with respect to lambda
        w = 1 / (sigma_mu_squared + one_over_h_sum)
        assert isinstance(w, np.ndarray)
        assert w.ndim == 2
        assert isinstance(bar_bar_err, np.ndarray)
        # (k x n) * (n x n_outcomes)
        grad_lambda = np.full((len(self.lambda_), len(self.outcomes)), np.nan)
        for i in range(len(self.outcomes)):
            grad_lambda[:, i] = -(w[:, i, None] * x_bar_bar[:, :, i]).T.dot(bar_bar_err[:, i])

        gradient[3 + len(self.beta):3 + 2 * len(self.beta), :] = grad_lambda

        # Gradient with respect to beta
        grad_beta = (self.xx_tilde.dot(beta) - self.xy_tilde) / sigma_epsilon_squared
        grad_beta -= np.einsum('i...,i...', x_bar_tilde, (h * within_teacher_err)[:, None, :])
        grad_beta += grad_lambda

        assert grad_beta.shape == (self.x_tilde.shape[1], len(self.outcomes))
        gradient[3:3+len(self.beta), :] = grad_beta

        # (n_teachers, n_outcomes).dot(n_teachers, n_outcomes))
        grad_alpha = -np.sum(bar_bar_err * w, 0)
        gradient[-1, :] = grad_alpha

        return ll, gradient

    def _get_ll_grad_scalar(self, outcome_num: int, sigma_mu_squared: float,
                            sigma_theta_squared: float, sigma_epsilon_squared: float,
                            get_grad=False):
        """
        Like _get_ll_grad, but variances must be provided,
        coefficients may not be provided, and only provides
        LL and gradient for one outcome. Used for optimizing variances.

        :param outcome_num:
        :param sigma_mu_squared:
        :param sigma_theta_squared:
        :param sigma_epsilon_squared:
        :param get_grad:
        :return:
        """

        assert sigma_mu_squared > 0
        assert sigma_theta_squared > 0
        assert sigma_epsilon_squared > 0

        # 41.6% of the time in this function
        h, h_sum, x_bar_tilde, y_bar_tilde, x_bar_bar, y_bar_bar \
            = self._setup_means_scalar(outcome_num, sigma_theta_squared, sigma_epsilon_squared)

        assert x_bar_tilde.shape == (self.n_classes, self.x_bar.shape[1])

        assert isinstance(h_sum, np.ndarray)
        assert np.all(h_sum > 0)

        one_over_h_sum = 1 / h_sum
        assert one_over_h_sum.shape == h_sum.shape
        assert one_over_h_sum.shape == (self.n_teachers,)

        assert x_bar_tilde.ndim == 2
        ll = (self.n_classes - self.n_students) * np.log(sigma_epsilon_squared)
        ll += np.sum(np.log(h)) - np.sum(np.log(h_sum))
        ll -= np.sum(np.log(sigma_mu_squared + one_over_h_sum))
        assert self.tilde_err_2.shape == (len(self.outcomes),)
        # Log-likelihoods are the same up to here
        ll -= self.tilde_err_2[outcome_num] / sigma_epsilon_squared

        s_mu_sq_plus_over_h = sigma_mu_squared + one_over_h_sum
        bar_bar_err = y_bar_bar - x_bar_bar.dot(self.beta[:, outcome_num] + self.lambda_[:, outcome_num]) \
            - self.alpha[outcome_num]
        assert bar_bar_err.ndim == 1

        ll -= np.sum(bar_bar_err**2 / s_mu_sq_plus_over_h)

        # Slowish
        within_teacher_err = y_bar_tilde - x_bar_tilde.dot(self.beta[:, outcome_num])
        assert within_teacher_err.shape == h.shape
        ll -= np.dot(h, within_teacher_err**2)
        assert isinstance(ll, float)

        ll /= -2
        assert np.isfinite(ll)
        if not get_grad:
            return ll

        # Gradient for log sigma mu squared
        assert s_mu_sq_plus_over_h.shape == (self.group_class_to_teacher.n_keys,)
        grad_s_mu = np.sum((bar_bar_err / s_mu_sq_plus_over_h)**2) - np.sum(1 / s_mu_sq_plus_over_h)
        assert isinstance(grad_s_mu, float)

        # Get gradient for log sigma theta squared
        first = 1 / h - within_teacher_err**2
        assert isinstance(first, np.ndarray)
        assert first.shape == h.shape
        second = -one_over_h_sum + one_over_h_sum**2 / s_mu_sq_plus_over_h \
                 - (one_over_h_sum / s_mu_sq_plus_over_h)**2 * bar_bar_err**2
        assert second.shape == one_over_h_sum.shape

        d_h_d_theta = -h**2
        assert d_h_d_theta.shape == h.shape
        d_h_sum = self.group_class_to_teacher.apply(np.sum, d_h_d_theta, broadcast=False,
                                                    shape=(self.n_teachers,))
        assert d_h_sum.shape == (self.n_teachers,)
        assert isinstance(d_h_d_theta, np.ndarray)
        assert isinstance(first, np.ndarray)

        # grad_s_theta = np.sum(d_h_d_theta * first, 0) + np.sum(d_h_sum * second, 0)
        grad_s_theta = d_h_d_theta.dot(first) + d_h_sum.dot(second)

        # Get gradient for log sigma epsilon squared
        d_h_d_eps = d_h_d_theta / self.n_students_per_class
        assert d_h_d_eps.shape == h.shape
        # Maybe this could be related to d_h_sum
        d_e_sum = self.group_class_to_teacher.apply(np.sum, d_h_d_eps, broadcast=False,
                                                    shape=(self.n_teachers,))
        assert d_e_sum.shape == (self.n_teachers,)

        grad_s_eps = (self.n_classes - self.n_students) / sigma_epsilon_squared + np.dot(d_h_d_eps, first)
        grad_s_eps += np.dot(d_e_sum, second) + self.tilde_err_2[outcome_num] / sigma_epsilon_squared**2

        return ll, np.array([grad_s_mu, grad_s_theta, grad_s_eps]) / -2

    def _update_variances(self):

        bounds = np.array([[1e-7, 2 * np.var(self.y_tilde)] for _ in range(3)], order='f')

        def get_ll_helper(params: np.ndarray, outcome_num, get_grad=False):
            return self._get_ll_grad_scalar(outcome_num, *params, get_grad=get_grad)

        for i in range(len(self.outcomes)):
            start = np.array([self.sigma_mu_squared[i], self.sigma_theta_squared[i],
                              self.sigma_epsilon_squared[i]])

            result = bfgs_outer(get_ll_helper, start.copy(), args=(i,),
                                disp=False, bounds=bounds)

            if self.verbose:
                print('\n')
                print(i)
                print('Before:',  start)
                print('After:', result['x'])
            # Sometimes BFGS doesn't work; not sure why; do something else
            if not result['success'] and np.all(np.isclose(result['x'], start)):
                result = minimize(get_ll_helper, start, args=(i,), bounds=bounds)
                if self.verbose:
                    print('BFGS didnt work')
                    print('After after: ', result['x'])

            self.sigma_mu_squared[i], self.sigma_theta_squared[i],\
                self.sigma_epsilon_squared[i] = result['x']

        return

    def _setup_means_scalar(self, outcome_num: int, sigma_theta_squared: float, sigma_epsilon_squared: float):
        """
        Returns precision-weighted means and anything needed to create them:
        :rtype: tuple
        :param sigma_theta_squared:
        :param sigma_epsilon_squared:
        :return:
        """
        h = 1 / (sigma_theta_squared + sigma_epsilon_squared / self.n_students_per_class)
        assert isinstance(h, np.ndarray)
        assert h.shape == (self.n_classes,) and np.all(h > 0)
        h_sum_long = self.group_class_to_teacher.apply(np.sum, h, shape=h.shape)
        assert isinstance(h_sum_long, np.ndarray)
        assert np.all(h_sum_long > 0)
        h_sum = h_sum_long[self.group_class_to_teacher.first_occurrences]
        assert isinstance(h_sum, np.ndarray)
        assert np.all(h_sum > 0) and h_sum.shape == (self.n_teachers,)
        precision_weights = h / h_sum_long
        assert precision_weights.shape == (self.n_classes,)

        assert self.y_bar.flags.c_contiguous
        y_bar_bar_long = \
            self.group_class_to_teacher.apply(np.sum, precision_weights * self.y_bar[:, outcome_num])

        y_bar_tilde = self.y_bar[:, outcome_num] - y_bar_bar_long

        product = precision_weights[:, None] * self.x_bar
        assert product.shape == (self.n_classes, self.x_bar.shape[1])

        x_bar_bar_long = self.group_class_to_teacher.apply(lambda x: np.sum(x, 0), product, shape=product.shape)
        x_bar_bar = x_bar_bar_long[self.group_class_to_teacher.first_occurrences, :]

        x_bar_tilde = self.x_bar - x_bar_bar_long

        assert x_bar_tilde.shape == (self.n_classes, self.x_tilde.shape[1],)
        assert x_bar_bar.shape == (self.n_teachers, self.x_tilde.shape[1],)
        y_bar_bar = y_bar_bar_long[self.group_class_to_teacher.first_occurrences]
        assert y_bar_bar.shape == (self.n_teachers,)

        assert isinstance(x_bar_tilde, np.ndarray)
        assert x_bar_tilde.shape == (self.n_classes, self.x_tilde.shape[1],)
        return h, h_sum, x_bar_tilde, y_bar_tilde, x_bar_bar, y_bar_bar

    def _setup_means(self, sigma_theta_squared: np.ndarray, sigma_epsilon_squared: np.ndarray):
        """
        Returns precision-weighted means and anything needed to create them:
        h, h_sum, etc. is used for precision weights, don't actually need to store h itself
        :rtype: tuple
        :param sigma_theta_squared:
        :param sigma_epsilon_squared:
        :return:
        """
        h = 1 / (sigma_theta_squared +
                 (1 / self.n_students_per_class)[:, None] * sigma_epsilon_squared[None, :])
        assert h.shape == (self.n_classes, len(self.outcomes))
        assert np.all(h > 0)
        h_sum_long = self.group_class_to_teacher.apply(lambda arr: np.sum(arr, 0), h, shape=h.shape)
        assert isinstance(h_sum_long, np.ndarray)
        assert np.all(h_sum_long > 0)
        h_sum = h_sum_long[self.group_class_to_teacher.first_occurrences, :]
        assert isinstance(h_sum, np.ndarray)
        assert np.all(h_sum > 0) and h_sum.shape == (self.n_teachers, len(self.outcomes))
        precision_weights = h / h_sum_long
        assert precision_weights.shape == (self.n_classes, len(self.outcomes))

        y_bar_bar_long = self.group_class_to_teacher.apply(lambda arr: np.sum(arr, 0),
                                                           precision_weights * self.y_bar, shape=self.y_bar.shape)
        assert y_bar_bar_long.shape == (self.n_classes, len(self.outcomes))
        y_bar_tilde = self.y_bar - y_bar_bar_long
        assert y_bar_tilde.shape == (self.n_classes, len(self.outcomes))

        product = precision_weights[:, None, :] * self.x_bar[:, :, None]
        assert product.shape == (self.n_classes, self.x_bar.shape[1], len(self.outcomes))

        x_bar_bar_long = self.group_class_to_teacher.apply(lambda x: np.sum(x, 0), product, shape=product.shape)
        x_bar_bar = x_bar_bar_long.T[:, :, self.group_class_to_teacher.first_occurrences].T
        x_bar_bar = np.asfortranarray(x_bar_bar)

        x_bar_tilde = self.x_bar[:, :, None] - x_bar_bar_long
        x_bar_tilde = np.asfortranarray(x_bar_tilde)
        assert x_bar_tilde.flags.f_contiguous

        assert x_bar_tilde.shape == (self.n_classes, self.x_tilde.shape[1], len(self.outcomes))
        assert x_bar_bar.shape == (self.n_teachers, self.x_tilde.shape[1], len(self.outcomes))
        y_bar_bar = y_bar_bar_long[self.group_class_to_teacher.first_occurrences, :]
        assert y_bar_bar.shape == (self.n_teachers, len(self.outcomes))

        assert isinstance(x_bar_tilde, np.ndarray)
        assert x_bar_tilde.shape == (self.n_classes, self.x_tilde.shape[1], len(self.outcomes))
        return h, h_sum, x_bar_tilde, y_bar_tilde, x_bar_bar, y_bar_bar, precision_weights,\
            h_sum_long

    def _update_coefficients(self):
        """
        # TODO: on repeated calls, shouldn't need to change h, h_sum, etc.
        Resets beta, lambda, alpha, precisions, and weighted means; keeps variances constant.
        :return:
        """
        self.h, self.h_sum, self.x_bar_tilde, self.y_bar_tilde, self.x_bar_bar, \
            self.y_bar_bar, precision_weights, self.h_sum_long \
            = self._setup_means(self.sigma_theta_squared, self.sigma_epsilon_squared)
        assert isinstance(self.x_bar_tilde, np.ndarray)

        # For beta
        sqrt_weights = 1 / np.sqrt(1 / self.h_sum + self.sigma_mu_squared)
        assert isinstance(sqrt_weights, np.ndarray)
        assert sqrt_weights.shape[1] == len(self.outcomes)
        assert np.all(np.isfinite(sqrt_weights))

        for i, h_ in enumerate(self.h[None, :, :].T):
            assert h_.shape == (self.n_classes, 1)
            x_mat = self.xx_tilde / self.sigma_epsilon_squared[i] + \
                self.x_bar_tilde[:, :, i].T.dot(self.x_bar_tilde[:, :, i] * h_)
            assert x_mat.shape == self.xx_tilde.shape
            y_mat = self.xy_tilde / self.sigma_epsilon_squared[i] + \
                self.x_bar_tilde[:, :, i].T.dot(self.y_bar_tilde[:, i, None] * h_)
            y_mat = y_mat[:, 0]

            try:
                self.beta[:, i] = np.linalg.solve(x_mat, y_mat)
            except np.linalg.LinAlgError:
                print('shape', x_mat.shape)
                print('rank', np.linalg.matrix_rank(x_mat))
                print('x shape', self.x_bar_tilde[:, :, i].shape)
                print('x rank', np.linalg.matrix_rank(self.x_bar_tilde[:, :, i]))

                assert False

            tilde_err = self.y_tilde - self.x_tilde.dot(self.beta)
            self.tilde_err_2 = np.sum(tilde_err**2, 0)

            # Now get beta + lambda
            resid = self.y_bar_bar[:, i] - self.x_bar_bar[:, :, i].dot(self.beta[:, i])
            y_w = resid * sqrt_weights[:, i]
            stacked = np.hstack((np.ones((self.n_teachers, 1)), self.x_bar_bar[:, :, i]))
            x_w = stacked * sqrt_weights[:, i, None]
            lambda_, _, rank, _ = np.linalg.lstsq(x_w, y_w)
            if rank != x_w.shape[1]:
                warnings.warn('x_w is rank deficient')
            self.alpha[i] = lambda_[0]
            self.lambda_[:, i] = lambda_[1:]

        return

    def get_hess(self, epsilon: float):
        k = len(self.beta)
        hessian = np.zeros((2 * k + 4, 2 * k + 4, len(self.outcomes)))

        def difference(upper_val, lower_val):
            """

            :param upper_val:
            :param lower_val:
            :rtype: np.ndarray
            :return:
            """
            diff = (upper_val - lower_val) / (2 * epsilon)
            assert isinstance(diff, np.ndarray)
            assert diff.shape == (hessian.shape[0], len(self.outcomes))
            return diff

        ll_up, upper = self._get_ll_grad(self.sigma_mu_squared + epsilon, get_grad=True)
        ll_lo, lower = self._get_ll_grad(self.sigma_mu_squared - epsilon, get_grad=True)
        hessian[0, :, :] = difference(upper, lower)

        assert isinstance(self.sigma_theta_squared, np.ndarray)
        ll_up, upper = self._get_ll_grad(sigma_theta_squared=self.sigma_theta_squared + epsilon,
                                         get_grad=True)
        ll_lo, lower = self._get_ll_grad(sigma_theta_squared=self.sigma_theta_squared - epsilon,
                                         get_grad=True)
        hessian[1, :, :] = difference(upper, lower)

        assert isinstance(self.sigma_epsilon_squared, np.ndarray)
        ll_up, upper = self._get_ll_grad(sigma_epsilon_squared=self.sigma_epsilon_squared + epsilon,
                                         get_grad=True)
        ll_lo, lower = self._get_ll_grad(sigma_epsilon_squared=self.sigma_epsilon_squared - epsilon,
                                         get_grad=True)
        hessian[2, :, :] = difference(upper, lower)

        eye = np.eye(len(self.beta))[:, :, None]

        # Partial with respect to beta
        for i in range(k):
            ll_up, upper = self._get_ll_grad(beta=self.beta + epsilon * eye[i],
                                             get_grad=True)
            ll_lo, lower = self._get_ll_grad(beta=self.beta - epsilon * eye[i],
                                             get_grad=True)
            hessian[2 + i, :, :] = difference(upper, lower)

        # Partial with respect to lambda, holding beta constant
        for i in range(k):
            ll_up, upper = self._get_ll_grad(lambda_=self.lambda_ + epsilon * eye[i], get_grad=True)
            ll_lo, lower = self._get_ll_grad(lambda_=self.lambda_ - epsilon * eye[i], get_grad=True)
            hessian[2 + k + i, :, :] = difference(upper, lower)

        # alpha
        ll_up, upper = self._get_ll_grad(alpha=self.alpha + epsilon, get_grad=True)
        ll_lo, lower = self._get_ll_grad(alpha=self.alpha - epsilon, get_grad=True)
        hessian[-1, :, :] = difference(upper, lower)

        for i in range(len(self.outcomes)):
            hessian[:, :, i] += hessian[:, :, i].T

        hessian /= 2
        return hessian

    def fit(self):
        if self.verbose:
            print('in fit')
        max_diff = 1
        i = 0

        while abs(max_diff) > 1e-5 and i < 10:
            beta_old = self.beta.copy()
            lambda_old = self.lambda_.copy()
            sigma_mu_squared_old = self.sigma_mu_squared
            sigma_theta_squared_old = self.sigma_theta_squared
            sigma_epsilon_squared_old = self.sigma_epsilon_squared
            self._update_coefficients()
            self._update_variances()
            max_diff = max(np.max(np.abs(self.beta - beta_old)),
                           np.max(np.abs(self.lambda_ - lambda_old)),
                           np.max(np.abs(self.sigma_mu_squared - sigma_mu_squared_old)),
                           np.max(np.abs(self.sigma_theta_squared - sigma_theta_squared_old)),
                           np.max(np.abs(self.sigma_epsilon_squared - sigma_epsilon_squared_old)))

            if self.verbose:
                print('Iteration ', i)
                print('sigma mu squared', self.sigma_mu_squared)
                print('max diff', max_diff)
            i += 1

        self._update_coefficients()

        assert isinstance(self.predictable_var, np.ndarray)
        predicted = np.sum(self.x_bar_bar * self.lambda_[None, :, :], 1)
        self.predictable_var = np.var(predicted, 0)
        if self.verbose:
            print('Number of tries', i)
            print('variances', self.sigma_mu_squared, self.sigma_theta_squared,
                  self.sigma_epsilon_squared)
            print('Predictable var', self.predictable_var)

        if self.get_bias_correction:
            self.hessian = self.get_hess(epsilon=1e-8)
            for i in range(len(self.outcomes)):
                var_lambda = np.linalg.inv(self.hessian[1:, 1:, i])
                x_bar_bar_demeaned = self.x_bar_bar[:, :, i] - np.mean(self.x_bar_bar[:, :, i], 0)
                assert x_bar_bar_demeaned.shape == (self.n_teachers, self.x_bar.shape[1])
                try:
                    bc = np.sum(x_bar_bar_demeaned.dot(np.linalg.cholesky(var_lambda))**2) \
                         / (self.n_teachers - 1)
                    self.bias_correction[i] = bc
                except np.linalg.LinAlgError:
                    warnings.warn('Asymptotic variance was not positive definite')
                    error_by_row = np.array([row.T.dot(var_lambda).dot(row)
                                             for row in x_bar_bar_demeaned])
                    self.bias_correction[i] = np.sum(np.maximum(error_by_row, 0)) / (self.n_teachers - 1)

                assert self.bias_correction[i] > 0
                del x_bar_bar_demeaned, var_lambda
        else:
            self.bias_correction = np.zeros(self.predictable_var.shape)

        self.total_var = self.sigma_mu_squared \
            + np.maximum(self.predictable_var - self.bias_correction, 0)
        return self

    def get_individual_scores(self, jackknife):
        if jackknife:
            raise NotImplementedError

        self.individual_scores = np.full((self.n_teachers, len(self.outcomes)), np.nan)
        residual = self.y_bar_bar - self.alpha
        assert residual.shape == (self.n_teachers, len(self.outcomes))
        tmp_2 = np.sum(self.x_bar_bar * self.beta[None, :, :], 1)
        predicted = np.full(residual.shape, np.nan)

        for i in range(len(self.outcomes)):
            xbb = self.x_bar_bar[:, :, i]
            tmp = xbb.dot(self.beta[:, i])
            print(np.max(np.abs(tmp - tmp_2[:, i])))
            residual[:, i] -= tmp
            predicted[:, i] = xbb.dot(self.lambda_[:, i])

        rho = self.sigma_mu_squared / (self.sigma_mu_squared + 1 / self.h_sum)
        assert rho.shape == (self.n_teachers, len(self.outcomes))
        assert predicted.shape == rho.shape
        self.individual_scores = (1 - rho) * residual + rho * predicted
        assert self.individual_scores.shape == rho.shape
        self.individual_scores = \
            pd.DataFrame(data=self.individual_scores, columns=self.outcomes,
                         index=self.class_level_teachers[self.group_class_to_teacher.first_occurrences])
        return self.individual_scores

    def get_asymp_var(self):
        # Delta method: variance of sigma mu squared + var(x bar bar * lambda)
        self.asymp_var = []
        self.var_teacher_effect_variance = []
        for i in range(len(self.outcomes)):
            predicted = self.x_bar_bar[:, :, i].dot(self.lambda_[:, i])[:, None]
            # cov(predicted, x bar bar)
            one = np.mean(predicted * self.x_bar_bar[:, :, i], 0)
            two = np.mean(predicted) * np.mean(self.x_bar_bar[:, :, i], 0)
            assert one.shape == two.shape
            g = np.concatenate(([1], 2 * (one - two)))
            asymp_var = np.linalg.inv(self.hessian[:, :, i])
            index = ['mu'] + ['lambda' + str(i) for i in range(len(self.lambda_))]
            self.asymp_var.append(pd.DataFrame(index=index, columns=index, data=asymp_var))
            self.var_teacher_effect_variance.append(g.T.dot(asymp_var).dot(g))

        return self.asymp_var


import json
from typing import Dict, List

from actstream.feeds import ModelJSONActivityFeed
from actstream.models import Action
from cursor_pagination import CursorPaginator
from django.contrib.auth import authenticate, login
from django.http import HttpRequest, HttpResponse, HttpResponseForbidden
from django.urls import reverse
from django.utils.feedgenerator import rfc3339_date
from mohawk.exc import InvalidCredentials
from rest_framework.exceptions import AuthenticationFailed

class W3CModelJSONActivityFeed(ModelJSONActivityFeed):
    """
    Returns JSON activity stream as per W3C Activity Stream 2.0 spec,
    with cursor-based pagination
    """

    raise_exception = True
    id_prefix = ["dit", "ConsentAPI"]

    def get_action_id(self, parts: List) -> str:
        """
        Returns an RFC3987 IRI ID for the given object and action
        """
        return ":".join([str(i) for i in self.id_prefix + parts])

    def format(self, action: Action) -> Dict:
        """
        Returns a formatted dictionary for the given action.
        """
        item = {
            "id": self.get_action_id(
                [
                    action.action_object.__class__.__name__,
                    action.action_object_object_id,
                    action.verb,
                ]
            ),
            "type": action.verb,
            "published": rfc3339_date(action.timestamp),
            "actor": self.format_actor(action),
            "name": str(action),
        }
        if action.description:
            item["content"] = action.description
        if action.target:
            item["target"] = self.format_target(action)
        if action.action_object:
            item["object"] = self.format_action_object(action)
        return item

    def format_item(self, action: Action, item_type="actor") -> Dict:
        """
        Returns a formatted dictionary for an individual item based on the action and item_type.
        """
        obj = getattr(action, item_type)
        obj_type = obj.__class__.__name__
        return {
            "id": self.get_action_id([obj_type, obj.pk]),
            "type": self.get_action_id([obj_type]),
            "name": str(obj),
        }

    def serialize(self, request: HttpRequest, *args, **kwargs) -> str:
        """
        Serialize activity stream to JSON, and handle pagination
        """
        items = self.items(request, *args, **kwargs)

        page_size = 10
        paginator = CursorPaginator(items, ordering=("-timestamp", "-id"))
        page = paginator.page(first=page_size, after=request.GET.get("cursor"))

        response = {
            "@context": [
                "https://www.w3.org/ns/activitystreams",
                {"dit": "https://www.trade.gov.uk/ns/activitystreams/v1"},
            ],
            "type": "Collection",
            "totalItems": items.count(),
            "orderedItems": [self.format(item) for item in page],
        }

        if page.has_next:
            url = reverse("actstream_model_feed_json", kwargs=kwargs)
            response["next"] = f"{url}?cursor={paginator.cursor(page[-1])}"

        return json.dumps(response, indent=4 if "pretty" in request.GET else None)

    def dispatch(self, request: HttpRequest, *args, **kwargs) -> HttpResponse:
        """
        Handle HTTP request
        """

        # Allow authenticated calls only
        #
        # The base class always returns a HttpResponse object with a 200 status
        # code regardless of the auth result, so we catch exceptions here and
        # return an appropriate status code instead

        try:
            user = authenticate(request)
        except (AuthenticationFailed, InvalidCredentials) as e:
            return HttpResponseForbidden(e, content_type="text/plain")
        else:
            if user is not None:
                login(request, user)

        return HttpResponse(
            self.serialize(request, *args, **kwargs), content_type="application/json"
        )


ref_genome = "/Users/holliswang/Bioinfomatics/Bioinfomatics_data/ref.genome/UCSC.genome.hg38/hg38.fa"
#positions = "/Users/holliswang/Desktop/testing.bed"

from Bio import SeqIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from collections import defaultdict

# read names and positions from bed file
positions = defaultdict(list)
with open("/Users/holliswang/Desktop/up_centroid_2000bp_1.txt") as f:
    for line in f:
        name, start, stop = line.split()
        positions[name].append((int(start), int(stop)))

# parse faste file and turn into dictionary
records = SeqIO.to_dict(SeqIO.parse(open(ref_genome), 'fasta'))

# search for short sequences
short_seq_records = []
for name in positions:
    for (start, stop) in positions[name]:
        long_seq_record = records[name]
        long_seq = long_seq_record.seq
        alphabet = long_seq.alphabet
        short_seq = str(long_seq)[start-1:stop]
        short_seq_record = SeqRecord(Seq(short_seq, alphabet), id=name, description='')
        short_seq_records.append(short_seq_record)

# write to file
with open('output.fasta', 'w') as f:
    SeqIO.write(short_seq_records, f, 'fasta')


# -*- coding: utf-8 -*-
from django.shortcuts import render, render_to_response
from django.template import loader,Context
from django.http import HttpResponse,Http404
from demo.models import TBook,TLoaninfo,TUser
from django.http import HttpResponseRedirect
import time,datetime
import random
def newBookResult(request):
    errors = []
    if 'bookname' in request.GET:
        bookname = request.GET['bookname']
    if 'author' in request.GET:
        author = request.GET['author']
    if 'press' in request.GET:
        press = request.GET['press']
    if 'qty' in request.GET:
        qty = request.GET['qty']

    if not bookname:
            errors.append('书名不能为空')
    if not author:
            errors.append('作者不能为空')
    if not press:
            errors.append('出版社不能为空')
    if not qty:
            errors.append('数量不能为空')

    if len(bookname) > 50:
            errors.append('书名不能超过50个字符！！')
    if len(author) > 50:
            errors.append('作者不能超过50个字符！！')
    if len(press) > 50:
            errors.append('出版社不能超过50个字符！！')
    if len(qty) > 50:
            errors.append('数量不能超过50个字符！！')

    if  len(errors)==0:
        errors = '新增图书成功！'
        tempBook=TBook()
        tempBook.bookid=random.randint(2,10000)
        tempBook.bookname= request.GET['bookname']
        tempBook.author= request.GET['author']
        tempBook.press= request.GET['press']
        tempBook.qty= request.GET['qty']
        tempBook.state=1
        tempBook.createdate=(datetime.datetime.now())
        tempBook.creater='admin'
        tempBook.save()
        posts = tempBook.__class__.objects.all()
        t =loader.get_template('books.html')
        c =Context({'posts':posts,'errors': errors})
        return HttpResponse(t.render(c))
    else:
        posts = TBook.objects.all()
        t =loader.get_template('newBooks.html')
        c =Context({'posts':posts,'errors': errors})
        return HttpResponse(t.render(c))


#!/usr/bin/env python3
import shutil
import os
import sys

"""
This file cleans the build directory and the Third_Party directory. Passing
the '-b' switch cleans the build directory. Passing the '-t' switch cleans the
the Third_Party directory.

Example:
    python cleanup.py -b -t
"""

# Absoulte path to this file
scripts_dir = os.path.dirname(os.path.realpath(__file__))

# Remove all build contents
def cleanBuild():
    print("--- Cleaning Build Directory ---")
    build_dir = scripts_dir + "/../build/"
    build_folders = next(os.walk(build_dir))[1]
    build_files = next(os.walk(build_dir))[2]

    for folders in build_folders:
        shutil.rmtree(build_dir + folders, ignore_errors=True)

    for files in build_files:
        os.remove(build_dir + files)

# Remove all folders in Third_Party
def cleanThirdParty():
    print("--- Cleaning Third_Party Directory ---")
    third_party_dir = scripts_dir + "/../third-party/"
    third_party_folders = next(os.walk(third_party_dir))[1]

    for folders in third_party_folders:
        shutil.rmtree(third_party_dir + folders, ignore_errors=True)

# Remove all docs folders
def cleanDocs():
    print("--- Cleaning Docs Directory ---")
    docs_dir = scripts_dir + "/../docs/"
    docs_dir_folders = next(os.walk(docs_dir))[1]

    for folder in docs_dir_folders:
        shutil.rmtree(docs_dir + folder, ignore_errors=True)

def cleanOutput():
    print("--- Cleaning Output Directory ---")
    output_dir = scripts_dir + "/../output/"
    output_dir_folders = next(os.walk(output_dir))[1]

    for folder in output_dir_folders:
        shutil.rmtree(output_dir + folder, ignore_errors=True)

# Main fucntion definition
def main():
    if "--build" in sys.argv:
        cleanBuild()

    if "--external" in sys.argv:
        cleanThirdParty()

    if "--docs" in sys.argv:
        cleanDocs()

    if "--output" in sys.argv:
        cleanOutput()

    if "--all" in sys.argv:
        cleanBuild()
        cleanDocs()
        cleanOutput()
        # cleanThirdParty()

if __name__ == "__main__":
    main()


from urllib.parse import urljoin

from .core import *
from .vparsers import *
from .utils import attributeerror_wrapper

class FloorParser(ValueParser):

    def parse(self, text):
        text = text.strip().upper()
        if "PARTER" in text:
            return 0
        return {
            "I": 1, "II": 2, "III": 3, "IV": 4, "V": 5, "VI": 6,
            "VII": 7, "VIII": 8, "IX": 9, "X": 10
        }.get(text)

class OsiedleLaurowaParser(SingleWebpageParser):
    url = "http://www.laurowa.pl/"
    method = "GET"

    schema = [
        DataUnit(pos=0, label="Budynek", parser=DOMTextExtractor(), id="building"),
        DataUnit(pos=1, label="Numer", parser=DOMTextExtractor(), id="number"),
        DataUnit(pos=2, label="Piętro", parser=FloorParser(DOMTextExtractor()), id="floor"),
        DataUnit(pos=3, label="Pow.", parser=AreaParser(DOMTextExtractor()), id="area"),
        DataUnit(pos=4, label="Pokoje", parser=IntParser(DOMTextExtractor()), id="rooms"),
        DataUnit(pos=5, label="Pow. balkonu", parser=AreaParser(DOMTextExtractor()), id="balcony"),
        DataUnit(pos=7, label="Plan", parser=LinkParser(DOMElementExtractor("a")), id="plan"),
        DataUnit(pos=7, label="Status", parser=StatusParser(DOMTextExtractor()), id="status")
    ]

    @attributeerror_wrapper(return_value=[])
    def find_records(self, soup):
        return soup.find("div", {"class": "places_container"})\
            .find_all("div", {"class": "table-row"})

    def split_record(self, record):
        return record.find_all("div", recursive=False)

    def modify_record(self, record, input_record=None):
        record["fid"] = self.create_fid(record)
        record["plan"] = urljoin(self.url, record["plan"])
        return record

    def create_fid(self, record):
        return "{building}/{number}".format(**record)


import json
from typing import Container
import strax
import fsspec
import pandas as pd
import straxen
import inspect
from urllib.parse import urlparse, parse_qs
from ast import literal_eval
from strax.config import OMITTED
import os
import tempfile
import tarfile

export, __all__ = strax.exporter()

_CACHES = {}

@export
def clear_config_caches():
    for cache in _CACHES.values():
        cache.clear()

@export
def config_cache_size_mb():
    return straxen.total_size(_CACHES)//1e6

def parse_val(val: str):
    '''Attempt to parse a string value as
    a python literal, falls back to returning just
    the original string if cant be parsed.
    '''
    try:
        val = literal_eval(val)
    except ValueError:
        pass
    except SyntaxError:
        pass
    return val

@export
class URLConfig(strax.Config):
    """Dispatch on URL protocol.
    unrecognized protocol returns identity
    inspired by dasks Dispatch and fsspec fs protocols.
    """
    _LOOKUP = {}
    SCHEME_SEP = '://'
    QUERY_SEP = '?'
    PLUGIN_ATTR_PREFIX = 'plugin.'

    def __init__(self, cache=0, **kwargs):
        """
        :param cache: number of values to keep in cache,
                      if set to True will cache all values
        :param **kwargs: additional keyword arguments accepted by strax.Option
        """
        self.final_type = OMITTED
        super().__init__(**kwargs)
        # Ensure backwards compatibility with Option validation
        # type of the config value can be different from the fetched value.
        if self.type is not OMITTED:
            self.final_type = self.type
            self.type = OMITTED  # do not enforce type on the URL
        if cache:
            cache_len = 100 if cache is True else int(cache)
            cache = straxen.CacheDict(cache_len=cache_len)
            _CACHES[id(self)] = cache

    @property
    def cache(self):
        return _CACHES.get(id(self), {})

    @classmethod
    def register(cls, protocol, func=None):
        """Register dispatch of `func` on urls
        starting with protocol name `protocol` """

        def wrapper(func):
            if isinstance(protocol, tuple):
                for t in protocol:
                    cls.register(t, func)
                return func

            if not isinstance(protocol, str):
                raise ValueError('Protocol name must be a string.')

            if protocol in cls._LOOKUP:
                raise ValueError(f'Protocol with name {protocol} already registered.')
            cls._LOOKUP[protocol] = func
            return func
        return wrapper(func) if func is not None else wrapper

    def dispatch(self, url, *args, **kwargs):
        """
        Call the corresponding method based on protocol in url.
        chained protocols will be called with the result of the
        previous protocol as input
        overrides are passed to any protocol whos signature can accept them.
        """

        # separate the protocol name from the path
        protocol, _, path = url.partition(self.SCHEME_SEP)

        # find the corresponding protocol method
        meth = self._LOOKUP.get(protocol, None)
        if meth is None:
            # unrecognized protocol
            # evaluate as string-literal
            return url

        if self.SCHEME_SEP in path:
            # url contains a nested protocol
            # first call sub-protocol
            arg = self.dispatch(path, **kwargs)
        else:
            # we are at the end of the chain
            # method should be called with path as argument
            arg = path

        # filter kwargs to pass only the kwargs
        #  accepted by the method.
        kwargs = straxen.filter_kwargs(meth, kwargs)

        return meth(arg, *args, **kwargs)

    @classmethod
    def split_url_kwargs(cls, url):
        """split a url into path and kwargs
        """
        path, _, _ = url.partition(cls.QUERY_SEP)
        kwargs = {}
        for k, v in parse_qs(urlparse(url).query).items():
            # values of query arguments are evaluated as lists
            # split logic depending on length
            n = len(v)
            if not n:
                kwargs[k] = None
            elif n == 1:
                kwargs[k] = parse_val(v[0])
            else:
                kwargs[k] = list(map(parse_val, v))
        return path, kwargs

    @classmethod
    def kwarg_from_url(cls, url: str, key: str):
        path, kwargs = cls.split_url_kwargs(url)
        return kwargs.get(key, None)

    @classmethod
    def format_url_kwargs(cls, url, **kwargs):
        '''Add keyword arguments to a URL.
        Sorts all arguments by key for hash consistency
        '''
        url, extra_kwargs = cls.split_url_kwargs(url)
        kwargs = dict(extra_kwargs, **kwargs)
        arg_list = []
        for k, v in sorted(kwargs.items()):
            if isinstance(v, list):
                # lists are passed as multiple arguments with the same key
                arg_list.extend([f"{k}={vi}" for vi in v])
            else:
                arg_list.append(f"{k}={v}")
        arg_str = "&".join(arg_list)
        arg_str = cls.QUERY_SEP + arg_str if arg_str else ''
        return url + arg_str

    def fetch_attribute(self, plugin, value):
        if isinstance(value, str) and value.startswith(self.PLUGIN_ATTR_PREFIX):
            # kwarg is referring to a plugin attribute, lets fetch it
            return getattr(plugin, value[len(self.PLUGIN_ATTR_PREFIX):], value)

        if isinstance(value, list):
            return [self.fetch_attribute(plugin, v) for v in value]

        # kwarg is a literal, add its value to the kwargs dict
        return value

    def fetch(self, plugin):
        """override the Config.fetch method
        this is called when the attribute is accessed
        """
        # first fetch the user-set value
        # from the config dictionary
        url = super().fetch(plugin)

        if not isinstance(url, str):
            # if the value is not a string it is evaluated
            # as a literal config and returned as is.
            return url

        if self.SCHEME_SEP not in url:
            # no protocol in the url so its evaluated
            # as string-literal config and returned as is
            return url

        # separate out the query part of the URL which
        # will become the method kwargs
        url, url_kwargs = self.split_url_kwargs(url)

        kwargs = {k: self.fetch_attribute(plugin, v)
                  for k, v in url_kwargs.items()}

        # construct a deterministic hash key
        key = strax.deterministic_hash((url, kwargs))

        # fetch from cache if exists
        value = self.cache.get(key, None)

        # not in cache, lets fetch it
        if value is None:
            value = self.dispatch(url, **kwargs)
            self.cache[key] = value

        return value

    @classmethod
    def protocol_descr(cls):
        rows = []
        for k, v in cls._LOOKUP.items():
            row = {
                'name': f"{k}://",
                'description': v.__doc__,
                'signature': str(inspect.signature(v)),
            }
            rows.append(row)
        return pd.DataFrame(rows)

    @classmethod
    def print_protocols(cls):
        df = cls.protocol_descr()
        print(df)

@URLConfig.register('cmt')
def get_correction(name: str,
                   run_id: str = None,
                   version: str = 'ONLINE',
                   detector: str = 'nt',
                   **kwargs):
    """Get value for name from CMT"""
    if run_id is None:
        raise ValueError('Attempting to fetch a correction without a run id.')
    return straxen.get_correction_from_cmt(run_id, (name, version, detector == 'nt'))

@URLConfig.register('resource')
def get_resource(name: str,
                 fmt: str = 'text',
                 **kwargs):
    """
    Fetch a straxen resource

    Allow a direct download using <fmt='abs_path'> otherwise kwargs are
    passed directly to straxen.get_resource.
    """
    if fmt == 'abs_path':
        downloader = straxen.MongoDownloader()
        return downloader.download_single(name)
    return straxen.get_resource(name, fmt=fmt)

@URLConfig.register('fsspec')
def read_file(path: str, **kwargs):
    """Support fetching files from arbitrary filesystems
    """
    with fsspec.open(path, **kwargs) as f:
        content = f.read()
    return content

@URLConfig.register('json')
def read_json(content: str, **kwargs):
    """Load json string as a python object
    """
    return json.loads(content)

@URLConfig.register('take')
def get_key(container: Container, take=None, **kwargs):
    """ return a single element of a container
    """
    if take is None:
        return container
    if not isinstance(take, list):
        take = [take]

    # support for multiple keys for
    # nested objects
    for t in take:
        container = container[t]

    return container

@URLConfig.register('format')
def format_arg(arg: str, **kwargs):
    """apply pythons builtin format function to a string"""
    return arg.format(**kwargs)

@URLConfig.register('itp_map')
def load_map(some_map, method='WeightedNearestNeighbors', **kwargs):
    """Make an InterpolatingMap"""
    return straxen.InterpolatingMap(some_map, method=method, **kwargs)

@URLConfig.register('bodega')
def load_value(name: str, bodega_version=None):
    """Load a number from BODEGA file"""
    if bodega_version is None:
        raise ValueError('Provide version see e.g. tests/test_url_config.py')
    nt_numbers = straxen.get_resource("XENONnT_numbers.json", fmt="json")
    return nt_numbers[name][bodega_version]["value"]

@URLConfig.register('tf')
def open_neural_net(model_path: str, **kwargs):
    # Nested import to reduce loading time of import straxen and it not
    # base requirement
    import tensorflow as tf
    if not os.path.exists(model_path):
        raise FileNotFoundError(f'No file at {model_path}')
    with tempfile.TemporaryDirectory() as tmpdirname:
        tar = tarfile.open(model_path, mode="r:gz")
        tar.extractall(path=tmpdirname)
        return tf.keras.models.load_model(tmpdirname)


#! /usr/bin/env python

import rospy
from std_srvs.srv import Empty, Trigger, TriggerResponse
import os

GPD_PATH = "/home/tpatten/Code/gpd/"
DATA_PATH = os.path.join(GPD_PATH, "hsrb_data")
CONFIG_PATH = os.path.join(GPD_PATH, "cfg")
CONFIG_FILE = "hsrb_params.cfg"
PCD_FILE = "temp.pcd"
GRASPS_FILE = "grasps.txt"
PROGRAM_NAME = os.path.join(GPD_PATH, "build/./detect_grasps_hsrb")
PROGRAM_ARGUMENTS = os.path.join(CONFIG_PATH, CONFIG_FILE) + " " + os.path.join(DATA_PATH, PCD_FILE) + " " + os.path.join(DATA_PATH, GRASPS_FILE)

class GPDService:
    def __init__(self):
        # Advertise service
        self.service = rospy.Service('/gpd_detect_grasps', Trigger, self.service_callback)
        rospy.loginfo('Service ready...')

    def service_callback(self, req):
        rospy.loginfo('GPD service called')
        ret = TriggerResponse()
        ret.success = False

        cmd = PROGRAM_NAME + " " + PROGRAM_ARGUMENTS
        print('CMD ' + cmd)
        cmd_res = os.system(cmd)
        print('CMD returned ' + str(cmd_res))

        if cmd_res == 0:
            rospy.loginfo('GPD service succeeded')
            ret.success = True
        else:
            rospy.logwarn('GPD service failed')
            return ret

        # Load the grasps file and fill the message field of the service response
        f = open(os.path.join(DATA_PATH, GRASPS_FILE), "r")
        grasps_text = f.readline()
        print(grasps_text)
        ret.message = grasps_text

        return ret

if __name__ == '__main__':
    rospy.init_node('GPD_service')
    rospy.loginfo('Starting GPD service')
    gpd_service = GPDService()
    rospy.spin()


from math import sqrt

def is_prime(number):
    if number <= 1:
        return False

    for x in range(2, int(sqrt(number)) + 1):
        if number % x == 0:
            return False
    return True

def question_10(n):
    total = 2
    for i in range(3, n + 1, 2):
        if is_prime(i):
            total += i
    return total

if __name__ == '__main__':
    print(question_10(2000000))


import timeit
import re

# py fileproc.py out_files_corexome/out_sh/out_new_rs_rs.txt coreexome_map
# py fileproc.py out_files_humanexome/out_sh/out_new_rs_rs.txt humanexome_map
# py fileproc.py out_files_infimun/out_sh/out_new_rs_rs.txt infiniumimmunoarray_map
# py fileproc.py out_files_msex/out_sh/out_new_rs_rs.txt msexome_map

# suggested change since flankerr.py creation: oldbrpos = [op for op in getbrpos[1] if op != '0:0' and op != 'pending' and op != 'flank_error']
#omrs_13.py is the first one to be done after flankerr.py

class NewRs_07:
# map table rsid not known to db, map table non-rsid in db consensus instead
# related (newrsbyalt): map table rsid not known to db, map table non-rsid in db alt_ids instead
    def newrs(self,brk=0,start=0):
        report = open('report_newrs_' + self.ts + '_' + self.tabname + '.txt',"w")
        count = 0
        for line in self.inp.read():
            count+=1
            if brk and count == brk:
                break
            if start and count < start:
                continue
            self.percent_comp(current=count,perc=10,total=brk,start=start)
            new_current = re.split('\tlookup=',line.rstrip())
            mapid,linkid = re.split('\t',new_current[0])
            mapid = re.split('= ',mapid)[1]
            linkid = re.split('= ',linkid)[1]
            mgchk = self.mergecheck(new_current[1])
            merges = mgchk['merges']
            becomes = self.mergedinto(evenarr=merges,potbef=mapid)
            stillmain = self.stillmain(linkid)
            mrgid = None
            if becomes:
                if len(becomes) > 1: # probably not necessary
                    print('map table id %s is merged into multiple (%s) according to dbsnp lookup. No action yet' % (mapid,'/'.join(becomes)))
                    continue
                mrgid = becomes[0]
                report.write('map table id %s merged to %s and linked to omics %s. correcting map table (%s -> %s)\n' % (mapid,mrgid,linkid,mapid,mrgid))
                self.mtab_change_id(xsting=mapid,chngto=mrgid)
                mrgid_knwn = self.checkomid(cid=mrgid) # is the mrgid unknown as the mapid was?
                if not mrgid_knwn[0] and not mrgid_knwn[1]:
                    report.write('new id %s is also unknown to omics so adding old map id %s as alternative id to %s and continuing\n' % (mrgid,mapid,linkid))
                    if stillmain:
                        self.add_alt(alt=mapid,main=linkid,ds=self.tabname)
                else:
                    report.write('new merged id %s is already known as consensus id or alternative id or both (%s). No action\n' % (mrgid,'-'.join([str(i) for i in mrgid_knwn])))
                    continue
            if not stillmain:
                report.write('link id %s is not in consensus table. Would have swapped it out with %s from map table (or merged %s)\n' % (linkid,mapid,mrgid))
                continue
            b38 = self.getb38(new_current[1])
            if not b38:
                if not mrgid:
                    report.write('b38 position of map table id %s (omics %s) not available (may be withdrawn). Will add to omics as alternative id instead\n' % (mapid,linkid))
                    self.add_alt(alt=mapid,main=linkid,ds=self.tabname)
                else:
                    print('b38 position of new merge id %s (omics %s) not available (may be withdrawn). Will add to omics as alternative id instead\n' % (mrgid,linkid))
                    self.add_alt(alt=mrgid,main=linkid,ds='dbsnp')
                continue
            contig = False
            if '_' in b38:
                report.write('map table id %s (omics %s) dbsnp look up has a contig id instead of regular chromosome identifier: %s. Will swap %s (or merge %s) into omics for %s but will not correct/update positions\n' % (mapid,linkid,b38,mapid,mrgid,linkid))
                contig = True
            getbrpos = self.checkbr_pos(mapid,b38)
            getompos = self.checkom_pos(linkid,chrpos=b38,build='38')
            if not contig:
                if not getbrpos[0]:
                    oldbrpos = [op for op in getbrpos[1] if op != '0:0']
                    for op in oldbrpos:
                        report.write('map table id %s (or merged %s) to be swapped in to consensus table in place of %s. %s in map table does not match dbsnp coordinate (%s vs %s). Correcting position in map table\n' % (mapid,mrgid,linkid,mapid,op,b38))
                        self.mtab_change_pos(anid=mapid,oldpos=op,newpos=b38)
                if not getompos[0]:
                    report.write('map table id %s (or merged %s) to be swapped in to consensus table in place of %s. %s in omics does not match dbsnp coordinate (%s vs %s). Adding dbsnp coordinate %s to omics\n' % (mapid,mrgid,linkid,linkid,b38,','.join(getompos[1]),b38))
                    badom = [bp for bp in getompos[1] if bp != '0:0']
                    self.addpos(linkid,chrpos=b38,ds='dbsnp',build='38')
                    for bp in badom:
                        report.write('flagging position %s in omics db because we have a dbsnp sourced coordinate %s for this variant (%s/%s)\n' % (bp,b38,linkid,mapid))
                        self.pos_flag(linkid,chrpos=bp,fl=-5)
            if not mrgid:
                report.write('swapping main id %s out of omics consensus table and new id %s from map table in\n' % (linkid,mapid))
                self.swapout_main(swin=mapid,swout=linkid,ds=self.tabname)
            else:
                report.write('swapping main id %s out of omics consensus table and new merged id %s from map table (originally %s) in\n' % (linkid,mrgid,mapid))
                self.swapout_main(swin=mrgid,swout=linkid,ds='dbsnp')
        print('Done. %s seconds' % (int(timeit.default_timer() - self.grandstart)))


from rest_framework import permissions

class IsAuthorOrReadOnly(permissions.BasePermission):
    def has_object_permission(self, request, view, obj):
        # Разрешения только на чтение разрешены для любого запроса
        if request.method in permissions.SAFE_METHODS:
            return True

        # Разрешение на запись разрешено только автору сообщения
        return obj.author == request.user


import numpy as np
data = np.load('test.npy').item()
import os
import matplotlib.pyplot as plt
for i in data:
    if i.upper() not in os.listdir('.'):
        os.makedirs(i.upper())
    data[i][0]
    f1 = open('./%s/PA.fasta' %i.upper(),'w')
    f1.write('> %s \n' %i.upper())
    f1.write(data[i][0][1].lower()+'\n')
    f1.close()
    f1 = open('./%s/PA.secstruct' %i.upper(),'w')
    f1.write(data[i][0][1].lower()+'\n')
    ss = ''
    for j in data[i][1][1]:
        if j == 0.0 :
            ss += '.'
        elif j == 1.0:
            ss += '('
        elif j == -1.0 :
            ss += ')'
    f1.write(ss +'\n')
    f1.close()
    constraints = np.argmax(data[i][-1]+np.transpose(data[i][-1],(0,2,1,3)),3)[0]
    f1 = open('./%s/constraints' %i.upper(),'w')
    f1.write('[ atompairs ]\n')
    for x in range(len(constraints)):
        for y in range(x+1,len(constraints)):
            if constraints[x,y] == 1:
                dist = '%s %s %s %s FLAT_HARMONIC %s 1 4\n' %('P',x+1,'P',y+1,12)
                f1.write( dist)
            elif constraints[x,y] == 0:
                dist = '%s %s %s %s FLAT_HARMONIC %s 1 4\n' %('P',x+1,'P',y+1,4)
                f1.write( dist)
    f, ax = plt.subplots(1,2)
    ax[0].imshow(constraints)
    a,b,c = (data[i][0][-1] < 8)*1,(data[i][0][-1] <= 15) & (data[i][0][-1] >= 8)*1,(data[i][0][-1] > 15)*1
    batch_y = np.stack((a,b,c),axis=2)
    ax[1].imshow(np.argmax(batch_y,2))
    ax[0].set_xlabel(i.upper())
    plt.savefig('./%s/%s.png' %(i.upper(),i.upper()))
    plt.clf()
    f1.close()


class Solution:
	def _printNumber(self,StrNum):
		'''
		this fuction is to print the number stored in the sting.
		input: ['0','1','0']
		output: 10
		'''
		for i in range(len(StrNum)):
			if StrNum[i]!='0':
				print(''.join(StrNum[i:]))
				break

	def _increment(self,StrNum):
		'''
		add 1 each time it was called
		this function is to simulate the addition of numbers
		when it reached the biggest number, we need to stop
		['0','0','0']->['0','0','1']->...->['9','9','9'] then stop
		carry->
		overflow->
		'''
		carry = 0
		overflow = 0
		n = len(StrNum)-1

		while n>=0:
			nsum = int(StrNum[n])+carry
			if n == len(StrNum)-1: # the last digit
				nsum+=1

			if nsum==10:
				if n==0:
					overflow = 1
				else:
					carry = 1
					StrNum[n]='0'

			else:
				StrNum[n] = '%d' %nsum
				break
			n = n-1
		return overflow

	def printNNumber(self,n):
		if n<=0:
			return 0
		else:
			N = ['0']*n
			print(N)
			# add 1 and print 1 number each time
			while self._increment(N)==0:
				self._printNumber(N)

s=Solution()
s.printNNumber(5)


from utils import translate_sentence, bleu, load_checkpoint, printSent
import torch
from data import getData
from train import train
from transfomer import Transformer

german, english, train_data, valid_data, test_data = getData()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
batch_size = 32

data = train_data[0:3]

for example in data:
    src = example.src
    trg = example.trg
    print(dir(example))
    print(">> ", src)
    print("   ", trg)

    # exit()
# Model hyperparameters
src_vocab_size = len(german.vocab)
trg_vocab_size = len(english.vocab)
embedding_size = 512
src_pad_idx = english.vocab.stoi["<pad>"]
print(src_pad_idx)
print(english.vocab.itos[src_pad_idx])

model = Transformer(device, embedding_size, src_vocab_size, trg_vocab_size, src_pad_idx).to(device)

load_model = False
save_model = True
learning_rate = 3e-4

optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

if load_model:
    load_checkpoint(torch.load("my_checkpoint.pth.tar"), model, optimizer)

# sentence = "ein pferd geht unter einer brücke neben einem boot."
#
# translated_sentence = translate_sentence(
#     model, sentence, german, english, device, max_length=50
# )
sentence1 = ['ein', 'pferd', 'geht', 'unter', 'einer', 'brücke', 'neben', 'einem', 'boot', '.']
translated_sentence = translate_sentence(model, sentence1, german, english, device, max_length=50)
# exit()
# print(f"Translated1 example sentence: \n {sentence}")
# print(f"Translated1 example sentence: \n {translated_sentence}")

# exit()

train(model, device, load_model, save_model, german, english, train_data, valid_data, test_data, batch_size)
# running on entire test data takes a while


import matplotlib.pyplot as plt
import numpy as np

a = np.linspace(1, 3, 33)
print(a)

x = np.random.randint(-100, 100, 1000)  # 1000개의 랜덤 값 추출
y = np.random.randint(-100, 100, 1000)  # 1000개의 랜덤 값 추출
size = np.random.randint(-100, 100, 1000)
mask1 = abs(x) > 50  # x에 저장된 값 중 절댓값이 50보다 큰 값 걸러 냄
mask2 = abs(y) > 50  # y에 저장된 값 중 절댓값이 50보다 큰 값 걸러 냄
x = x[mask1 + mask2]  # mask1과 mask2 중 하나라도 만족하는 값 저장
y = y[mask1 + mask2]  # mask1과 mask2 중 하나라도 만족하는 값 저장
plt.scatter(x, y, s=size, c=x, cmap='jet', alpha=0.7)
plt.colorbar()
plt.show()


import os
from server import Server
from server import app
from controllers.differentials import DifferentialController
from controllers.taxonomy import TaxonomyController
from controllers.adaptation_node import AdaptationNodeController
from controllers.differential_adaptation import DifferentialAdaptationController
from controllers.api_element_type import ApiElementTypeController

app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///conflict_solver.db'

def main():
    print('*************************************')
    print('          CONFLICT SOLVER')
    print('*************************************')

    print('Reading settings...')
    port = 5000
    if os.getenv("CONFLICT_SOLVER_PORT") is not None:
        port = int(os.getenv("CONFLICT_SOLVER_PORT"))

    print('Initiating server')
    server = Server(port=port)

    print('Loading controllers')
    server.addControllers([
            (DifferentialController, '/differential'),
            (TaxonomyController, '/taxonomy'),
            (AdaptationNodeController, '/adaptation'),
            (DifferentialAdaptationController, '/differential_adaptation'),
            (ApiElementTypeController, '/api_element_type')
        ])

    print('Starting server')
    server.serve()

if __name__ == '__main__':
    main()


class SampleClass():
    def __init__(self):
        self.__priv = ('i am Private')
        self._prot = ('i am Protected')
        self.pub = ('i am Public')

    def __myPriveMethod(self):
        print("this is private Method")

obj = SampleClass()
print(obj.pub)
#its a convenstion but not forced rule.
print(obj._prot)
#can not access private attibutes but still there is a way to do this
#print(obj.__priv)

#private method also can not be accessed
#obj.__myPrivateMethod()
obj._SampleClass__myPriveMethod()
print(obj._SampleClass__priv)


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-x86_64/egg/sample_graphs/execution_stop.py
# Compiled at: 2016-07-13 17:17:02
from datamodel.base import graph, exceptions
from datamodel.nodes.lib import value
from robograph.datamodel.nodes.lib import apply

def execution_stop(number):

    def stop_here(value):
        if value >= 0:
            raise exceptions.StopGraphExecutionSignal('arg is positive')
        raise exceptions.StopGraphExecutionSignal('arg is negative')

    v = value.Value(value=number)
    a = apply.Apply(function=stop_here)
    g = graph.Graph('execution_stop', [a, v])
    g.connect(a, v, 'argument')
    return g


#!/usr/bin/python3
""" New engine for our program. """
from sqlalchemy import create_engine
from os import getenv
from models.base_model import Base
from models.city import City
from models.state import State
from models.user import User
from models.place import Place
from models.amenity import Amenity
from models.review import Review
from sqlalchemy.orm import sessionmaker, scoped_session

class DBStorage:
    """ Class for the new engine. """
    __engine = None
    __session = None

    def __init__(self):
        """ Initializer for the instances of the class. """
        self.__engine = create_engine("mysql+mysqldb://{}:{}@{}/{}"
                                      .format(getenv("HBNB_MYSQL_USER"),
                                              getenv("HBNB_MYSQL_PWD"),
                                              getenv("HBNB_MYSQL_HOST"),
                                              getenv("HBNB_MYSQL_DB")),
                                      pool_pre_ping=True)
        if getenv("HBNB_ENV") == "test":
            Base.metadata.drop_all(bind=self.__engine)

    def all(self, cls=None):
        """ Query on the current session all objects by class. """
        if cls is not None:
            res = self.__session.query(eval(cls)).all()
        else:
            res = self.__session.query(City).all()
            res += self.__session.query(State).all()
            res += self.__session.query(User).all()
            res += self.__session.query(Place).all()
            res += self.__session.query(Amenity).all()
            res += self.__session.query(Review).all()
        output = {}
        for elem in res:
            key = '{}.{}'.format(type(elem).__name__, elem.id)
            output[key] = elem
        return output

    def new(self, obj):
        """ Add the object to the current database session. """
        self.__session.add(obj)

    def save(self):
        """ Commit all changes of the current database session. """
        self.__session.commit()

    def delete(self, obj=None):
        """ Deletes obj from the current database session. """
        if obj is not None:
            self.__session.delete(obj)

    def reload(self):
        """ Creates all tables in the database and the session. """
        Base.metadata.create_all(self.__engine)
        sess = sessionmaker(bind=self.__engine, expire_on_commit=False)
        Session = scoped_session(sess)
        self.__session = Session()

    def close(self):
        """ Close function. """
        self.__session.close()


import os
from pathlib import Path
from typing import Dict, Any, Iterable, List, Tuple

from virtool_workflow.analysis import utils, analysis_info
from virtool_workflow.storage.utils import copy_paths
from virtool_workflow.execute import run_shell_command, FunctionExecutor
from virtool_workflow.fixtures.workflow_fixture import fixture

@fixture
def trimming_output_path(cache_path: Path):
    """The output path for the trimming command."""
    path = cache_path/"reads"
    path.mkdir(exist_ok=True, parents=True)
    return path

@fixture
async def trimming_input_paths(analysis_args: analysis_info.AnalysisArguments,
                               run_in_executor: FunctionExecutor) -> utils.ReadPaths:
    """Copy sample data to raw_path and read_paths and return the read_paths."""
    sample_paths = utils.make_read_paths(analysis_args.sample_path, analysis_args.paired)
    raw_read_paths = {path: analysis_args.raw_path / path.name for path in sample_paths}
    await copy_paths(raw_read_paths.items(), run_in_executor)

    await copy_paths(
        zip(sample_paths, analysis_args.read_paths),
        run_in_executor
    )

    return analysis_args.read_paths

def compose_trimming_command(
        output_path: Path,
        trimming_parameters: Dict[str, Any],
        number_of_processes: int,
        input_paths: Iterable[Path]
):
    """
    Compose a shell command to run skewer on the reads located by read_paths.

    :param output_path: The Path to a directory where the output from skewer should be stored
    :param trimming_parameters: The trimming parameters
        (see virtool_workflow.analysis.trimming_parameters)
    :param number_of_processes: The number of allowable processes to be used by skewer
    :param input_paths: The paths to the reads data
        (see virtool_workflow.analysis.reads_path)
    """
    command = [
        "skewer",
        "-r", str(trimming_parameters["max_error_rate"]),
        "-d", str(trimming_parameters["max_indel_rate"]),
        "-m", str(trimming_parameters["mode"]),
        "-l", str(trimming_parameters["min_length"]),
        "-q", str(trimming_parameters["end_quality"]),
        "-Q", str(trimming_parameters["mean_quality"]),
        "-t", str(number_of_processes),
        "-o", str(output_path/"reads"),
        "-n",
        "-z",
        "--quiet",
    ]

    if trimming_parameters["max_length"]:
        command += [
            "-L", str(trimming_parameters["max_length"]),
            "-e"
        ]

    command += [str(path) for path in input_paths]

    return command

@fixture
def trimming_command(
        trimming_output_path: Path,
        trimming_parameters: Dict[str, Any],
        number_of_processes: int,
        trimming_input_paths: utils.ReadPaths
) -> List[str]:
    """A fixture for the skewer trimming command."""
    return compose_trimming_command(trimming_output_path,
                                    trimming_parameters,
                                    number_of_processes,
                                    trimming_input_paths)

@fixture
async def trimming_output(
        trimming_command: List[str],
        trimming_input_paths: utils.ReadPaths,
        trimming_output_path: Path
) -> Tuple[Path, str]:
    """Perform the trimming command and return the trimming_output_path."""
    env = dict(os.environ, LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu")

    out, err = await run_shell_command(trimming_command, env=env)

    if err:
        raise RuntimeError("trimming command failed", err, trimming_command)

    return trimming_output_path, out


import datetime
import json
import os
import time

from flask import send_from_directory  # 文件下载
from flask import (current_app, flash, jsonify, redirect, render_template,
                   request, session, url_for)
from flask_login import (current_user, login_manager, login_required,
                         login_user, logout_user)
from werkzeug import secure_filename  # 安全的文件名

from . import main
from .. import auth, db
from ..decorators import admin_required
from ..email import send_email
from ..models import Article, ArticleType, Data, User
from ..net_tools import ali_oss
from .forms import ArticleForm

@main.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        userkey = request.form.get('ID')
        user = User.query.filter_by(userkey=userkey).first()
        if user is not None:
            login_user(user, True)  # true 记住用户
            return redirect(url_for('main.user', username=user.username))
        flash('请输入正确的令牌')
    return render_template('index.html')

@main.route('/blogs', methods=['POST', 'GET'])
def blogs():
    if request.method == "POST":
        data = json.loads(request.get_data().decode('utf8'))
        article = Article.query.filter_by(
            article_type_id=data['type']).order_by(
                Article.finish_time.desc()).paginate(data['page'],
                                                     1, error_out=False)
        try:
            data = article.items[0].to_json()
        except IndexError as e:
            return jsonify({'status': '400', 'data': str(e)})
        except Exception as e:
            return jsonify({'status': '400', 'data': str(e)})
        else:
            return jsonify({
                'status': '200',
                'data': article.items[0].to_json()
            })

    articletypes = ArticleType.query.order_by(ArticleType.id.asc()).all()
    print(articletypes)
    counts = [articletype.articles.count() for articletype in articletypes]

    return render_template("blogs.html", counts=counts)

@main.route('/blog/<int:id>')
def blog(id):
    blog = Article.query.get_or_404(id)
    return render_template('blog.html', blog=blog)

@main.route('/userlogout')
@login_required
def userlogout():
    logout_user()
    return redirect(url_for('main.index'))

@main.route('/user/<username>', methods=['GET', 'POST'])
@login_required
def user(username):
    user = User.query.filter_by(username=username).first()
    if username != current_user.username:
        return render_template('404.html'), 404

    data = user.data.order_by(Data.timestamp.asc()).all()
    host = ali_oss.get_host()+'/'
    return render_template('user.html', data=data, user=user, host=host)

@main.route('/get-token', methods=['POST', 'GET'])
def get_token():
    user_dir = current_user.email+"/"
    if request.method == 'POST':
        return ali_oss.get_token(user_dir)

@main.route('/call-back', methods=['POST', 'GET'])
def call_back():
    if request.method == 'POST':

        host = ali_oss.get_host()

        sec_filename = request.form.get('filename')
        filename = sec_filename.split('/')[-1]
        email = sec_filename.split('/')[0]
        user = User.query.filter_by(email=email).first()
        bucket = request.form.get('bucket')
        filetype = request.form.get('mimeType')  # 文件类型

        dt = datetime.datetime.utcnow()

        data = Data(filename=filename, sec_filename=sec_filename,
                    filetype=filetype,
                    author=user)
        db.session.add(data)

        url = host+'/' + sec_filename
        return json.dumps({'filename': filename, 'url': url})
    return 'success'

@main.route('/_sendmessage')
@login_required
def sendmessage():
    text = request.args.get('text', '')
    device_type = request.args.get('device_type', '')
    if text is not None:
        data = Data(text=text,
                    author=current_user._get_current_object())
        db.session.add(data)
    return jsonify(result='success')

@main.route('/_sendfile', methods=['GET', 'POST'])
@login_required
def sendfile():

    if request.method == "POST":
        try:
            dt = datetime.datetime.utcnow()
            sec_key = dt.strftime("%Y-%m-%d-%H-%M-%S")
            file = request.files['file']
            device_type = request.args.get('device_type', '')
            filename = file.filename
            sec_filename = '['+sec_key + ']' + filename

            user = User.query.filter_by(username=current_user.username).first()
            persional_folder = UPLOAD_FOLDER+user.email
            print(persional_folder)
            file.save(os.path.join(persional_folder, sec_filename))
            data = Data(filename=filename, sec_filename=sec_filename,
                        device_type=device_type,
                        author=current_user._get_current_object())
            db.session.add(data)
        except:
            file = None
        return jsonify({'name': filename, 'url': sec_filename})

@main.route('/uploads/<filename>')
@login_required
def uploaded_file(filename):
    user = User.query.filter_by(username=current_user.username).first()
    persional_folder = UPLOAD_FOLDER+user.email
    return send_from_directory(persional_folder,
                               filename)

@main.route('/register')
def turn():
    return redirect(url_for('auth.register'))

@main.route('/secret')
@login_required
def secret():
    return 'you can'

@main.route('/about_me')
def about():
    return render_template('about_me.html')


import gym
import sys
from agents.agent import DDPG
from pendulum_task import PendulumTask

num_episodes = 10
task = PendulumTask()
agent = DDPG(task)

display_graph = True
display_freq = 5

for i_episode in range(1, num_episodes + 1):
    state = agent.reset_episode()  # start a new episode

    # prior to the start of each episode, clear the datapoints
    # x, y1, y2 = [], [], []

    while True:
        action = agent.act(state)
        next_state, reward, done = task.step(action)
        agent.step(action, reward, next_state, done)
        state = next_state
        # within the episode loop
        if (i_episode % display_freq == 0) and (display_graph == True):
            #    x.append(task.sim.time) # time
            #    y1.append(reward) # y-axis 1 values
            #    y2.append(task.sim.pose[2]) # y-axis 2 values

            print(f'Episode number {i_episode}')
            print(
                f'action {action}, reward {reward}, next_state {next_state}, done {done}')
        #    print(f'Plot values - time {task.sim.time}, reward {reward}, z {task.sim.pose[2]}')
        # if done:
        #    print("\rEpisode = {:4d}, score = {:7.3f} (best = {:7.3f}), noise_scale = {}".format(
        #        i_episode, agent.score, agent.best_score, agent.noise_scale), end="")  # [debug]
        #    break
        if done:
            print("\rEpisode = {:4d}, reward = {:7.3f}".format(
                i_episode, reward), end="")  # [debug]
        #    if (episode % display_freq == 0) and (display_graph == True):
        #               plt_dynamic(x, y1, y2)
        break

    sys.stdout.flush()


import numpy as np
import time

# below the timer function is adopted from the code in lecture5
def timer(f):
    def inner(*args):
        t0 = time.time()
        output = f(*args)
        elapsed = time.time() - t0
        print("Time Elapsed", elapsed)
        return output
    return inner

def my_ave(areas):
    my_sum = 0
    for a in areas:
        my_sum += a
    my_sum = my_sum / len(areas)
    #print("Myavg :",my_sum)
    return my_sum

def np_ave (areas):
    return np.mean(areas)

file = open('circles.txt')
list_str= file.read().splitlines()
radius = [ float(x) for x in list_str ]

areas = [0.0]*len(radius)
areas_np = np.zeros(len(radius))
for i in range(len(radius)):
    areas[i] = np.pi * radius[i]**2
    areas_np[i] = np.pi * radius[i]**2

# below the demos are adopted from the code in lecture5

# Time allocation with lists
my_func = timer(my_ave)
l1 = my_func(areas)

# Time allocation with numpy array
np_func = timer(np_ave)
l2 = np_func(areas_np)

file.close()


# -*- coding: utf-8 -*-
"""
Kernel tricks
-------------

This example shows some niche, but useful functionalities of :class:`cellrank.tl.kernels.Kernel`.
"""

import cellrank as cr

adata = cr.datasets.pancreas_preprocessed("../example.h5ad")
adata

# %% First, we create some kernels that will be used to compute the cell-to-cell transition matrix.
vk = cr.tl.kernels.VelocityKernel(adata)
ck = cr.tl.kernels.ConnectivityKernel(adata)
pk = cr.tl.kernels.PalantirKernel(adata)

# %%
# Kernels can be easily combined with each other. All of the constants within parentheses
# (if they are needed) will be normalized to 1 automatically. This effect will be shown after the transition matrix
# has been computed.
10 * vk + 15 * ck

# %%
# We can also build more complex kernel expression. Note that multiple usage of the same kernel instance in the
# expression caches it's transition matrix, so it's only computed once.
k = ((vk + vk + 42 * vk) + ck * 2) + ck * (3 * vk + 4 * pk)
k

# %%
# For complex expression like the one above, one can get the unique base kernels as follows.
k.kernels

# %%
# Kernels can also be inverted (i.e. the ``backward`` attribute is set to the opposite value to the current one).
# Note that this operation does *NOT* create a copy and modifies the kernel inplace, most importantly, the computed
# transition matrix is removed. This makes it more easier to recompute the transition matrix of kernels,
# since the handles point to the original objects.
#
# Note that in the 2nd :func:`print` statement, we access the private attribute -
# that's because accessing :paramref:`cellrank.tl.kernels.Kernel.transition_matrix` computes the transition matrix
# with default values. This happens only with basic kernels and not the kernel expressions and only if they are not part
# of a larger expression.
ck.compute_transition_matrix()
print(ck.transition_matrix is not None)

inv_ck = ~ck
print(ck._transition_matrix is not None)
print(inv_ck is ck)

# %%
# We can also easily copy the kernel objects.
vk.compute_transition_matrix(
    mode="deterministic", softmax_scale=4, show_progress_bar=False
)
print(vk.transition_matrix is not None)

inv_vk = ~(vk.copy())
print(vk._transition_matrix is not None)
print(vk is inv_vk)

# %%
# After the transition matrix is computed, we can inspect the parameters used in its computation.
ck.compute_transition_matrix()
ck.params

# %%
# The transition matrix can be written to :class:`anndata.AnnData` object. It is saved in the ``.obsp`` attribute.
# This also writes the parameters to the ``.uns`` attribute.
ck.write_to_adata(key="transition_matrix")
adata.obsp["transition_matrix"]

# %%
# Precomputed kernel doesn't require :mod:`anndata` object - a dummy one will be created if none is supplied.
# This can be useful in conjunction with :meth:`cellrank.tl.estimators.BaseEstimator.set_final_states` to calculate
# absorption probabilities using :meth:`cellrank.tl.estimators.BaseEstimator.compute_absorption_probabilities`.
pk = cr.tl.kernels.PrecomputedKernel(adata.obsp["transition_matrix"])
pk.adata


#!/usr/bin/env python3
from gpiozero import Button
from signal import pause
import RPi.GPIO as GPIO
from time import sleep

button = Button(2)
SOL = 0
SOL_ACK = 17
SOL_SYN = 4

GPIO.setmode(GPIO.BCM)
GPIO.setup(SOL, GPIO.OUT)
GPIO.setup(SOL_SYN, GPIO.OUT)
GPIO.setup(SOL_ACK, GPIO.OUT)

def triggerSol():
    GPIO.output(SOL, 1)
    sleep(0.05)
    GPIO.output(SOL, 0)

    print("I'm triggered.")

def triggerSYN():
    GPIO.output(SOL_SYN, 1)
    sleep(0.05)
    GPIO.output(SOL_SYN, 0)
    print("I'm triggered SYN.")

def triggerACK():
    GPIO.output(SOL_ACK, 1)
    sleep(0.05)
    GPIO.output(SOL_ACK, 0)

    print("I'm triggered ACK.")
#button.when_pressed = triggerSol
#pause()


import shutil
import json
from typing import Optional

import git
import requests

import model
import util
from db import session

LOCKFILE = '/var/lock/ksi-task-new'

def createGit(git_path: str, git_branch: str, author_id: int,
              title: str) -> str:
    repo = git.Repo(util.git.GIT_SEMINAR_PATH)
    repo.git.checkout("master")
    repo.remotes.origin.pull()

    # Vytvorime novou gitovskou vetev
    repo.git.checkout("HEAD", b=git_branch)

    # Zkopirujeme vzorovou ulohu do adresare s ulohou
    # Cilovy adresar nesmi existovat (to vyzaduje copytree)
    target_path = util.git.GIT_SEMINAR_PATH + git_path
    shutil.copytree(util.git.GIT_SEMINAR_PATH + util.git.TASK_MOOSTER_PATH,
                    target_path)

    # Predzpracovani dat v repozitari
    with open(target_path+'/task.json', 'r') as f:
        data = json.loads(f.read())
    data['author'] = author_id
    with open(target_path+'/task.json', 'w') as f:
        f.write(json.dumps(data, indent=4, ensure_ascii=False))

    with open(target_path+'/assignment.md', 'w') as f:
        s = title + ".\n\n" + \
            "# " + title + "\n\n" + \
            "Název úlohy musí být uvozen `#`, nikoliv podtrhnut rovnítky.\n\n"
        f.write(s)

    # Commit
    repo.index.add([git_path])
    repo.index.commit("Nova uloha: "+title)

    # Push
    # Netusim, jak udelat push -u, tohle je trosku prasarna:
    g = git.Git(util.git.GIT_SEMINAR_PATH)
    g.execute(["git", "push", "-u", "origin", git_branch+':'+git_branch])

    # Pull request
    seminar_repo = util.config.seminar_repo()
    github_token = util.config.github_token()

    if None not in (seminar_repo, github_token):
        # PR su per-service, teda treba urobit POST request na GitHub API
        url_root = "https://api.github.com/repos/fi-ksi/" + util.config.seminar_repo()

        headers = {
            "Accept": "application/vnd.github+json",
            "Authorization": "token " + util.config.github_token()
        }

        pull_id = requests.post(
            url_root + "/pulls",
            headers=headers,
            data=json.dumps({
                "title": "Nova uloha: " + title,
                "head": git_branch,
                "base": "master"
            })
        ).json()['number']

        author: Optional[model.User] = session.query(model.User).\
            filter(model.User.id == int(author_id)).\
            first()

        if author.github:
            requests.post(
                url_root + f"/issues/{pull_id}/assignees",
                headers=headers,
                data=json.dumps({
                    "assignees": [author.github]
                })
            )

    return repo.head.commit.hexsha


# Sean McIlvenny
# CTEC 121 / Winter 2019
# Module 4 / Problem Set 5
# Problem 2 (50 points)

"""
Using the graphics library, develop a Python program to draw the a set of LEGOS. You will find a picture of the LEGO's in a file named LEGOS.png.

Be sure to create one LEGO and then use the .clone() method whenever possible to create the other five.
"""

from graphics import *

def main():
    win = GraphWin("Lego's",800,800)

    blueBase = Rectangle(Point(130,145),Point(330,210))
    blueBase.setFill('blue')
    blueBase.setWidth(2)
    blueBase.draw(win)
    blueridgeOne = Rectangle(Point(140,145),Point(155,135))
    blueridgeOne.setFill('blue')
    blueridgeOne.setWidth(2)
    blueridgeOne.draw(win)
    blueridgeTwo = blueridgeOne.clone()
    blueridgeTwo.move(40,0)
    blueridgeTwo.draw(win)
    blueridgeThree = blueridgeTwo.clone()
    blueridgeThree.move(40,0)
    blueridgeThree.draw(win)
    blueridgeFour = blueridgeThree.clone()
    blueridgeFour.move(40,0)
    blueridgeFour.draw(win)
    blueridgeFive = blueridgeFour.clone()
    blueridgeFive.move(40,0)
    blueridgeFive.draw(win)

    yellowBase = blueBase.clone()
    yellowBase.move(0,200)
    yellowBase.setFill('yellow')
    yellowBase.draw(win)
    yellowridgeOne = blueridgeOne.clone()
    yellowridgeOne.move(0,200)
    yellowridgeOne.setFill('yellow')
    yellowridgeOne.draw(win)
    yellowridgeTwo = blueridgeTwo.clone()
    yellowridgeTwo.move(0,200)
    yellowridgeTwo.setFill('yellow')
    yellowridgeTwo.draw(win)
    yellowridgeThree = yellowridgeTwo.clone()
    yellowridgeThree.move(40,0)
    yellowridgeThree.draw(win)
    yellowridgeFour = yellowridgeThree.clone()
    yellowridgeFour.move(40,0)
    yellowridgeFour.draw(win)
    yellowridgeFive = yellowridgeFour.clone()
    yellowridgeFive.move(40,0)
    yellowridgeFive.draw(win)

    greenbase = blueBase.clone()
    greenbase.move(300,0)
    greenbase.setFill('green')
    greenbase.draw(win)
    greenridgeOne = blueridgeOne.clone()
    greenridgeOne.move(300,0)
    greenridgeOne.setFill('green')
    greenridgeOne.draw(win)
    greenridgeTwo = greenridgeOne.clone()
    greenridgeTwo.move(40,0)
    greenridgeTwo.draw(win)
    greenridgeThree = greenridgeTwo.clone()
    greenridgeThree.move(40,0)
    greenridgeThree.draw(win)
    greenridgeFour = greenridgeThree.clone()
    greenridgeFour.move(40,0)
    greenridgeFour.draw(win)
    greenridgeFive = greenridgeFour.clone()
    greenridgeFive.move(40,0)
    greenridgeFive.draw(win)

    redbase = yellowBase.clone()
    redbase.move(300,0)
    redbase.setFill('red')
    redbase.draw(win)
    redridgeOne = yellowridgeOne.clone()
    redridgeOne.move(300,0)
    redridgeOne.setFill('red')
    redridgeOne.draw(win)
    redridgeTwo = redridgeOne.clone()
    redridgeTwo.move(40,0)
    redridgeTwo.draw(win)
    redridgeThree = redridgeTwo.clone()
    redridgeThree.move(40,0)
    redridgeThree.draw(win)
    redridgeFour = redridgeThree.clone()
    redridgeFour.move(40,0)
    redridgeFour.draw(win)
    redridgeFive = redridgeFour.clone()
    redridgeFive.move(40,0)
    redridgeFive.draw(win)

    blackbase = redbase.clone()
    blackbase.move(0,200)
    blackbase.setFill('black')
    blackbase.setOutline('red')
    blackbase.draw(win)
    blackone = redridgeOne.clone()
    blackone.setFill('black')
    blackone.setOutline('red')
    blackone.move(0,200)
    blackone.draw(win)
    blacktwo = blackone.clone()
    blacktwo.move(40,0)
    blacktwo.draw(win)
    blackthree = blacktwo.clone()
    blackthree.move(40,0)
    blackthree.draw(win)
    blackfour = blackthree.clone()
    blackfour.move(40,0)
    blackfour.draw(win)
    blackfive = blackfour.clone()
    blackfive.move(40,0)
    blackfive.draw(win)

    tealbase = yellowBase.clone()
    tealbase.move(0,200)
    tealbase.setFill('teal')
    tealbase.draw(win)
    tealone = yellowridgeOne.clone()
    tealone.move(0,200)
    tealone.setFill('teal')
    tealone.draw(win)
    tealtwo = tealone.clone()
    tealtwo.move(40,0)
    tealtwo.draw(win)
    tealthree = tealtwo.clone()
    tealthree.move(40,0)
    tealthree.draw(win)
    tealfour = tealthree.clone()
    tealfour.move(40,0)
    tealfour.draw(win)
    tealfive = tealfour.clone()
    tealfive.move(40,0)
    tealfive.draw(win)

    input('Close program')
    win.close()

main()


from qiskit.converters import circuit_to_dag, dag_to_circuit
from qiskit.pulse import Schedule
from qiskit import QuantumCircuit
from qiskit.circuit.gate import Gate

empty_circuit = QuantumCircuit(1)

dummy_gate = Gate("my_dummy_gate", 1, [])
circuit_with_calibrated_pulse_definition = QuantumCircuit(1)
circuit_with_calibrated_pulse_definition.append(dummy_gate, qargs=[0])
circuit_with_calibrated_pulse_definition.add_calibration(dummy_gate, [0], Schedule())

empty_dag = circuit_to_dag(empty_circuit)
calibrated_dag = circuit_to_dag(circuit_with_calibrated_pulse_definition)
composed_dag = empty_dag.compose(calibrated_dag, inplace=False)

print("Empty DAG:     ", empty_dag.calibrations)
print("Calibrated DAG:", calibrated_dag.calibrations)
print("Composed DAG:  ", composed_dag.calibrations) # Should be the same as calibrated_dag


# 加载依赖
import pymongo

# 创建连接
myclient = pymongo.MongoClient("mongodb://localhost:27017/")

# 定位到数据库mongo_python和集合sites

mp = myclient["mongo_python"]

sites = mp["sites"]
#alexa--->网站的全球排名指标

documents = [
  { "name": "Taobao", "alexa": "100", "url": "https://www.taobao.com" },
  { "name": "QQ", "alexa": "101", "url": "https://www.qq.com" },
  { "name": "Facebook", "alexa": "10", "url": "https://www.facebook.com" },
  { "name": "知乎", "alexa": "103", "url": "https://www.zhihu.com" },
  { "name": "Github", "alexa": "109", "url": "https://www.github.com" }
]

ret = sites.insert_many(documents)

print(ret)

##打印插入的文档的id值


import csv
import os.path
from unidecode import unidecode

#function is used to print the headers for the csv in such a way that sub keys
# will be tied to their parent keys
# ie. key.keyA.keyAi
def prettyprint(section):
	section_dict = {}
	for key, val in section.__dict__.items():
		if not type(val).__name__ == 'instance':
			section_dict[key] = val
		else:
			for k1, v1 in val.__dict__.items():
				if not type(v1).__name__ == 'instance':
					section_dict['%s.%s' % (key, k1)] = v1
				else:
					for k2, v2 in v1.__dict__.items():
						if not type(v2).__name__ == 'instance':
							section_dict['%s.%s.%s' % (key, k1, k2)] = v2
						else:
							for k3, v3 in v2.__dict__.items():
								section_dict['%s.%s.%s.%s' % (key, k1, k2, k3)] = v3

	return section_dict

#function to write a full section to the csv
def writesection(i, ein, year, section, fields):

	keys = []
	values = []
	for k,v in fields.items():
		keys.append(k)
		values.append(v)

	outputfile = 'output/%s_%s.csv' % (section.part,year)

	#create the file if not already created
	if not os.path.isfile(outputfile):

		with open(outputfile, 'wb') as csvfile:

			w = csv.writer(csvfile, delimiter=',')

			keys.insert(0, 'taxyear')
			keys.insert(0, 'ein')
			w.writerow(keys)

			values.insert(0, year)
			values.insert(0, ein)

			try:
				##Will catch unicode errors for things like accents
				w.writerow(values)
			except UnicodeEncodeError:
				for i in range(len(values)):
					if isinstance (values[i], unicode):
						values[i] = unidecode(values[i])
				w.writerow(values)

	#otherwise, append
	else:

		with open(outputfile, 'a') as csvfile:

			w = csv.writer(csvfile, delimiter=',')

			values.insert(0, year)
			values.insert(0, ein)

			try:
				##Will catch unicode errors for things like accents
				w.writerow(values)
			except UnicodeEncodeError:
				for i in range(len(values)):
					if isinstance (values[i], unicode):
						values[i] = unidecode(values[i])
				w.writerow(values)

#function to write just the lists of specific sections
def writelist(i, ein, year, lst):

	keys = []

	for l in range(len(lst)):

		outputfile = 'output/%s_%s.csv' % (lst[l].part,year)

		values = []

		if not os.path.isfile(outputfile):

			for k, v in lst[l].__dict__.items():
				if k != 'part':
					if type(v).__name__ == 'instance':
						for k1, v2 in v.__dict__.items():
							keys.append('%s.%s' % (k, k1))
							values.append(v2)
					else:
						keys.append(k)
						values.append(v)

			keys.insert(0, 'taxyear')
			keys.insert(0, 'ein')

			values.insert(0, year)
			values.insert(0, ein)

			with open(outputfile, 'wb') as csvfile:

				w = csv.writer(csvfile, delimiter=',')
				w.writerow(keys)

				try:
					##Will catch unicode errors for things like accents
					w.writerow(values)
				except UnicodeEncodeError:
					for i in range(len(values)):
						if isinstance (values[i], unicode):
							values[i] = unidecode(values[i])
					w.writerow(values)

		else:

			with open(outputfile, 'r') as csvfile:

				reader = csv.reader(csvfile)
				headers = next(reader)

			with open(outputfile, 'a') as csvfile:

				w = csv.writer(csvfile, delimiter=',')

				for k in headers:
					if k != 'ein' and k != 'taxyear':
						# print lst, k
						if '.' in k:
							key1 = k.rpartition('.')[0]
							key2 = k.rpartition('.')[2]
							val = getattr(getattr(lst[l], key1), key2)
							values.append(val)
						else:
							val = getattr(lst[l], k)
							values.append(val)

				values.insert(0, year)
				values.insert(0, ein)

				try:
					##Will catch unicode errors for things like accents
					w.writerow(values)
				except UnicodeEncodeError:
					for i in range(len(values)):
						if isinstance (values[i], unicode):
							values[i] = unidecode(values[i])
					w.writerow(values)