import csv
from sys import platform
import pickle
import math
import os
import json

data_stored = False
data = []

file_path_MacOS = 'tools/SampleData.csv'
file_path_Windows = 'SampleData.csv'
file_path = ''

# Open SampleData file store data into data variable
def openSampleData():
    global data_stored

    # Check OS for file path
    if platform == "darwin":
        file_path = file_path_MacOS
    elif platform == "win32":
        file_path = file_path_Windows
    else:
        print("ERROR: Operating system not supported for file open")
        file_path = file_path_MacOS

    try:
        with open(file_path, 'r', encoding='utf-8-sig') as file:
            csv_reader = csv.reader(file)
            data = [row for row in csv_reader]
            data_stored = True
        return data
    except FileNotFoundError:
        print("File tools/SampleData.csv not found.")
        return []

# Returns a full row sample packet
def getSamplePacket():
    # Aquire data into data variable
    if not data_stored:
        data = openSampleData()

    return data[0]

# Returns a cleaned up sample packet (first portion of data in the packet)
def getCleanedSamplePacket():
    # Aquire data into data variable
    if not data_stored:
        data = openSampleData()

    packet = data[0]            # Value is an array
    packet_string = packet[0]   # Value is the string inside the array (full packet data)

    # Parse to get first portion of data
    start_index = packet_string.find("BrainBitSignalData(")
    end_index = packet_string.find(")")

    return packet_string[start_index:end_index + 1]

def parseUserData(user_id):
    # Initialize variables to accumulate values
    o1_sum = 0
    o2_sum = 0
    t3_sum = 0
    t4_sum = 0
    sample_count = 0

    user_id = 'temp'

    create_user_directory(user_id)  # Creating user directory if it has not been created
    in_filename = f"{user_id}_raw.pkl" # Will change to User ID specific file when connection to front end is made
    out_filename = os.path.join("user_data", user_id, f"{user_id}_parsed.pkl")
    #in_filename = "SampleData.pkl" # Comment in for debug purposes

    # Edit this in to print the data
    # with open(filename, 'rb') as file:
    #     data = pickle.load(file)
    #     print(data)
    #     while data:
    #         data = pickle.load(file)
    #         print(data)

    try:
        with open(in_filename, 'rb') as file: # Opens file in read binary mode
            data = pickle.load(file) # Will return one packet
            while data: # While data is still being read from the .pkl file
                o1_sum += sum(sample.O1 for sample in data) # Sum will add up all the values for o1 given for ONE packet and the will add that sum to a running total for O1
                o2_sum += sum(sample.O2 for sample in data)
                t3_sum += sum(sample.T3 for sample in data)
                t4_sum += sum(sample.T4 for sample in data)
                sample_count += len(data) # Will count the amount of samples given per packet (can be 2 4 or 6 samples per packet)
                data = pickle.load(file) # Load the new packet
    except FileNotFoundError:
        print(f"File '{in_filename}' not found.")
    except Exception as e:
        print(e) # Exists to notify when data can no longer be read

    # Calculate the averages
    if sample_count > 0:
        o1_avg = o1_sum / sample_count
        o2_avg = o2_sum / sample_count
        t3_avg = t3_sum / sample_count
        t4_avg = t4_sum / sample_count
        averages = [o1_avg, o2_avg, t3_avg, t4_avg]
        # Store the list of averages using pickle
        with open(out_filename, 'wb') as file:
            pickle.dump(averages, file)

        # Comment in for debug purposes
        # print(f"Average O1: {o1_avg}")
        # print(f"Average O2: {o2_avg}")
        # print(f"Average T3: {t3_avg}")
        # print(f"Average T4: {t4_avg}")
    else:
        print("No data available to calculate averages.")

# User matching using euclidean distance ( user_data_x is a list for averages [O1, O2, T3, T4] )
def calculate_user_match(user_data_1, user_data_2):
    # Calculate difference between both users data
    o1_difference = user_data_1[0] - user_data_2[0]
    o2_difference = user_data_1[1] - user_data_2[1]
    t3_difference = user_data_1[2] - user_data_2[2]
    t4_difference = user_data_1[3] - user_data_2[3]

    # Calculate euclidean distance using difference calculations
    euclidean_distance = math.sqrt( (o1_difference)**2 +
                                    (o2_difference)**2 +
                                    (t3_difference)**2 +
                                    (t4_difference)**2 )

    # lower distance = better match
    return euclidean_distance

# Return list of subdirectories from a root directory
def get_subdirectories(root_dir):
    subdir = []

    # Go through each file/folder in the root directory
    for item in os.listdir(root_dir):
        # Check if the item is a sub-directory
        full_path = os.path.join(root_dir, item)
        if os.path.isdir(full_path):
            subdir.append(item)

    return subdir

# Returns data from pkl file (check for file exist done before function call)
def get_data_from_pkl(pkl_path):
    pkl_data = None

    with open(pkl_path, 'rb') as file:
            pkl_data = pickle.load(file)

    return pkl_data

# Create json file to save matches if not already created
def create_matches_json():
    path_to_json = os.path.join('user_data', 'user_matches.json')

    # Create empty json if doesnt exist
    if not os.path.exists(path_to_json):
        matches = {}

        # Write the empty dictionary to the file
        with open(path_to_json, 'w') as file:
            json.dump(matches, file)

# Loads existing matches from json and returns them as dictionary
def load_existing_matches():
    path_to_json = os.path.join('user_data', 'user_matches.json')
    matches = {}

    # Create empty json if doesnt exist
    if os.path.exists(path_to_json):
        with open(path_to_json, 'r') as file:
            matches = json.load(file)
    else:
        print("ERROR: JSON does not exists ", path_to_json)
        return None

    return matches

# Add a match to the json dictionary
def add_to_matches(matches, cur_user_id, another_user, euclidean_distance):
    # Check if cur_user_id is existing as a key
    if cur_user_id in matches:
        # The other user is being added or if already exists, euclidean is updated
        matches[cur_user_id][another_user] = euclidean_distance  # Add to list of users (matches)
    else:
        # Create a new dictionary for this user_id
        matches[cur_user_id] = {}
        # Add other user as a match
        matches[cur_user_id][another_user] = euclidean_distance

# Save the matches dictionary to the matches json
def save_matches_to_json(matches):
    path_to_json = os.path.join('user_data', 'user_matches.json')

    # Write the updated dictionary back to the file
    with open(path_to_json, 'w') as file:
        json.dump(matches, file, indent=4)

# Function will go through other users data to find matches for current user
def find_user_matches(current_user_id):
    root_directory = 'user_data'
    subdir = get_subdirectories(root_directory)

    current_user_data_path = os.path.join(root_directory, current_user_id, f'{current_user_id}_parsed.pkl')
    current_user_data = None

    create_matches_json()   # Create json file to save matches if not already created
    existing_matches = load_existing_matches()  # Load existing saved matches dict from json

    # Check if current user has parsed data
    if os.path.exists(current_user_data_path):
        # Save current user's data
        current_user_data = get_data_from_pkl(current_user_data_path)
    else:
        print("ERROR: Current user path, ", current_user_data_path, " not found.")
        return None

    # Loop through each subdir in user_data directory ( user_id is a subdirectory name )
    for other_user_id in subdir:
        # Create full path to user parsed data
        other_user_path = os.path.join(root_directory, other_user_id, f'{other_user_id}_parsed.pkl')

        # Run calculation to find matches
        if os.path.exists(other_user_path) and other_user_id != current_user_id:
            other_user_data = get_data_from_pkl(other_user_path) # get other user's data

            # Call function to calculate euclidean distance between both users
            euclidean_distance = calculate_user_match(current_user_data, other_user_data)

            # If criteria for a match is met and it is not a rejected match, add the match
            if euclidean_distance < 5 and not is_rejected_match(current_user_id, other_user_id):
                add_to_matches(existing_matches, current_user_id, other_user_id, euclidean_distance)

        elif other_user_id != current_user_id:
            print("ERROR: ", other_user_path, " not found.")

    # saves all changes back to user_matches.json
    save_matches_to_json(existing_matches)

# Creating a new user directory, user_id is a string
def create_user_directory(user_id):
    path_to_user_dir = os.path.join('user_data', user_id)
    os.makedirs(path_to_user_dir, exist_ok=True)    # If directory already exists, it will not be replaced

#
#   Reject User utils
#

# Allowing a user to reject a match
def reject_match(user_id, user_to_reject_id):
    # Create json to save rejected matches if not already created
    create_rejected_matches_json()

    # Load existing rejected matches from json
    rejected_matches = load_existing_rejected_matches()

    # Add the new rejected matches to variable
    add_to_rejected_matches(rejected_matches, user_id, user_to_reject_id)

    # Saved rejected matches variable to json
    save_rejected_matches_to_json(rejected_matches)

    # Update matches json based on the rejection
    update_matches_from_reject(user_id, user_to_reject_id)

# Check rejected_matches.json to see if a user is rejected or other user rejected current user,return bool
def is_rejected_match(user1_id, user2_id):
    rejected_matches = load_existing_rejected_matches()
    if user2_id in rejected_matches.get(user1_id, []) or user1_id in rejected_matches.get(user2_id, []):
        return True
    else:
        return False

# Removes the match from both users
def update_matches_from_reject(user1_id, user2_id):
    matches = load_existing_matches()

    # Check if user is a key
    if user1_id in matches:
        # Check if user to be removed is an existing value
        if user2_id in matches[user1_id]:
            matches[user1_id].pop(user2_id)

    # Check if user is a key
    if user2_id in matches:
        # Check if user to be removed is an existing value
        if user1_id in matches[user2_id]:
            matches[user2_id].pop(user1_id)

    save_matches_to_json(matches)

# Create json file to save rejected matches if not already created
def create_rejected_matches_json():
    path_to_json = os.path.join('user_data', 'rejected_matches.json')

    # Create empty json if doesnt exist
    if not os.path.exists(path_to_json):
        rejected_matches = {}

        # Write the empty dictionary to the file
        with open(path_to_json, 'w') as file:
            json.dump(rejected_matches, file)

# Loads existing rejected matches from json and returns them as dictionary
def load_existing_rejected_matches():
    path_to_json = os.path.join('user_data', 'rejected_matches.json')
    rejected_matches = {}

    # Create empty json if doesnt exist
    if os.path.exists(path_to_json):
        with open(path_to_json, 'r') as file:
            rejected_matches = json.load(file)
    else:
        print("ERROR: JSON does not exists ", path_to_json)
        return None

    return rejected_matches

# Add a rejected_match to the json dictionary
def add_to_rejected_matches(rejected_matches, cur_user_id, rejected_user):
    # Check if cur_user_id is existing as a key
    if cur_user_id in rejected_matches:
        # Check if rejection already exists
        if rejected_user not in rejected_matches[cur_user_id]:
            # The other user is being added or if exists nothing changes
            rejected_matches[cur_user_id].append(rejected_user)
    else:
        # Add other user as a rejected_match
        rejected_matches[cur_user_id] = [rejected_user]

# Save the rejected matches dictionary to the matches json
def save_rejected_matches_to_json(rejected_matches):
    path_to_json = os.path.join('user_data', 'rejected_matches.json')

    # Write the updated dictionary back to the file
    with open(path_to_json, 'w') as file:
        json.dump(rejected_matches, file, indent=4)


import json
import logging
from util import rest_client

logger = logging.getLogger("grafeas.auth_util")

class Subject(object):
    def __init__(self, kind, id, account_id, email=None):
        self.kind = kind
        self.id = id
        self.account_id = account_id
        self.email = email

    def __str__(self):
        return "{{kind:{}, id:{}, account:{}, email:{}}}".format(self.kind, self.id, self.account_id, self.email)

    def to_dict(self):
        return {
            'kind': self.kind,
            'id': self.id,
            'account_id': self.account_id,
            'email': self.email
        }

def get_subject(decoded_auth_token):
    if 'sub_type' not in decoded_auth_token:
        kind = 'user'
    else:
        sub_type = decoded_auth_token['sub_type']
        if sub_type == 'ServiceId':
            kind = 'service-id'
        else:
            raise ValueError("Invalid IAM token: unsupported '{}' subject type".format(sub_type))

    iam_id = decoded_auth_token.get('iam_id')
    if iam_id is None:
        raise ValueError("Invalid IAM token: missing 'iam_id' field")

    account = decoded_auth_token.get('account')
    if not account:
        raise ValueError("Invalid IAM token: missing 'account' field")

    account_id = account.get('bss')
    if not account_id:
        raise ValueError("Invalid IAM token: missing 'account.bss' field")

    email = decoded_auth_token.get('email')
    # email could be missing. For example, in subjects of 'service-id' kind

    return Subject(kind, iam_id, account_id, email)

def get_identity_token(iam_base_url, api_key):
    with rest_client.RestClient() as client:
        client.add_header("Accept", "application/json")
        client.add_header("Content-Type", "application/x-www-form-urlencoded")

        body = {
            "grant_type": "urn:ibm:params:oauth:grant-type:apikey",
            "apikey": api_key
        }

        response = client.post(
            url="{}/identity/token".format(iam_base_url),
            data=body)

        if response.status_code != 200:
            response.raise_for_status()

        content = json.loads(response.content.decode('utf-8'))
        return content['access_token']


import json

def read_data(file_path):
    with open(file_path, "r", encoding='utf-8-sig') as f:
        data = json.load(f)
    datas = []

    for item in data['data']:
        if 'topic' in item:
            topic = item['topic']

        sentences = []
        if 'utterance' in item:
            sentence = item['utterance']
        datas.append((sentence, topic))
    return datas

read_data("combined_data.json")


import numpy as np
from abc import ABCMeta, abstractmethod
from glmnet import LogitNet
from scipy.stats import gaussian_kde, rankdata, norm
from sklearn.covariance import ledoit_wolf

from abcpy.graphtools import GraphTools

class Approx_likelihood(metaclass=ABCMeta):
    """This abstract base class defines the approximate likelihood
    function.
    """

    @abstractmethod
    def __init__(self, statistics_calc):
        """
        The constructor of a sub-class must accept a non-optional statistics
        calculator; then, it must call the __init__ method of the parent class. This ensures that the
        object is initialized correctly so that the _calculate_summary_stat private method can be called when computing
        the distances.

        Parameters
        ----------
        statistics_calc : abcpy.statistics.Statistics
            Statistics extractor object that conforms to the Statistics class.
        """
        self.statistics_calc = statistics_calc

        # Since the observations do always stay the same, we can save the
        #  summary statistics of them and not recalculate it each time
        self.stat_obs = None
        self.data_set = None
        self.dataSame = False

    @abstractmethod
    def loglikelihood(self, y_obs, y_sim):
        """To be overwritten by any sub-class: should compute the approximate loglikelihood
        value given the observed data set y_obs and the data set y_sim simulated from
        model set at the parameter value.

        Parameters
        ----------
        y_obs: Python list
            Observed data set.
        y_sim: Python list
            Simulated data set from model at the parameter value.

        Returns
        -------
        float
            Computed approximate loglikelihood.
        """

        raise NotImplemented

    def likelihood(self, y_obs, y_sim):
        """Computes the likelihood by taking the exponential of the loglikelihood method.

        Parameters
        ----------
        y_obs: Python list
            Observed data set.
        y_sim: Python list
            Simulated data set from model at the parameter value.

        Returns
        -------
        float
            Computed approximate likelihood.

        """
        return np.exp(self.loglikelihood(y_obs, y_sim))

    def _calculate_summary_stat(self, y_obs, y_sim):
        """Helper function that extracts the summary statistics s_obs and s_sim from y_obs and
        y_y_sim using the statistics object stored in self.statistics_calc. This stores s_obs for the purpose of checking
        whether that is repeated in next calls to the function, and avoiding computing the statitistics for the same
        dataset several times.

        Parameters
        ----------
        y_obs : array-like
            d1 contains n_obs data sets.
        y_sim : array-like
            d2 contains n_sim data sets.

        Returns
        -------
        tuple
            Tuple containing numpy.ndarray's with the summary statistics extracted from d1 and d2.
        """
        if not isinstance(y_obs, list):
            raise TypeError('Observed data is not of allowed types')

        if not isinstance(y_sim, list):
            raise TypeError('simulated data is not of allowed types')

        # Check whether y_obs is same as the stored dataset.
        if self.data_set is not None:
            # check that the the observations have the same length; if not, they can't be the same:
            if len(y_obs) != len(self.data_set):
                self.dataSame = False
            elif len(np.array(y_obs[0]).reshape(-1, )) == 1:
                self.dataSame = self.data_set == y_obs
            else:  # otherwise it fails when y_obs[0] is array
                self.dataSame = all(
                    [(np.array(self.data_set[i]) == np.array(y_obs[i])).all() for i in range(len(y_obs))])

        if self.stat_obs is None or self.dataSame is False:
            self.stat_obs = self.statistics_calc.statistics(y_obs)
            self.data_set = y_obs

        # Extract summary statistics from the simulated data
        stat_sim = self.statistics_calc.statistics(y_sim)

        if self.stat_obs.shape[1] != stat_sim.shape[1]:
            raise ValueError("The dimension of summaries in the two datasets is different; check the dimension of the"
                             " provided observations and simulations.")

        return self.stat_obs, stat_sim

class SynLikelihood(Approx_likelihood):

    def __init__(self, statistics_calc):
        """This class implements the approximate likelihood function which computes the approximate
        likelihood using the synthetic likelihood approach described in Wood [1].
        For synthetic likelihood approximation, we compute the robust precision matrix using Ledoit and Wolf's [2]
        method.

        [1] S. N. Wood. Statistical inference for noisy nonlinear ecological
        dynamic systems. Nature, 466(7310):1102–1104, Aug. 2010.

        [2] O. Ledoit and M. Wolf, A Well-Conditioned Estimator for Large-Dimensional Covariance Matrices,
        Journal of Multivariate Analysis, Volume 88, Issue 2, pages 365-411, February 2004.

        Parameters
        ----------
        statistics_calc : abcpy.statistics.Statistics
            Statistics extractor object that conforms to the Statistics class.
        """

        super(SynLikelihood, self).__init__(statistics_calc)

    def loglikelihood(self, y_obs, y_sim):
        """Computes the loglikelihood.

        Parameters
        ----------
        y_obs: Python list
            Observed data set.
        y_sim: Python list
            Simulated data set from model at the parameter value.

        Returns
        -------
        float
            Computed approximate loglikelihood.

        """

        stat_obs, stat_sim = self._calculate_summary_stat(y_obs, y_sim)

        # Compute the mean, robust precision matrix and determinant of precision matrix
        mean_sim = np.mean(stat_sim, 0)
        lw_cov_, _ = ledoit_wolf(stat_sim)
        robust_precision_sim = np.linalg.inv(lw_cov_)
        sign_logdet, robust_precision_sim_logdet = np.linalg.slogdet(robust_precision_sim)  # we do not need sign
        # print("DEBUG: combining.")
        # we may have different observation; loop on those now:
        # likelihoods = np.zeros(stat_obs.shape[0])
        # for i, single_stat_obs in enumerate(stat_obs):
        #     x_new = np.einsum('i,ij,j->', single_stat_obs - mean_sim, robust_precision_sim, single_stat_obs - mean_sim)
        #     likelihoods[i] = np.exp(-0.5 * x_new)
        # do without for loop:
        diff = stat_obs - mean_sim.reshape(1, -1)
        x_news = np.einsum('bi,ij,bj->b', diff, robust_precision_sim, diff)
        logliks = -0.5 * x_news
        logfactor = 0.5 * self.stat_obs.shape[0] * robust_precision_sim_logdet
        return np.sum(logliks) + logfactor  # compute the sum of the different loglikelihoods for each observation

class SemiParametricSynLikelihood(Approx_likelihood):

    def __init__(self, statistics_calc, bw_method_marginals="silverman"):
        """
        This class implements the approximate likelihood function which computes the approximate
        likelihood using the semiparametric Synthetic Likelihood (semiBSL) approach described in [1]. Specifically, this
        represents the likelihood as a product of univariate marginals and the copula components (exploiting Sklar's
        theorem).
        The marginals are approximated from simulations using a Gaussian KDE, while the copula is assumed to be a Gaussian
        copula, whose parameters are estimated from data as well.

        This does not yet include shrinkage strategies for the correlation matrix.

        [1] An, Z., Nott, D. J., & Drovandi, C. (2020). Robust Bayesian synthetic likelihood via a semi-parametric approach.
        Statistics and Computing, 30(3), 543-557.

        Parameters
        ----------
        statistics_calc : abcpy.statistics.Statistics
            Statistics extractor object that conforms to the Statistics class.
        bw_method_marginals : str, scalar or callable, optional
            The method used to calculate the estimator bandwidth, passed to `scipy.stats.gaussian_kde`. Following the docs
            of that method, this can be 'scott', 'silverman', a scalar constant or a callable. If a scalar, this will be
            used directly as `kde.factor`. If a callable, it should take a `gaussian_kde` instance as only parameter
            and return a scalar. If None (default), 'silverman' is used. See the Notes in `scipy.stats.gaussian_kde`
            for more details.
        """
        super(SemiParametricSynLikelihood, self).__init__(statistics_calc)
        # create a dict in which store the denominator of the correlation matrix for the different n values;
        # this saves from repeating computations:
        self.corr_matrix_denominator = {}
        self.bw_method_marginals = bw_method_marginals  # the bw method to use in the gaussian_kde

    def loglikelihood(self, y_obs, y_sim):
        """Computes the loglikelihood. This implementation aims to be equivalent to the `BSL` R package,
        but the results are slightly different due to small differences in the way the KDE is performed

        Parameters
        ----------
        y_obs: Python list
            Observed data set.
        y_sim: Python list
            Simulated data set from model at the parameter value.

        Returns
        -------
        float
            Computed approximate loglikelihood.
        """

        stat_obs, stat_sim = self._calculate_summary_stat(y_obs, y_sim)
        n_obs, d = stat_obs.shape
        if d < 2:
            raise RuntimeError("The dimension of the statistics need to be at least 2 in order to apply semiBSL.")

        # first: estimate the marginal KDEs for each coordinate
        logpdf_obs = np.zeros_like(stat_obs)  # this will contain the estimated pdf at the various observation points
        u_obs = np.zeros_like(stat_obs)  # this instead will contain the transformed u's using the estimated CDF
        for j in range(d):
            # estimate the KDE using the data in stat_sim for coordinate j. This leads to slightly different results
            # from the R package implementation due to slightly different way to estimate the factor as well as
            # different way to evaluate the kernel (they use a weird interpolation there).
            kde = gaussian_kde(stat_sim[:, j], bw_method=self.bw_method_marginals)
            logpdf_obs[:, j] = kde.logpdf(stat_obs[:, j])
            for i in range(n_obs):  # loop over the different observations
                u_obs[i, j] = kde.integrate_box_1d(-np.infty, stat_obs[i, j])  # compute the CDF
        etas_obs = norm.ppf(u_obs)

        # second: estimate the correlation matrix for the gaussian copula using gaussian rank correlation
        R_hat = self._estimate_gaussian_correlation(stat_sim)
        R_hat_inv = np.linalg.inv(R_hat)
        R_sign_det, R_inv_logdet = np.linalg.slogdet(R_hat_inv)  # sign not used

        # third: combine the two to compute the loglikelihood;
        # for each observation:
        # logliks = np.zeros(n_obs)
        # for i in range(n_obs):
        #     logliks[i] = np.sum(logpdf_obs[i])  # sum along marginals along dimensions
        #     # add the copula density:
        #     logliks[i] += 0.5 * R_inv_logdet
        #     logliks[i] -= 0.5 * np.einsum("i,ij,j->", etas_obs[i], R_hat_inv - np.eye(d), etas_obs[i])

        # do jointly:
        loglik = np.sum(logpdf_obs)  # sum along marginals along dimensions
        # add the copula density:
        copula_density = -0.5 * np.einsum("bi,ij,bj->b", etas_obs, R_hat_inv - np.eye(d), etas_obs)
        loglik += np.sum(copula_density) + 0.5 * n_obs * R_inv_logdet

        return loglik

    def _estimate_gaussian_correlation(self, x):
        """Estimates the correlation matrix using data in `x` in the way described in [1]. This implementation
        gives the same results as the `BSL` R package.

        Parameters
        ----------
        x: np.ndarray
            Data set.

        Returns
        -------
        np.ndarray
            Estimated correlation matrix for the gaussian copula.
        """
        n, d = x.shape
        r = np.zeros_like(x)
        for j in range(d):
            r[:, j] = rankdata(x[:, j])

        rqnorm = norm.ppf(r / (n + 1))

        if n not in self.corr_matrix_denominator.keys():
            # compute the denominator:
            self.corr_matrix_denominator[n] = np.sum(norm.ppf((np.arange(n) + 1) / (n + 1)) ** 2)
        denominator = self.corr_matrix_denominator[n]

        R_hat = np.einsum('ki,kj->ij', rqnorm, rqnorm) / denominator

        return R_hat

class PenLogReg(Approx_likelihood, GraphTools):

    def __init__(self, statistics_calc, model, n_simulate, n_folds=10, max_iter=100000, seed=None):
        """This class implements the approximate likelihood function which computes the approximate
        likelihood up to a constant using penalized logistic regression described in
        Dutta et. al. [1]. It takes one additional function handler defining the
        true model and two additional parameters n_folds and n_simulate correspondingly defining number
        of folds used to estimate prediction error using cross-validation and the number
        of simulated dataset sampled from each parameter to approximate the likelihood
        function. For lasso penalized logistic regression we use glmnet of Friedman et.
        al. [2].

        [1] Thomas, O., Dutta, R., Corander, J., Kaski, S., & Gutmann, M. U. (2020).
        Likelihood-free inference by ratio estimation. Bayesian Analysis.

        [2] Friedman, J., Hastie, T., and Tibshirani, R. (2010). Regularization
        paths for generalized linear models via coordinate descent. Journal of Statistical
        Software, 33(1), 1–22.

        Parameters
        ----------
        statistics_calc : abcpy.statistics.Statistics
            Statistics extractor object that conforms to the Statistics class.
        model : abcpy.models.Model
            Model object that conforms to the Model class.
        n_simulate : int
            Number of data points to simulate for the reference data set; this has to be the same as n_samples_per_param
            when calling the sampler. The reference data set is generated by drawing parameters from the prior and
            samples from the model when PenLogReg is instantiated.
        n_folds: int, optional
            Number of folds for cross-validation. The default value is 10.
        max_iter: int, optional
            Maximum passes over the data. The default is 100000.
        seed: int, optional
            Seed for the random number generator. The used glmnet solver is not
            deterministic, this seed is used for determining the cv folds. The default value is
            None.
        """

        super(PenLogReg, self).__init__(statistics_calc)  # call the super init to initialize correctly

        self.model = model
        self.n_folds = n_folds
        self.n_simulate = n_simulate
        self.seed = seed
        self.rng = np.random.RandomState(seed)
        self.max_iter = max_iter
        # Simulate reference data and extract summary statistics from the reference data
        self.ref_data_stat = self._simulate_ref_data(rng=self.rng)[0]

    def loglikelihood(self, y_obs, y_sim):
        """Computes the loglikelihood.

        Parameters
        ----------
        y_obs: Python list
            Observed data set.
        y_sim: Python list
            Simulated data set from model at the parameter value.

        Returns
        -------
        float
            Computed approximate loglikelihood.
        """
        stat_obs, stat_sim = self._calculate_summary_stat(y_obs, y_sim)

        if not stat_sim.shape[0] == self.n_simulate:
            raise RuntimeError("The number of samples in the reference data set is not the same as the number of "
                               "samples in the generated data. Please check that `n_samples` in the `sample()` method"
                               "for the sampler is equal to `n_simulate` in PenLogReg.")

        # Compute the approximate likelihood for the y_obs given theta
        y = np.append(np.zeros(self.n_simulate), np.ones(self.n_simulate))
        X = np.array(np.concatenate((stat_sim, self.ref_data_stat), axis=0))
        # define here groups for cross-validation:
        groups = np.repeat(np.arange(self.n_folds), np.int(np.ceil(self.n_simulate / self.n_folds)))
        groups = groups[:self.n_simulate].tolist()
        groups += groups  # duplicate it as groups need to be defined for both datasets
        m = LogitNet(alpha=1, n_splits=self.n_folds, max_iter=self.max_iter, random_state=self.seed, scoring="log_loss")
        m = m.fit(X, y, groups=groups)
        result = -np.sum((m.intercept_ + np.sum(np.multiply(m.coef_, stat_obs), axis=1)), axis=0)

        return result

    def _simulate_ref_data(self, rng=np.random.RandomState()):
        """
        Simulate the reference data set. This code is run at the initialization of
        Penlogreg

        Parameters
        ----------
        rng: Random number generator, optional
            Defines the random number generator to be used. If None, a newly initialized one is used

        Returns
        -------
        list
            The simulated list of datasets.

        """

        ref_data_stat = [[None] * self.n_simulate for i in range(len(self.model))]
        self.sample_from_prior(rng=rng)
        for model_index, model in enumerate(self.model):
            ind = 0
            while ref_data_stat[model_index][-1] is None:
                data = model.forward_simulate(model.get_input_values(), 1, rng=rng)
                # this is wrong, it applies the computation of the statistic independently to the element of data[0]:
                # print("data[0]", data[0].tolist())
                # data_stat = self.statistics_calc.statistics(data[0].tolist())
                # print("stat of data[0]", data_stat)
                # print("data", data)
                data_stat = self.statistics_calc.statistics(data)
                # print("stat of data", data_stat)
                ref_data_stat[model_index][ind] = data_stat
                ind += 1
            ref_data_stat[model_index] = np.squeeze(np.asarray(ref_data_stat[model_index]))
        return ref_data_stat


import json

from django.contrib.auth.models import User
from rest_framework import status
from rest_framework.generics import ListAPIView, ListCreateAPIView, DestroyAPIView
from rest_framework.response import Response
from datetime import datetime
import pytz
from rest_framework.views import APIView
from bookmeeting.api.public import week_range
from .api_serializer import MeetingRoomSerializer, UserSerializer, BookingInfoSerializer
from bookmeeting.models import MeetingRoom, BookingInfo
import logging
from django.db import transaction

logger = logging.getLogger('book-meeting')

class MeetRoomView(ListAPIView):
    serializer_class = MeetingRoomSerializer
    queryset = MeetingRoom.objects.all()

class UserListView(ListAPIView):
    serializer_class = UserSerializer
    queryset = User.objects.exclude(username__in=['admin', 'oamanager'])

class BookView(ListCreateAPIView, DestroyAPIView):
    serializer_class = BookingInfoSerializer

    def get_queryset(self):
        queryset = BookingInfo.objects.all()
        return queryset

    def perform_create(self, serializer):
        # 保存外键表的方法
        serializer.save(meeting_room=MeetingRoom.objects.get(name=self.request.data.get('meeting_room')))

    def get(self, request, *args, **kwargs):
        return self.list(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        # TODO 预定的时候会出现覆盖的情况，目前没想好怎么处理
        today = datetime.today()
        wek = week_range(today)
        # 只允许预定本周的会议室
        st, ed = wek[0].strftime('%Y-%m-%d 00:00:00'), wek[1].strftime('%Y-%m-%d 23:59:59')

        request_data = request.data
        start_time = request_data.get('start_time')
        end_time = request_data.get('end_time')
        book_user = request_data.get('user')
        location = request_data.get('meeting_room')
        subject = request_data.get('subject')
        members = request_data.get('member')

        # 回滚操作
        # def __rollback(id):
        #     d = BookingInfo.objects.get(id=id)
        #     d.delete()

        # invitation = request_data.get('invitation') 必须邀请
        if start_time < st or start_time > ed or end_time < st or end_time > ed:
            return Response({
                'message': '只允许预定本周的会议室',
                'status': 901
            })

        save = self.create(request, *args, **kwargs)
        if save.status_code == 201:
            logger.debug('meeting room book success, now send invitation to members.')
            from bookmeeting.api.mail import send_invitation
            logger.debug('request data: {}'.format(request_data))
            for you in members.split(';'):
                if len(you) == 0: continue
                logger.debug('send invitation info: {}'.format(
                    json.dumps({
                        'you': you, 'book_user': book_user, 'subject': subject, 'start_time': start_time,
                        'end_time': end_time,
                        'member': members, 'meeting_room': location
                    }, indent=4)))
                try:
                    send_invitation(you, book_user, subject, start_time, end_time, members, location)
                except Exception as e:
                    save_id = save.data.get('id')
                    logger.error('send invitation error, book success, error:{}'.format(e))
                    logger.error('rollback database id: {}'.format(save_id))
                    return Response({
                        'message': '预定成功，邮件失败',
                        'status': 209
                    })
        return save

    def delete(self, request, *args, **kwargs):
        request_data = request.data
        id = request_data.get('id', None)
        book_user = request_data.get('bookUser', None)
        if id:
            b = BookingInfo.objects.get(id=id)
            if b.user != book_user:
                return Response({
                    'message': '您无权删除他人的预定',
                    'status': 900
                })
            else:
                b.delete()
                return Response({
                    'message': '取消预定成功',
                    'status': status.HTTP_204_NO_CONTENT
                })

class BookCollectionView(APIView):
    serializer_class = BookingInfoSerializer

    @property
    def time_deadline(self):
        today = datetime.today()
        end = {
            'start': '08:00:00',
            'forenoon': '12:00:00',
            'afternoon': '18:00:00',
            'evening': '23:59:59'
        }
        today_fmt = '{}-{}-{}'.format(today.year, today.month, today.day)
        return {
            'start': datetime.strptime('{} {}'.format(today_fmt, end.get('start')), "%Y-%m-%d %H:%M:%S").replace(
                tzinfo=pytz.timezone('Asia/Shanghai')),

            'forenoon': datetime.strptime('{} {}'.format(today_fmt, end.get('forenoon')), "%Y-%m-%d %H:%M:%S").replace(
                tzinfo=pytz.timezone('Asia/Shanghai')),

            'afternoon': datetime.strptime('{} {}'.format(today_fmt, end.get('afternoon')),
                                           "%Y-%m-%d %H:%M:%S").replace(
                tzinfo=pytz.timezone('Asia/Shanghai')),

            'evening': datetime.strptime('{} {}'.format(today_fmt, end.get('evening')), "%Y-%m-%d %H:%M:%S").replace(
                tzinfo=pytz.timezone('Asia/Shanghai')),
        }

    @staticmethod
    def fmt(minutes):
        hour = int(minutes / 60)
        minutes = int(minutes % 60)
        return '{}小时{}分'.format(hour, minutes)

    def get(self, request):
        meeting_room = request.GET.get('meetingroom', None)
        if not meeting_room:
            return Response({
                'error': 'meetingroom arguments miss.'
            })
        forenoon = BookingInfo.objects.filter(start_time__gte=self.time_deadline.get('start'),
                                              end_time__lte=self.time_deadline.get('forenoon'),
                                              meeting_room__name=meeting_room)
        afternoon = BookingInfo.objects.filter(start_time__gte=self.time_deadline.get('forenoon'),
                                               end_time__lte=self.time_deadline.get('afternoon'),
                                               meeting_room__name=meeting_room)
        evening = BookingInfo.objects.filter(start_time__gte=self.time_deadline.get('afternoon'),
                                             end_time__lte=self.time_deadline.get('evening'),
                                             meeting_room__name=meeting_room)
        forenoon_book, afternoon_book, evening_book = 0, 0, 0
        for f in forenoon: forenoon_book += f.duration
        for a in afternoon: afternoon_book += a.duration
        for e in evening: evening_book += e.duration

        response = Response({
            'forenoon_book': self.fmt(forenoon_book),
            'forenoon_rest': self.fmt(240 - forenoon_book),
            'afternoon_book': self.fmt(afternoon_book),
            'afternoon_rest': self.fmt(360 - afternoon_book),
            'evening_book': self.fmt(evening_book),
            'evening_rest': self.fmt(360 - evening_book)
        })
        return response


from transformers import AdapterTrainer, TrainingArguments
from transformers import AutoModelForCausalLM, GPT2Tokenizer

import os
import json
import logging
import argparse
from datasets import Dataset
from datasets import load_dataset
import datasets
import transformers
print(datasets.__version__)
print(transformers.__version__)

import torch
print(torch.cuda.is_available())
print(torch.cuda.device_count())
print(torch.cuda.current_device())
print(torch.cuda.device(0))
print(torch.cuda.get_device_name(0))

logging.basicConfig(filename=os.path.splitext(
    os.path.basename(__file__))[0]+'.log', level=logging.INFO)

parser = argparse.ArgumentParser(
    description='tool for training adapters for project_cc')

parser.add_argument('-path_to_processed', type=str,
                    default='/netscratch/fonseca/project_cc/datasets/processed/emotions_ds.json', help='path to processed dataset')
parser.add_argument('-mood', type=str, choices=['happy', 'surprised', 'sad', 'angry', 'neutral'],
                    help="""list of possible moods.
                    ACHTUNG: If you select more than one they'll be trained sequentially""")
parser.add_argument('--out_dir', type=str, default=os.path.dirname(__file__),
                    help='Path to output file ')

args = parser.parse_args()

path_to_processed = args.path_to_processed
mood = args.mood
out_path = args.out_dir + (mood)

def jsonl_generator(shards):
    for shard in shards:
        with open(shard) as json_obj:
            for line in json_obj.readlines():
                yield json.loads(line)

def emotions_generator():
    with open(path_to_processed) as json_obj:
        for line in json_obj.readlines():
            yield json.loads(line)

# jsonl_paths = []
# for root, dir, files in os.walk(path_to_processed):
#     for file in files:
#         jsonl_paths.append(os.path.join(root, file))

def add_special_tokens_(model, tokenizer):
    """ Add special tokens to the tokenizer and the model if they have not already been added. """
    orig_num_tokens = len(tokenizer.encoder)
    num_added_tokens = tokenizer.add_special_tokens(
        ATTR_TO_SPECIAL_TOKEN)  # doesn't add if they are already there

    model.resize_token_embeddings(len(tokenizer))
    # if num_added_tokens - orig_num_tokens > 0:
    #     print("resizing my tokens embeddings")

# Dataset processing for training and trainig functions
# Source:
# https://colab.research.google.com/github/Adapter-Hub/adapter-transformers/blob/master/notebooks/06_Text_Generation.ipynb#scrollTo=ioLpFbOfnPE6
# https://adapterhub.ml/blog/2021/04/adapters-for-generative-and-seq2seq-models-in-nlp/

# Tokenize the entries in the dataset

def encode_batch(batch):
    """Encodes a batch of input data using the model tokenizer."""
    encoding = tokenizer(batch["text"], truncation=True, max_length=1024)
    # encoding = tokenizer(batch["text"])
    # For language modeling the labels need to be the input_ids
    # encoding["labels"] = encoding["input_ids"]
    return encoding

# Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size.
def group_texts(examples):
    # Concatenate all texts.
    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
    total_length = len(concatenated_examples[list(examples.keys())[0]])
    # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
    # customize this part to your needs.
    total_length = (total_length // block_size) * block_size
    # Split by chunks of max_len.
    result = {
        k: [t[i: i + block_size] for i in range(0, total_length, block_size)]
        for k, t in concatenated_examples.items()
    }
    result["labels"] = result["input_ids"].copy()
    return result

# for line in jsonl_generator(jsonl_paths):
    # for k, v in line.items():
    #     if type(v) == float:
    #         logging.info('nan lines:\n{} : {}'.format(k, v))

print(path_to_processed)
moody_dataset = Dataset.from_generator(emotions_generator)
# moody_dataset = load_dataset('json',data_files=path_to_processed, split=['train','test'])
# print(type(moody_dataset))

cur_ds = moody_dataset.filter(
    lambda example: example["EmotionTag"] == mood)
print(cur_ds.column_names)

cur_ds = cur_ds.train_test_split(test_size=0.2)

del moody_dataset

print('Loading pre_trained gpt2 tokenizer')
tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
print("tokenizer len: {}".format(len(tokenizer)))
# The GPT-2 tokenizer does not have a padding token. In order to process the data
# in batches we set one here
# tokenizer.pad_token = tokenizer.eos_token

SPECIAL_TOKENS = ["<bos>", "<eos>", "<speaker1>", "<speaker2>", "<pad>"]
ATTR_TO_SPECIAL_TOKEN = {'bos_token': '<bos>', 'eos_token': '<eos>', 'pad_token': '<pad>',
                         'additional_special_tokens': ['<speaker1>', '<speaker2>']}
tokenizer.add_special_tokens(ATTR_TO_SPECIAL_TOKEN)
print("tokenizer len: {}".format(len(tokenizer)))
print('set pad token')
column_names = cur_ds["train"].column_names

dataset = cur_ds.map(encode_batch, remove_columns=column_names, batched=True)

# reate chunks with a length of block_size.
print('Creating chunks')
block_size = 50
dataset = dataset.map(group_texts, batched=True,)

dataset.set_format(type="torch", columns=[
                   "input_ids", "attention_mask", "labels"])

print(len(dataset['test'][0]['input_ids']))
###############################################################

# config = AutoConfig.from_json_file("./model/config.json")
model = AutoModelForCausalLM.from_pretrained("/netscratch/fonseca/project_cc/model")
a = model.config
print(a)
add_special_tokens_(model, tokenizer)
b = model.config
print(b)
assert a == b
# add new adapter
model.add_adapter(mood)
# activate adapter for training
model.train_adapter(mood)

##############################################################
training_args = TrainingArguments(
    output_dir="./examples",
    do_train=True,
    remove_unused_columns=False,
    learning_rate=5e-4,
    num_train_epochs=15,
)

trainer = AdapterTrainer(
    model=model,
    args=training_args,
    tokenizer=tokenizer,
    train_dataset=dataset["train"],
    eval_dataset=dataset["test"],
)
try:
    trainer.train()
except:
    # print("didn't train")
    logging.WARNING('Did not train')

model.save_adapter("adapter_"+mood, mood)


import tensorflow as tf
import numpy as np
import model3 as M
import network
import util
import config
import datasets
from text.symbols import symbols
from tqdm import tqdm
import time

def create_gta(model, trainset, savepath):
	for i in range(trainset.maxiters):
		x, m, ids, m_lens = trainset.get_next()
		_, gta, _ = model(x, m)
		gta = gta.numpy()
		for j in range(len(ids)):
			mel = gta[j][:,:m_lens[j]]
			mel = (mel + 4) / 8
			ID = ids[j]
			np.save(f'{savepath}{ID}.npy', mel, allow_pickle=False)
		print(f'{i}/{iters}')

model = network.Tacotron(embed_dim=config.tts_embed_dims,
						num_chars=len(symbols),
						enc_dim=config.tts_encoder_dims,
						dec_dim=config.tts_decoder_dims,
						n_mels=config.num_mels,
						fft_bins=config.num_mels,
						postnet_dim=config.tts_postnet_dims,
						enc_K=config.tts_encoder_K,
						lstm_dim=config.tts_lstm_dims,
						postnet_K=config.tts_postnet_K,
						num_highway=config.tts_num_highways,
						dropout=config.tts_dropout)

saver = M.Saver(model)
saver.restore(config.model_folder)

# @tf.function
def grad_loss(x,m):
	with tf.GradientTape() as tape:
		m1_hat, m2_hat, att = model(x, m)
		m1_ls = tf.reduce_mean(tf.abs(m1_hat - m))
		m2_ls = tf.reduce_mean(tf.abs(m2_hat - m))
		loss = m1_ls + m2_ls
	t1 = time.time()
	grads = tape.gradient(loss, model.trainable_variables)
	t2 = time.time()
	# print('Time grad:', t2-t1)
	return grads, loss

def tts_train_loop(optim, trainset, epochs):
	for e in range(epochs):
		bar = tqdm(range(trainset.maxiters))
		for i in bar:
			x, m, ids, _ = trainset.get_next()
			grads, loss = grad_loss(x,m)
			optim.apply_gradients(zip(grads, model.trainable_variables))

			lsmsg = 'Epoch:%d Iter:%d Loss:%.4f'%(e, i, loss)
			bar.set_description(lsmsg)
			# print()

for r,lr,maxepoch,bsize in config.schedule:
	optim = tf.optimizers.Adam(lr)
	model.set_r(r)
	trainset = datasets.get_tts_dataset(config.data_path, bsize, r)

	tts_train_loop(optim, trainset, maxepoch)

print('Training complete')
print('Create GroundTruth Aligned Dataset..')
trainset = datasets.get_tts_dataset(config.data_path, 8, model.get_r())
create_gta(model, trainset, config.gta_path)


import uuid
from enum import Enum, unique

from sqlalchemy import (
    Column, MetaData, Table,
    String, DateTime, Enum as PgEnum, Integer,
    ForeignKey, Index, UniqueConstraint
)
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy_utils import LtreeType

metadata = MetaData()

@unique
class ShopUnitType(Enum):
    CATEGORY = "CATEGORY"
    OFFER = "OFFER"

units_table = Table(
    "units",
    metadata,
    Column("id", UUID(as_uuid=True), primary_key=True, default=uuid.uuid4),
    Column("name", String, nullable=False),
    Column("unit_code", String, nullable=False),
    Column("date", DateTime, nullable=False),
    Column("parent_id", UUID(as_uuid=True), ForeignKey("units.id"), nullable=True),
    Column("type", PgEnum(ShopUnitType, name="type"), nullable=False),
    Column("price", Integer, nullable=True),
    Column("path", LtreeType, nullable=False),
    Index("ix__units__path", "path", postgresql_using="gist"),
    Index("ix__unique_unit_in_category", "name", "unit_code", "parent_id",
          unique=True,
          postgresql_where="parent_id IS NOT NULL"
          ),
    Index("ix__unique_unit_in_root", "name", "unit_code",
          unique=True,
          postgresql_where="parent_id IS NULL"
          ),
)


from itertools import permutations

def solution(user_id, banned_id):
    answer = []

    for users in list(permutations(user_id, len(banned_id))):
        check = True
        for i in range(len(banned_id)):
            if len(users[i]) != len(banned_id[i]):
                check = False
                break
            for j in range(len(users[i])):
                if users[i][j] == banned_id[i][j] or banned_id[i][j] == '*':
                    continue
                else:
                    check = False
                    break
        if check:
            users = list(users)
            users.sort()
            if users not in answer:
                answer.append(users)
    return len(answer)


"""Represent queries."""
from __future__ import annotations

import strawberry

from pypmanager.helpers import get_general_ledger
from pypmanager.loader_transaction.const import ColumnNameValues

from .models import LedgerRow

@strawberry.type
class Query:
    """GraphQL query."""

    @strawberry.field
    def all_general_ledger(self) -> list[LedgerRow]:
        """Return all general ledger rows."""
        df_general_ledger = get_general_ledger()
        df_general_ledger = df_general_ledger.sort_values(
            [
                ColumnNameValues.TRANSACTION_DATE,
                ColumnNameValues.NAME,
            ],
            ascending=False,
        )
        output_dict = df_general_ledger.reset_index().to_dict(orient="records")

        output_data: list[LedgerRow] = []

        for row in output_dict:
            output_data.append(
                LedgerRow(
                    date=row[ColumnNameValues.TRANSACTION_DATE],
                    broker=row[ColumnNameValues.BROKER],
                    source=row[ColumnNameValues.SOURCE],
                )
            )

        return output_data


from django.contrib.auth.decorators import login_required
from django.conf.urls import url, patterns
from .views import *

urlpatterns = patterns('Egresados.apps.home.views',
	url(r'^$', CreateEgresados.as_view(), name = 'crear_egresados'),
	url(r'^message/$', MessageView.as_view(), name = 'message'),
	url(r'^lista/$', login_required(ListEgresados.as_view()), name = 'lista_egresados'),
	url(r'^egresado/(?P<numero_documento>\d+)/$', UpdateEgresados.as_view(), name = 'actualizar_egresado'),
	url(r'^listado/$', ImprimirListado.as_view(), name = 'listado_egresado')
)


import torch
import torch.nn as nn
import math
import torch.utils.model_zoo as model_zoo
import torchvision.models as models
from operations import *

__all__ = ['ResNet', 'resnet50', 'resnet101','resnet152']

model_urls = {
    'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
    'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
    'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
    'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
    'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
}

class Bottleneck(nn.Module):
	expansion = 4

	def __init__(self, inplanes, planes, stride = 1, downsample = None, search = False, op_code = 'conv3d', conv_config = None):
		super(Bottleneck, self).__init__()
		self.conv1 = nn.Conv3d(inplanes, planes, kernel_size=1, bias=False)
		self.bn1 = nn.BatchNorm3d(planes)

		if op_code == 'conv3d':
			expand_ratio = 1
			self.convT = Conv3D(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1)
		elif op_code == 'conv2d':
			expand_ratio = 1
			self.convT = Conv2D(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1)
		elif op_code == 'convp3d':
			expand_ratio = 1
			self.convT = ConvP3D(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1)
		elif op_code == 'convst':
			if search:
				expand_ratio = 2
			else:
				expand_ratio = 1
			self.convT = ConvST(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1, conv_config = conv_config)

		elif op_code == 'convst_dropout':
			if search:
				expand_ratio = 1
			else:
				expand_ratio = 1
			self.convT = ConvST_Dropout(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1, conv_config = conv_config)
		elif op_code == 'conv1_2d_dropout':
			if search:
				expand_ratio = 1
			else:
				expand_ratio = 1
			self.convT = Conv1_2D_Dropout(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1, conv_config = conv_config)
		elif op_code == 'conv1_2_3d_dropout':
			if search:
				expand_ratio = 1
			else:
				expand_ratio = 1
			self.convT = Conv1_2_3D_Dropout(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1, conv_config = conv_config)

		elif op_code == 'mixconv1d':
			if search:
				expand_ratio = 3
			else:
				expand_ratio = 1
			self.convT = MixConv1D(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1, conv_config = conv_config)
		elif op_code == 'mixconv2d':
			if search:
				expand_ratio = 3
			else:
				expand_ratio = 1
			self.convT = MixConv2D(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1, conv_config = conv_config)
		elif op_code == 'mixconvall':
			if search:
				expand_ratio = 7
			else:
				expand_ratio = 1
			self.convT = MixConvAll(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1, conv_config = conv_config)
		elif op_code == 'conv1_2d':
			if search:
				expand_ratio = 2
			else:
				expand_ratio = 1
			self.convT = Conv1_2D(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1, conv_config = conv_config)
		elif op_code == 'conv2_3d':
			if search:
				expand_ratio = 2
			else:
				expand_ratio = 1
			self.convT = Conv2_3D(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1, conv_config = conv_config)
		elif op_code == 'conv1_2_3d':
			if search:
				expand_ratio = 3
			else:
				expand_ratio = 1
			self.convT = Conv1_2_3D(planes, planes*expand_ratio, kernel_size = 3, stride = stride, padding = 1, conv_config = conv_config)
		else:
			raise NotImplementedError

		self.conv3 = nn.Conv3d(planes*expand_ratio, planes * self.expansion, kernel_size=1, bias=False)
		self.bn3 = nn.BatchNorm3d(planes * self.expansion)
		self.relu = nn.ReLU(inplace=True)
		self.downsample = downsample
		self.stride = stride

	def forward(self, x):
		residual = x

		out = self.conv1(x)
		out = self.bn1(out)
		out = self.relu(out)

		out = self.convT(out)

		out = self.conv3(out)
		out = self.bn3(out)

		if self.downsample is not None:
			residual = self.downsample(residual)

		out += residual
		out = self.relu(out)

		return out

class ResNet(nn.Module):

	def __init__(self, block, layers, num_classes=1000, search = False, op_code = 'conv3d', conv_config = None, conv_index = None):
		self.inplanes = 64

		self.search = search
		self.op_code = op_code
		if conv_config is None:
			conv_config = [None] * 200
		self.conv_config = conv_config

		super(ResNet, self).__init__()
		self.conv1 = nn.Conv3d(3, 64, kernel_size=(1,7,7), stride=(1,2,2), padding=(0,3,3),
							   bias=False)
		self.bn1 = nn.BatchNorm3d(64)
		self.relu = nn.ReLU(inplace=True)
		self.maxpool = nn.MaxPool3d(kernel_size=(1,3,3), stride=(1,2,2), padding=(0,1,1))
		self.layer1 = self._make_layer(block, 64,  layers[0], stride=1, search=search, op_code=op_code, conv_config=conv_config[:layers[0]])
		self.layer2 = self._make_layer(block, 128, layers[1], stride=2, search=search, op_code=op_code, conv_config=conv_config[layers[0]:layers[0]+layers[1]])
		self.layer3 = self._make_layer(block, 256, layers[2], stride=2, search=search, op_code=op_code, conv_config=conv_config[layers[0]+layers[1]:layers[0]+layers[1]+layers[2]])
		self.layer4 = self._make_layer(block, 512, layers[3], stride=2, search=search, op_code=op_code, conv_config=conv_config[layers[0]+layers[1]+layers[2]:layers[0]+layers[1]+layers[2]+layers[3]])
		self.avgpool = nn.AvgPool2d(7, stride=1)
		self.fc = nn.Linear(512 * block.expansion, num_classes)

		for m in self.modules():
			if isinstance(m, nn.Conv3d):
				nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
			elif isinstance(m, nn.BatchNorm3d):
				nn.init.constant_(m.weight, 1)
				nn.init.constant_(m.bias, 0)

	def _make_layer(self, block, planes, blocks, stride=1, search=False, op_code='conv3d', conv_config=None):
		downsample = None
		if stride != 1 or self.inplanes != planes * block.expansion:
			downsample = nn.Sequential(
				nn.Conv3d(self.inplanes, planes * block.expansion,
						  kernel_size=1, stride=(1,stride,stride), bias=False),
				nn.BatchNorm3d(planes * block.expansion),
			)

		layers = []
		layers.append(block(self.inplanes, planes, stride, downsample, search=search, op_code=op_code, conv_config=conv_config[0]))
		self.inplanes = planes * block.expansion
		for i in range(1, blocks):
			layers.append(block(self.inplanes, planes, stride=1, downsample=None,  search=search, op_code=op_code, conv_config=conv_config[i]))

		return nn.Sequential(*layers)

	def forward(self, x):
		x = self.conv1(x)
		x = self.bn1(x)
		x = self.relu(x)
		x = self.maxpool(x)

		x = self.layer1(x)
		x = self.layer2(x)
		x = self.layer3(x)
		x = self.layer4(x)
		x = x.transpose(1,2).contiguous()
		x = x.view((-1,)+x.size()[2:])

		x = self.avgpool(x)
		x = x.view(x.size(0), -1)
		x = self.fc(x)

		return x

def resnet50(**kwargs):
	"""Constructs a ResNet-50 based model.
	"""
	model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs)
	checkpoint = model_zoo.load_url(model_urls['resnet50'])

	model_dict = model.state_dict()
	new_dict = {}
	#conv_index = kwargs['conv_index']

	### ImageNet pre-trained weight:
	for k, v in checkpoint.items():
		if 'conv' in k or 'downsample.0.weight' in k:
			v = v.unsqueeze(2)
		if 'bn2' in k:
			if kwargs['op_code'] == 'conv3d' or kwargs['op_code'] == 'conv2d':
				k = k.replace('bn2', 'convT.op.1')
			elif kwargs['op_code'] == 'convp3d':
				k = k.replace('bn2', 'convT.op.2')
			elif kwargs['op_code'] == 'convst' or kwargs['op_code'] == 'conv1_2d' or kwargs['op_code'] == 'conv2_3d':
				k = k.replace('bn2', 'convT.out.0')
				if kwargs['search']:
					v = v.repeat(2)
			elif kwargs['op_code'] == 'conv1_2_3d':
				k = k.replace('bn2', 'convT.out.0')
				if kwargs['search']:
					v = v.repeat(3)

			elif kwargs['op_code'] == 'convst_dropout':
				k = k.replace('bn2', 'convT.norm')
				if kwargs['search']:
					v = v.repeat(2)
			elif kwargs['op_code'] == 'conv1_2d_dropout':
				k = k.replace('bn2', 'convT.norm')
				if kwargs['search']:
					v = v.repeat(2)
			elif kwargs['op_code'] == 'conv1_2_3d_dropout':
				k = k.replace('bn2', 'convT.norm')
				if kwargs['search']:
					v = v.repeat(3)

			elif kwargs['op_code'] == 'mixconv1d':
				k = k.replace('bn2', 'convT.out.0')
				if kwargs['search']:
					v = v.repeat(3)
			elif kwargs['op_code'] == 'mixconv2d':
				k = k.replace('bn2', 'convT.out.0')
				if kwargs['search']:
					v = v.repeat(3)
			elif kwargs['op_code'] == 'mixconvall':
				k = k.replace('bn2', 'convT.out.0')
				if kwargs['search']:
					v = v.repeat(7)
		if 'conv2' in k:
			if kwargs['op_code'] == 'conv3d' or kwargs['op_code'] == 'conv2d' or kwargs['op_code'] == 'convp3d':
				k = k.replace('conv2', 'convT.op.0')
				if kwargs['op_code'] == 'conv3d':
					v = v.repeat(1,1,3,1,1) / 3.0
				if kwargs['op_code'] == 'convp3d':
					new_dict[k.replace('convT.op.0','convT.op.1')] = model_dict[k.replace('convT.op.0','convT.op.1')]

			elif kwargs['op_code'] == 'convst' or kwargs['op_code'] == 'convst_dropout' or kwargs['op_code'] == 'conv2_3d':
				k3d = k.replace('conv2', 'convT.convs.0.0')
				k2d = k.replace('conv2', 'convT.convs.1.0')
				if kwargs['search']:
					v3d = v.repeat(1,1,3,1,1) / 3.0
					v2d = v
				else:
					cout3d, _, _, _, _ = model_dict[k3d].size()
					cout2d, _, _, _, _ = model_dict[k2d].size()
					v3d = v[:cout3d,:,:,:,:].repeat(1,1,3,1,1) / 3.0
					v2d = v[cout3d:cout3d+cout2d,:,:,:,:]
				new_dict[k3d] = v3d
				new_dict[k2d] = v2d
				continue
			elif kwargs['op_code'] == 'conv1_2d' or kwargs['op_code'] == 'conv1_2d_dropout':
				k2d = k.replace('conv2', 'convT.convs.0.0')
				k1d = k.replace('conv2', 'convT.convs.1.0')
				if kwargs['search']:
					v2d = v
					v1d = torch.sum(torch.sum(v, dim=3, keepdim=True), dim=4, keepdim=True).repeat(1,1,3,1,1) / 3.0
				else:
					cout2d, _, _, _, _ = model_dict[k2d].size()
					cout1d, _, _, _, _ = model_dict[k1d].size()
					v2d = v[:cout2d,:,:,:,:]
					v1d = torch.sum(torch.sum(v[cout2d:cout2d+cout1d,:,:,:,:], dim=3, keepdim=True), dim=4, keepdim=True).repeat(1,1,3,1,1) / 3.0
				new_dict[k2d] = v2d
				new_dict[k1d] = v1d
				continue
			elif kwargs['op_code'] == 'conv1_2_3d' or kwargs['op_code'] == 'conv1_2_3d_dropout':
				k3d = k.replace('conv2', 'convT.convs.0.0')
				k2d = k.replace('conv2', 'convT.convs.1.0')
				k1d = k.replace('conv2', 'convT.convs.2.0')
				if kwargs['search']:
					v3d = v.repeat(1,1,3,1,1) / 3.0
					v2d = v
					v1d = torch.sum(torch.sum(v, dim=3, keepdim=True), dim=4, keepdim=True).repeat(1,1,3,1,1) / 3.0
				else:
					cout3d, _, _, _, _ = model_dict[k3d].size()
					cout2d, _, _, _, _ = model_dict[k2d].size()
					try:
						cout1d, _, _, _, _ = model_dict[k1d].size()
					except:
						pass
					v3d = v[:cout3d,:,:,:,:].repeat(1,1,3,1,1) / 3.0
					v2d = v[cout3d:cout3d+cout2d,:,:,:,:]
					try:
						v1d = torch.sum(torch.sum(v[cout3d+cout2d:cout3d+cout2d+cout1d,:,:,:,:], dim=3, keepdim=True), dim=4, keepdim=True).repeat(1,1,3,1,1) / 3.0
					except:
						pass
				new_dict[k3d] = v3d
				new_dict[k2d] = v2d
				try:
					if k1d in model_dict:
						new_dict[k1d] = v1d
				except:
					pass
				continue
			elif kwargs['op_code'] == 'mixconv1d':
				### v: cout,cin,1,3,3
				k311 = k.replace('conv2', 'convT.convs.0.0')
				k131 = k.replace('conv2', 'convT.convs.1.0')
				k113 = k.replace('conv2', 'convT.convs.2.0')
				if kwargs['search']:
					#v = torch.mean(torch.mean(v, dim=3, keepdim=True), dim=4, keepdim=True).repeat(1,1,3,1,1)
					v = torch.sum(torch.sum(v, dim=3, keepdim=True), dim=4, keepdim=True).repeat(1,1,3,1,1) / 3.0  ### use sum to keep scale?
					v311 = v
					v131 = v.permute(0,1,3,2,4)
					v113 = v.permute(0,1,4,3,2)
				else:
					v = torch.sum(torch.sum(v, dim=3, keepdim=True), dim=4, keepdim=True).repeat(1,1,3,1,1) / 3.0
					cout311, _, _, _, _ = model_dict[k311].size()
					cout131, _, _, _, _ = model_dict[k131].size()
					cout113, _, _, _, _ = model_dict[k113].size()
					v311 = v[:cout311,:,:,:,:]
					v131 = v[cout311:cout311+cout131,:,:,:,:].permute(0,1,3,2,4)
					v113 = v[cout311+cout131:cout311+cout131+cout113,:,:,:,:].permute(0,1,3,4,2)
				new_dict[k311] = v311
				new_dict[k131] = v131
				new_dict[k113] = v113
			elif kwargs['op_code'] == 'mixconv2d':
				k331 = k.replace('conv2', 'convT.convs.0.0')
				k133 = k.replace('conv2', 'convT.convs.1.0')
				k313 = k.replace('conv2', 'convT.convs.2.0')
				if kwargs['search']:
					v331 = v.permute(0,1,4,3,2)
					v133 = v
					v313 = v.permute(0,1,3,2,4)
				else:
					cout331, _, _, _, _ = model_dict[k331].size()
					cout133, _, _, _, _ = model_dict[k133].size()
					cout313, _, _, _, _ = model_dict[k313].size()
					v331 = v[:cout331,:,:,:,:].permute(0,1,4,3,2)
					v133 = v[cout331:cout331+cout133,:,:,:,:]
					v313 = v[cout331+cout133:cout331+cout133+cout313,:,:,:,:].permute(0,1,3,2,4)
				new_dict[k331] = v331
				new_dict[k133] = v133
				new_dict[k313] = v313

			elif kwargs['op_code'] == 'mixconvall':
				raise NotImplementedError

		if 'conv3' in k:
			if kwargs['search']:
				if kwargs['op_code'] == 'convst' or kwargs['op_code'] == 'conv1_2d' or kwargs['op_code'] == 'conv2_3d':
					### 2 times
					v = v.repeat(1,2,1,1,1)
				elif kwargs['op_code'] == 'mixconv1d' or kwargs['op_code'] == 'mixconv2d' or kwargs['op_code'] == 'conv1_2_3d':
					v = v.repeat(1,3,1,1,1)
				elif kwargs['op_code'] == 'mixconvall':
					v = v.repeat(1,7,1,1,1)

		if k in model_dict:
			new_dict[k] = v

	model.load_state_dict(new_dict)

	return model

def resnet101(**kwargs):
	"""Constructs a ResNet-101 model.
	Args:
		groups
	"""
	model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs)
	checkpoint = model_zoo.load_url(model_urls['resnet101'])
	layer_name = list(checkpoint.keys())
	for ln in layer_name:
		if 'conv' in ln or 'downsample.0.weight' in ln:
			checkpoint[ln] = checkpoint[ln].unsqueeze(2)
		# if 'conv2' in ln:
		# 	n_out, n_in, _, _, _ = checkpoint[ln].size()
		# 	checkpoint[ln] = checkpoint[ln][:n_out // alpha * (alpha - 1), :n_in//beta,:,:,:]
	model.load_state_dict(checkpoint,strict = False)

	return model

def load_state_dict_supernet(net, super_net_dict, conv_index):
	### for convST
	print('length of conv_index:', len(conv_index))
	index = 0
	net_dict = net.state_dict()
	flag1, flag2, flag3 = False, False, False
	print(list(super_net_dict.keys()))
	for k in net_dict:
		print(index, k)
		if 'convT.convs.0.0' in k: # 3d
			if flag1 and flag2 and flag3:
				index += 1
				flag1, flag2, flag3 = False, False, False
			indices = conv_index[index]['kxkxk']
			indices.sort()
			net_dict[k] = super_net_dict[k][indices,:,:,:,:]
			flag1 = True

		if 'convT.convs.1.0' in k: # 2d
			indices = conv_index[index]['1xkxk']
			indices.sort()
			net_dict[k] = super_net_dict[k][indices,:,:,:,:]
			flag2 = True

		if 'convT.out.0' in k: # bn
			if 'num_batches_tracked' in k:
				continue
			channels = super_net_dict[k].shape[0] // 2
			indices0 = conv_index[index]['kxkxk']
			indices1 = [ i+channels for i in conv_index[index]['1xkxk']]
			indices = indices0 + indices1
			net_dict[k] = super_net_dict[k][indices]
			flag3 = True

	net.load_state_dict(net_dict)
	return net


from bs4 import BeautifulSoup
from pprint import pprint
import requests
from queue import Queue
import pandas as pd
import time

def Technology(url):
    news_df = pd.DataFrame()
    html_content = requests.get(url)
    soup = BeautifulSoup(html_content.content, 'lxml')

    ul = soup.find('ul', {"class": "article-list"})

    d = dict()
    for all_li in ul.find_all('li'):
        link = all_li.find("a").get("href")
        print(link)
        d['HEADLINE'] = all_li.text.lstrip().rstrip()
        d['LINK'] = link
        d['IMAGE_LINK'] = all_li.find('img').get('src')

        html_data = requests.get(link)
        soup = BeautifulSoup(html_data.content, 'lxml')
        div_full_content = soup.find('div', {"id": "pcl-full-content"})
        news_content = ''

        for paragraph in div_full_content.find_all('p'):
            news_content += paragraph.text

        d['NEWS'] = news_content

        news_df = news_df.append(d, ignore_index=True)
        d.clear()

    news_df.to_csv('TechnologyNews.csv')

def LifestyleEntertainmentSportsOpinionCities(url, tag):
    print(url)
    df = pd.DataFrame()
    html_content = requests.get(url)
    soup = BeautifulSoup(html_content.content, 'lxml')

    ul = soup.find('div', {"class": "nation"})

    d = dict()
    for all_li in ul.find_all('div'):
        if all_li.get('class') is not None and all_li.get('class')[0].startswith("articles"):

            link = all_li.find("a").get("href")
            d['HEADLINE'] = all_li.text.lstrip().rstrip()
            d['LINK'] = link
            d['IMAGE_LINK'] = all_li.find('img').get('src')

            html_data = requests.get(link)
            soup = BeautifulSoup(html_data.content, 'lxml')
            div_full_content = soup.find('div', {"id": "pcl-full-content"})
            news_content = ''
            print(link)
            try:
                for paragraph in div_full_content.find_all('p'):
                    news_content += paragraph.text
            except AttributeError:
                div_articleBody = soup.find('div', {"itemprop": "articleBody"})
                for paragraph in div_articleBody.find_all('p'):
                    news_content += paragraph.text

            d['NEWS'] = news_content
            # print(d)
            df = df.append(d, ignore_index=True)
            d.clear()

        elif all_li.get('class') is None:
            break

    df.to_csv(tag + 'News.csv')

def main():
    NavbarLinksQueue = Queue()

    # Index page
    index_page_content = requests.get('https://indianexpress.com/')

    index_page_soup = BeautifulSoup(index_page_content.content, 'lxml')
    UL_Navbar = index_page_soup.find('ul', {"id": "navbar"})

    counter = 0
    # Get 8 important links from Navbar
    for ul_li in UL_Navbar.find_all('li'):
        if 1 <= counter <= 8:
            NavbarLinksQueue.put(ul_li.find('a').get('href'))
        counter += 1

    while not NavbarLinksQueue.empty():
        split_links = NavbarLinksQueue.get().split('/')[2]
        if split_links == 'technology':
            Technology(f'https://indianexpress.com/section/{split_links}/')
        elif split_links == 'lifestyle' or split_links == 'entertainment' or \
                split_links == 'sports' or split_links == 'opinions' or split_links == 'cities':
            LifestyleEntertainmentSportsOpinionCities('https://indianexpress.com/section/{}/'.format(split_links),
                                                      split_links)

if __name__ == '__main__':
    start = time.time()
    main()
    print('Time taken', time.time() - start)


from django.db import models
import os
from django.conf import settings

#### PyKE import ####
from .models_pyke.knowledge import pyke_load_engine, pyke_calculate_bmr, pyke_calculate_EnergyAmount_kcal, pyke_calculate_Nutrients
pyke_load_engine()

######################################################################
# Initialize the data once when Django loads up
# -- will improve in next iteration
######################################################################

from .models_ortools import readFoodData
csv_file = open(os.path.join(settings.BASE_DIR, 'foodrec/Dataset/foodDataBase_Final.csv'))
readFoodData(csv_file)

class Food(models.Model):
    FOOD_GROUPS = sorted(
                    [
                        ("BEVERAGES", "BEVERAGES"),
                        ("CEREAL AND CEREAL PRODUCTS", "CEREAL AND CEREAL PRODUCTS"),
                        ("EGG AND EGG PRODUCTS", "EGG AND EGG PRODUCTS"),
                        ("FAST FOODS", "FAST FOODS"),
                        ("FISH AND FISH PRODUCTS", "FISH AND FISH PRODUCTS"),
                        ("FRUIT AND FRUIT PRODUCTS", "FRUIT AND FRUIT PRODUCTS"),
                        ("HEALTHIER CHOICE SYMBOL (HCS) PRODUCTS", "HEALTHIER CHOICE SYMBOL (HCS) PRODUCTS"),
                        ("MEAT AND MEAT PRODUCTS", "MEAT AND MEAT PRODUCTS"),
                        ("MILK AND MILK PRODUCTS", "MILK AND MILK PRODUCTS"),
                        ("MISCELLANEOUS", "MISCELLANEOUS"),
                        ("MIXED ETHNIC DISHES, ANALYZED IN SINGAPORE", "MIXED ETHNIC DISHES, ANALYZED IN SINGAPORE"),
                        ("NUTS AND SEEDS, PULSES AND PRODUCTS", "NUTS AND SEEDS, PULSES AND PRODUCTS"),
                        ("OILS AND FATS", "OILS AND FATS"),
                        ("OTHER MIXED ETHNIC DISHES", "OTHER MIXED ETHNIC DISHES"),
                        ("SUGARS, SWEETS AND CONFECTIONERY", "SUGARS, SWEETS AND CONFECTIONERY"),
                        ("VEGETABLE AND VEGETABLE PRODUCTS", "VEGETABLE AND VEGETABLE PRODUCTS"),
                    ])

    MEALS = [
                ("BREAKFAST", "BREAKFAST"),
                ("LUNCH", "LUNCH"),
                ("DINNER", "DINNER"),
            ]

    FoodIndex = models.IntegerField()
    FoodName = models.CharField(max_length=100, blank=False)
    FoodGroup = models.CharField(max_length=100, choices=FOOD_GROUPS, default='MISCELLANEOUS')
    CarbohydrateAmount_g = models.FloatField()
    EnergyAmount_kcal = models.FloatField()
    ProteinAmount_g = models.FloatField()
    TotalFatAmount_g = models.FloatField()

    IsVegan = models.BooleanField()
    IsVegetarian = models.BooleanField()
    IsHalal = models.BooleanField()

    FoodMealRanking = models.CharField(max_length=100, choices=MEALS)

    def __repr__(self):
        return self.FoodName

    def __str__(self):
        return self.FoodName

    class Meta:
        ordering = ['FoodName']

#######################################################
# Non-Django classes for now
#######################################################
class Profile:
    # tuple (value, display_text) for GENDER
    GENDER = (
        ('male', 'Male'),
        ('female', 'Female'),
    )

    ACTIVITY = (
        ('sedentary', 'Sedentary'),
        ('lightly_active', 'Lightly Active'),
        ('moderately_active', 'Moderately Active'),
        ('very_active', 'Very Active'),
    )

    DIET = (
        ('standard', 'Standard'),
        ('ketogenic', 'Ketogenic'),
    )

    def __init__(self, gender=None, age=None, height=None, weight=None, activity=None, diet=None):
        self.gender = gender
        self.age = age
        self.height = height
        self.weight = weight
        self.activity = activity
        self.diet = diet
        self.nutrientNeeds = NutrientNeeds(diet=diet,profile=self)    # Initialize an empty NutriendNeeds object

class NutrientNeeds:
    def __init__(self, CarbohydrateAmount_g=None, EnergyAmount_kcal=None, ProteinAmount_g=None, TotalFatAmount_g=None, diet=None, profile=None):
        self.CarbohydrateAmount_g = CarbohydrateAmount_g
        self.EnergyAmount_kcal = EnergyAmount_kcal
        self.ProteinAmount_g = ProteinAmount_g
        self.TotalFatAmount_g = TotalFatAmount_g
        self.diet = diet
        self.profile = profile

    # Calculate the Nutrient Needs
    def calculate(self):
        # Firstly, calculate the BMR
        bmr =float('%.2f' % (pyke_calculate_bmr(self.profile.gender, self.profile.weight, self.profile.height, self.profile.age)))

        # Calculate the Energy
        # EnergyAmount_kcal
        self.EnergyAmount_kcal = pyke_calculate_EnergyAmount_kcal(bmr, self.profile.activity)

        # Calculate the nutrients
        # CarbohydrateAmount_g, TotalFatAmount_g, ProteinAmount_g
        # * Standard diet = balance ratio of Carbs:Fat:Protein = 40%:30%:30%
        # * Keto diet = 75% Fat, 20% Protein, 5% Carb or <50grams
        self.CarbohydrateAmount_g, self.TotalFatAmount_g, self.ProteinAmount_g = pyke_calculate_Nutrients(self.EnergyAmount_kcal, self.diet)


##
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##

"""
Run test:
>> mpirun -n 2 python -m pytest --with-mpi -q python/pycylon/test/test_cylon_table_conversion.py
"""

import os
import pytest
from pycylon.io import read_csv
from pycylon import Table
from pycylon.net import MPIConfig
from pycylon import CylonContext
from pycylon.io import CSVReadOptions
import pandas as pd
import numpy as np

@pytest.mark.mpi
def test_conversion_check():
    mpi_config = MPIConfig()
    ctx: CylonContext = CylonContext(config=mpi_config, distributed=True)

    rank, size = ctx.get_rank(), ctx.get_world_size()
    assert 0 <= rank < size and size > 0

    assert size == 2

    table1_path = f'data/input/user_usage_tm_{rank + 1}.csv'
    table2_path = f'data/input/user_device_tm_{rank + 1}.csv'

    assert os.path.exists(table1_path)
    assert os.path.exists(table2_path)

    csv_read_options = CSVReadOptions().use_threads(True).block_size(1 << 30)

    tb1: Table = read_csv(ctx, table1_path, csv_read_options)

    tb2: Table = read_csv(ctx, table2_path, csv_read_options)

    tb3: Table = tb1.distributed_join(table=tb2, join_type='inner', algorithm='sort', left_on=[3],
                                      right_on=[0])

    # pdf: pd.DataFrame = tb3.to_pandas()
    npy: np.ndarray = tb3.to_numpy(order='C')

    # Cylon table rows must be equal to the rows of pandas dataframe extracted from the table
    # assert tb3.rows == pdf.shape[0]
    # Cylon table columns must be equal to the columns of pandas dataframe extracted from the table
    # assert tb3.columns == pdf.shape[1]
    # Cylon table rows must be equal to the rows of numpy ndarray extracted from the table
    assert tb3.row_count == npy.shape[0]
    # Cylon table columns must be equal to the columns of numpy ndarray extracted from the table
    assert tb3.column_count == npy.shape[1]

    print(f"Rank[{ctx.get_rank()}]: Table.Rows={tb3.row_count}, Table.Columns={tb3.column_count}, "
          f"Numpy Array Shape = {npy.shape}")

    print(f"Array Config Rank[{ctx.get_rank()}], {npy.flags} {npy.dtype}")

    # Note: Not needed when using PyTest with MPI
    # ctx.finalize()


# coding:utf-8

from sogou_mrc.utils.tokenizer import SpacyTokenizer
from sogou_mrc.dataset.base_dataset import BaseReader, BaseEvaluator
import json
from collections import OrderedDict, Counter
from tqdm import tqdm
import logging
import re
import collections
import string
import sys

class SquadV2Reader(BaseReader):
    def __init__(self):
        self.tokenizer = SpacyTokenizer()

    def read(self, file_path):
        logging.info("Reading file at %s", file_path)
        logging.info("Processing the dataset.")
        instances = self._read(file_path)
        instances = [instance for instance in tqdm(instances)]
        return instances

    def _read(self, file_path):
        with open(file_path) as dataset_file:
            dataset_json = json.load(dataset_file)
            dataset = dataset_json['data']
        for article in dataset:
            for paragraph in article['paragraphs']:
                context = paragraph["context"]
                context_tokens, context_token_spans = self.tokenizer.word_tokenizer(context)
                for question_answer in paragraph['qas']:
                    question = question_answer["question"].strip()
                    question_tokens, _ = self.tokenizer.word_tokenizer(question)

                    answers, span_starts, span_ends = [], [], []
                    if "answers" in question_answer:
                        answers = [answer['text'] for answer in question_answer['answers']]
                        span_starts = [answer['answer_start']
                                       for answer in question_answer['answers']]
                        span_ends = [start + len(answer)
                                     for start, answer in zip(span_starts, answers)]
                    if 'is_impossible' in question_answer and question_answer['is_impossible']:
                        span_starts = [0]
                        span_ends = [0]

                    answer_char_spans = zip(span_starts, span_ends) if len(
                        span_starts) > 0 and len(span_ends) > 0 else None
                    answers = answers if len(answers) > 0 else [""]
                    is_impossible = None if 'is_impossible' not in question_answer else question_answer[
                                                                                'is_impossible'] * 1
                    qid = question_answer['id']
                    yield self._make_instance(context, context_tokens, context_token_spans,
                                              question, question_tokens, answer_char_spans, answers, is_impossible, qid)

    def _make_instance(self, context, context_tokens, context_token_spans, question, question_tokens,
                       answer_char_spans=None, answers=None, is_impossible=None, qid=None):
        answer_token_starts, answer_token_ends = [], []
        if answers is not None:
            for answer_char_start, answer_char_end in answer_char_spans:
                answer_token_span = []
                for idx, span in enumerate(context_token_spans):
                    if not (answer_char_end <= span[0] or answer_char_start >= span[1]):
                        answer_token_span.append(idx)

                if  len(answer_token_span) == 0: break #print(is_impossible) #break #continue
                answer_token_starts.append(answer_token_span[0])
                answer_token_ends.append(answer_token_span[-1])
        abstractive_answer_mask = [0]
        if is_impossible is not None and is_impossible:
            answer_token_starts=[]
            answer_token_ends=[]
            answer_token_starts.append(0)
            answer_token_ends.append(0)
            abstractive_answer_mask = [1]

        return OrderedDict({
            "context": context,
            "context_tokens": context_tokens,
            "context_token_spans": context_token_spans,
            "is_impossible": is_impossible,
            "question": question,
            'qid': qid,
            "question_tokens": question_tokens,
            "answer": answers[0] if answers is not None else [],
            "answer_start": answer_token_starts[0] if len(answer_token_starts) > 0 is not None else None,
            "answer_end": answer_token_ends[0] if len(answer_token_ends) > 0 is not None else None,
            "abstractive_answer_mask": abstractive_answer_mask
        })

class SquadV2Evaluator(BaseEvaluator):
    def __init__(self, file_path, monitor='best_f1', na_prob_thresh=1.0):
        self.na_prob_thresh = na_prob_thresh
        self.monitor = monitor
        self.ground_dict = dict()
        self.id_list = []
        with open(file_path) as dataset_file:
            dataset_json = json.load(dataset_file)
            dataset = dataset_json['data']
        self.has_ans_qids = set()
        self.no_ans_qids = set()
        for article in dataset:
            for paragraph in article['paragraphs']:
                for question_answer in paragraph['qas']:
                    id = question_answer["id"]
                    gold_answers = [a['text'] for a in question_answer['answers']
                                    if SquadV2Evaluator.normalize_answer(a['text'])]
                    if not gold_answers:
                        gold_answers = ['']
                        self.no_ans_qids.add(id)
                    else:
                        self.has_ans_qids.add(id)
                    self.ground_dict[id] = gold_answers

    def get_monitor(self):
        return self.monitor

    def get_score(self, input):
        preds, na_proba = input
        exact_raw, f1_raw = self.get_raw_score(preds)
        na_prob_thresh = self.na_prob_thresh
        exact_thresh = self.apply_no_ans_threshold(exact_raw, na_proba, na_prob_thresh)
        f1_thresh = self.apply_no_ans_threshold(f1_raw, na_proba, na_prob_thresh)
        out_eval = SquadV2Evaluator.make_eval_dict(exact_thresh, f1_thresh)
        if len(self.has_ans_qids) > 0:
            has_ans_eval = SquadV2Evaluator.make_eval_dict(exact_thresh, f1_thresh, qid_list=self.has_ans_qids)
            SquadV2Evaluator.merge_eval(out_eval, has_ans_eval, 'HasAns')
        if len(self.no_ans_qids) > 0:
            no_ans_eval = SquadV2Evaluator.make_eval_dict(exact_thresh, f1_thresh, qid_list=self.no_ans_qids)
            SquadV2Evaluator.merge_eval(out_eval, no_ans_eval, 'NoAns')
        self.find_all_best_thresh(out_eval, preds, exact_raw, f1_raw, na_proba)
        return dict(out_eval)

    def find_all_best_thresh(self, main_eval, preds, exact_raw, f1_raw, na_probs):
        best_exact, exact_thresh = self.find_best_thresh(preds, exact_raw, na_probs)
        best_f1, f1_thresh = self.find_best_thresh(preds, f1_raw, na_probs)
        main_eval['best_exact'] = best_exact
        main_eval['best_exact_thresh'] = exact_thresh
        main_eval['best_f1'] = best_f1
        main_eval['best_f1_thresh'] = f1_thresh

    @staticmethod
    def merge_eval(main_eval, new_eval, prefix):
        for k in new_eval:
            main_eval['%s_%s' % (prefix, k)] = new_eval[k]

    @staticmethod
    def make_eval_dict(exact_scores, f1_scores, qid_list=None):
        if not qid_list:
            total = len(exact_scores)
            return collections.OrderedDict([
                ('exact', 100.0 * sum(exact_scores.values()) / total),
                ('f1', 100.0 * sum(f1_scores.values()) / total),
                ('total', total),
            ])
        else:
            total = len(qid_list)
            return collections.OrderedDict([
                ('exact', 100.0 * sum(exact_scores[k] for k in qid_list) / total),
                ('f1', 100.0 * sum(f1_scores[k] for k in qid_list) / total),
                ('total', total),
            ])

    def find_best_thresh(self, preds, scores, na_probs):
        num_no_ans = len(self.no_ans_qids)
        cur_score = num_no_ans
        best_score = cur_score
        best_thresh = 0.0
        qid_list = sorted(na_probs, key=lambda k: na_probs[k])
        for i, qid in enumerate(qid_list):
            if qid not in scores: continue
            if qid in self.has_ans_qids:
                diff = scores[qid]
            else:
                if preds[qid]:
                    diff = -1
                else:
                    diff = 0
            cur_score += diff
            if cur_score > best_score:
                best_score = cur_score
                best_thresh = na_probs[qid]
        return 100.0 * best_score / len(scores), best_thresh

    def apply_no_ans_threshold(self, scores, na_probs, na_prob_thresh):
        new_scores = {}
        for qid, s in scores.items():
            pred_na = na_probs[qid] > na_prob_thresh
            if pred_na:
                new_scores[qid] = float(not qid in self.has_ans_qids)
            else:
                new_scores[qid] = s
        return new_scores

    def get_raw_score(self, preds):
        # assert len(self.ground_dict) == len(preds)
        exact_scores = {}
        f1_scores = {}
        for qid in self.ground_dict:
            if qid not in preds:
                print('Missing prediction for %s' % qid)
                continue
            a_pred = preds[qid]
            golden_answers = self.ground_dict[qid]
            # Take max over all gold answers
            exact_scores[qid] = max([SquadV2Evaluator.compute_exact(a, a_pred) for a in golden_answers])
            f1_scores[qid] = max([SquadV2Evaluator.compute_f1(a, a_pred) for a in golden_answers])
        return exact_scores, f1_scores

    @staticmethod
    def normalize_answer(s):
        """Lower text and remove punctuation, articles and extra whitespace."""

        def remove_articles(text):
            regex = re.compile(r'\b(a|an|the)\b', re.UNICODE)
            return re.sub(regex, ' ', text)

        def white_space_fix(text):
            return ' '.join(text.split())

        def remove_punc(text):
            exclude = set(string.punctuation)
            return ''.join(ch for ch in text if ch not in exclude)

        def lower(text):
            return text.lower()

        return white_space_fix(remove_articles(remove_punc(lower(s))))

    @staticmethod
    def compute_exact(a_gold, a_pred):
        return int(SquadV2Evaluator.normalize_answer(a_gold) == SquadV2Evaluator.normalize_answer(a_pred))

    @staticmethod
    def get_tokens(s):
        if not s: return []
        return SquadV2Evaluator.normalize_answer(s).split()

    @staticmethod
    def compute_f1(a_gold, a_pred):
        gold_toks = SquadV2Evaluator.get_tokens(a_gold)
        pred_toks = SquadV2Evaluator.get_tokens(a_pred)
        common = collections.Counter(gold_toks) & collections.Counter(pred_toks)
        num_same = sum(common.values())
        if len(gold_toks) == 0 or len(pred_toks) == 0:
            # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
            return int(gold_toks == pred_toks)
        if num_same == 0:
            return 0
        precision = 1.0 * num_same / len(pred_toks)
        recall = 1.0 * num_same / len(gold_toks)
        f1 = (2 * precision * recall) / (precision + recall)
        return f1


from typing import Any, Text, Dict, List

from rasa_sdk import Action, Tracker
from rasa_sdk.executor import CollectingDispatcher
from rasa_sdk.events import SlotSet, FollowupAction
from db_api.product import search_products, get_product_by_id
from db_api.product_variant import search_product_variants, get_product_variant_by_id
from db_api.schema import Product

def get_color_product(product_id):
    variants = search_product_variants(product_id)
    colors = []
    for variant in variants:
        if variant.color and variant.color not in colors:
            colors.append(variant.color)
    return colors

class ActionRequestColor(Action):
    def name(self) -> Text:
        return "action_request_color"

    async def run(self, dispatcher: CollectingDispatcher, tracker: Tracker,
                  domain: Dict[Text, Any]) -> List[Dict[Text, Any]]:
        event = []
        product = None
        product_other = False

        pre_product_id = tracker.get_slot("product_id")
        if pre_product_id:
            pre_product = get_product_by_id(pre_product_id)
        else:
            pre_product = None

        object_types = list(tracker.get_latest_entity_values(entity_type="object_type"))

        if len(object_types) == 0:
            if pre_product:
                product = pre_product
            else:
                dispatcher.utter_message(text="Bạn muốn xem màu sản phẩm nào ạ")
        else:
            # Nếu có object type kèm theo, kiểm tra xem có phải đang nhắc tới sản phẩm trước đó hay ko
            object_type_level = Product.get_level_of_object_type(object_types[0])

            if object_type_level > 0:
                # Nếu có nhắc tới, thì trả về màu sp trước đó, còn nếu là object type chung chung mà không
                # nhắc tới sp trước đó thì chuyển sang action request product
                if pre_product and pre_product.check_is_general_of_product(object_types[0]):
                    product = pre_product
                else:
                    dispatcher.utter_message(text="Có phải bạn đang tìm kiếm sản phẩm không ạ")
                    event.append(FollowupAction("action_request_product"))
                    return event
            else:
                # Nếu là một sp khác thì trả về sản phẩm tương ứng
                products = search_products(product_type=object_types[0])
                if products:
                    product = products[0]
                    event.append(SlotSet("object_type", object_types))
                    event.append(SlotSet("product_id", product.id))
                    product_other = True
                else:
                    dispatcher.utter_message(text="Bạn muốn hỏi màu sản phẩm nào ạ")
                    return event

        if product is None:
            dispatcher.utter_message(text="Bạn muốn hỏi màu sản phẩm nào ạ")
            return event

        colors = get_color_product(product.id)
        if len(colors) == 0:
            dispatcher.utter_message(response="utter_fall_back_1")
        else:
            color_text = ", ".join(colors).strip(',')
            text = f"Dạ, Sản phâm {product.title} bên mình đang còn màu: {color_text} ạ"
            dispatcher.utter_message(text)
            if product_other:
                variant_dict = {}
                for color in colors:
                    variants = search_product_variants(product_id=product.id, color=color)
                    if variants:
                        variant_dict[color] = variants[0]
                elements = []
                for color, variant in variant_dict.items():
                    element = variant.to_messenger_element()
                    elements.append(element)

                if elements:
                    message = {
                        "attachment": {
                            "type": "template",
                            "payload": {
                                "template_type": "generic",
                                "elements": elements
                            }
                        }
                    }
                    dispatcher.utter_message(json_message=message)
        return event


import uuid

import dash_auth
import pandas as pd
from dash import dash, html, dcc
import dash_bootstrap_components as dbc
from collections import defaultdict
from dash_extensions.enrich import Output, DashProxy, Input, MultiplexerTransform

import db
import job

app = DashProxy(external_stylesheets=[dbc.themes.SOLAR, dbc.icons.BOOTSTRAP], transforms=[MultiplexerTransform()])

VALID_USERNAME_PASSWORD_PAIRS = {
    'hello': 'world'
}

# globals
df_full = pd.DataFrame()
df_preprocessed = pd.DataFrame()
df_feature_extraction = pd.DataFrame()
df_feature_selection = pd.DataFrame()
run_jobs = pd.DataFrame(columns=["algorithm", "config", "best_score", "time"])
connection = db.connection
current_job = job.Job(datasets=[], preprocessing_steps=[], feature_extraction_method='',
                      feature_selection_percent=0)

# from sklearn import datasets
# df_feature_selection = datasets.load_iris()
# df_feature_selection['category'] = [1 for _ in range df_feature_selection.sh]
df1 = pd.read_csv('./datasets/Tweets1.csv')
df2 = pd.read_csv('./datasets/Reddit_Data.csv')
df3 = pd.read_csv('./datasets/apple-twitter-sentiment-texts.csv')
df3 = df3[['clean_text', 'category']]
df1.dropna(axis=0, inplace=True)
df2.dropna(axis=0, inplace=True)
df3.dropna(axis=0, inplace=True)

auth = dash_auth.BasicAuth(
    app,
    VALID_USERNAME_PASSWORD_PAIRS
)

def render_alert(message: str, link: str, color: str, url: str):
    return dbc.Alert(
        [
            html.I(className="bi bi-x-octagon-fill me-2"),
            html.P(message), dcc.Link(link, url)
        ],
        color=color,
        className="d-flex align-items-center",
    )


#!/usr/bin/env python

"""
crawl thread function
"""

import os
import time
import logging
import threading

from logger import Logger
from webpage_save import Saver
from webpage_parse import Parser
from url_table import UrlTable

SPIDER_ROOT = os.path.join(os.path.dirname(__file__), '.')

lock = threading.Lock()

crawled_url = UrlTable()

class CrawlThread(threading.Thread):
    """
    based on threading.Thread
    """

    def __init__(self, config, queue):
        """
        constructor
        """
        threading.Thread.__init__(self)
        self.config = config
        self.queue = queue
        self.crawl_timeout = int(config['crawl_timeout'])
        self.crawl_interval = int(config['crawl_interval'])

        Logger.config(
            log_file=os.path.join(SPIDER_ROOT, 'log/crawl_thread.log'),
            use_stdout=True,
            log_level=logging.DEBUG)
        self.logger = Logger.get_logger(tag="CrawlThead")
        self.logger.info('Init logger')

    def run(self):
        """
        threading.Thread api run
        @parameter url, crawled_url, parent_depth
        """
        while not self.queue.empty():
            # get queue payload
            payload = self.queue.get()
            url = payload[0]
            parent_depth = payload[1]
            self.logger.debug("working with queue: %s, and with %s" % (payload, self.name))

            # page crawl max_depth check
            link_depth = parent_depth + 1
            if link_depth > int(self.config['max_depth']):
                self.logger.error("Touch the max depth: %s when crawled url %s" %
                                  (self.config['max_depth'], url))

            global crawled_url
            url_table = crawled_url.get()

            # check crawled_url
            # 多线程中的数据互斥访问, 使用threading.Lock()
            if lock.acquire():
                if url in url_table:
                    self.logger.error("This url have been crawled : %s" % url)

                # Parse url
                parser = Parser(self.crawl_timeout)
                a_elements = parser.parse(url, self.config['target_url'])

                url_dict = {}
                href = []
                if a_elements:
                    for link in a_elements:
                        self.queue.put([link, link_depth])
                        href.append(link)
                    url_dict[url] = href
                    page_save = Saver(self.config['output_directory'])
                    page_save.save(url_dict)
                    self.logger.debug("saved crawled data:%s" % url_dict)

                self.queue.task_done()

                crawled_url.set(url)
                self.logger.debug("updated crawled_url: %s" % crawled_url.get())
                lock.release()

            time.sleep(self.crawl_interval)
        self.queue.join()