from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

DOCUMENTATION = r'''
---
module: zvm_clone_disk

short_description: z/VM Clone Disk Ansible Module

version_added: "0.0.1"

description: an Ansible module for z/VM virtual machine minidisks

options:
    name:
        description: this is the target who will own the newly cloned disk
        required: true
        type: str
    diskaddr:
        description: the address of the newly cloned disk in the targets configuration
        required: true
        type: str
    srcname:
        description: source virtual machine that owns the disk to be cloned
        required: true
        type: str
    srcaddr:
        description: source virtual machine disk address to be cloned
        required: true
        type: str
    diskalloc:
        description: how should extents for the new disk be chosen? Select by Region, by Volume, by Volume Group, or a specific device?
        type: str
        choices:
            - AUTOR
            - AUTOV
            - AUTOG
            - DEVNO
    diskloc:
        description: name of the region where diskalloc managed extents should get pulled from - the specific region, volume, or group.
        type: str
    diskmode:
        description: minidisk access mode
        type: str
        choices:
            - R
            - RR
            - W
            - WR
            - M
            - MR
            - MW
    diskrpw:
        description: read-access password to access the disk
        type: str
    diskwpw:
        description: write-access password to access the disk
        type: str
    diskmpw:
        description: multi-write-access password to access the disk
        type: str

seealso:
    - name: IBM z/VM SMAPI documentation
      description: Reference for SMAPI application developers
      link: https://www.ibm.com/docs/en/zvm/7.3?topic=reference-socket-application-programming-interfaces

author:
    - Jay Brenneman (@rjbrenn)
'''

EXAMPLES = r'''
# clone GOLDMAST user disk 250 to existing NEWGUY disk 100
- name: clone NEWGUY 100 disk
  zvm_clone_disk:
    name: NEWGUY
    diskaddr: 100
    srcname: GOLDMAST
    srcaddr: 250

# Create NEWGUY 100 in BIGPOOL0 and clone GOLDMAST 250 into it
- name: make NEWGUY 100 and clone into it
  zvm_clone_disk:
    name: NEWGUY
    diskaddr: 100
    srcname: GOLDMAST
    srcaddr: 250
    diskalloc: AUTOG
    diskloc: BIGPOOL0
    diskmode: W
'''

RETURN = r'''
return_stdout:
    description: the stdout responses from smcli as a dict
    returned: always
    type: list
    elements: str
    sample: "...Done"
return_stderr:
    description: the stderr responses from smcli as a dict
    returned: always
    type: list
    elements: str
    sample: "Defining thingone in the directory...
             Failed
             Return Code: 400
             Reason Code: 8
             Description: ULGSMC5400E Image or profile name already defined
             API issued : Image_Create_DM"
return_code:
    description: The results of doing the thing
    type: int
    returned: always
    sample: 0
reason_code:
    description: The reason for the result
    type: int
    returned: always
    sample: 0
'''

from ansible.module_utils.basic import AnsibleModule

def run_module():
    module_args = dict(
        name=dict(type='str', required=True),
        diskaddr=dict(type='str', required=True),
        srcname=dict(type='str', required=True),
        srcaddr=dict(type='str', required=True),
        diskalloc=dict(type='str', required=False, choices=['AUTOR', 'AUTOV', 'AUTOG', 'DEVNO']),
        diskloc=dict(type='str', required=False),
        diskmode=dict(type='str', required=False, choices=['R', 'RR', 'W', 'WR', 'M', 'MR', 'MW']),
        diskrpw=dict(type='str', required=False),
        diskwpw=dict(type='str', required=False),
        diskmpw=dict(type='str', required=False)
    )

    result = dict(
        changed=False,
        return_stdout=[],
        return_stderr=[],
        return_code=-9,
        reason_code=-9
    )

    module = AnsibleModule(
        argument_spec=module_args,
        supports_check_mode=True
    )

    if module.check_mode:
        module.exit_json(**result)

    cmd_string = []

    cmd_string.append("/opt/zthin/bin/smcli")
    cmd_string.append("Image_Disk_Copy_DM")
    cmd_string.append("-T")
    cmd_string.append(module.params['name'])
    cmd_string.append("-t")
    cmd_string.append(module.params['diskaddr'])
    cmd_string.append("-S")
    cmd_string.append(module.params['srcname'])
    cmd_string.append("-s")
    cmd_string.append(module.params['srcaddr'])
    if module.params['diskalloc'] is not None:
        cmd_string.append("-a")
        cmd_string.append(module.params['diskalloc'])
    if module.params['diskloc'] is not None:
        cmd_string.append("-n")
        cmd_string.append(module.params['diskloc'])
    if module.params['diskmode'] is not None:
        cmd_string.append("-m")
        cmd_string.append(module.params['diskmode'])
    if module.params['diskrpw'] is not None:
        cmd_string.append("-r")
        cmd_string.append(module.params['diskrpw'])
    if module.params['diskwpw'] is not None:
        cmd_string.append("-w")
        cmd_string.append(module.params['diskwpw'])
    if module.params['diskmpw'] is not None:
        cmd_string.append("-x")
        cmd_string.append(module.params['diskmpw'])

    # check=false because some errors are expected if we are adding a disk that already exists
    # on subsequent runs of the module - idempotently
    smcli_results = module.run_command(cmd_string, check_rc=False)
    result['return_stdout'] = smcli_results[1]
    result['return_stderr'] = smcli_results[2]
    result['return_code'] = smcli_results[0]

    if result['return_code'] == 0:
        result['changed'] = True
        module.exit_json(**result)

    if result['return_code'] >= 1:
        notreally_errs = ["Image disk already defined"]
        #
        # notreally_errs is a list of VM error messages that are OK within this contect
        noterror_count = 0

        #
        # an SMAPI error message comes back like this. The important parts are the
        # return code and the reason code
        #    "return_stdout": [
        #        "Defining thingone in the directory... ",
        #        "Failed",
        #        "  Return Code: 400",
        #        "  Reason Code: 8",
        #        "  Description: ULGSMC5400E Image or profile name already defined",
        #        "  API issued : Image_Create_DM"
        #    ]

        #
        # this is gross but I looked at alternatives and they were less readable
        # scann return stdout and stderr for
        for i in result['return_stdout']:
            for j in notreally_errs:
                if i.find(j) != -1:
                    noterror_count += 1
        if noterror_count > 0:
            #
            # optimistically guessing this is not an error condition
            # it might still actually be an error condition if there is a real error among the not-errors
            # will figure that out later if it ends up happening
            # exit no error
            result['return_stdout'].append("skipping an error because its probably OK in this situation")
            module.exit_json(**result)
        # here we are in error condition
        errormsg = "failing return code from smcli is: " + str(result['return_code'])
        module.fail_json(errormsg, **result)

    errormsg = "unknown return code from smcli: " + str(result['return_code'])
    result['return_stderr'] = ["how did we get here???"]
    module.fail_json(errormsg, **result)

def main():
    run_module()

if __name__ == '__main__':
    main()


import torch
import numpy as np
import os
import pandas as pd
import pickle
from sklearn.model_selection import train_test_split
from scipy.special import expit
import urllib
import zipfile

# Class reproducing the ACIC 2016 dataset
class ACIC2016(object):
    def __init__(self, setting=1, exp_num=1, one_hot_factors=True, device='cpu', dataset='train', path_data="../data", tensor=True, seed=0, train_size=0.6, val_size=0.2, scale=True):

        self.path_data = path_data
        self.one_hot_factors = one_hot_factors
        self.covariates = pd.read_csv(os.path.join(path_data,'ACIC2016/covariates.csv'))
        self.info = pd.read_csv(os.path.join(path_data,'ACIC2016/info.csv'))
        self.dgp_data = pd.read_csv(os.path.join(path_data,f'ACIC2016/dgp_data/setting{setting}_dataset{exp_num}.csv'))
        self.X_df = self._process_covariates(self.covariates) # turn factor variables into one-hot binary variables

        attrs = {}
        attrs['x'] = np.array(self.X_df)
        attrs['t'] = np.array(self.dgp_data['z']).reshape((-1, 1))
        attrs['y'] = np.array(self.dgp_data['y']).reshape((-1,1))
        attrs['y0'] = np.array(self.dgp_data['y.0']).reshape((-1, 1))
        attrs['y1'] = np.array(self.dgp_data['y.1']).reshape((-1, 1))
        attrs['mu0'] = np.array(self.dgp_data['mu.0']).reshape((-1, 1))
        attrs['mu1'] = np.array(self.dgp_data['mu.1']).reshape((-1, 1))
        attrs['cate_true'] = attrs['mu1'] - attrs['mu0']
        attrs['ps'] = np.array(self.dgp_data['e']).reshape((-1, 1))

        n_samples = self.X_df.shape[0]
        self.rng = np.random.default_rng(seed=seed)
        original_indices = self.rng.permutation(n_samples)
        n_train = int(train_size * n_samples)
        n_val = int(val_size * n_samples)
        itr = original_indices[:n_train] # train set
        iva = original_indices[n_train:n_train + n_val] # val set
        itrva = original_indices[:n_train + n_val] # train+val set
        ite = original_indices[n_train + n_val:]  # test set

        # Check that each factor is in train, val and test sets
        for covariate_name in self.covariates.columns:
            if not pd.to_numeric(self.covariates[covariate_name], errors='coerce').notnull().all():
                factors_tr = pd.unique(self.covariates[covariate_name].iloc[itr])
                factors_va = pd.unique(self.covariates[covariate_name].iloc[iva])
                factors_te = pd.unique(self.covariates[covariate_name].iloc[ite])
                assert set(factors_va).issubset(set(factors_tr))
                assert set(factors_te).issubset(set(factors_tr))

        if dataset == 'train':
            original_indices = itr
        elif dataset == 'val':
            original_indices = iva
        elif dataset == "train_val":
            original_indices = itrva
        else:
            original_indices = ite
        self.original_indices = original_indices

        # Find binary covariates
        self.binary = []
        for ind in range(attrs['x'].shape[1]):
            self.binary.append(len(np.unique(attrs['x'][:,ind])) == 2)
        self.binary = np.array(self.binary)

        # Normalise - continuous data
        self.scale = scale
        self.xm = np.zeros(self.binary.shape)
        self.xs = np.ones(self.binary.shape)
        if self.scale:
            self.xm[~self.binary] = np.mean(attrs['x'][itrva][:,~self.binary], axis=0)
            self.xs[~self.binary] = np.std(attrs['x'][itrva][:,~self.binary], axis=0)
        attrs['x'] -= self.xm
        attrs['x'] /= self.xs

        # Subsample data and convert to torch.Tensor with the right device
        for key, value in attrs.items():
            value = value[original_indices]
            if tensor:
                value = torch.Tensor(value).to(device)
            setattr(self, key, value)

    def __getitem__(self, index, attrs=None):
        if attrs is None:
            attrs = ['x', 'y', 't', 'mu0', 'mu1', 'cate_true', 'ps']
        res = []
        for attr in attrs:
            res.append(getattr(self, attr)[index])
        return (*res,)

    def __len__(self):
        return len(self.original_indices)

    def _process_covariates(self, covariates):
        covariates_done = {}
        for ind,covariate_name in enumerate(covariates.columns):
            if not 'x_' in covariate_name:
                continue
            if pd.to_numeric(covariates[covariate_name], errors='coerce').notnull().all():
                covariates_done[covariate_name] = covariates[covariate_name]
            else:
                if self.one_hot_factors:
                    for item in sorted(pd.unique(covariates[covariate_name])):
                       covariates_done[covariate_name + '_' + item] = (covariates[covariate_name] == item).astype(int)
                else:
                    covariates_done[covariate_name] = pd.Series([0]*len(covariates[covariate_name]))
                    for idx, item in sorted(enumerate(pd.unique(covariates[covariate_name]))):
                        covariates_done[covariate_name][covariates[covariate_name] == item] = idx

        return pd.DataFrame(covariates_done)

class IHDP(object):
    def __init__(self, exp_num, device='cpu', dataset='train', path_data="../data", tensor=True, normalise_y=True, split_original=False, rescale_cates=True, train_size=0.6, val_size=0.2, scale=True):
        self.path_data = path_data

        attrs = {}
        data = np.loadtxt(os.path.join(self.path_data,'IHDP/csv/ihdp_npci_' + str(exp_num) + '.csv'), delimiter=',')
        t, y, y_cf = data[:, 0][:, np.newaxis], data[:, 1][:, np.newaxis], data[:, 2][:, np.newaxis]
        mu_0, mu_1, x = data[:, 3][:, np.newaxis], data[:, 4][:, np.newaxis], data[:, 5:]
        x[:, 13] -= 1 # binary variable with values 1 and 2
        attrs['t'] = t
        attrs['y'] = y
        attrs['y_cf'] = y_cf
        attrs['mu0'] = mu_0
        attrs['mu1'] = mu_1
        attrs['cate_true'] = mu_1 - mu_0
        attrs['x'] = x

        if split_original: # unused in experiments - divide data as in code for CEVAE : https://github.com/AMLab-Amsterdam/CEVAE/blob/master/datasets.py
            idxtrain, ite = train_test_split(np.arange(x.shape[0]), test_size=0.1, random_state=1)
            itr, iva = train_test_split(idxtrain, test_size=0.3, random_state=1)
        else:
            n_samples = x.shape[0]
            rng = np.random.default_rng(seed=0)
            original_indices = rng.permutation(n_samples)
            n_train = int(train_size * n_samples)
            n_val = int(val_size * n_samples)
            itr = original_indices[:n_train] # train set
            iva = original_indices[n_train:n_train + n_val] # val set
            idxtrain = original_indices[:n_train + n_val] # train + val set
            ite = original_indices[n_train + n_val:]  # test set

        if rescale_cates: # rescale CATEs as in Curth et al. 2021 - see paper for ref
            sd_cate = np.sqrt(np.var(attrs['cate_true'][idxtrain]))
            if sd_cate > 1:
                error = y - t*mu_1 - (1-t)*mu_0
                mu_0 = mu_0 / sd_cate
                mu_1 = mu_1 / sd_cate
                y  = t*mu_1 + (1-t)*mu_0 + error
                attrs['y'] = y
                attrs['mu0'] = mu_0
                attrs['mu1'] = mu_1
                attrs['cate_true'] = mu_1 - mu_0

        ytr = y[itr]
        ym, ys = np.mean(ytr), np.std(ytr)
        self.ym = ym
        self.ys = ys
        if dataset == 'train':
            original_indices = itr
        elif dataset == 'val':
            original_indices = iva
        elif dataset == "train_val":
            original_indices = idxtrain
        else:
            original_indices = ite
        self.original_indices = original_indices

        # Find binary covariates
        self.binary = []
        for ind in range(attrs['x'].shape[1]):
            self.binary.append(len(np.unique(attrs['x'][:, ind])) == 2)
        self.binary = np.array(self.binary)

        # Normalise - continuous variables only
        self.scale = scale
        self.xm = np.zeros(self.binary.shape)
        self.xs = np.ones(self.binary.shape)
        if self.scale:
            self.xm[~self.binary] = np.mean(attrs['x'][idxtrain][:, ~self.binary], axis=0)
            self.xs[~self.binary] = np.std(attrs['x'][idxtrain][:, ~self.binary], axis=0)
        attrs['x'] -= self.xm
        attrs['x'] /= self.xs

        # Subsample data and convert to torch.Tensor with the right device
        for key, value in attrs.items():
            value = value[original_indices]
            if tensor:
                value = torch.Tensor(value).to(device)
            setattr(self, key, value)

        if normalise_y:
            self.y = (self.y - ym) / ys

    def __getitem__(self, index, attrs=None):
        if attrs is None:
            attrs = ['x', 'y', 't', 'mu0', 'mu1', 'cate_true']
        res = []
        for attr in attrs:
            res.append(getattr(self, attr)[index])
        return (*res,)

    def __len__(self):
        return len(self.original_indices)

class News():

    def __init__(self, exp_num, dataset='train', tensor=True, device="cpu", train_size=0.6, val_size=0.2,
                 data_folder=None, scale=True, seed=0):

        if data_folder is None:
            data_folder = '../data'

        # Create data if it does not exist
        if not os.path.isdir(os.path.join(data_folder, 'News/numpy_dicts/')):
            self._create_data(data_folder)

        with open(os.path.join(data_folder, 'News/numpy_dicts/data_as_dicts_with_numpy_seed_{}'.format(exp_num)),
                  'rb') as file:
            data = pickle.load(file)
        data['cate_true'] = data['mu1'] - data['mu0']

        # Create and store indices
        x = data['x']
        n_samples = x.shape[0]
        rng = np.random.default_rng(seed=seed)
        original_indices = rng.permutation(n_samples)
        n_train = int(train_size * n_samples)
        n_val = int(val_size * n_samples)
        itr = original_indices[:n_train] # train set
        iva = original_indices[n_train:n_train + n_val] # val set
        idxtrain = original_indices[:n_train + n_val] # train + val set
        ite = original_indices[n_train + n_val:]  # test set

        if dataset == 'train':
            original_indices = itr
        elif dataset == 'val':
            original_indices = iva
        elif dataset == "train_val":
            original_indices = idxtrain
        else:
            original_indices = ite
        self.original_indices = original_indices

        # Subsample data and convert to torch.Tensor with the right device
        for key, value in data.items():
            value = value[original_indices]
            if tensor:
                value = torch.Tensor(value).to(device)
            setattr(self, key, value)

    @staticmethod
    def _create_data(data_folder):

        print('News : no data, creating it')
        print('Downloading zipped csvs')
        urllib.request.urlretrieve('http://www.fredjo.com/files/NEWS_csv.zip', os.path.join(data_folder, 'News/csv.zip'))

        print('Unzipping csvs with sparse data')
        with zipfile.ZipFile(os.path.join(data_folder, 'News/csv.zip'), 'r') as zip_ref:
            zip_ref.extractall(os.path.join(data_folder, 'News'))

        print('Densifying the sparse data')
        os.mkdir(os.path.join(data_folder, 'News/numpy_dicts/'))

        for f_index in range(1, 50 + 1):
            mat = pd.read_csv(os.path.join(data_folder,'News/csv/topic_doc_mean_n5000_k3477_seed_{}.csv.x'.format(f_index)))
            n_rows, n_cols = int(mat.columns[0]), int(mat.columns[1])
            x = np.zeros((n_rows, n_cols)).astype(int)
            for i, j, val in zip(mat.iloc[:, 0], mat.iloc[:, 1], mat.iloc[:, 2]):
                x[i - 1, j - 1] = val
            data = {}
            data['x'] = x
            meta = pd.read_csv(
                os.path.join(data_folder, 'News/csv/topic_doc_mean_n5000_k3477_seed_{}.csv.y'.format(f_index)),
                names=['t', 'y', 'y_cf', 'mu0', 'mu1'])
            for col in ['t', 'y', 'y_cf', 'mu0', 'mu1']:
                data[col] = np.array(meta[col]).reshape((-1, 1))
            with open(os.path.join(data_folder, 'News/numpy_dicts/data_as_dicts_with_numpy_seed_{}'.format(f_index)), 'wb') as file:
                pickle.dump(data, file)

        print('Done!')

    def __getitem__(self, index, attrs=None):
        if attrs is None:
            attrs = ['x', 'y', 't', 'mu0', 'mu1', 'cate_true']
        res = []
        for attr in attrs:
            res.append(getattr(self, attr)[index])
        return (*res,)

    def __len__(self):
        return len(self.original_indices)


"""
Classes used by the rest api and other utilities that handle data formatting
and gaps in a series of data.
"""
import datetime

from collections import OrderedDict

from esmond.util import atdecode, atencode
from tastypie.exceptions import BadRequest

class QueryUtil(object):
    """Class holding common query methods used by multiple resources
    and data structures to validate incoming request elements."""

    _timerange_limits = {
        30: datetime.timedelta(days=30),
        60: datetime.timedelta(days=30),
        300: datetime.timedelta(days=30),
        3600: datetime.timedelta(days=365),
        86400: datetime.timedelta(days=365*10),
    }

    timeseries_request_types = ['RawData', 'BaseRate', 'Aggs']
    bulk_request_types = ['timeseries', 'interface']

    @staticmethod
    def decode_datapath(datapath):
        return [ atdecode(step) for step in datapath ]

    @staticmethod
    def encode_datapath(datapath):
        return [ atencode(step) for step in datapath ]

    @staticmethod
    def valid_timerange(obj, in_ms=False):
        """Check the requested time range against the requested aggregation
        level and limit if too much data was requested.

        The in_ms flag is set to true if a given resource (like the
        /timeseries/ namespace) is doing business in milliseconds rather
        than seconds."""
        if in_ms:
            s = datetime.timedelta(milliseconds=obj.begin_time)
            e = datetime.timedelta(milliseconds=obj.end_time)
        else:
            s = datetime.timedelta(seconds=obj.begin_time)
            e = datetime.timedelta(seconds=obj.end_time)

        divs = { False: 1, True: 1000 }

        try:
            if e - s > QueryUtil._timerange_limits[obj.agg/divs[in_ms]]:
                return False
        except KeyError:
            raise BadRequest('invalid aggregation level: %s' %
                    obj.agg)

        return True

    @staticmethod
    def format_data_payload(data, in_ms=False, coerce_to_bins=None):
        """Massage results from cassandra for json return payload.

        The in_ms flag is set to true if a given resource (like the
        /timeseries/ namespace) is doing business in milliseconds rather
        than seconds.

        If coerce_to_bins is not None, truncate the timestamp to the
        the bins spaced coerce_to_bins ms apart. This is useful for
        fitting raw data to bin boundaries."""

        divs = { False: 1000, True: 1 }

        results = []

        for row in data:
            ts = row['ts']

            if coerce_to_bins:
                ts -= ts % coerce_to_bins

            d = [ts/divs[in_ms], row['val']]

            # Further options for different data sets.
            if row.has_key('is_valid'): # Base rates
                if row['is_valid'] == 0: d[1] = None
            elif row.has_key('cf'): # Aggregations
                pass
            else: # Raw Data
                pass

            results.append(d)

        return results

class Fill(object):
    """Set of methods to verify that a series of binned data contains
    the correct number of datapoints over a given time range, and if
    not, fill the missing bins with an invalid value before returning
    the data to the client.

    Normally persister will backfill gaps in the data in the DB but gaps
    could be returned under the following circumstances:

    1 - The most common scenario is when requesting up to the minute
    binned data where the most recent value might not have been written
    yet.

    2 - The query might span when a new device started having data
    recorded.

    3 - A gap (like if a device was offline for a period of time) might
    exceed the "heartbeat limit" and generate a span too long to reasonably
    fill with invalid data points.
    """
    @staticmethod
    def expected_bin_count(start_bin, end_bin, freq):
        """Get expected number of bins in a given range of bins."""
        return ((end_bin - start_bin) / freq) + 1

    @staticmethod
    def get_expected_first_bin(begin, freq):
        """Get the first bin of a given frequency based on the begin ts
        of a timeseries query."""
        # Determine the first bin in the series based on the begin
        # timestamp in the timeseries request.
        #
        # Bin math will round down to last bin but timerange queries will
        # return the next bin.  That is, given a 30 second bin, a begin
        # timestamp of 15 seconds past the minute will yield a bin calc
        # of on the minute, but but the time range query will return
        # 30 seconds past the minute as the first result.
        #
        # A begin timestamp falling directly on a bin will return
        # that bin.
        bin = (begin/freq)*freq
        if bin < begin:
            return bin+freq
        elif bin == begin:
            return bin
        else:
            # Shouldn't happen
            raise RuntimeError

    @staticmethod
    def get_bin_alignment(begin, end, freq):
        """Generate a few values needed for checking and filling a series if
        need be."""
        start_bin = Fill.get_expected_first_bin(begin,freq)
        end_bin = (end/freq)*freq
        expected_bins = Fill.expected_bin_count(start_bin, end_bin, freq)

        return start_bin, end_bin, expected_bins

    @staticmethod
    def generate_filled_series(start_bin, end_bin, freq, data):
        """Genrate a new 'filled' series if the returned series has unexpected
        gaps.  Initialize a new range based in the requested time range as
        an OrderedDict, then iterate through original series to retain original
        values.
        """
        # Generate the empty "proper" timerange
        filled_range = []
        s = start_bin + 0 # copy it
        while s <= end_bin:
            filled_range.append((s,None))
            s += freq

        # Make it a ordered dict
        fill = OrderedDict(filled_range)

        # Go through the original data and plug in
        # good values

        for dp in data:
            fill[dp[0]] = dp[1]

        for i in fill.items():
            yield list(i)

    @staticmethod
    def verify_fill(begin, end, freq, data):
        """Top-level function to inspect a returned series for gaps.
        Returns the original series of the count is correct, else will
        return a new filled series."""
        begin, end, freq = int(begin), int(end), int(freq)
        start_bin,end_bin,expected_bins = Fill.get_bin_alignment(begin, end, freq)
        #print 'got :', len(data)
        #print 'need:', Fill.expected_bin_count(start_bin,end_bin,freq)
        if len(data) == Fill.expected_bin_count(start_bin,end_bin,freq):
            #print 'verify: not filling'
            return data
        else:
            #print 'verify: filling'
            return list(Fill.generate_filled_series(start_bin,end_bin,freq,data))


#!/usr/bin/python3
import pandas as pd
import requests
import numpy as np
import os
from nsetools import Nse
from datetime import timezone
from datetime import date
from datetime import datetime,timedelta
from scipy.optimize import curve_fit
import matplotlib.pyplot as plt
from statistics import mean
import random

class stock_analsys:
    global_constants = {}
    #this tolerance is for verifying if the candle length and the candle body length are close....test for MARUBOZU
    global_constants['ratio_MARUBOZU'] = 0.9
    global_constants['ratio_DOJI'] = 0.3
    global_constants['ratio_MEAN'] = 0.9
    global_constants['ratio_trend_REV'] = 3.
    global_constants['trend_days'] = 20
    global_constants['curve_fit_poly_degree'] = 5
    global_constants['prev_trend_days'] = [4,8,11]
    global_constants['supp_res_days'] = 50 #number of days to be considered for finding the support resistance
    global_constants['num_days_prev_vol'] = 10
    global_out_dict = {}

    def __init__(self,symbs,delta_days=500):
        self.make_plots = False
        self.ff = 'poly10'
        self.fitting_funcs = {'poly2':self.poly2,'poly3':self.poly3,'poly4':self.poly4,'poly5':self.poly5,'expo':self.expo,'poly6':self.poly6,'poly7':self.poly7,'poly8':self.poly8,'poly9':self.poly9,'poly10':self.poly10,'poly11':self.poly11,'poly12':self.poly12,'poly13':self.poly13,'sin_mix':self.sin_mix}
        self.delta_days = delta_days
        self.stocks = []
        self.marubozu = []
        self.engulf = []
        self.harami = []
        self.doji = []
        self.reversal = []
        self.trend = {}
        self.supp = []
        self.resis = []
        self.vol = []
        for i in self.global_constants['prev_trend_days']:
            self.trend['TREND'+str(i)] = []
        self.symbs = symbs
        self.multi_stock()

    def single_stock(self,stock):
        self.make_plots = True
        self.stock_name = stock
        self.yahoo_fetch_history()
        self.process_data_all()
        self.analyze_candleS()
        self.find_trend()

    def multi_stock(self):
        self.TREND = {}
        iCount = 0
        for key in self.symbs:
            for stock in self.symbs[key]:
                iCount += 1
                print(stock)
                self.MARUBOZU = 'FALSE'
                self.ENGULF = 'FALSE'
                self.DOJI = 'FALSE'
                self.REVSERSAL = 'FALSE'
                self.HARAMI = 'FALSE'
                self.VOL = 'FALSE'
                for i in self.global_constants['prev_trend_days']:
                    self.TREND['TREND'+str(i)] = []

                self.stock_name = stock
                self.yahoo_fetch_history()
                self.process_data_all()
                self.analyze_candleS()
                self.find_trend()

                self.stocks.append(stock)
                self.marubozu.append(self.MARUBOZU)
                self.engulf.append(self.ENGULF)
                self.doji.append(self.DOJI)
                self.harami.append(self.HARAMI)
                self.reversal.append(self.REVSERSAL)
                self.vol.append(self.VOL)
                for i in self.global_constants['prev_trend_days']:
                    self.trend['TREND'+str(i)].extend(self.TREND['TREND'+str(i)])

        self.global_out_dict['STOCK'] = self.stocks
        self.global_out_dict['MARUBOZU'] = self.marubozu
        self.global_out_dict['DOJI'] = self.doji
        for i in self.global_constants['prev_trend_days']:
            self.global_out_dict['TREND'+str(i)] = self.trend['TREND'+str(i)]
        self.global_out_dict['TREND_REVERSAL'] = self.reversal
        self.global_out_dict['ENGULF'] = self.engulf
        self.global_out_dict['HARAMI'] = self.harami
        #self.global_out_dict['SUPPORT'] = self.supp
        #self.global_out_dict['RESISTANCE'] = self.resis
        self.global_out_dict['GOOD volume'] = self.vol
        self.df = pd.DataFrame(self.global_out_dict)
        self.df.to_csv(os.getcwd()+'/OUT.csv')

    def yahoo_fetch_history(self):
        now = datetime.now()
        past = now - timedelta(days = self.delta_days)
        period1 = int(datetime.timestamp(past))
        period2 = int(datetime.timestamp(now))
        url = 'https://query1.finance.yahoo.com/v7/finance/download/'+self.stock_name+'.NS'
        url += '?period1='+str(period1)+'&period2='+str(period2)+'&interval=1d&events=history&includeAdjustedClose=true'
        r = requests.get(url, allow_redirects=False)
        with open(os.getcwd()+'/dat.csv', 'wb') as f:f.write(r.content)
        #reading into temp ff to remove "null" rows and then write back to dat.csv
        ff = pd.read_csv('dat.csv')
        ff = ff.dropna()
        ff.to_csv(os.getcwd()+'/dat.csv')
        self.dat = pd.read_csv('dat.csv')

    def process_data_all(self):
        self.dat['candle_len'] = (self.dat['High']-self.dat['Low']).to_list()
        #in case of DOJI candle we assign 0.001 as a very small body length...to avoid any DIV error!
        self.dat['body_len'] = list(map(lambda x: abs(x) if x!=0.0 else 0.001,(self.dat['Close']-self.dat['Open']).to_list()))
        self.dat['body_mean'] = [abs(x) for x in ((self.dat['Close']+self.dat['Open'])*.5).to_list()]
        self.dat['open_close_diff'] = (self.dat['Close'] - self.dat['Open']).to_list()
        self.dat['ratio_bodylen_cndlen'] = (self.dat['body_len']/self.dat['candle_len']).to_list()
        self.dat['upper_wick_len'] = list(map(lambda hi,clos,opn,diff: hi-clos if diff>0.0 else hi-opn,self.dat['High'],self.dat['Close'],self.dat['Open'],self.dat['open_close_diff']))
        self.dat['lower_wick_len'] = list(map(lambda low,clos,opn,diff: opn-low if diff>0.0 else clos-low,self.dat['Low'],self.dat['Close'],self.dat['Open'],self.dat['open_close_diff']))

    def create_bins(self,lb,ub,bins):
        print(type(lb))
        bins = []
        width = ((ub-lb)/bins)
        for low in range(lb, lb + bins*width + 1, width):
            bins.append((low, low+width))
        bins2 = pd.IntervalIndex.from_tuples(bins)
        return bins2

    def analyze_candleS(self):
        now = -1
        prev = now -1

        supp_res_data_frame = self.dat.iloc[-70:,:]

        #supp_res_data_frame['supp_bins'] = pd.qcut(supp_res_data_frame['Low'],q=5,precision=0)
        supp_res_data_frame['supp_bins'] = pd.cut(supp_res_data_frame['Low'].to_list(),bins=5,right=False)
        #supp_res_data_frame['supp_bins'].value_counts().plot(kind='barh')

        #print(supp_res_data_frame)
        #print(supp_res_data_frame['supp_bins'].value_counts())

        #to avoid DIV error
        if(self.dat['lower_wick_len'].to_list()[now]>0):
            lower_wick_len = self.dat['lower_wick_len'].to_list()[now]
        else:
            lower_wick_len = .1

        marubozu_cond1 = self.dat['ratio_bodylen_cndlen'].to_list()[now]>self.global_constants['ratio_MARUBOZU']
        engulf_cond1 = (self.dat['Low'].to_list()[now]/self.dat['Low'].to_list()[prev])<=1.#Current Low shld be lower than prev Low
        engulf_cond2 = (self.dat['High'].to_list()[now]/self.dat['High'].to_list()[prev])>=1.#Curn high > Prev high
        engulf_cond3 = (self.dat['body_len'].to_list()[now]/self.dat['body_len'].to_list()[prev])>1.#

        doji_cond1 = self.dat['ratio_bodylen_cndlen'].to_list()[now]<self.global_constants['ratio_DOJI']
        doji_cond2 = self.dat['upper_wick_len'].to_list()[now]/lower_wick_len>self.global_constants['ratio_MEAN']
        trend_cond1 = self.dat['upper_wick_len'].to_list()[now]/lower_wick_len>self.global_constants['ratio_trend_REV']
        trend_cond2 = self.dat['lower_wick_len'].to_list()[now]/lower_wick_len>self.global_constants['ratio_trend_REV']

        #checking if the Curr vol > avg of the last 10 days volume
        good_volumes = self.dat['Volume'].to_list()[now]>mean(self.dat['Volume'].to_list()[-self.global_constants['num_days_prev_vol']:-1])

        harami_cond1 = (self.dat['body_len'].to_list()[now]/self.dat['body_len'].to_list()[prev])<1.
        #Curr body len<Prev body len
        if(self.dat['open_close_diff'].to_list()[prev]>0):
            harami_cond2 =(self.dat['Open'].to_list()[prev]<self.dat['body_mean'].to_list()[now]<self.dat['Close'].to_list()[prev])
        else:
            harami_cond2 =(self.dat['Close'].to_list()[prev]<self.dat['body_mean'].to_list()[now]<self.dat['Open'].to_list()[prev])

        if(marubozu_cond1):
            self.MARUBOZU = 'TRUE'
        if(engulf_cond1 and engulf_cond2 and engulf_cond3):
            self.ENGULF = 'TRUE'
        if(doji_cond1 and doji_cond2):
            self.DOJI = 'TRUE'
        if((trend_cond1 or trend_cond2) and doji_cond1):
            self.REVSERSAL = 'TRUE'
        if(harami_cond1 and harami_cond2):
            self.HARAMI = 'TRUE'
        if(good_volumes):
            self.VOL = 'TRUE'

    #https://machinelearningmastery.com/curve-fitting-with-python/
    def find_trend(self):
        #print(self.stock_name)
        high_price = self.dat["High"].to_list()
        low_price = self.dat["Low"].to_list()
        high_price = high_price[-int(self.global_constants['trend_days']):-1]
        low_price = low_price[-int(self.global_constants['trend_days']):-1]
        mean_values = list(map(lambda x,y: (x+y)/2.,high_price,low_price))
        x_data = np.arange(1,int(self.global_constants['trend_days']),1)
        if(len(mean_values)!=0):
            popt, _ = curve_fit(self.fitting_funcs[self.ff],x_data,mean_values)
            mean_fitted = self.fitting_funcs[self.ff](x_data,*popt)
            if(self.make_plots):
                plt.plot(x_data,mean_values,'--b')
                plt.plot(x_data,mean_fitted,'-r')
                plt.show()
            for i in self.global_constants['prev_trend_days']:
                x_tick = self.global_constants['trend_days']-i
                mean_func_left_intrvl = self.fitting_funcs[self.ff](x_data[x_tick],*popt)
                mean_func_right_intrvl = self.fitting_funcs[self.ff](x_data[-1],*popt)
                mean_slope = (mean_func_right_intrvl-mean_func_left_intrvl)/i
                #print(mean_func_left_intrvl)
                #print(mean_func_right_intrvl)
                #print(mean_slope)
                if(mean_slope>0.0):
                    self.trend['TREND'+str(i)].append('BULLISH')
                else:
                    self.trend['TREND'+str(i)].append('BEARISH')
        else:
            for i in self.global_constants['prev_trend_days']:
                self.trend['TREND'+str(i)].append('UNKOWN')
    def sin_mix(self,x, a, b, c, d):
	    return a * np.sin(b - x) + c * x**2 + d
    def expo(self,x, a, b, c):
	    return a * np.exp (-b * x) + c
    def poly2(self,x, a, b, c):
	    return (a * x) + (b * x**2) + c
    def poly3(self,x, a, b, c, d):
	    return (a * x) + (b * x**2) + (c * x**3) + d
    def poly4(self,x, a, b, c, d, e):
	    return (a * x) + (b * x**2) + (c * x**3) + (d * x**4) + e
    def poly5(self,x, a, b, c, d, e, f):
	    return (a * x) + (b * x**2) + (c * x**3) + (d * x**4) + (e * x**5) + f
    def poly6(self,x, a, b, c, d, e, f,g):
	    return (a * x) + (b * x**2) + (c * x**3) + (d * x**4) + (e * x**5) + (f * x**6) + g
    def poly7(self,x, a, b, c, d, e, f,g,h):
	    return (a * x) + (b * x**2) + (c * x**3) + (d * x**4) + (e * x**5) + (f * x**6) + (g * x**7) +h
    def poly8(self,x, a, b, c, d, e, f,g,h,i):
	    return (a * x) + (b * x**2) + (c * x**3) + (d * x**4) + (e * x**5) + (f * x**6) + (g * x**7) + (h * x**8) +i
    def poly9(self,x, a, b, c, d, e, f,g,h,i,j):
	    return (a * x) + (b * x**2) + (c * x**3) + (d * x**4) + (e * x**5) + (f * x**6) + (g * x**7) + (h * x**8) + (i * x**9) + j
    def poly10(self,x, a, b, c, d, e, f,g,h,i,j,k):
	    return (a * x) + (b * x**2) + (c * x**3) + (d * x**4) + (e * x**5) + (f * x**6) + (g * x**7) + (h * x**8) + (i * x**9) + (j * x**10) + k
    def poly11(self,x, a, b, c, d, e, f,g,h,i,j,k,l):
	    return (a * x) + (b * x**2) + (c * x**3) + (d * x**4) + (e * x**5) + (f * x**6) + (g * x**7) + (h * x**8) + (i * x**9) + (j * x**10) + (k * x**11) + l
    def poly12(self,x, a, b, c, d, e, f,g,h,i,j,k,l,m):
	    return (a * x) + (b * x**2) + (c * x**3) + (d * x**4) + (e * x**5) + (f * x**6) + (g * x**7) + (h * x**8) + (i * x**9) + (j * x**10) + (k * x**11) + (l * x**12) + m
    def poly13(self,x, a, b, c, d, e, f,g,h,i,j,k,l,m,n):
	    return (a * x) + (b * x**2) + (c * x**3) + (d * x**4) + (e * x**5) + (f * x**6) + (g * x**7) + (h * x**8) + (i * x**9) + (j * x**10) + (k * x**11) + (l * x**12) + (m * x**13) + n

    def test_func(self):
        rnd = random.sample(range(1, 50), 10)
        rnd.append(1000)
        rnd.append(300)
        rnd.append(500)
        rnd.append(700)
        xx = pd.cut(rnd,bins=3)
        print(rnd)
        print(xx.value_counts())

stock_symbols = {}
stock_symbols['NIFTY F&O'] = pd.read_csv('sos_scheme.csv')['Symbol'].to_list()
cls_instance = stock_analsys(stock_symbols)
#cls_instance.single_stock('IOC')
#cls_instance.single_stock('DRREDDY')
#cls_instance.test_func()

#stock_symbols['NIFTY auto'] = pd.read_csv('ind_niftyautolist.csv')['Symbol'].to_list()
#stock_symbols['NIFTY bank'] = pd.read_csv('ind_niftybanklist.csv')['Symbol'].to_list()
#stock_symbols['NIFTY it'] = pd.read_csv('ind_niftyitlist.csv')['Symbol'].to_list()
#stock_symbols['NIFTY metal'] = pd.read_csv('ind_niftymetallist.csv')['Symbol'].to_list()
#stock_symbols['NIFTY pharma'] = pd.read_csv('ind_niftypharmalist.csv')['Symbol'].to_list()
#stock_symbols['NIFTY oilGas'] = pd.read_csv('ind_niftyoilgaslist.csv')['Symbol'].to_list()
#cls_instance.single_stock('KOTAKBANK')
#cls_instance.single_stock('RELIANCE')
#nse = Nse()
#ll = nse.get_index_list()
#ll = nse.get_index_quote("nifty it")


from dataclasses import dataclass

# from smallcalc.calc_lexer import INTEGER, LITERAL

@dataclass
class IntegerNode:
    value: int

    def __init__(self, value):
        self.value = int(value)
        self.type = 'integer'

    def asdict(self):
        return {
            'type': self.type,
            'value': self.value
        }

@dataclass
class LiteralNode:
    value: str

    def __init__(self, value):
        self.value = str(value)
        self.type = 'literal'

    def asdict(self):
        return {
            'type': self.type,
            'value': self.value
        }

class ExpressionNode:
    value: str

    def __init__(self, left, right, operator):
        self.type = 'binary'
        self.left = left
        self.right = right
        self.operator = operator

    def asdict(self):
        return {
            'type': self.type,
            'left': self.left.asdict(),
            'right': self.right.asdict(),
            'operator': self.operator.asdict()
        }


from flask import Flask, jsonify, request, render_template, redirect, url_for, session
from flask_cors import CORS
from sqlalchemy_cockroachdb import run_transaction
from sqlalchemy import create_engine, null
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from sqlalchemy import Column, Integer, String, ForeignKey, Boolean, Date
from sqlalchemy.dialects.postgresql import ARRAY
import requests
from datetime import datetime

app = Flask(__name__, template_folder='../templates', static_folder='../static')
app.config['SECRET_KEY'] = 'flirtify_esd_micro'
app.config['SESSION_COOKIE_DOMAIN'] = '127.0.0.1'
app.config['SESSION_COOKIE_PATH'] = '/'
CORS(app)

# Configure the SQLAlchemy engine to use CockroachDB
engine = create_engine('cockroachdb+psycopg2://jeremy:GvtUwDUhQOYrlDC7jEbblg@flirtify-4040.6xw.cockroachlabs.cloud:26257/flirtify?sslmode=require')
# Create a SQLAlchemy session factory to manage database connections
Session = sessionmaker(bind=engine)

# Create a base class for database models
Base = declarative_base()

class Match(Base):
    __tablename__ = 'match'
    match_id = Column(Integer, primary_key=True, autoincrement=True)

    user_id1 = Column(Integer, nullable = False)
    user_id2 = Column(Integer, nullable = False)

    user1_match = Column(Boolean, nullable = True)
    user2_match = Column(Boolean, nullable = True)

    datePrefs = Column(ARRAY(String))
    dateIdea = Column(ARRAY(String))

    dateMatched = Column(Date, nullable = True)

    def json(self):
        return {"match_id": self.match_id, "user_id1": self.user_id1, "user_id2": self.user_id2, "user1_match": self.user1_match, "user2_match": self.user2_match, "datePrefs": self.datePrefs, "dateIdea": self.dateIdea, "dateMatched":self.dateMatched}

session = Session()

@app.route('/home')
def home():
    return render_template('home.html')

@app.route('/matches')
def matches():
    return render_template('matches.html')

@app.route('/chat')
def chat():
    return render_template('message.html')

#add table
@app.route('/add_table')
def add_table():
    # Create the database tables
    Base.metadata.create_all(engine)
    return "table created successfully"

#drop table
@app.route('/drop_table')
def drop_table():
    # Drop the table
    def drop_table(txn):
        txn.execute("DROP TABLE match")
    run_transaction(engine, drop_table)

    return 'Table dropped successfully.'

# retrieve all matches
@app.route("/match")
def get_all():
    matches = session.query(Match).all()
    if len(matches):
        return jsonify(
            {
                "code": 200,
                "data": {
                    "matches": [match.json() for match in matches]
                }
            }
        )
    return jsonify(
        {
            "code": 404,
            "message": "There are no matches."
        }
    ), 404

# specific match by match_id
@app.route("/match/<string:match_id>")
def find_by_match_id(match_id):
    match = session.query(Match).filter_by(match_id=match_id).first()

    if match:
        return jsonify(
            {
                "code": 200,
                "data": match.json()
            }
        )
    return jsonify(
        {
            "code": 404,
            "message": "Match not found."
        }
    ), 404

# return successful matches by user id
@app.route("/successful_match/<string:user_id>")
def find_successful_matches(user_id):
    #filter out by userid
    m1 = session.query(Match).filter(Match.user_id1 == user_id)
    m2 = session.query(Match).filter(Match.user_id2 == user_id)

    #all matches by userid
    matches = m1.union(m2).filter(Match.user1_match == "true", Match.user2_match == "true").all()

    # if matches means swiped - or have created match
    if matches:

        all_matches = []

        for match in matches:
            match_details = {
                'match_id'      : str(match.match_id),
                'dateIdea'      : match.dateIdea,
                'datePrefs'     : match.datePrefs,
                'user1_match'   : match.user1_match,
                'user2_match'   : match.user2_match,
                'user_id1'      : str(match.user_id1),
                'user_id2'      : str(match.user_id2),
                'dateMatched'   : match.dateMatched
                }
            all_matches.append(match_details)

        response = jsonify(
            {
                "code": 200,
                "data": all_matches
            }
        )
    else:
        response = jsonify(
        {
            "code": 404,
            "message": "Match not found."
        }
        ), 404

    return response

# get all matches for a unique user
@app.route("/match/user/<int:user_id>")
def find_matches_by_user_id(user_id):
    m1 = session.query(Match).filter(Match.user_id1 == user_id)
    m2 = session.query(Match).filter(Match.user_id2 == user_id)
    matches = m1.union(m2).all()

    if matches:
        return jsonify(
            {
                "code": 200,
                "data": [match.json() for match in matches]
            }
        )
    return jsonify(
        {
            "code": 404,
            "message": "Matches not found for user_id: {}".format(user_id)
        }
    ), 404

# @app.route("/create_match", methods=['POST'])
# def create_match():

#     #hidden form data

#     #person deciding
#     user_chooser_id = request.form['user_chooser_id']
#     print(user_chooser_id)
#     #person that is recommended
#     user_suggested_id = request.form['user_suggested_id']

#     #yes or no decision
#     decision_form = request.form['decision']
#     if decision_form == "1" or decision_form == 1:
#         decision = True
#     else:
#         decision = False

#     chooser_as_user2_match = session.query(Match).filter(Match.user_id2 == user_chooser_id).first()

#     suggested_as_user1_match = session.query(Match).filter(Match.user_id1 == user_suggested_id).first()

#     #checking if match with this 2 users exists
#     if (chooser_as_user2_match and suggested_as_user1_match):
#         chooser_as_user2_match.user2_match = decision

#         try:
#             # # add a date field here
#             # chooser_as_user2_match.dateMatch = date.today.strftime('%d/%m/%Y')

#             # # can also populate the datePref and dateIdea here
#             # # call the two urls

#             session.commit()
#             return jsonify(
#                 {
#                     "code": 201,
#                     "data": chooser_as_user2_match.json()
#                 }
#             ), 201

#         except:
#             return jsonify(
#                 {
#                     "code": 500,
#                     "message": "An error occurred updating the match. Please try again."
#                 }
#             ), 500
#     else:
#         new_match = Match(
#             user_id1 = user_chooser_id,
#             user_id2 = user_suggested_id,
#             user1_match = bool(decision)
#         )
#         try:
#             session.add(new_match)
#             session.commit()

#             return jsonify(
#                 {
#                     "code": 201,
#                     "data": new_match.json()
#                 }
#             ), 201

#         except:
#             return jsonify(
#                 {
#                     "code": 500,
#                     "message": "An error occurred creating the match. Please try again."
#                 }
#             ), 500

@app.route("/create_match/<string:user_chooser_id>/<string:user_suggested_id>/0", methods=['POST'])
def create_match_reject(user_chooser_id,user_suggested_id):

    chooser_as_user2_match = session.query(Match).filter(Match.user_id2 == user_chooser_id, Match.user_id1 == user_suggested_id).first()

    #checking if match with this 2 users exists
    if (chooser_as_user2_match):
        chooser_as_user2_match.user2_match = False

        try:
            # # add a date field here
            # chooser_as_user2_match.dateMatch = date.today.strftime('%d/%m/%Y')

            # # can also populate the datePref and dateIdea here
            # # call the two urls

            session.commit()

            return_data = {
                'dateIdea': chooser_as_user2_match.dateIdea,
                'dateMatched': chooser_as_user2_match.dateMatched,
                'datePrefs': chooser_as_user2_match.datePrefs,
                'match_id': str(chooser_as_user2_match.match_id),
                'user1_match': chooser_as_user2_match.user1_match,
                'user2_match': chooser_as_user2_match.user2_match,
                'user_id1': str(chooser_as_user2_match.user_id1),
                'user_id2': str(chooser_as_user2_match.user_id2),
                }
            return jsonify(
                {
                    "code": 201,
                    "data": return_data
                }
            ), 201

        except:
            return jsonify(
                {
                    "code": 500,
                    "message": "An error occurred updating the match. Please try again."
                }
            ), 500
    else:
        new_match = Match(
            user_id1 = user_chooser_id,
            user_id2 = user_suggested_id,
            user1_match = bool(False)
        )
        try:
            session.add(new_match)
            session.commit()

            return_data = {
                        'dateIdea': new_match.dateIdea,
                        'dateMatched': new_match.dateMatched,
                        'datePrefs': new_match.datePrefs,
                        'match_id': str(new_match.match_id),
                        'user1_match': new_match.user1_match,
                        'user2_match': new_match.user2_match,
                        'user_id1': str(new_match.user_id1),
                        'user_id2': str(new_match.user_id2),
                        }

            return jsonify(
                {
                    "code": 200,
                    "data": return_data
                }
            ), 200

        except:
            return jsonify(
                {
                    "code": 500,
                    "message": "An error occurred creating the match. Please try again."
                }
            ), 500

@app.route("/create_match/<string:user_chooser_id>/<string:user_suggested_id>/1", methods=['POST'])
def create_match_accept(user_chooser_id,user_suggested_id):

    chooser_as_user2_match = session.query(Match).filter(Match.user_id2 == user_chooser_id, Match.user_id1 == user_suggested_id).first()

    # suggested_as_user1_match = session.query(Match).filter(Match.user_id1 == user_suggested_id).first()

    #checking if match with this 2 users exists
    if (chooser_as_user2_match):
        chooser_as_user2_match.user2_match = True
        matchid = chooser_as_user2_match.match_id

        # only if both swiped right on each other then
        ifMatch = False
        if (chooser_as_user2_match.user2_match == True and chooser_as_user2_match.user1_match == True):
            ifMatch = True
            chooser_as_user2_match.dateMatched = datetime.today() #.strftime('%Y/%m/%d')

        try:
            session.commit()

            # only if match then populate datePref and dateIdea
            if ifMatch:

                try:
                    requests.post("http://match:5002/populate_dateprefs/{}".format(matchid))

                    try:
                        requests.post("http://match:5002/date_recommendation/{}".format(matchid))

                    except:
                        return jsonify(
                            {
                                "code": 502,
                                "message": "An error occurred populating the date recommendation. Please try again."
                            }
                        ), 502

                except:
                    return jsonify(
                        {
                            "code": 502,
                            "message": "An error occurred populating the date recommendation. Please try again."
                        }
                    ), 502

        except:
            return jsonify(
                {
                    "code": 501,
                    "message": "An error occurred creating the date preference. Please try again."
                }
            ), 501

        return_data = {
                'dateIdea': chooser_as_user2_match.dateIdea,
                'dateMatched': chooser_as_user2_match.dateMatched,
                'datePrefs': chooser_as_user2_match.datePrefs,
                'match_id': str(matchid),
                'user1_match': chooser_as_user2_match.user1_match,
                'user2_match': chooser_as_user2_match.user2_match,
                'user_id1': str(chooser_as_user2_match.user_id1),
                'user_id2': str(chooser_as_user2_match.user_id2),
                }

        # only if both swiped right on each other then
        ifMatch = False
        if (chooser_as_user2_match.user2_match == True and chooser_as_user2_match.user1_match == True):
            ifMatch = True
            chooser_as_user2_match.dateMatched = datetime.today() #.strftime('%Y/%m/%d')

            try:
                session.commit()

                # only if match then populate datePref and dateIdea
                if ifMatch:

                    try:
                        requests.post("http://match:5002/populate_dateprefs/{}".format(matchid))

                        try:
                            requests.post("http://match:5002/date_recommendation/{}".format(matchid))

                        except:
                            return jsonify(
                                {
                                    "code": 502,
                                    "message": "An error occurred populating the date recommendation. Please try again."
                                }
                            ), 502

                    except:
                        return jsonify(
                            {
                                "code": 501,
                                "message": "An error occurred creating the date preference. Please try again."
                            }
                        ), 501

                return_data = {
                        'dateIdea': chooser_as_user2_match.dateIdea,
                        'dateMatched': chooser_as_user2_match.dateMatched,
                        'datePrefs': chooser_as_user2_match.datePrefs,
                        'match_id': str(matchid),
                        'user1_match': chooser_as_user2_match.user1_match,
                        'user2_match': chooser_as_user2_match.user2_match,
                        'user_id1': str(chooser_as_user2_match.user_id1),
                        'user_id2': str(chooser_as_user2_match.user_id2),
                        }

                return jsonify(
                    {
                        "code":     200,
                        "data":     return_data,
                        "matched":  ifMatch
                    }
                ), 200

            except:
                return jsonify(
                    {
                        "code": 500,
                        "message": "An error occurred updating the match. Please try again."
                    }
                ), 500

    else:
        new_match = Match(
            user_id1 = user_chooser_id,
            user_id2 = user_suggested_id,
            user1_match = bool(True)
        )
        try:
            session.add(new_match)
            session.commit()

            return_data = {
                        'dateIdea': new_match.dateIdea,
                        'dateMatched': new_match.dateMatched,
                        'datePrefs': new_match.datePrefs,
                        'match_id': str(new_match.match_id),
                        'user1_match': new_match.user1_match,
                        'user2_match': new_match.user2_match,
                        'user_id1': str(new_match.user_id1),
                        'user_id2': str(new_match.user_id2),
                        }

            return jsonify(
                {
                    "code":     201,
                    "data":     return_data,
                    "matched":  False
                }
            ), 201

        except:
            return jsonify(
                {
                    "code": 503,
                    "message": "An error occurred creating the match. Please try again."
                }
            ), 503

# DELETE MATCH - In event of banning
@app.route("/match/<string:match_id>", methods=['DELETE'])
def delete_match(match_id):
    match = session.query(Match).filter_by(match_id=match_id).first()
    if match:
        session.delete(match)
        session.commit()
        return jsonify(
            {
                "code": 200,
                "data": {
                    "match_id": match_id
                },
                "message": "Match successfully deleted."
            }
        )
    return jsonify(
        {
            "code": 404,
            "data": {
                "match_id": match_id
            },
            "message": "Match not found."
        }
    ), 404

@app.route("/match/ban/<string:user_id>", methods=['DELETE'])
def delete_all_matches(user_id):
    m1 = session.query(Match).filter(Match.user_id1 == user_id)
    m2 = session.query(Match).filter(Match.user_id2 == user_id)
    matches = m1.union(m2).all()

    if matches:
        for match in matches:
            session.delete(match)
        session.commit()
        return jsonify(
            {
                "code": 200,
                "message": "Deleted all matches related to user_id: {}".format(user_id)
            }
        )
    return jsonify(
        {
            "code": 404,
            "message": "Matches not found for user_id: {}".format(user_id)
        }
    ), 404

# new -- populate datePrefs
@app.route('/populate_dateprefs/<int:match_id>', methods=['POST'])
def populate_datepref(match_id):
    match = session.query(Match).filter_by(match_id=match_id).first()

    user1 = match.user_id1
    user2 = match.user_id2

    dateprefs = []

    for userid in [user1, user2]:
        user_data = requests.get("http://user:8000/user/{}".format(userid))

        if user_data.status_code == 200 and 'application/json' in user_data.headers.get('content-type'):
            json_data = user_data.json()['data']

            for variable in json_data:
                # Check if the "name" key is present in the dictionary
                if variable == 'desiredfirstdate':
                    dateprefs.append(json_data['desiredfirstdate'])
        else:
            return jsonify(
                {
                    "code": 404,
                    "message": "An error occurred creating the date preference. Date preference was not populated"
                }
            )

    if dateprefs != []:
        try:
            match.datePrefs = dateprefs
            # session.query(Match).filter_by(match_id=match_id).first().update({'datePrefs': dateprefs})
            session.commit()

            return jsonify(
                {
                    "code": 201,
                    "data": match.json()
                }
            ), 201

        except:
            return jsonify(
                {
                    "code": 500,
                    "message": "An error occurred creating the date preference. Please try again."
                }
            ), 500

# new -- populate datePrefs end

@app.route('/date_recommendation/<int:match_id>', methods=['POST'])
def populate_dateIdea(match_id):
    match = session.query(Match).filter_by(match_id=match_id).first()

    datePrefs = match.datePrefs
    dateIdeas = []

    for prefDate in datePrefs:
        # send a formdata for each of the match
        url = f"http://dateideaapi:5005/dateidea/{prefDate}"
        response = requests.get(url)

        # Handle the API response
        if response.status_code == 200:
            # Extract the idea from the JSON response
            idea = response.json()

            # Append the idea to the list of date ideas
            dateIdeas.append(idea)
        else:
            # Handle the error
            return jsonify(
                {
                    "message": "There was an error generating the date idea."
                }
            ), 404

        try:
            # Return the success message with the populated date ideas
            match.dateIdea = dateIdeas
            # session.query(Match).filter_by(match_id=match_id).first().update({'datePrefs': dateprefs})
            session.commit()

            return jsonify(
                {
                    "code": 201,
                    "data": match.json()
                }
            ), 201

        except:
            return jsonify(
                {
                    "code": 500,
                    "message": "An error occurred creating the date idea. Please try again."
                }
            ), 500

# @app.context_processor
# def inject_navbar():
#     return dict(navbar="navbar.html")

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5002, debug=True)


from io import StringIO
from contextlib import contextmanager
from psycopg2.errorcodes import UNIQUE_VIOLATION
from psycopg2 import errors

# --- Reset Database ---
DROP_TABLE_COMPANIES = "DROP TABLE IF EXISTS companies;"
DROP_TABLE_BARS = "DROP TABLE IF EXISTS bars;"
DROP_TABLE_NOT_FOUND = "DROP TABLE IF EXISTS not_found;"

# --- Create Tables ---
CREATE_COMPANIES = """
    CREATE TABLE IF NOT EXISTS companies (
        ticker TEXT PRIMARY KEY,
        name TEXT,
        exchange TEXT,
        sector TEXT
    );
"""
CREATE_COMPANIES_ = """
    CREATE TABLE IF NOT EXISTS companies (
        id SERIAL PRIMARY KEY,
        ticker TEXT,
        name TEXT,
        city TEXT,
        country TEXT,
        currency TEXT,
        exchange TEXT,
        sector TEXT,
        industry TEXT,
        UNIQUE (ticker)
    );
"""
CREATE_EXCHANGES = """
    CREATE TABLE IF NOT EXISTS exchanges (
        id SERIAL PRIMARY KEY,
        name TEXT,
        UNIQUE (name)
    );
"""
CREATE_SECTORS = """
    CREATE TABLE IF NOT EXISTS sectors (
        id SERIAL PRIMARY KEY,
        name TEXT,
        UNIQUE (name)
    );
"""
CREATE_INDUSTRIES = """
    CREATE TABLE IF NOT EXISTS industries (
        id SERIAL PRIMARY KEY,
        name TEXT,
        UNIQUE (name)
    );
"""
CREATE_BARS = """
    CREATE TABLE IF NOT EXISTS bars (
        id SERIAL PRIMARY KEY,
        ticker TEXT,
        date DATE,
        open NUMERIC,
        high NUMERIC,
        low NUMERIC,
        close NUMERIC,
        adj_close NUMERIC,
        volume BIGINT,
        UNIQUE (ticker, date)
    );
"""
CREATE_NOT_FOUND = """
    CREATE TABLE IF NOT EXISTS not_found (
        id SERIAL PRIMARY KEY,
        ticker TEXT,
        UNIQUE (ticker)
    );
"""

# --- SELECT FROM DATABASE ---
SELECT_LAST_TICKER = """
    SELECT MAX(date) AS last_row FROM bars
    WHERE ticker = %s;
"""
SELECT_FIRST_TICKER = """
    SELECT MIN(date) AS first_row FROM bars
    WHERE ticker = %s;
"""

SELECT_COMPANIES = """
    SELECT ticker FROM companies
    LIMIT %s;
"""

SELECT_COMPANIES_WHERE_EXCHANGE = """
    SELECT ticker FROM companies
    WHERE exchange = %s
    LIMIT %s;
"""

SELECT_COMPANIES_WHERE_TICKER = """
    SELECT ticker FROM companies
    WHERE ticker = %s;
"""

SELECT_TICKERS_NOT_FOUND = """
    SELECT ticker from not_found;
"""

SELECT_EXCHANGES = """
    SELECT DISTINCT exchange from companies
    ORDER BY exchange ASC
"""

# --- INSERT INTO DATABASE ---
INSERT_COMPANIE_RETURN_TICKER = """
    INSERT INTO companies
    (ticker, name, exchange, sector)
    VALUES (%s, %s, %s, %s)
    RETURNING ticker;
"""
INSERT_COMPANIE_RETURN_ID = """
    INSERT INTO companies
    (ticker, name, city, country, currency, exchange, sector, industry)
    VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
    RETURNING id;
"""
INSERT_TICKER_INTO_NOT_FOUND = """
    INSERT INTO not_found
    (ticker)
    VALUES (%s);
"""

# --- DELETE FROM DATABASE ---
DELETE_TICKER_NOT_FOUND = """
    DELETE FROM not_found
    WHERE ticker = '%s';
"""

@contextmanager
def get_cursor(connection):
    with connection:
        with connection.cursor() as cursor:
            yield cursor

def create_tables(connection):
    with get_cursor(connection) as cursor:
        # cursor.execute(CREATE_COMPANIES_)
        # cursor.execute(CREATE_BARS)
        cursor.execute(CREATE_NOT_FOUND)

def drop_tables(connection):
    with get_cursor(connection) as cursor:
        # cursor.execute(DROP_TABLE_COMPANIES)
        # cursor.execute(DROP_TABLE_BARS)
        cursor.execute(DROP_TABLE_NOT_FOUND)

def get_last_ts(connection, ticker):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_LAST_TICKER, (ticker,))
        return cursor.fetchone()[0]

def get_first_ts(connection, ticker):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_FIRST_TICKER, (ticker,))
        return cursor.fetchone()[0]

def get_not_found_tickers(connection):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_TICKERS_NOT_FOUND)
        return cursor.fetchall()

def get_tickers(connection, exchange, limit):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_COMPANIES_WHERE_EXCHANGE, (exchange, limit))
        return cursor.fetchall()

def get_all_tickers(connection, limit):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_COMPANIES, (limit, ))
        return cursor.fetchall()

def get_exchanges(connection):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_EXCHANGES)
        return cursor.fetchall()

def company_ticker_exist(connection, ticker):
    with get_cursor(connection) as cursor:
        cursor.execute(SELECT_COMPANIES_WHERE_TICKER, (ticker,))
        if cursor.fetchone():
            return True
    return False

def add_company(
    connection, ticker, name, city,
    country, currency, exchange, sector,
    industry
):
    with get_cursor(connection) as cursor:
        try:
            cursor.execute(
                INSERT_COMPANIE_RETURN_ID,
                (
                    ticker, name, city, country,
                    currency, exchange, sector,
                    industry
                )
            )
        except errors.lookup(UNIQUE_VIOLATION):
            return None

        return cursor.fetchone()[0]

def add_ticker_not_found(connection, ticker):
    with get_cursor(connection) as cursor:
        try:
            cursor.execute(
                INSERT_TICKER_INTO_NOT_FOUND,
                (ticker,)
            )
        except errors.lookup(UNIQUE_VIOLATION):
            print(f"Ticker: {ticker} exists")

def bulk_insert_bars(connection, df, table: str):
    # Initialize a string buffer
    sio = StringIO()
    # Write the Pandas DataFrame as a csv to the buffer
    sio.write(df.to_csv(index=None, header=None))
    # Be sure to reset the position to the start of the stream
    sio.seek(0)

    with get_cursor(connection) as cursor:
        try:
            cursor.copy_from(sio, table, columns=df.columns, sep=',')
            connection.commit()
            return True
        except errors.lookup(UNIQUE_VIOLATION):
            return 'Error: Rows already exists!'

def delete_ticker_not_found(connection, ticker):
    with get_cursor(connection) as cursor:
        cursor.execute(DELETE_TICKER_NOT_FOUND, (ticker,))

if __name__ == '__main__':
    from connection_pool import get_connection
    with get_connection() as connection:
        drop_tables(connection)
        create_tables(connection)
        # get_last_ts(connection, 'NHY.OL')


import os.path as osp
from functools import partial
from typing import Callable, Sequence

import numpy as np
import rcsenv
from init_args_serializer import Serializable

from pyrado.environments.rcspysim.base import RcsSim
from pyrado.tasks.base import Task
from pyrado.tasks.desired_state import DesStateTask
from pyrado.tasks.final_reward import FinalRewMode, FinalRewTask
from pyrado.tasks.masked import MaskedTask
from pyrado.tasks.parallel import ParallelTasks
from pyrado.tasks.predefined import create_task_space_discrepancy_task
from pyrado.tasks.reward_functions import AbsErrRewFcn, ExpQuadrErrRewFcn, RewFcn
from pyrado.tasks.utils import proximity_succeeded
from pyrado.utils.data_types import EnvSpec

rcsenv.addResourcePath(rcsenv.RCSPYSIM_CONFIG_PATH)
rcsenv.addResourcePath(osp.join(rcsenv.RCSPYSIM_CONFIG_PATH, "PlanarInsert"))

def create_insert_task(env_spec: EnvSpec, state_des: np.ndarray, rew_fcn: RewFcn, success_fcn: Callable):
    # Define the indices for selection. This needs to match the observations' names in RcsPySim.
    idcs = ["Effector_X", "Effector_Z", "Effector_B", "Effector_Xd", "Effector_Zd", "Effector_Bd"]

    # Get the masked environment specification
    spec = EnvSpec(
        env_spec.obs_space, env_spec.act_space, env_spec.state_space.subspace(env_spec.state_space.create_mask(idcs))
    )

    # Create a wrapped desired state task with the goal behind the wall
    fdst = FinalRewTask(
        DesStateTask(spec, state_des, rew_fcn, success_fcn),
        mode=FinalRewMode(state_dependent=True, time_dependent=True),
    )

    # Mask selected states
    return MaskedTask(env_spec, fdst, idcs)

class PlanarInsertSim(RcsSim, Serializable):
    """
    Planar 5- or 6-link robot environment where the task is to push the wedge-shaped end-effector through a small gap
    """

    def __init__(self, task_args: dict, collision_config: dict = None, max_dist_force: float = None, **kwargs):

        """
        Constructor

        .. note::
            This constructor should only be called via the subclasses.

        :param task_args: arguments for the task construction
        :param max_dist_force: maximum disturbance force, pass `None` for no disturbance
        :param kwargs: keyword arguments forwarded to `RcsSim`
                       collisionConfig: specification of the Rcs CollisionModel
        """
        Serializable._init(self, locals())

        # Forward to RcsSim's constructor
        RcsSim.__init__(
            self,
            envType="PlanarInsert",
            task_args=task_args,
            physicsConfigFile="pPlanarInsert.xml",
            collisionConfig=collision_config,
            **kwargs,
        )

        if kwargs.get("collisionConfig", None) is None:
            collision_config = {
                "pairs": [
                    {"body1": "Effector", "body2": "Link3"},
                    {"body1": "Effector", "body2": "Link2"},
                    {"body1": "UpperWall", "body2": "Link4"},
                    {"body1": "LowerWall", "body2": "Link4"},
                    {"body1": "LowerWall", "body2": "Link3"},
                    {"body1": "LowerWall", "body2": "Link2"},
                ],
                "threshold": 0.05,
            }
        else:
            collision_config = kwargs.get("collisionConfig")

        # Setup disturbance
        self._max_dist_force = max_dist_force

    def _create_task(self, task_args: dict) -> Task:
        # Define the task including the reward function
        state_des = task_args.get("state_des", None)
        if state_des is None:
            # Get the goal position in world coordinates
            p = self.get_body_position("Goal", "", "")
            state_des = np.array([p[0], p[2], 0, 0, 0, 0])  # X, Z, B, Xd, Zd, Bd

        # Create the individual subtasks
        task_reach_goal = create_insert_task(
            self.spec,
            state_des,
            rew_fcn=ExpQuadrErrRewFcn(
                Q=np.diag([2e1, 2e1, 1e-1, 1e-2, 1e-2, 1e-2]), R=2e-2 * np.eye(self.act_space.flat_dim)
            ),
            success_fcn=partial(proximity_succeeded, thold_dist=0.07, dims=[0, 1, 2]),  # position and angle
        )
        task_ts_discrepancy = create_task_space_discrepancy_task(
            self.spec, AbsErrRewFcn(q=0.1 * np.ones(2), r=np.zeros(self.act_space.shape))
        )
        return ParallelTasks([task_reach_goal, task_ts_discrepancy])

    @classmethod
    def get_nominal_domain_param(cls):
        return dict(
            link1_mass=2.0,
            link2_mass=2.0,
            link3_mass=2.0,
            link4_mass=2.0,
            upperwall_pos_offset_x=0.0,
            upperwall_friction_coefficient=0.5,
            effector_friction_coefficient=0.7,
        )

    def _disturbance_generator(self) -> (np.ndarray, None):
        if self._max_dist_force is None:
            return None
        # Sample angle and force uniformly
        angle = np.random.uniform(-np.pi, np.pi)
        force = np.random.uniform(0, self._max_dist_force)
        return np.array([force * np.sin(angle), 0, force * np.cos(angle)])

class PlanarInsertIKActivationSim(PlanarInsertSim, Serializable):
    """Planar 5- or 6-link robot environment controlled by  setting the input to an Rcs IK-based controller"""

    name: str = "pi-ika"

    def __init__(self, state_des: np.ndarray = None, **kwargs):
        """
        Constructor

        :param state_des: desired state for the task, pass `None` to use the default goal
        :param kwargs: keyword arguments forwarded to `RcsSim`
                       graphFileName: str = 'gPlanarInsert5Link.xml' or 'gPlanarInsert6Link.xml'
                       checkJointLimits: bool = True,
                       collisionAvoidanceIK: bool = True,
                       observeForceTorque: bool = True,
                       observePredictedCollisionCost: bool = False,
                       observeManipulabilityIndex: bool = False,
                       observeCurrentManipulability: bool = True,
                       observeDynamicalSystemGoalDistance: bool = False,
                       observeDynamicalSystemDiscrepancy: bool = False,
        """
        Serializable._init(self, locals())

        dt = kwargs.get("dt", 0.01)  # 100 Hz is the default
        task_spec_ik = [
            dict(x_des=np.array([dt * 0.1])),
            dict(x_des=np.array([dt * 0.1])),
            dict(x_des=np.array([dt * 5 / 180 * np.pi])),
        ]

        # Forward to the PlanarInsertSim's constructor, nothing more needs to be done here
        PlanarInsertSim.__init__(
            self,
            task_args=dict(state_des=state_des),
            actionModelType="ik_activation",
            taskSpecIK=task_spec_ik,
            **kwargs,
        )

class PlanarInsertTASim(PlanarInsertSim, Serializable):
    """Planar 5- or 6-link robot environment controlled by setting the task activation of a Rcs control task"""

    name: str = "pi-ta"

    def __init__(self, mps: Sequence[dict] = None, state_des: np.ndarray = None, **kwargs):
        """
        Constructor

        :param mps: movement primitives for the cartesian x and z velocity and the angular velocity around y-axis
        :param state_des: desired state for the task, pass `None` to use the default goal
        :param kwargs: keyword arguments forwarded to `RcsSim`
                       graphFileName: str = 'gPlanarInsert5Link.xml' or 'gPlanarInsert6Link.xml'
                       taskCombinationMethod: str = 'sum',  # or 'mean', 'softmax', 'product'
                       checkJointLimits: bool = True,
                       collisionAvoidanceIK: bool = True,
                       observeForceTorque: bool = True,
                       observePredictedCollisionCost: bool = False,
                       observeManipulabilityIndex: bool = False,
                       observeCurrentManipulability: bool = True,
                       observeDynamicalSystemGoalDistance: bool = False,
                       observeDynamicalSystemDiscrepancy: bool = False,
        """
        Serializable._init(self, locals())

        # Define the movement primitives
        dt = kwargs.get("dt", 0.01)  # 100 Hz is the default
        if mps is None:
            mps = [
                # Xd
                {"function": "lin", "errorDynamics": 2.0, "goal": dt * 0.1},  # [m/s]
                {"function": "lin", "errorDynamics": 2.0, "goal": -dt * 0.1},  # [m/s]
                # Zd
                {"function": "lin", "errorDynamics": 2.0, "goal": dt * 0.1},  # [m/s]
                {"function": "lin", "errorDynamics": 2.0, "goal": -dt * 0.1},  # [m/s]
                # Bd
                {"function": "lin", "errorDynamics": 2.0, "goal": dt * 10.0 / 180 * np.pi},  # [rad/s]
                {"function": "lin", "errorDynamics": 2.0, "goal": -dt * 10.0 / 180 * np.pi},  # [rad/s]
            ]

        # Forward to the PlanarInsertSim's constructor, nothing more needs to be done here
        PlanarInsertSim.__init__(
            self, task_args=dict(state_des=state_des), actionModelType="ds_activation", tasks=mps, **kwargs
        )


# -*-: coding utf-8 -*-
""" Skeleton Snips skill. """
import os
import sys
import spotipy
from spotipy.oauth2 import SpotifyOAuth

class SpotifyController:
    """ Skeleton Snips skill. """

    def __init__(self, client_id, secret_id, redirect_uri, tts_service=None):
        """
        """
        self.client_id = client_id
        self.secret_id = secret_id
        self.redirect_url = redirect_uri
        self.tts = tts_service
        # self.hostname = hostname
        # self.light_ids = light_ids

    def __get_access_token(self):
        scope = 'user-modify-playback-state user-read-playback-state'
        auth = SpotifyOAuth(
            self.client_id, self.secret_id, self.redirect_url,
            scope=scope, cache_path='/home/pi/.cache--snipscontroller'
        )
        token = auth.get_cached_token()
        if token is None:
            auth_url = auth.get_authorize_url()
            try:
                import webbrowser
                webbrowser.open(auth_url)
                print("Opened %s in your browser" % auth_url)
            except:
                self.tts.speak("Please navigate here: %s" % auth_url)
                print("Please navigate here: %s" % auth_url)
                print

            response = raw_input("Enter the URL you were redirected to: ")
            code = auth.parse_response_code(response)
            token = auth.get_access_token(code)
        else:
            if auth.is_token_expired(token):
                token = auth.refresh_access_token(token['refresh_token'])

        return token['access_token']

    def __adjust_volume(self, volume_change):
        sp = spotipy.Spotify(auth=self.__get_access_token())
        device = sp.current_playback()['device']
        volume = device['volume_percent']
        new_volume = min(100, max(0, volume + volume_change))
        sp.volume(new_volume)

    def stop_playing(self):
        """ Turn on something. """
        sp = spotipy.Spotify(auth=self.__get_access_token())
        sp.pause_playback()

    def resume_playing(self):
        """ Turn of something. """
        sp = spotipy.Spotify(auth=self.__get_access_token())
        sp.start_playback()

    def next_track(self):
        """ Turn of something. """
        sp = spotipy.Spotify(auth=self.__get_access_token())
        sp.next_track()

    def previous_track(self):
        """ Turn of something. """
        sp = spotipy.Spotify(auth=self.__get_access_token())
        sp.previous_track()

    def decrease_volume(self):
        self.__adjust_volume(-12)

    def increase_volume(self):
        self.__adjust_volume(12)

    def give_current_track_info(self):
        sp = spotipy.Spotify(auth=self.__get_access_token())
        current_playback = sp.current_playback()
        if current_playback['item']:
            artist = current_playback['item']['artists'][0]['name'] #.encode('utf-8')
            title = current_playback['item']['name']
            self.tts.speak(u"This is {} by {}".format(title, artist))


from openpyxl.reader.excel import load_workbook
from random_batch import *
import numpy as np
import torch
import random

def load_data(excel_yes, excel_no):
    """读取数据"""
    wb_yes = load_workbook(excel_yes)
    ws_yes = wb_yes.active
    wb_no = load_workbook(excel_no)
    ws_no = wb_no.active
    """读取数据"""
    list_yes = []
    for i in range(1, 2555):
        embedding_yes = ws_yes.cell(row=i, column=2).value
        embedding_yes = embedding_yes.lstrip('[')
        embedding_yes = embedding_yes.rstrip(']')
        em = str.split(embedding_yes)
        for j in range(len(em)):
            em[j] = eval(em[j])
        em = np.array(em, dtype='float32')
        em = torch.from_numpy(em)
        list_yes.append(em)
    list_no = []
    for i in range(1, 11369):
        embedding_no = ws_no.cell(row=i, column=2).value
        embedding_no = embedding_no.lstrip('[')
        embedding_no = embedding_no.rstrip(']')
        em = str.split(embedding_no)
        for j in range(len(em)):
            em[j] = eval(em[j])
        em = np.array(em, dtype='float32')
        em = torch.from_numpy(em)
        list_no.append(em)
    x_t = list_yes + list_no
    y_t = [1] * 2554 + [0] * 11368
    return x_t, y_t

net1 = torch.load('net3_26_1.pkl')
net1.eval()
x_test, y_test= load_data('Yes_Sep.xlsx', 'No_Sep.xlsx')
test_data, test_label = random_batch(x_test, y_test, len(x_test))
y_p = net1(test_data[0])
y_h = test_label[0]
summ = 0
y_p_l = torch.zeros((y_h.shape[0]))
for i in range (0, y_h.shape[0]):
    if y_p[i,0] >= 0.5:
        y_p_l[i] = 0
    else:
        y_p_l[i] = 1
    if y_p_l[i] == y_h[i]:
        summ = summ + 1
acc = summ / y_h.shape[0]
print(acc)

wb = load_workbook('Yes_Sep.xlsx')
ws = wb.active
for i in range (1,2555):
    ws.cell(i,3,value = str(y_p_l[i-1].item()))
wb.save('Yes_Sep.xlsx')

wb = load_workbook('No_Sep.xlsx')
ws = wb.active
for i in range (1,11369):
    ws.cell(i,3,value =  str(y_p_l[i - 1 + 2554].item()))
wb.save('No_Sep.xlsx')


# -*- coding: utf-8 -*-
"""
Created on Tue Dec  1 14:59:15 2020

@author: Ozaki
"""

import pygame
import sys
from hall import Hall
from livingroom import Livingroom
from privateroom import Privateroom
from item_base import Item_Base
from ending import Ending

def main():
    width = 1024
    height = 768
    pygame.init()
    screen = pygame.display.set_mode((width + 80, height))
    clock = pygame.time.Clock()

    lock_flag = [False, False, False]

    item_get = [False, False, False, False]
    item_use = [False, False, False, False]
    item_ctrl = Item_Base(screen, item_get, item_use, width, height)

    room_state = 0
    next_room_state = 0
    room_ctrl = [Hall(screen, lock_flag, item_get, item_use),
                 Livingroom(screen, lock_flag, item_get, item_use),
                 Privateroom(screen, lock_flag, item_get, item_use),
                 Ending(screen, lock_flag, item_get, item_use)]

    while True:

        """ マウスカーソル位置 """
        x, y = pygame.mouse.get_pos()
        room_ctrl[room_state].pos_event(x, y)

        """ クリックイベント，キーイベント """
        for event in pygame.event.get():
            if event.type == pygame.MOUSEBUTTONDOWN and event.button == 1:
                if x < width:
                    room_ctrl[room_state].click_event(x, y)
                else:
                    item_ctrl.click_event(int(y / 64))
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_LEFT:
                    room_state = (room_state - 1) % -3
                elif event.key == pygame.K_RIGHT:
                    room_state = (room_state + 1) % 3
                elif event.key == pygame.K_a:
                    lock_flag[0] = not lock_flag[0]
                elif event.key == pygame.K_b:
                    lock_flag[1] = not lock_flag[1]
                elif event.key == pygame.K_c:
                    lock_flag[2] = not lock_flag[2]
                elif event.key == pygame.K_0:
                    item_get[0] = not item_get[0]
                    item_use[0] = False
                elif event.key == pygame.K_1:
                    item_get[1] = not item_get[1]
                    item_use[1] = False
                elif event.key == pygame.K_2:
                    item_get[2] = not item_get[2]
                    item_use[2] = False
                elif event.key == pygame.K_3:
                    item_get[3] = not item_get[3]
                    item_use[3] = False

            if event.type == pygame.QUIT:
                pygame.quit()
                sys.exit()

        """ 処理 """
        room_ctrl[room_state].do()
        item_ctrl.do()

        """ 描画 """
        screen.fill((0, 0, 0))
        room_ctrl[room_state].draw()
        item_ctrl.draw()
        pygame.display.update()

        """ 部屋移動 """
        next_room_state = room_ctrl[room_state].next_state()
        if room_state != next_room_state:
            surface = pygame.Surface((width, height), pygame.SRCALPHA)
            alpha = 0
            for i in range(12):
                screen.fill((0,0,0))
                surface.fill((0, 0, 0, alpha))
                room_ctrl[room_state].draw()
                item_ctrl.do()
                item_ctrl.draw()
                screen.blit(surface, (0, 0))
                pygame.display.update()
                clock.tick(30)
                alpha = alpha + 20
            room_state = next_room_state
            for i in range(12):
                screen.fill((0,0,0))
                surface.fill((0, 0, 0, alpha))
                room_ctrl[room_state].draw()
                item_ctrl.do()
                item_ctrl.draw()
                screen.blit(surface, (0, 0))
                pygame.display.update()
                clock.tick(30)
                alpha = alpha - 20

        clock.tick(30)

if __name__ == '__main__':
    main()


import numpy
import requests
import re
import time
import sys
import csv
import pandas as pd
from collections import Counter
from lxml import html
from tqdm import *

page_cstart = 0
href = []
authors = []
year = []
citation = []
journal = []
ref_web = []
while 1:
  page_url = 'https://scholar.google.com/citations?user=__XZ88LXYAAAAJ=en&cstart=' + str(page_cstart) + '&pagesize=100'
  page = requests.get(page_url)
  tree = html.fromstring(page.content)
  m = tree.xpath('//td[@class="gsc_a_e"]/text()')
  if len(m)>0 and m[0] == 'There are no articles in this profile.':
    break
  page_cstart = page_cstart + 100
  href = href + tree.xpath('//a[@class="gsc_a_at"]/text()')
  authors_temp = tree.xpath('//div[@class="gs_gray"]')
  for i in range(len(authors_temp)):
    if i%2 == 0:
      if authors_temp[i].text:
        authors.append(authors_temp[i].text)
      else:
        authors.append('')
    else:
      if authors_temp[i].text:
        journal.append(authors_temp[i].text)
      else:
        journal.append('')
  year_temp = tree.xpath('//span[@class="gsc_a_h"]')
  year_temp = year_temp[1:]
  for x in year_temp:
    if x.text:
      year.append(x.text)
    else:
      year.append('')
  citation_temp = tree.xpath('//a[contains(@class,"gsc_a_ac")]')
  for x in citation_temp:
    if x.text:
      citation.append(x.text)
    else:
      citation.append('')
  websites = tree.findall('.//a')
  for x in websites:
    try:
      if x.attrib['class'] == 'gsc_a_at':
        ref_web.append(x.attrib['href'])
    except KeyError:
      pass

ref_authors = []
ref_description = []
for ref_page in tqdm(ref_web):
  ref_url = 'https://scholar.google.com/' + ref_page
  page = requests.get(ref_url)
  tree = html.fromstring(str(page.content).replace('<br>',' '))
  ref_div = tree.findall('.//div')
  for i in range(len(ref_div)):
    if ref_div[i].text == 'Authors':
      ref_authors.append(ref_div[i+1].text)
    if ref_div[i].text == 'Description':
      ref_description.append(ref_div[i+1].text)

#### set With missing values in citation to 0
citation_index = [i for i in range(len(citation)) if citation[i]=='\xa0']
my_citation = numpy.array(citation)
my_citation[citation_index] = 0

#### extract name of journals
my_journal = []
for j_temp in journal:
  my_journal.append(re.split(' [0-9]', j_temp)[0])
my_journal = [element.upper() for element in my_journal]

#### Get
outf = [href, authors, year, my_citation.tolist(), my_journal]
my_df = pd.DataFrame(outf)
my_df.to_csv("output.csv")

with open("output.csv", 'wb') as f:
  writer = csv.writer(f)
  writer.writerows(outf)


import scrapy
from parsing_search.user_agents import USER_AAGENTS
import random
import time
import re
from parsing_search.items import UrlItem

class QuotesSpider(scrapy.Spider):
    name = "parsing_search"
    count = 1
    url_set = set()

    pattern = re.compile(r'p=[0-9]{1,2}')

    def start_requests(self):

        urls = [
            'https://yandex.ru/search/?text=%D0%A0%D1%8E%D0%BA%D0%B7%D0%B0%D0%BA%20%D0%BC%D0%BE%D1%81%D0%BA%D0%B2%D0%B0&lr=1',
            'https://yandex.ru/search/?text=%D0%BF%D0%BE%D1%81%D1%82%D0%B5%D0%BB%D1%8C%D0%BD%D0%BE%D0%B5%20%D0%B1%D0%B5%D0%BB%D1%8C%D0%B5%20%D0%BA%D1%83%D0%BF%D0%B8%D1%82%D1%8C%20%D0%B2%20%D0%BC%D0%BE%D1%81%D0%BA%D0%B2%D0%B5&lr=1',
            'https://yandex.ru/search/?text=%D0%B8%D0%BD%D1%82%D0%B5%D1%80%D0%BD%D0%B5%D1%82%20%D0%BC%D0%B0%D0%B3%D0%B0%D0%B7%D0%B8%D0%BD%D1%8B%20%D0%9D%D0%B8%D0%B6%D0%BD%D0%B8%D0%B9%20%D0%BD%D0%BE%D0%B2%D0%B3%D0%BE%D1%80%D0%BE%D0%B4&lr=1',
            'https://yandex.ru/search/?text=%D0%BF%D1%80%D0%BE%D0%B4%D0%B0%D0%B6%D0%B0%20%D0%BC%D0%B5%D1%85%D0%B0%20&lr=1',
            'https://yandex.ru/search/?text=%D1%81%D0%BF%D0%BE%D1%80%D1%82%D0%B8%D0%B2%D0%BD%D0%B0%D1%8F%20%D0%BE%D0%B4%D0%B5%D0%B6%D0%B4%D0%B0&lr=1',
            'https://yandex.ru/search/?text=%D1%82%D1%83%D1%80%D0%B8%D1%81%D1%82%D0%B8%D1%87%D0%B5%D1%81%D0%BA%D0%B8%D0%B5%20%D1%84%D0%B8%D1%80%D0%BC%D1%8B%20%D1%80%D0%BE%D1%81%D1%81%D0%B8%D0%B8&lr=1',
            'https://yandex.ru/search/?text=%D1%81%D0%B0%D0%B9%D1%82%D1%8B%20%D0%BA%D0%BE%D0%BC%D0%BF%D0%B0%D0%BD%D0%B8%D0%B9%20%D1%80%D0%BE%D1%81%D1%81%D0%B8%D0%B8&lr=1',

        ]

        for url in urls:
            self.count += 1
            user_ag = random.choice(USER_AAGENTS)
            yield scrapy.Request(url=url,
                                 callback=self.parse,
                                 headers={'User-Agent': user_ag},
                                 cb_kwargs={'main_request': url, 'page': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,\
                                                                          15, 16, 17, 18, 19, 20, 21, 22, 23, 24],
                                            'count': self.count}
                                 )

    def parse(self, response, **kwargs) -> object:
        if response.status == 200 and len(response.cb_kwargs.get("page")) != 0:

            if self.pattern.search(response.url):
                if len(response.cb_kwargs.get("page")) > 1:
                    index = random.randrange(0, len(response.cb_kwargs.get("page")))
                else:
                    index = 0

                new_url = re.sub(self.pattern, f'p={response.cb_kwargs.get("page").pop(index)}', response.url)

            else:
                index = random.randrange(0, 24)
                new_url = response.url + f'&p={response.cb_kwargs.get("page").pop(index)}'

            user_ag = random.choice(USER_AAGENTS)
            yield scrapy.Request(url=new_url,
                                 callback=self.parse,
                                 headers={'User-Agent': user_ag},
                                 meta={'dont_merge_cookies': True},
                                 cb_kwargs=response.cb_kwargs
                                 )

        else:
            yield UrlItem(
                url=set()
            )


import numpy as np
import pandas as pd

df = df = pd.read_csv("/Users/anilcan/Desktop/BAU/Data Science/PyFiles/LinearReg/covid-homework/covidTurkey.csv")

# BACKWARD ELIMINATION

x = df.iloc[:, [3,5,6,7,8]].values
y = df.iloc[:, 4].values

from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 1/3, random_state = 0)
from sklearn.linear_model import LinearRegression
regressor = LinearRegression()
regressor.fit(x_train, y_train)

print(regressor.score(x_test, y_test)) # # 0.9991942639750137

import statsmodels.api as sm
x = np.append(arr = np.ones((45,1)).astype(float), values = x, axis = 1)
x_opt = x[:, [0,1,2,3,4,5]]
regressor_ols = sm.OLS(y, x_opt).fit()
#print(regressor_ols.summary())

s1 = 0.05

num_vars = len(x_opt[0])
for i in range(0, num_vars):
    regressor_ols = sm.OLS(y, x_opt).fit()
    print("i: ", i)
    print(regressor_ols.summary())
    maxVar = max(regressor_ols.pvalues).astype(float)
    if maxVar > s1:
        for j in range(0, num_vars - i):
            if(regressor_ols.pvalues[j].astype(float) == maxVar):
                x_opt = np.delete(x_opt, j, 1)

# choices are x1,x2,x3

x1 = df.iloc[:, [3,5,6]].values

from sklearn.model_selection import train_test_split
x1_train, x1_test, y_train, y_test = train_test_split(x1, y, test_size = 1/3, random_state = 0)
from sklearn.linear_model import LinearRegression
regressor = LinearRegression()
regressor.fit(x1_train, y_train)

print(regressor.score(x1_test, y_test)) # 0.9992122709590304