import json
import time
import logging
import requests

from tqdm import tqdm
from multiprocessing import Pool

# this package is only valid in our lab's private internet
from gpt import GPT

logger = logging.getLogger('my_logger')
logger.setLevel(logging.DEBUG)
log_file = './cli_free.log'
file_handler = logging.FileHandler(log_file)
file_handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)

# set your own pace, name of output file and url
NUM_OF_SESSIONS = 20
NUM_OF_PROCESSES = 30
output_file = 'sd-8125.json'
URL = "http://127.0.0.1:8125/generate"

role_of_chatGPT = "You are an artificial intelligence assistant."

def chat(num_of_processes):
    logger.info(f'Using {num_of_processes} processes.')

    for i in tqdm(range(NUM_OF_SESSIONS)):
        url = URL
        data = {
            "history": None,
            "temperature": 0.7,
            "max_new_tokens": 512
        }
        response = requests.post(url, json=data)

        gpt = GPT(user_name='kongchuyi', new_version='0.1.0')

        num_of_rounds = 0
        one_session_dict = {}
        one_session_list = []

        while True:
            try:
                server_history = eval(response.text).get("history")
            except NameError:
                break
            q =  server_history[-1][-1]
            if "</s>" in q:
                q = q.replace("</s>", "")

            answer = None
            err = 1
            while err or (not answer) or (answer == 'APIKey Error') or type(answer)!= str or ('context_length_exceeded' in answer) or ('invalid_api_key' in answer) or ('account_deactivated' in answer) or ('rate_limit_exceeded' in answer) or ('insufficient_quota' in answer) or ('获取请求参数user_name和parameters失败' in answer) or ('请求官方API失败。Error code:200') in answer:
                try:
                    _, answer = gpt.call(q, role_of_chatGPT)
                    err = 0
                except BaseException as e:
                    print(f'\nThe {i+1}-session: \n {type(e)} \n {e}')

            num_of_rounds += 1
            one_session_list.append({"from": "human", "value": q})
            one_session_list.append({"from": "gpt", "value": answer})

            ans_list = ['Assistant']
            ans_list.append(answer)
            server_history.append(ans_list)
            data['history'] = server_history
            response = requests.post(url, json=data)

        one_session_dict['session_id'] = i+1
        one_session_dict['num_of_rounds'] = num_of_rounds
        one_session_dict['conversations'] = one_session_list

        with open(output_file, 'a', encoding='utf-8') as f:
            json.dump(one_session_dict, f, ensure_ascii=False)
            f.write('\n')

if __name__ == '__main__':
    start_time = time.time()

    with Pool() as pool:
        pool.map(chat, range(NUM_OF_PROCESSES))

    end_time = time.time()
    print(f'Elapsted {end_time-start_time} seconds.')


'''BrainDecoderToolbox2/BdPy data class

This file is a part of BdPy

API list
--------

- Data modification
    - add
    - update
    - add_metadata
    - rename_metadata
    - set_metadatadescription
- Data access
    - select
    - get
    - get_metadata
    - show_metadata
- File I/O
    - load
    - save
'''

__all__ = ['BData']

import os
import sys
import warnings
import time
import datetime
import inspect
import re

import h5py
import numpy as np
import scipy.io as sio

from .metadata import MetaData
from .featureselector import FeatureSelector

# BData class ##########################################################

class BData(object):
    '''BrainDecoderToolbox2/BdPy data class

    The instance of class `BData` contains `dataset` and `metadata` as instance
    variables.

    Parameters
    ----------
    file_name : str, optional
        File which contains BData (default: None)
    file_type : {'Matlab', 'HDF5', 'None'}, optional
        File type (default: None)

    If `file_name` was not given, BData.__init__() creates an empty
    dataset and metadata.

    Attributes
    ----------
    dataset : numpy array (dtype=float)
        Dataset array
    metadata : metadata object
        Meta-data object
    '''

    def __init__(self, file_name=None, file_type=None):
        self.__dataset = np.ndarray((0, 0), dtype=float)
        self.__metadata = MetaData()
        self.__header = {}
        self.__vmap = {}

        if file_name is not None:
            self.load(file_name, file_type)

    # Properties -------------------------------------------------------

    # dataset
    @property
    def dataset(self):
        return self.__dataset

    @dataset.setter
    def dataset(self, value):
        self.__dataset = value

    # metadata
    @property
    def metadata(self):
        return self.__metadata

    @metadata.setter
    def metadata(self, value):
        self.__metadata = value

    # header
    @property
    def header(self):
        return self.__header

    # dataSet
    @property
    def dataSet(self):
        return self.__dataset

    @dataSet.setter
    def dataSet(self, value):
        self.__dataset = value

    # metaData
    @property
    def metaData(self):
        return self.__metadata

    @metaData.setter
    def metaData(self, value):
        self.__metadata = value

    # Misc -------------------------------------------------------------

    def __obsoleted_method(alternative):
        '''Decorator for obsoleted functions'''
        def __obsoleted_method_in(func):
            import functools
            @functools.wraps(func)
            def wrapper(*args, **kwargs):
                funcname = func.__name__
                warnings.warn("'%s' is obsoleted and kept for compatibility. Use '%s' instead." % (funcname, alternative),
                              UserWarning, stacklevel=2)
                return func(*args, **kwargs)
            return wrapper
        return __obsoleted_method_in

    # Data modification ------------------------------------------------

    def add(self, x, name):
        '''Add `x` to dataset as `name`.`

        Parameters
        ----------
        x : array
            Data matrix to be added in dataset
        name : str
            Name of the data `x`

        Returns
        -------
        None
        '''

        if x.ndim == 1:
            x = x[:, np.newaxis]

        colnum_has = self.dataset.shape[1] # Num of existing columns in 'dataset'
        colnum_add = x.shape[1]            # Num of columns to be added

        # Add 'x' to dataset
        if not self.dataset.size:
            self.dataset = x
        else:
            # TODO: Add size check of 'x' and 'self.dataset'
            self.dataset = np.hstack((self.dataset, x))

        # Add new metadata
        column_description = '1 = %s' % name
        column_value = [np.nan for _ in range(colnum_has)] + [1 for _ in range(colnum_add)]

        self.metadata.set(name, column_value, column_description,
                          lambda x, y: np.hstack((y[:colnum_has], x[-colnum_add:])))

    @__obsoleted_method('add')
    def add_dataset(self, x, attribute_key):
        '''Add `x` to dataset with attribute meta-data key `attribute_key`

        Parameters
        ----------
        x : array
            Data matrix to be added in dataset
        attribute_key : str
            Key of attribute meta-data, which specifies the columns containing `x`

        Returns
        -------
        None
        '''
        return self.add(x, attribute_key)

    def update(self, key, dat):
        '''Update dataset

        Parameters
        ----------
        key : str
           Name of columns to be updated
        dat : array_like
           New data array

        Returns
        -------
        None
        '''
        mdind = [a == 1 for a in self.get_metadata(key)]
        self.dataset[:, np.array(mdind)] = dat

    def add_metadata(self, key, value, description='', where=None, attribute=None):
        '''Add meta-data with `key`, `description`, and `value` to metadata

        Parameters
        ----------
        key : str
            Meta-data key
        value : array
            Meta-data array
        description : str, optional
            Meta-data description
        where : str, optional
            Meta-data key masking the columns in the dataset

        Returns
        -------
        None
        '''

        # TODO: Add attribute specifying
        # TODO: Add size check of metadata/value

        if attribute is not None:
            warnings.warn("Keyword argument 'attribute' is obsoleted and kept for compatibility. Use 'where' instead.",
                          UserWarning, stacklevel=2)
            if where is not None:
                warnings.warn("Value of 'attribute' is overridden by 'where'.",
                              UserWarning, stacklevel=2)
            else:
                where = attribute

        if where is not None:
            attr_ind = self.metadata.get(where, 'value') == 1
            add_value = np.array([np.nan for _ in range(self.metadata.get_value_len())])
            add_value[attr_ind] = value
        else:
            add_value = value

        self.metadata.set(key, add_value, description)

    def merge_metadata(self, key, sources, description='', where=None, method='logical_or'):
        '''Merage metadata rows.'''

        if not method == 'logical_or':
            raise NotImplementedError('Only `logical_or` is implemented')

        if where is None:
            raise ValueError('You need to specify `where`.')

        mdv_lst = [self.get_metadata(s, where=where) for s in sources]
        mdv_new = np.nansum(np.vstack(mdv_lst), axis=0)
        mdv_new[mdv_new > 1] = 1

        self.add_metadata(key, mdv_new, description, where=where)

    def rename_metadata(self, key_old, key_new):
        '''Rename meta-data key

        Parameters
        ----------
        key_old, key_new : str
            Old and new meta-data keys

        Returns
        -------
        None
        '''
        self.metadata.key[self.metadata.key.index(key_old)] = key_new
        return None

    def set_metadatadescription(self, key, description):
        '''Set description of metadata specified by `key`

        Parameters
        ----------
        key : str
            Meta-data key
        description : str
            Meta-data description

        Returns
        -------
        None
        '''

        self.metadata.set(key, None, description,
                          lambda x, y: y)

    @__obsoleted_method('set_metadatadescription')
    def edit_metadatadescription(self, metakey, description):
        '''Set description of metadata specified by `key`

        Parameters
        ----------
        key : str
            Meta-data key
        description : str
            Meta-data description

        Returns
        -------
        None
        '''
        self.set_metadatadescription(metakey, description)

    def update_header(self, header):
        '''Update header.'''
        self.__header.update(header)

    def applyfunc(self, func, where=None, **kargs):
        '''Apply `func` to the dataset.'''

        if where is None:
            # FIXME
            fout = func(self.dataset, **kargs)

            if isinstance(fout, tuple):
                self.dataset = fout[0]
            else:
                self.dataset = fout
        else:
            # FIXME
            if not isinstance(where, list):
                where = [where]

            data_selector = '|'.join([w + ' = 1' for w in where])

            x, x_ind = self.select(data_selector, return_index=True)

            fout = func(x, **kargs)

            if isinstance(fout, tuple):
                # Index mapping
                ind_map = fout[1]

                ds = np.zeros((len(ind_map), self.dataset.shape[1]))

                index = np.zeros(self.dataset.shape[1], dtype=bool)
                index[x_ind] = True

                #import pdb; pdb.set_trace()

                ds[:, index] = fout[0]
                ds[:, ~index] = self.dataset[np.ix_(ind_map, ~index)]

                self.dataset = ds
            else:
                # No index mapping
                self.dataset[:, x_ind] = fout

        return self

    # Data access ------------------------------------------------------

    def select(self, condition, return_index=False, verbose=True):
        '''Select data (columns) from dataset.

        Parameters
        ----------
        condition : str
            Condition specifying columns.
        retrun_index : bool, optional
            If True, return index of selected columns (default: False).
        verbose : bool, optional
            If True, display verbose messages (default: True).

        Returns
        -------
        array-like
            Selected data
        list, optional
            Selected index

        Note
        ----
        The following operators are acceptable in `condition`.

        - | (or)
        - & (and)
        - = (equal)
        - @ (conditional)
        '''

        expr_rpn = FeatureSelector(condition).rpn

        stack = []
        buf_sel = []

        for i in expr_rpn:
            if i == '=':
                r = stack.pop()
                l = stack.pop()

                stack.append(np.array([n == r for n in l], dtype=bool))

            elif i == 'top':
                # Dirty solution

                # Need fix on handling 'None'

                n = int(stack.pop()) # Num of elements to be selected
                v = stack.pop()

                order = self.__get_order(v)

                stack.append(order)
                buf_sel.append(n)

            elif i in ['|', '&', '-']:
                r = stack.pop()
                l = stack.pop()

                if r.dtype != 'bool':
                    # 'r' should be an order vector
                    num_sel = buf_sel.pop()
                    r = self.__get_top_elm_from_order(r, num_sel)
                    #r = np.array([ n < num_sel for n in r ], dtype = bool)

                if l.dtype != 'bool':
                    # 'l' should be an order vector
                    num_sel = buf_sel.pop()
                    l = self.__get_top_elm_from_order(l, num_sel)
                    #l = np.array([ n < num_sel for n in l ], dtype = bool)

                if i == '|':
                    result = np.logical_or(l, r)
                elif i == '&':
                    result = np.logical_and(l, r)
                elif i == '-':
                    result = np.logical_and(l, np.logical_not(r))

                stack.append(result)

            elif i == '@':
                # FIXME
                # In the current version, the right term of '@' is assumed to
                # be a boolean, and the left is to be an order vector.

                r = stack.pop() # Boolean
                l = stack.pop() # Float

                l[~r] = np.inf

                selind = self.__get_top_elm_from_order(l, buf_sel.pop())

                stack.append(np.array(selind))

            else:
                if isinstance(i, str):
                    if i.isdigit():
                        # 'i' should be a criteria value
                        i = float(i)
                    else:
                        # 'i' should be a meta-data key
                        i = self.__metadata_key_to_bool_vector(i)

                stack.append(i)

        selected_index = stack.pop()

        # If buf_sel still has an element, `select_index` should be an order vector.
        # Select N elements based on the order vector.
        if buf_sel:
            num_sel = buf_sel.pop()
            selected_index = [n < num_sel for n in selected_index]

        # Very dirty solution
        selected_index = np.array(selected_index) == True

        if return_index:
            return self.dataset[:, np.array(selected_index)], selected_index
        else:
            return self.dataset[:, np.array(selected_index)]

    @__obsoleted_method('select')
    def select_dataset(self, condition, return_index=False, verbose=True):
        '''Select data (columns) from dataset.

        Parameters
        ----------
        condition : str
            Condition specifying columns.
        retrun_index : bool, optional
            If True, return index of selected columns (default: False).
        verbose : bool, optional
            If True, display verbose messages (default: True).

        Returns
        -------
        array-like
            Selected data
        list, optional
            Selected index

        Note
        ----
        The following operators are acceptable in `condition`.

        - | (or)
        - & (and)
        - = (equal)
        - @ (conditional)
        '''
        return self.select(condition, return_index, verbose)

    @__obsoleted_method('select')
    def select_feature(self, condition, return_index=False, verbose=True):
        '''Select data (columns) from dataset.

        Parameters
        ----------
        condition : str
            Condition specifying columns.
        retrun_index : bool, optional
            If True, return index of selected columns (default: False).
        verbose : bool, optional
            If True, display verbose messages (default: True).

        Returns
        -------
        array-like
            Selected data
        list, optional
            Selected index

        Note
        ----
        The following operators are acceptable in `condition`.

        - | (or)
        - & (and)
        - = (equal)
        - @ (conditional)
        '''
        return self.select(condition, return_index, verbose)

    def get(self, key=None):
        '''Get dataset

        When `key` is not given, `get_dataset` returns `dataset`. When `key` is
        given, `get_dataset` returns data specified by `key`
        '''

        if key is None:
            return self.dataset
        else:
            query = '%s = 1' % key
            return self.select(query, return_index=False, verbose=False)

    @__obsoleted_method('get')
    def get_dataset(self, key=None):
        '''Get dataset

        When `key` is not given, `get_dataset` returns `dataset`. When `key` is
        given, `get_dataset` returns data specified by `key`
        '''
        return self.get(key)

    def get_metadata(self, key, where=None):
        '''Get value of meta-data specified by `key`

        Parameters
        ----------
        key : str
            Meta-data key.

        where : str, optional
            Columns which mask meta-data array.

        Returns
        -------
        array-like
        '''

        md = self.metadata.get(key, 'value')

        if where != None:
            # Mask the metadata array with columns specified with `where`
            ind = self.metadata.get(where, 'value') == True
            md = md[ind]

        return md

    def show_metadata(self):
        '''Show all the key and description in metadata'''

        # Get max length
        max_key = max([len(k) for k in self.metadata.key])
        max_desc = max([len(k) for k in self.metadata.description])

        # Disp header
        print('| ' + 'Key' + ' ' * (max_key - 3) + ' | ' + 'Description' + ' ' * (max_desc - 11) + ' |')
        print('|-' + '-' * max_key + '-|-' + '-' * max_desc + '-|')

        # Disp key and description
        for k, d in zip(self.metadata.key, self.metadata. description):
            print('| ' + k + ' ' * (max_key - len(k)) + ' | ' + d + ' ' * (max_desc - len(d)) + ' |')

    # Value-label map --------------------------------------------------------

    def get_labels(self, key):
        '''Get `key` as labels.'''
        if not key in self.__vmap:
            raise ValueError('Key not found in vmap: %s' % key)
        value = self.select(key).flatten()
        label = []
        for x in value:
            if np.isnan(x):
                v = 'n/a'
            else:
                v = self.__vmap[key][x]
            label.append(v)
        return label

    def get_label(self, key):
        '''Get `key` as labels.'''
        return self.get_labels(key)

    def get_vmap(self, key):
        '''Returns vmap of `key`.'''
        if key in self.__vmap:
            return self.__vmap[key]
        else:
            raise ValueError('%s not found in vmap' % key)

    def get_vmap_keys(self):
        return self.__vmap.keys()

    def add_vmap(self, key, vmap):
        '''Add vmap.'''
        if not key in self.__metadata.key:
            raise ValueError('%s not found in metadata.' % key)

        if type(vmap) is not dict:
            raise TypeError('`vmap` should be a dictionary.')
        for k in vmap.keys():
            if type(k) is str:
                raise TypeError('Keys of `vmap` should be numerical.')

        if key in self.__vmap:
            # Check vmap consistency
            if self.__check_vmap_consistency(vmap, self.__vmap[key]):
                vmap.update(self.__vmap[key])
                vmap_add = self.__get_act_vmap(key, vmap)
                self.__vmap[key].update(vmap_add)
            else:
                raise ValueError('Invalid vmap: labels are inconsistent between old and new vmaps.')
        else:
            vmap_add = self.__get_act_vmap(key, vmap)
            self.__vmap.update({key: vmap_add})

        return None

    def __get_act_vmap(self, key, vmap):
        values = np.unique(self.get(key))
        try:
            vmap_add = {}
            for val in values:
                if np.isnan(val):
                    continue
                vmap_add.update({val: vmap[val]})
        except KeyError:
            raise ValueError('Invalid vmap: label for %f not found.' % val)
        return vmap_add

    def __check_vmap_consistency(self, vmap_new, vmap_old):
        for key in vmap_new.keys():
            if not key in vmap_old:
                continue
            if vmap_old[key] != vmap_new[key]:
                print('Inconsistent label:')
                print('    Key: %f' % key)
                print('    Old label: %s' % vmap_old[key])
                print('    New label: %s' % vmap_new[key])
                return False
        return True

    # File I/O---------------------------------------------------------

    def load(self, load_filename, load_type=None):
        '''Load 'dataset' and 'metadata' from a given file'''

        if load_type is None:
            load_type = self.__get_filetype(load_filename)

        if load_type == "Matlab":
            self.__load_mat(load_filename)
        elif load_type == "HDF5":
            self.__load_h5(load_filename)
        else:
            raise ValueError("Unknown file type: %s" % (load_type))

    def save(self, file_name, file_type=None):
        '''Save 'dataset' and 'metadata' to a file'''

        # Store data creation information
        t_now = time.time()
        t_now_str = datetime.datetime.fromtimestamp(t_now).strftime('%Y-%m-%d %H:%M:%S')

        callstack = []
        callstack_code = []
        f = inspect.currentframe()
        while True:
            f = f.f_back
            if f is None: break
            fname = os.path.abspath(f.f_code.co_filename)
            fline = f.f_lineno
            callstack.append('%s:%d' % (fname, fline))
            if os.path.exists(fname):
                with open(fname, 'r') as fl:
                    fcode = fl.read()
            else:
                fcode = ''
            callstack_code.append(fcode)

        self.__header.update({'ctime': t_now_str,
                              'ctime_epoch': t_now,
                              'callstack': callstack,
                              'callstack_code': callstack_code})

        if file_type is None:
            file_type = self.__get_filetype(file_name)

        if file_type == "Matlab":
            raise RuntimeError('Saving BData as a mat file is no longer supported. Please save the data as HDF5 (.h5).')
        elif file_type == "HDF5":
            self.__save_h5(file_name, header=self.__header)

        else:
            raise ValueError("Unknown file type: %s" % (file_type))

    # Private methods --------------------------------------------------

    def __metadata_key_to_bool_vector(self, key):
        '''Convert meta-dat key(s) to boolean vector.'''
        key_esc = re.escape(key).replace('\*', '.*') + '$'  # `fullmatch` is available in Python >= 3.4
        keys = [k for k in self.metadata.key if re.match(key_esc, k)]
        if len(keys) == 0:
            raise RuntimeError('Meta-data %s not found' % key)
        vals = np.vstack([
            self.get_metadata(k)
            for k in keys])
        vals = (vals == 1)
        vec = np.sum(vals, axis=0).astype(bool)
        return vec

    def __get_order(self, v, sort_order='descend'):

        # 'np.nan' comes to the last of an acending series, and thus the top of a decending series.
        # To avoid that, convert 'np.nan' to -Inf.
        v[np.isnan(v)] = -np.inf

        sorted_index = np.argsort(v)[::-1] # Decending order
        order = np.arange(len(v))
        for i, x in enumerate(sorted_index):
            order[x] = i

        return np.array(order, dtype=float)

    def __get_top_elm_from_order(self, order, n):
        '''Get a boolean index of top `n` elements from `order`'''
        sorted_index = np.argsort(order)
        for i, x in enumerate(sorted_index):
            order[x] = i

        index = np.array([r < n for r in order], dtype=bool)

        return index

    def __save_h5(self, file_name, header=None):
        '''Save data in HDF5 format (*.h5)'''
        with h5py.File(file_name, 'w') as h5file:
            # dataset
            h5file.create_dataset('/dataset', data=self.dataset)

            # metadata
            md_keys = self.metadata.key
            md_desc = self.metadata.description
            md_vals = self.metadata.value

            h5file.create_group('/metadata')
            h5file.create_dataset('/metadata/key', data=[self.__to_bytes(x) for x in md_keys])
            h5file.create_dataset('/metadata/description', data=[self.__to_bytes(x) for x in md_desc])
            h5file.create_dataset('/metadata/value', data=md_vals)

            # header
            if header is not None:
                h5file.create_group('/header')
                for k, v in header.items():
                    if isinstance(v, list):
                        h5file.create_dataset('/header/' + k, data=[self.__to_bytes(x) for x in v])
                    else:
                        h5file.create_dataset('/header/' + k, data=self.__to_bytes(v)) # FIXME: save unicode str as is

            # vmap
            h5file.create_group('/vmap')
            for mk, vm in self.__vmap.items():
                h5file.create_group('/vmap/' + mk)
                for k, v in vm.items():
                    h5file.create_dataset('/vmap/' + mk + '/' + str(k), data=self.__to_bytes(v)) # FIXME: save unicode str as is

    def __load_mat(self, load_filename):
        '''Load dataset and metadata from Matlab file'''

        dat = sio.loadmat(load_filename)

        if 'metaData' in dat:
            md_keys = [str(i[0]).strip() for i in np.asarray(dat["metaData"]['key'][0, 0])[0].tolist()]
            md_descs = [str(i[0]).strip() for i in np.asarray(dat["metaData"]['description'][0, 0])[0].tolist()]
            md_values = np.asarray(dat["metaData"]['value'][0, 0])
        else:
            md_keys = [str(i[0]).strip() for i in np.asarray(dat["metadata"]['key'][0, 0])[0].tolist()]
            md_descs = [str(i[0]).strip() for i in np.asarray(dat["metadata"]['description'][0, 0])[0].tolist()]
            md_values = np.asarray(dat["metadata"]['value'][0, 0])

        if 'dataSet' in dat:
            self.dataset = np.asarray(dat["dataSet"])
        else:
            self.dataset = np.asarray(dat["dataset"])

        if 'header' in dat:
            self.__header = dat['header']

        self.__metadata.key = md_keys
        self.__metadata.value = md_values
        self.__metadata.description = md_descs

    def __load_h5(self, load_filename):
        '''Load dataset and metadata from HDF5 file'''

        dat = h5py.File(load_filename, 'r')

        if 'metaData' in dat:
            md_keys = [self.__to_unicode(x) for x in dat["metaData"]['key'][:].tolist()]
            md_descs = [self.__to_unicode(x) for x in dat["metaData"]['description'][:].tolist()]
            md_values = np.asarray(dat["metaData"]['value'], dtype=np.float)
        else:
            md_keys = [self.__to_unicode(x) for x in dat["metadata"]['key'][:].tolist()]
            md_descs = [self.__to_unicode(x) for x in dat["metadata"]['description'][:].tolist()]
            md_values = np.asarray(dat["metadata"]['value'], dtype=np.float)

        if 'dataSet' in dat:
            self.dataset = np.asarray(dat["dataSet"], dtype=np.float)
        else:
            self.dataset = np.asarray(dat["dataset"], dtype=np.float)

        if 'header' in dat:
            for k, v in dat['header'].items():
                k = self.__to_unicode(k)
                if isinstance(v[()], np.ndarray):
                    v = [self.__to_unicode(x) for x in v[()]]
                else:
                    v = self.__to_unicode(v[()])
                self.__header.update({k: v})

        if 'vmap' in dat:
            for mk in dat['vmap'].keys():
                vmap = {}
                for k in dat['vmap'][mk].keys():
                    vmap.update({float(k): self.__to_unicode(dat['vmap'][mk][k][()])})
                # TODO: fix this dirty solution
                if sys.version_info.major == 2:
                    mk = mk.encode('utf-8')
                self.__vmap.update({mk: vmap})

        self.__metadata.key = md_keys
        self.__metadata.value = md_values
        self.__metadata.description = md_descs

    def __to_unicode(self, s):
        '''Convert s (bytes) to unicode str.'''
        if sys.version_info.major == 3:
            if isinstance(s, bytes):
                return s.decode('utf-8')
        return s

    def __to_bytes(self, s):
        '''Convert s (unicode str) to bytes.'''
        if sys.version_info.major == 3:
            if isinstance(s, str):
                return s.encode('utf-8')
        return s

    def __get_filetype(self, file_name):
        '''Return the type of `file_name` based on the file extension'''

        _, ext = os.path.splitext(file_name)

        if ext == ".mat":
            file_type = "Matlab"
        elif ext == ".h5":
            file_type = "HDF5"
        else:
            raise ValueError("Unknown file extension: %s" % (ext))

        return file_type


import logging
import configparser
import datetime
import argparse
import tweepy
import os
import urllib.request
from .imgviewer import show_image

logger = logging.getLogger()
config = configparser.ConfigParser()
config.read("config.ini")
BEGIN_TIME = datetime.datetime.now()
ROWS, COLUMNS = os.popen("stty size", "r").read().split()

def get_thread(twitter, url):
    status = twitter.get_status(url.split("/")[-1], tweet_mode="extended")
    userid = status.author.id
    yield status

    while True:
        found = False
        statusid = status.id
        curs = tweepy.Cursor(
            twitter.user_timeline,
            user_id=userid,
            since_id=statusid,
            tweet_mode="extended",
            count=100,
        ).items()
        for index, status in enumerate(curs, 0):
            if status.in_reply_to_status_id == statusid:
                yield status
                found = True
        if not found:
            break

def twitterconnect():
    consumer_key = config["twitter"]["consumer_key"]
    secret_key = config["twitter"]["secret_key"]
    access_token = config["twitter"]["access_token"]
    access_token_secret = config["twitter"]["access_token_secret"]

    auth = tweepy.OAuthHandler(consumer_key, secret_key)
    auth.set_access_token(access_token, access_token_secret)
    return tweepy.API(auth)

def main():
    args = parse_args()
    twitter = twitterconnect()

    if not args.url:
        logger.error("Use the -u flag to specify an url. Exiting.")
        exit()
    else:
        url = args.url

    for status in get_thread(twitter, url):
        print(f"{'-' * int(COLUMNS)}")
        # Display images if available
        if args.display_images:
            if hasattr(status, "extended_entities"):
                if "media" in status.extended_entities:
                    for media in status.extended_entities["media"]:
                        urllib.request.urlretrieve(
                            media["media_url"], "/tmp/image.png"
                        )
                        show_image("/tmp/image.png")

        print(status.full_text)

def parse_args():
    parser = argparse.ArgumentParser(
        description="Bot posting images from lastfm_cg to twitter or mastodon."
    )
    parser.add_argument(
        "--debug",
        help="Display debugging information",
        action="store_const",
        dest="loglevel",
        const=logging.DEBUG,
        default=logging.INFO,
    )
    parser.add_argument("-u", "--url", help="Tweet URL.", type=str)
    parser.add_argument(
        "-i",
        "--display_images",
        help="Display images if available.",
        dest="display_images",
        action="store_true",
    )
    parser.set_defaults(display_images=False)
    args = parser.parse_args()
    logging.basicConfig(level=args.loglevel)
    return args

if __name__ == "__main__":
    main()


import csv
from multiprocessing import dummy
from pprint import PrettyPrinter
import math
from operator import itemgetter

printer = PrettyPrinter()

df1 = open("E:\File Tugas\Semester II\Tugas Computation\knn.csv")
type(df1)

csvreader = csv.reader(df1)

header = []
header = next(csvreader)
# printer.pprint(header)

rows = []
for row in csvreader:
    rows.append(row)
# printer.pprint(rows)

target = ["D8-X", 6, 8, 3, 0, 1, 0, 0, 2]

resultSum = []
resultSqrt = []

for row in rows:
    i = 1
    j = 1
    total = 0
    for r in row:
        if j == 1:
            j++1
            pass
        else:
            dummy = pow((int(target[i]) - int(r)), 2)
            total == dummy
            i++1
    row.append(total)
    total = math.sqrt(total)
    total = float('{0:.6f}'.format(total))
    row.append(total)

#printer.pprint(rows)

rows = sorted(rows, key = itemgetter(-1))

k = input("Enter the desired k: ")
i = 0
while i < int(k):
    print(f"{target[0]} to {rows[i][0]} = {rows[i][-1]}")
    i += 1


import logging

from flask import Flask

from .config.configuration import Development

app = Flask(__name__)

def configure_logging():
	FORMAT = """[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)s]    %(message)s\n"""
	logging.basicConfig(format=FORMAT)
	logging.disable(logging.INFO)

def create_app():
	app.config.from_object(Development)

	from .api import api_bp
	app.register_blueprint(api_bp)

	configure_logging()

	return app


from django.shortcuts import render
from rest_framework import generics, permissions
from .serializers import RejectedReasonSelectSerializer
from .models import RejectedReason

from rest_framework import generics
from .models import RejectedReason, RejectedProject
from .serializers import RejectedProjectSerializer, RejectedReasonSelectSerializer
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import filters
import django_filters
from core.filters import ClientMultiSelectFilter,ReasonMultiSelectFilter
from core.pagination import StandardResultsSetPagination
# Create your views here.
from client.models import Client
from rest_framework.response import Response
from rest_framework.decorators import api_view
class RejectedReasonSelectView(generics.ListAPIView):
    """
    View to list all rejected reasons in the system.

    * Requires token authentication.
    """
    queryset = RejectedReason.objects.all()
    serializer_class = RejectedReasonSelectSerializer
    # permission_classes = (permissions.IsAuthenticated,)

class RejectedProjectListView(generics.ListAPIView):
    queryset = RejectedProject.objects.select_related('client', 'reason').all()
    serializer_class = RejectedProjectSerializer
    filter_backends = [django_filters.rest_framework.DjangoFilterBackend,filters.SearchFilter,filters.OrderingFilter,ClientMultiSelectFilter,ReasonMultiSelectFilter]
    filterset_fields = ['name','client','reason','created_at','updated_at',]
    search_fields = ['name','client__name','last_comment','reason__name',]
    pagination_class = StandardResultsSetPagination
    pass

@api_view(['GET'])
def rejectedProjectsAPIDescription(request):
    clients = Client.objects.all()
    client_options = [{'value':client.id,'label':client.name} for client in clients]
    resasons = RejectedReason.objects.all()
    reason_options = [{'value':reason.id,'label':reason.name} for reason in resasons]

    ret = {
        'extra':{
            'client_options':client_options,
            'reason_options':reason_options,
        },
        'filters':{
            'name':{
                'type':'text',
                'name': 'שם',
                'slug': 'name',
            },
            'client':{
                'type':'multi-select',
                'options':'client_options',
                'name': 'לקוח',
                'slug': 'client',
            },
            'reason':{
                'type':'multi-select',
                'options':'reason_options',
                'name': 'סיבה',
                'slug': 'reason',
            },
            'created_at':{
                'type':'date',
                'name': 'נוצר בתאריך',
                'slug': 'created_at',
            },
            'updated_at':{
                'type':'date',
                'name': 'עודכן בתאריך',
                'slug': 'updated_at',
            },
        },
        'fields': {
            'name':{
                'lable': 'שם',
                'sortable': True,
                'type': 'text',
            },
            'client__name': {
                'lable': 'לקוח',
                'sortable': True,
                'type': 'text',
            },
            'reason__name': {
                'lable': 'סיבה',
                'sortable': True,
                'type': 'text',
            },
            'created_at': {
                'lable': 'נוצר בתאריך',
                'sortable': True,
                'type': 'date',
            },
            'updated_at': {
                'lable': 'עודכן בתאריך',
                'sortable': True,
                'type': 'date',
            },
        },
        'actions':{
        },
    }
    from django.http import JsonResponse
    return JsonResponse({'api-description': ret})


from __future__ import annotations

import dataclasses

from pretty_modbus.exceptions import (
    NoSuchSlaveLayoutError,
    MissingSubLayoutError,
    VariableNotFoundError,
)

@dataclasses.dataclass
class SlaveContextLayout:
    holding_registers: Optional[registers.RegisterLayout] = None
    input_registers: Optional[registers.RegisterLayout] = None
    coils: Optional[coils.CoilLayout] = None
    discrete_inputs: Optional[coils.CoilLayout] = None

_TYPES = [
    "input_registers",
    "holding_registers",
    "coils",
    "discrete_inputs",
]

class ServerContextLayout:
    def __init__(self, slaves: dict[Key, SlaveContextLayout]) -> None:
        self._slaves = slaves

    def find(self, var: str) -> tuple[Key, str]:
        """Find the type and the containing unit of ``var``.

        Args:
            var: The variable to find

        Returns:
            Tuple ``(type, unit)``

        Raises:
            VariableNotFoundError:
                If there is no layout that contains the variable
        """
        for unit, _ in self._slaves.items():
            for type_ in _TYPES:
                layout = self._get(unit, type_)
                if layout is None:
                    continue
                if var in layout:
                    return unit, type_
        raise VariableNotFoundError(var)

    def where(self, var: str, unit: Optional[Key] = None) -> str:
        """Return where a variable is stored.

        Returns:
            ``"input_registers"``, ``"holding_registers"``, ``"coils"`` or
            ``"discrete_inputs"`` or ``None``, depending on where and if
            the variable is stored

        Raises:
            VariableNotFoundError:
                If there is no layout that contains the variable
        """
        for type_ in _TYPES:
            layout = self._get(unit, type_)
            if layout is None:
                continue
            if var in layout:
                return type_
        raise VariableNotFoundError(var)

    def _get(
        self, unit: Key, type: str
    ) -> Union[registers.RegisterLayout, coils.CoilLayout]:
        slave = self._slaves.get(unit)
        if slave is None:
            raise NoSuchSlaveLayoutError(unit)
        layout = getattr(slave, type, None)
        return layout

    def _get_fallible(
        self, unit: Key, type: str
    ) -> registeres.RegisterLayout | coils.CoilLayout:
        layout = self._get(unit, type)
        if layout is None:
            raise MissingSubLayoutError(unit, type)
        return layout

    def get_holding_register_layout(self, unit: Key) -> registers.RegisterLayout:
        return self._get_fallible(unit, "holding_registers")

    def get_input_register_layout(self, unit: Key) -> registers.RegisterLayout:
        return self._get_fallible(unit, "input_registers")

    def get_coil_layout(self, unit: Key) -> coils.CoilLayout:
        return self._get_fallible(unit, "coils")

    def get_discrete_input_layout(self, unit: Key) -> coils.CoilLayout:
        return self._get_fallible(unit, "discrete_inputs")


# Copyright (c) 2020-2023.

# Author(s):

#   Harry Dove-Robinson <admin@wx-star.com>
#   for Here GOES Radiotelescope (Harry Dove-Robinson & Heidi Neilson)

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import astropy.units as u
import numpy as np
from astropy import coordinates
from astropy.time import Time

from heregoes.util import nearest_2d, njit, orbital, rad2deg

class ABINavigation:
    """
    This is a class for GOES-R ABI navigation routines using an ABIObject to access netCDF data.
    The routines may be constrained to a single pixel by `index`, or to a single location by providing `lat_deg` and `lon_deg` (degrees).
    All calculations return 32-bit floating point NumPy arrays which should be accurate enough for most applications at this scale.

    The following quantities are always generated upon instantiation:
        - Instrument scanning angle (`y_rad`, `x_rad`)
        - Latitude and longitude of pixels from instrument scanning angle (`lat_deg`, `lon_deg`)
            OR index of a single pixel (`index`) from latitude and longitude (degrees)

    These quantities may be generated upon access after instantiation:
        - Angles of the satellite vector at the pixel (`sat_za`, `sat_az`)
        - Angles of the Sun vector at the pixel (`sun_za`, `sun_az`)
        - The effective projected area of the pixel in square meters (`area_m`)

    Arguments:
        - `abi_data`: The ABIObject formed on a GOES-R ABI L1b Radiance netCDF file as returned by `heregoes.load()`
        - `hae_m`: The Height Above Ellipsoid (HAE) in meters of the ABI array to correct for terrain height. Default 0.0 (no correction)
        - `time`: The time for which the Sun position is valid. The product midpoint time is used if not provided
        - `precise_sun`: Whether to calculate solar position using Equation of Time with Pyorbital (`False`, default) or real ephemeris with Astropy (`True`)
        - `degrees`: Whether to return calculated Sun/satellite vector angles in radians or degrees. Default `False`
    """

    def __init__(
        self,
        abi_data,
        index=slice(None, None),
        lat_deg=None,
        lon_deg=None,
        hae_m=0.0,
        time=None,
        precise_sun=False,
        degrees=False,
    ):
        self.abi_data = abi_data
        self.index = index
        self.hae_m = np.atleast_1d(hae_m).astype(np.float32)
        self.time = time
        self.precise_sun = precise_sun
        self.degrees = degrees

        self._sat_za = None
        self._sat_az = None
        self._sun_za = None
        self._sun_az = None
        self._area_m = None

        if self.index == slice(None, None):
            self.x_rad, self.y_rad = np.meshgrid(
                self.abi_data["x"][:],
                self.abi_data["y"][:],
            )

        else:
            self.x_rad = np.atleast_1d(self.abi_data["x"][self.index[1]])
            self.y_rad = np.atleast_1d(self.abi_data["y"][self.index[0]])

        if lat_deg is None or lon_deg is None:
            self.lat_deg, self.lon_deg = self.navigate(
                self.y_rad,
                self.x_rad,
                lon_origin=self.abi_data[
                    "goes_imager_projection"
                ].longitude_of_projection_origin,
                r_eq=self.abi_data["goes_imager_projection"].semi_major_axis,
                r_pol=self.abi_data["goes_imager_projection"].semi_minor_axis,
                sat_height=self.abi_data[
                    "goes_imager_projection"
                ].perspective_point_height,
            )

            if self.hae_m.shape != self.lat_deg.shape:
                self.hae_m = np.full(self.lat_deg.shape, self.hae_m, dtype=np.float32)

            # correct for terrain parallax if HAE is provided
            if (self.hae_m != 0.0).any():
                self.y_rad, self.x_rad = self.reverse_navigate(
                    self.lat_deg,
                    self.lon_deg,
                    lon_origin=self.abi_data[
                        "goes_imager_projection"
                    ].longitude_of_projection_origin,
                    r_eq=self.abi_data["goes_imager_projection"].semi_major_axis,
                    r_pol=self.abi_data["goes_imager_projection"].semi_minor_axis,
                    sat_height=self.abi_data[
                        "goes_imager_projection"
                    ].perspective_point_height,
                    feature_height=self.hae_m,
                )
                self.lat_deg, self.lon_deg = self.navigate(
                    self.y_rad,
                    self.x_rad,
                    lon_origin=self.abi_data[
                        "goes_imager_projection"
                    ].longitude_of_projection_origin,
                    r_eq=self.abi_data["goes_imager_projection"].semi_major_axis,
                    r_pol=self.abi_data["goes_imager_projection"].semi_minor_axis,
                    sat_height=self.abi_data[
                        "goes_imager_projection"
                    ].perspective_point_height,
                )

        else:
            self.lat_deg = np.atleast_1d(lat_deg)
            self.lon_deg = np.atleast_1d(lon_deg)

            derived_y_rad, derived_x_rad = self.reverse_navigate(
                self.lat_deg,
                self.lon_deg,
                lon_origin=self.abi_data[
                    "goes_imager_projection"
                ].longitude_of_projection_origin,
                r_eq=self.abi_data["goes_imager_projection"].semi_major_axis,
                r_pol=self.abi_data["goes_imager_projection"].semi_minor_axis,
                sat_height=self.abi_data[
                    "goes_imager_projection"
                ].perspective_point_height,
                feature_height=self.hae_m,
            )
            self.index = nearest_2d(
                self.y_rad, self.x_rad, derived_y_rad, derived_x_rad
            )
            self.y_rad = derived_y_rad
            self.x_rad = derived_x_rad

        if self.time is None:
            self.time = self.abi_data.midpoint_time

    @property
    def sat_za(self):
        if self._sat_za is None:
            self._calc_sat()

        return self._sat_za

    @sat_za.setter
    def sat_za(self, value):
        self._sat_za = value

    @property
    def sat_az(self):
        if self._sat_az is None:
            self._calc_sat()

        return self._sat_az

    @sat_az.setter
    def sat_az(self, value):
        self._sat_az = value

    @property
    def sun_za(self):
        if self._sun_za is None:
            self._calc_sun()

        return self._sun_za

    @sun_za.setter
    def sun_za(self, value):
        self._sun_za = value

    @property
    def sun_az(self):
        if self._sun_az is None:
            self._calc_sun()

        return self._sun_az

    @sun_az.setter
    def sun_az(self, value):
        self._sun_az = value

    @property
    def area_m(self):
        if self._area_m is None:
            self._area_m = self.pixel_area(
                self.y_rad,
                self.x_rad,
                self.abi_data["goes_imager_projection"].semi_major_axis,
                self.abi_data["goes_imager_projection"].perspective_point_height,
                self.abi_data.resolution_ifov,
            )

        return self._area_m

    @area_m.setter
    def area_m(self, value):
        self._area_m = value

    def _calc_sat(self):
        self._sat_az, self._sat_za = orbital.get_observer_look(
            sat_lon=np.atleast_1d(self.abi_data["nominal_satellite_subpoint_lon"][:]),
            sat_lat=np.atleast_1d(self.abi_data["nominal_satellite_subpoint_lat"][:]),
            sat_alt=np.atleast_1d(self.abi_data["nominal_satellite_height"][:]),
            jdays2000=orbital.jdays2000(self.time),
            lon=self.lon_deg,
            lat=self.lat_deg,
            alt=self.hae_m / 1000.0,
        )

        # normalize azimuth to North-clockwise convention between 0 and 2pi
        self._sat_az = orbital.norm_az(self._sat_az)
        # pyorbital functions output elevation/altitude, convert to zenith angle
        self._sat_za = orbital.el2za(self._sat_za)

        if self.degrees:
            self._sat_az = rad2deg(self._sat_az)
            self._sat_za = rad2deg(self._sat_za)

    def _calc_sun(self):
        if self.precise_sun:
            earth_position = coordinates.EarthLocation.from_geodetic(
                lat=self.lat_deg * u.deg,
                lon=self.lon_deg * u.deg,
                height=self.hae_m * u.m,
                ellipsoid="GRS80",
            )
            sun_position = coordinates.get_sun(Time(self.time)).transform_to(
                coordinates.AltAz(obstime=Time(self.time), location=earth_position)
            )
            self._sun_az = np.atleast_1d(sun_position.az.rad).astype(np.float32)
            self._sun_za = np.atleast_1d(sun_position.alt.rad).astype(np.float32)

        else:
            self._sun_za, self._sun_az = orbital.get_alt_az(
                jdays2000=orbital.jdays2000(self.time),
                lon=self.lon_deg,
                lat=self.lat_deg,
            )

        # normalize azimuth to North-clockwise convention between 0 and 2pi
        self._sun_az = orbital.norm_az(self._sun_az)
        # pyorbital functions output elevation/altitude, convert to zenith angle
        self._sun_za = orbital.el2za(self._sun_za)

        if self.degrees:
            self._sun_az = rad2deg(self._sun_az)
            self._sun_za = rad2deg(self._sun_za)

    @staticmethod
    @njit.heregoes_njit
    def navigate(y_rad, x_rad, lon_origin, r_eq, r_pol, sat_height):
        # navigates instrument scanning angle to latitude and longitude
        # following 7.1.2.8.1 in the PUG Volume 4: https://www.goes-r.gov/users/docs/PUG-GRB-vol4.pdf

        H = sat_height + r_eq

        lambda_0 = np.deg2rad(lon_origin)

        a = np.square(np.sin(x_rad)) + (
            np.square(np.cos(x_rad))
            * (
                np.square(np.cos(y_rad))
                + (((np.square(r_eq)) / (np.square(r_pol))) * np.square(np.sin(y_rad)))
            )
        )
        b = -2.0 * H * np.cos(x_rad) * np.cos(y_rad)
        c = np.square(H) - np.square(r_eq)

        r_s = (-b - np.sqrt((np.square(b)) - (4.0 * a * c))) / (2.0 * a)

        s_x = r_s * np.cos(x_rad) * np.cos(y_rad)
        s_y = -r_s * np.sin(x_rad)
        s_z = r_s * np.cos(x_rad) * np.sin(y_rad)

        lat_deg = np.rad2deg(
            np.arctan(
                ((np.square(r_eq)) / (np.square(r_pol)))
                * ((s_z / np.sqrt((np.square((H - s_x))) + (np.square(s_y)))))
            )
        )
        lon_deg = np.rad2deg(lambda_0 - np.arctan(s_y / (H - s_x)))

        return (
            np.atleast_1d(lat_deg).astype(np.float32),
            np.atleast_1d(lon_deg).astype(np.float32),
        )

    @staticmethod
    @njit.heregoes_njit
    def reverse_navigate(
        lat_deg, lon_deg, lon_origin, r_eq, r_pol, sat_height, feature_height
    ):
        # navigates latitude and longitude to instrument scanning angle
        # following 7.1.2.8.2 in the PUG Volume 4: https://www.goes-r.gov/users/docs/PUG-GRB-vol4.pdf

        phi = np.deg2rad(lat_deg)
        lambda_ = np.deg2rad(lon_deg)

        e = 0.0818191910435

        H = sat_height + r_eq

        lambda_0 = np.deg2rad(lon_origin)

        phi_c = np.arctan((np.square(r_pol) / np.square(r_eq)) * np.tan(phi))

        r_c = r_pol / np.sqrt(1 - np.square(e) * np.square(np.cos(phi_c)))
        r_c -= feature_height

        s_x = H - r_c * np.cos(phi_c) * np.cos(lambda_ - lambda_0)
        s_y = -r_c * np.cos(phi_c) * np.sin(lambda_ - lambda_0)
        s_z = r_c * np.sin(phi_c)

        y_rad = np.arctan(s_z / s_x)
        x_rad = np.arcsin(
            -s_y / np.sqrt(np.square(s_x) + np.square(s_y) + np.square(s_z))
        )

        return (
            np.atleast_1d(y_rad).astype(np.float32),
            np.atleast_1d(x_rad).astype(np.float32),
        )

    @staticmethod
    @njit.heregoes_njit
    def pixel_area(y_rad, x_rad, semi_major_axis, perspective_point_height, ifov):
        # returns the effective area of a pixel in meters

        r = semi_major_axis
        sh = perspective_point_height
        beta = ifov

        # https://doi.org/10.1017/CBO9781139029346.005 eqs. 3.10 a-b
        # cross-track (N-S)
        alpha_c = y_rad
        delta = np.arcsin(((sh + r) / r) * np.sin(alpha_c)) - alpha_c
        Lc = 2 * (((r * np.sin(delta)) / np.sin(alpha_c)) * np.tan(beta / 2.0))

        # https://doi.org/10.1017/CBO9781139029346.005 eqs. 3.7 - 3.9
        # along-track (W-E)
        alpha_a = x_rad
        alpha_a1 = alpha_a - beta / 2.0
        alpha_a2 = alpha_a + beta / 2.0
        L1 = r * (np.arcsin(((sh + r) / r) * np.sin(alpha_a1)) - alpha_a1)
        L2 = r * (np.arcsin(((sh + r) / r) * np.sin(alpha_a2)) - alpha_a2)
        La = L2 - L1

        return np.atleast_1d(Lc * La).astype(np.float32)


#!/usr/bin/env python
# -*- coding:utf-8 -*-

import  requests
import json
if __name__ == "__main__":
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36'

    }
    id_list = []  # 存储企业的id
    all_data_list = []  # 存储所有的企业详情数据
    #批量获取不同企业的id值
    url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsList'
    #参数的封装
    for page in range(1,6):
        page = str(page)
        data = {
            'on': 'true',
            'page': page,
            'pageSize': '15',
            'productName':'',
            'conditionType': '1',
            'applyname':'',
            'applysn':'',
        }
        json_ids = requests.post(url=url,headers=headers,data=data).json()
        for dic in json_ids['list']:
            id_list.append(dic['ID'])

    #获取企业详情数据
    post_url = 'http://125.35.6.84:81/xk/itownet/portalAction.do?method=getXkzsById'
    for id in id_list:
        data = {
            'id':id
        }
        detail_json = requests.post(url=post_url,headers=headers,data=data).json()
        # print(detail_json,'-------------ending-----------')
        all_data_list.append(detail_json)

    #持久化存储all_data_list
    fp = open('./allData.json','w',encoding='utf-8')
    json.dump(all_data_list,fp=fp,ensure_ascii=False)
    print('over!!!')

选修进阶/爬虫开发和APP逆向爬虫/新版爬虫课程/第二章：requests模块基础/06.requests实战之药监总局相关数据爬取.py


import datetime
import random

import numpy as np
import pandas as pd

from option_util import get_minute_list, millis, load_nifty_min_data, load_india_vix_day_data, get_nifty_spot_price, \
    get_nearest_thursday
from util import get_pickle_data, write_pickle_data

minute_list = get_minute_list('%H:%M:%S')

vix_data_dic = get_pickle_data('india-vix-day-candle')
day_keys = list(vix_data_dic.keys())
# print(len(strike_prices) * len(day_keys) * len(minute_list))
# print(len(strike_prices), len(day_keys), len(minute_list))
trade_intervals = ["0920", "0940", "1000", "1020", "1040",
                   "1100", "1120", "1140", "1200", "1220", "1240", "1300", "1320", "1340", "1400"]
df_base_min_list = []
df_base_day_list = []
df_base_day_str_list = []
df_base_india_vix_list = []

df_minute_list = []
df_day_list = []
df_strike_list = []
df_value_list = []
df_index_list = []

df_other_day_list = []
df_other_time_intervals = []
df_other_atm_option_strike_list = []
df_other_atm_premium_list = []
df_other_option_type_list = []

def populate_list():
    count = 0
    start_time = 1546300800000
    start_strike_price = 34500
    for day in range(750):
        day_time = start_time + (day * 86400000)
        strike_list = []
        for minute in minute_list:
            df_base_min_list.append(minute)
            df_base_day_list.append(day_time)
        for strike_index in range(6):
            in_date_format = datetime.datetime.fromtimestamp(day_time / 1000.0)
            strike_price = get_nearest_thursday(in_date_format, "PE" if strike_index % 2 == 0 else "CE", strike_index)
            strike_list.append(strike_price)
            for minute in minute_list:
                # print(random.randrange(100, 300, 3))
                df_value_list.append(random.randrange(100, 300, 3))
                df_day_list.append(day_time)
                df_strike_list.append(strike_price)
                df_minute_list.append(minute)
                df_index_list.append(count)
                count = count + 1
                # print(count)
        for trade_interval in trade_intervals:
            for option_type in ["PE", "CE"]:
                atm_option_type_strike = [strike for strike in strike_list if option_type in strike][
                    random.randrange(0, 3)]
                atm_premium = random.randrange(100, 300, 3)
                df_other_day_list.append(day_time)
                df_other_time_intervals.append(trade_interval)
                df_other_atm_option_strike_list.append(atm_option_type_strike)
                df_other_atm_premium_list.append(atm_premium)
                df_other_option_type_list.append(option_type)

    print(f'len of other day list:{len(df_other_day_list)}')

    base_minute_df = pd.DataFrame(
        {'day': df_base_day_list, 'minute': df_base_min_list},
        df_base_day_list)

    every_minute_df = pd.DataFrame(
        {'day': df_day_list, 'strike': df_strike_list, 'minute': df_minute_list, 'value': df_value_list},
        df_strike_list)

    all_atm_df = pd.DataFrame(
        {'day': df_other_day_list, 'interval': df_other_time_intervals,
         'atm_option_strike': df_other_atm_option_strike_list,
         'atm_premium': df_other_atm_premium_list, "option_type": df_other_option_type_list},
        df_other_day_list)

    write_pickle_data('every_minute_df', every_minute_df)
    write_pickle_data('all_atm_df', all_atm_df)
    write_pickle_data('base_minute_df', base_minute_df)
    print("done with iteration")

def test_data():
    compare_df = pd.DataFrame(
        {'day': [1, 2, 3]},
        ['BANKNIFTYJAN34500PE', 'BANKNIFTYJAN34500PE', 'BANKNIFTYJAN34500PE'])
    start = millis()
    my_test_df = get_pickle_data('my_test_df')
    print(len(my_test_df))
    start = millis()
    # print(len(my_test_df))
    # print(my_test_df.loc[1750000])Ω
    # df_out_1 = my_test_df.query(
    #     'index == 0 or index==1 or index == 3 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1 or index == 0 or index==1')
    # df_out = my_test_df.query('index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530" or '
    #                           'index == "2022-02-16T00:00:00+0530" or index == "2022-02-17T00:00:00+0530"'
    #                           )

    # print(my_test_df.index)
    # my_test_df['flag'] = np.where(
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530") |
    #     (my_test_df.index == "2022-02-16T00:00:00+0530")
    #     , 'Y', '0')

    hellons = my_test_df[(my_test_df.index == 'BANKNIFTYJAN34500PE') |
                         # (my_test_df.index == 'BANKNIFTYAPR34000CE') |
                         # (my_test_df.index == 'BANKNIFTYAPR34000CE') |
                         # (my_test_df.index == 'BANKNIFTYAPR34000CE') |
                         (my_test_df.index == 'BANKNIFTYMAR34500CE')
                         ]
    print("length", compare_df.iloc[1].day == 2)
    # test_df = my_test_df[compare_df.loc[my_test_df.index].day == 1]
    # print(my_test_df.iloc[1].index)
    # print(my_test_df.iloc[True].index)
    # check_diff = compare_df[my_test_df.iloc[1].day.values[0] == 1]
    check_diff = compare_df[(True)]
    print(("time taken to read first", (millis() - start)))

    my_test_df['check'] = my_test_df['value'] + 1
    print(("time taken to read first", (millis() - start)))
    # print(my_test_df)
    # helloons = df_out.values
    print("to filter by query", (millis() - start))

    print("to filter by query", (millis() - start))
    # for i in range(len(my_test_df)):
    for i in []:
        # start = millis()
        # df_row = my_test_df[my_test_df.day == "2022-02-16T00:00:00+0530"]
        df_row = my_test_df.loc[i]
        # print(("time taken to read file", (millis() - start)))
        print(i)
    print("done")
    start = millis()
    print(("time taken to read file", (millis() - start)))

def new_test_data():
    start = millis()

    print(f'time after trading days:{millis() - start}')
    all_strike_df = get_pickle_data('every_minute_df')
    base_minute_df = get_pickle_data('base_minute_df')
    all_atm_df = get_pickle_data('all_atm_df')
    print(f'time after getting pickle data:{millis() - start}')
    df_only_9_20_pe = all_atm_df[(all_atm_df.interval == "0920") & (all_atm_df.option_type == "PE")]
    df_only_9_20_ce = all_atm_df[(all_atm_df.interval == "0920") & (all_atm_df.option_type == "CE")]
    print(f'time after getting 920 strike symbol:{millis() - start}')
    print(len(df_only_9_20_pe), len(df_only_9_20_pe))
    _0920_pe_strike_df = pd.merge(all_strike_df, df_only_9_20_pe, how='inner', left_on=['day', 'strike'],
                                  right_on=['day', 'atm_option_strike'])
    _0920_ce_strike_df = pd.merge(all_strike_df, df_only_9_20_ce, how='inner', left_on=['day', 'strike'],
                                  right_on=['day', 'atm_option_strike'])
    print(f'time after getting 920 strike ticker:{millis() - start}')
    print(len(_0920_ce_strike_df), len(_0920_ce_strike_df))
    _0920_straddle_df = pd.merge(base_minute_df, _0920_pe_strike_df, how='left', left_on=['day', 'minute'],
                                 right_on=['day', 'minute'])
    _0920_straddle_df = pd.merge(_0920_straddle_df, _0920_ce_strike_df, how='inner', left_on=['day', 'minute'],
                                 right_on=['day', 'minute'])
    print(len(_0920_straddle_df), len(_0920_straddle_df))
    print(f'time after merging strike ticker to base:{millis() - start}')
    # df_0920_atm_strike.to_csv("df_0920.csv")
    # write_pickle_data('_0920_straddle_df', _0920_straddle_df)
    print(f'time before loop:{millis() - start}')
    values = _0920_straddle_df.values
    done_days = []
    day_profit = 0
    day_profit_list = []
    for i in range(len(values)):
        date_time = values[i][0]
        if date_time not in done_days:
            day_profit = 0
            done_days.append(date_time)
            day_profit_list.append(day_profit)
        else:
            in_date_format = datetime.datetime.fromtimestamp(date_time / 1000.0)
            # get_nifty_spot_price(, close_candle['time'],
            #                      self.day_tracker.nifty_min_data_dic,
            #                      self.day_tracker.config.column_to_consider)
            day_profit = day_profit + 9
        # print()
    print(f'time:{millis() - start}', len(day_profit_list))
    # merge = pd.merge(trade_df_list[0], trade_df_list[1], how='inner', left_index=True, right_index=True)

# test_data()
# populate_list()
new_test_data()
# print(len(minute_list))
# get_nearest_thursday()
# print("done!!!", random.random())


from django.core.management import BaseCommand

from posts.models import Tag

class Command(BaseCommand):
    help = 'Создаем тэги'

    def handle(self, *args, **kwargs):
        data = [
            {'name': 'Завтрак', 'color': '#E26C2D', 'slug': 'breakfast'},
            {'name': 'Обед', 'color': '#49B64E', 'slug': 'dinner'},
            {'name': 'Ужин', 'color': '#8775D2', 'slug': 'supper'}]
        Tag.objects.bulk_create(Tag(**tag) for tag in data)
        self.stdout.write(self.style.SUCCESS('Все данные залиты в бд'))


import numpy as np
import matplotlib.pyplot as plt
def objfun(x):
    y = (3*x[0,0]-4)**2 + (4*x[1,0]**2-2)**2
    return y
def gradobjfun(x):
    y = np.array([[6*(3*x[0,0]-4)],[16*x[1,0]*(4*x[1,0]**2-2)]])
    return y
def phi(alpha, xk):
    dk = -gradobjfun(xk)
    y = objfun(xk+alpha*dk)
    return y
if __name__ == "__main__":
    alpha = np.arange(0,2.1,0.1)
    xk = np.array([[0],[0]])
    phialpha = [phi(i, xk) for i in alpha]
    print(phialpha)
    plt.plot(alpha, phialpha, "*")
    plt.show()


from Quanser.q_misc import BasicStream
import time
import numpy as np
import cv2

# -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
# Image Parameters
imageWidth = 640
imageHeight = 480
imageChannels = 3
buffer_size = imageHeight*imageWidth*imageChannels # 4 bytes for float32 data, 3 channels for RGB
image_data = np.zeros((imageHeight, imageWidth, imageChannels), dtype=np.uint8)

# -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
# Create a BasicStream object configured as a Server agent, with buffer sizes enough to send/receive the image above.
myServer = BasicStream('tcpip://192.168.2.12:18001', agent='s', send_buffer_size=buffer_size, recv_buffer_size=buffer_size)
prev_con = False

# -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
## Timing Parameters and methods
startTime = time.time()
def elapsed_time():
    return time.time() - startTime

sampleRate = 30.0
sampleTime = 1/sampleRate
simulationTime = 30.0
print('Sample Time: ', sampleTime)

# -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
# Main Loop
try:
    while elapsed_time() < simulationTime:

        if not myServer.connected:
            myServer.checkConnection()

        if myServer.connected and not prev_con:
            print('Connection to Client was successful.')
        prev_con = myServer.connected

        if myServer.connected:

            # Start timing this iteration
            start = time.time()

            # Receive data from client
            image_data, bytes_received = myServer.receive(image_data)

            if bytes_received < len(image_data.tobytes()):
                print('Client stopped sending data over.')
                break
            # End timing this iteration
            end = time.time()

            # Calculate the computation time, and the time that the thread should pause/sleep for
            computationTime = end - start
            sleepTime = sampleTime - ( computationTime % sampleTime )

            # Pause/sleep for sleepTime in milliseconds
            msSleepTime = int(1000*sleepTime)
            if msSleepTime <= 0:
                msSleepTime = 1 # this check prevents an indefinite sleep as cv2.waitKey waits indefinitely if input is 0

            cv2.imshow('Server Image Received', image_data)
            cv2.waitKey(msSleepTime)

except KeyboardInterrupt:
    print("User interrupted!")

finally:
    # Terminate Server
    myServer.terminate()
    print('All the right turns in all the right places.')


from configs.config import config
import torch
from torch.autograd import Variable
from torchvision import transforms
from PIL import Image, ImageOps
import numpy as np
import cv2, imgviz

# torch.backends.cudnn.deterministic = True

transformer = transforms.Compose([
                                 # this transfrom converts BHWC -> BCHW and
                                 # also divides the image by 255 by default if values are in range 0..255.
                                 transforms.ToTensor(),
                                ])

crop_size = config['data']['hr_crop_size']
op_stride = config['model']['output_stride']
torch_resizer = transforms.Compose([transforms.Resize(size=(crop_size//op_stride, crop_size//op_stride),
                                                interpolation=transforms.InterpolationMode.NEAREST)])
torch_imgresizer = transforms.Compose([transforms.Resize(size=(crop_size//op_stride, crop_size//op_stride),
                                                interpolation=transforms.InterpolationMode.BILINEAR)])
def collate(batch):
    '''
    custom Collat funciton for collating individual fetched data samples into batches.
    '''

    img = [ b['img'] for b in batch ] # w, h
    lbl = [ b['lbl'] for b in batch ]
    a = [ b['geo_augs'] for b in batch ]
    b = [ b['noise_augs'] for b in batch ]

    return {'img': img, 'lbl': lbl, 'geo_augs': a, 'noise_augs': b}

normalize = lambda x, alpha, beta : (((beta-alpha) * (x-np.min(x))) / (np.max(x)-np.min(x))) + alpha
standardize = lambda x : (x - np.mean(x)) / np.std(x)

def std_norm(img, norm=True, alpha=0, beta=1):
    '''
    Standardize and Normalizae data sample wise
    alpha -> -1 or 0 lower bound
    beta -> 1 upper bound
    '''
    img = standardize(img)
    if norm:
        img = normalize(img, alpha, beta)

    return img

def _mask_transform(mask):
    target = np.array(mask).astype('int32')
    return target

def masks_transform(masks, numpy=False):
    '''
    masks: list of PIL images
    '''
    targets = []
    for m in masks:
        targets.append(_mask_transform(m))
    targets = np.array(targets)
    if numpy:
        return targets
    else:
        return torch.from_numpy(targets).long().to('cuda' if torch.cuda.is_available() else 'cpu')

def images_transform(images):
    '''
    images: list of PIL images
    '''
    inputs = []
    for img in images:
        inputs.append(transformer(img))
    inputs = torch.stack(inputs, dim=0).float().to('cuda' if torch.cuda.is_available() else 'cpu')
    return inputs

def get_crop_bbox(img_size, crop_size):
    """Randomly get a crop bounding box."""
    assert len(img_size) == len(crop_size)
    assert len(img_size) == 2
    margin_h = max(img_size[0] - crop_size[0], 0)
    margin_w = max(img_size[1] - crop_size[1], 0)
    offset_h = np.random.randint(0, margin_h + 1)
    offset_w = np.random.randint(0, margin_w + 1)
    crop_y1, crop_y2 = offset_h, offset_h + crop_size[0]
    crop_x1, crop_x2 = offset_w, offset_w + crop_size[1]

    return crop_y1, crop_y2, crop_x1, crop_x2

def decode_preds(mask, img, font_size=60):

    mask = cv2.resize(mask, (img.shape[1], img.shape[0]),
                       interpolation = cv2.INTER_NEAREST)

    op = imgviz.label2rgb(label=mask,
                          image=(img*255).astype(np.uint8),
                          alpha=0.5,
                          label_names=config['sub_classes'],
                          font_size=font_size,
                          thresh_suppress=0,
                          colormap=np.asarray(config['pallet_sub_class']),
                          loc="rb",
                          font_path=None,
                          )
    return op


import streamlit as st
import plotly.express as px
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from wordcloud import WordCloud
from nltk.corpus import stopwords

# Use the full page instead of a narrow central column
st.set_page_config(layout="wide")

# Set page title
st.title('Reviews da Amazon - Firestick TV')

@st.cache
def load_data():
    df = pd.read_csv("data/reviews_v2.csv")
    df = df[df["reviews"].notna()]
    df["class"] = df["stars"].apply(lambda x : "Positiva" if x >=4 else "Negativa")
    df["dates"] = pd.to_datetime(df["dates"])
    return df

@st.cache
def load_pred():
    df = pd.read_csv("data/best_pred.csv")
    df["pred"] = df["pred"].apply(lambda x : "Positiva" if x == 1 else "Negativa")
    df["dates"] = pd.to_datetime(df["dates"])
    return df

@st.cache
def load_rep():
    df = pd.read_csv("data/reviews_rep.csv")
    df
    return df

@st.cache
def load_metrics():
    df = pd.read_csv("data/metrics.csv").drop("predictions", axis=1)
    return df

#Loading data
data_load_state = st.text('Loading data...')
data = load_data()
data_pred = load_pred()
data_rep = load_rep()
data_metrics = load_metrics()
data_load_state.text("Done! (using st.cache)")

# st.subheader('Raw data')
# st.write(data)

col1, col2 = st.columns(2)

### Plot Time Series - Reviews ###
col1.header("Reviews no tempo - Original (todos os dados)")
df_count_reviews = pd.DataFrame({'count' : data.groupby( [ "dates", "class"] ).size()}).reset_index()
fig = px.line(df_count_reviews, x="dates", y="count",
              color='class',
                color_discrete_map={
                 "Positiva": "#00FF00",
                 "Negativa": "red"
                },
                labels={
                        "dates": "Data",
                        "count": "Número de reviews",
                        "class": "Tipo de review"
                        },
                        title="Reviews no tempo")
col1.plotly_chart(fig)

col2.header("Reviews no tempo - Predições do BERT (conjunto de teste)")
df_count_reviews = pd.DataFrame({'count' : data_pred.groupby( [ "dates", "pred"] ).size()}).reset_index()
fig2 = px.line(df_count_reviews, x="dates", y="count",
                color='pred',
                color_discrete_map={
                 "Positiva": "#00FF00",
                 "Negativa": "red"
                },
                labels={
                        "dates": "Data",
                        "count": "Número de reviews",
                        "pred": "Tipo de review"
                        },
                        title="Reviews no tempo")

fig2.update_layout(legend_traceorder="reversed")
col2.plotly_chart(fig2)

col3, col4 = st.columns(2)

### Plot Wordcloud ###
col3.header("Wordcloud")
data_load_state = st.text('Loading data...')
lista_rev = data["reviews"].tolist()
big_string = (" ").join(lista_rev)
stopwords = stopwords.words('portuguese')
wordcloud = WordCloud(stopwords=stopwords, background_color="white", max_words=200, contour_width=3, width=800, height=400).generate(big_string)
fig3 = plt.figure(figsize=[20,10])
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
col3.pyplot(fig3)
data_load_state.text("")

#Review mais representativos
col4.header("Reviews mais representativos")
col4.subheader("Negativos")
col4.markdown('- '+ data_rep["negativo"][0])
col4.markdown('- '+ data_rep["negativo"][1])
col4.subheader("Positivos")
col4.markdown('- '+ data_rep["positivo"][0])
col4.markdown('- '+ data_rep["positivo"][1])

#Métricas dos classificadores
st.header("Métricas dos classificadores")
st.table(data_metrics)


import torch.nn as nn
import torch.nn.functional as F

class MLP(nn.Module):

    def __init__(self, input_dim, output_dim, hidden_dim=8):
        super(MLP, self).__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, hidden_dim)
        self.fc3 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class CNN(nn.Module):

    def __init__(self, channels, height, width, output_dim):
        super(CNN, self).__init__()
        self.conv1 = nn.Conv2d(channels, 8, kernel_size=5, stride=2, padding=2)
        self.conv2 = nn.Conv2d(8, 8, kernel_size=5, stride=2, padding=2)
        self.num_flat_features = 8 * (height // 4) * (width // 4)
        self.fc1 = nn.Linear(self.num_flat_features, 64)
        self.fc2 = nn.Linear(64, output_dim)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.relu(self.conv2(x))
        x = F.relu(self.fc1(x.view(-1, self.num_flat_features)))
        x = self.fc2(x)
        return x


# Load data
import pandas as pd
import networkx as nx
import numpy as np
from tqdm import tqdm
import pickle
from scipy import sparse
import matplotlib.pyplot as plt
import torch
#from torch_geometric import utils, data

def haversine_start(df, car1, car2, max_dist = 1500):
    def _edge_weight(x, max_dist):
        return max((max_dist-x)/max_dist,0)
    dfc1 = df[df.car == car1]
    dfc2 = df[df.car == car2]

    point1 = dfc1[['leave_location_lat','leave_location_long']].values[0]
    point2 = dfc2[['leave_location_lat','leave_location_long']].values[0]
    #return point1

    # convert decimal degrees to radians
    lat1, lon1 = map(np.radians, point1)
    lat2, lon2 = map(np.radians, point2)

    # Deltas
    delta_lon = lon2 - lon1
    delta_lat = lat2 - lat1

    # haversine formula
    a = np.sin(delta_lat/2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(delta_lon/2)**2
    c = 2 * np.arcsin(np.sqrt(a))
    r = 6371000 # Radius of earth in m
    return _edge_weight(c * r, max_dist)

def haversine_add(current_locs, to_add, max_dist = 1500):
    def _edge_weight(x, max_dist):
        return max((max_dist-x)/max_dist,0)
    def _haversine(point1, lat2, lon2):

         # convert decimal degrees to radians
        lat1, lon1 = map(np.radians, point1)

        # Deltas
        delta_lon = lon2 - lon1
        delta_lat = lat2 - lat1

        # haversine formula
        a = np.sin(delta_lat/2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(delta_lon/2)**2
        c = 2 * np.arcsin(np.sqrt(a))
        r = 6371000 # Radius of earth in m
        return c * r

    leave_lat_add, leave_long_add = to_add.leave_location_lat, to_add.leave_location_long

    lat2, lon2 = map(np.radians, [leave_lat_add,leave_long_add])

    new_weights = [_edge_weight(_haversine(loc, lat2, lon2), max_dist) for loc in current_locs]

    return new_weights

def delete_rc(mat, i):
    # row
    n = mat.indptr[i+1] - mat.indptr[i]
    if n > 0:
        mat.data[mat.indptr[i]:-n] = mat.data[mat.indptr[i+1]:]
        mat.data = mat.data[:-n]
        mat.indices[mat.indptr[i]:-n] = mat.indices[mat.indptr[i+1]:]
        mat.indices = mat.indices[:-n]
    mat.indptr[i:-1] = mat.indptr[i+1:]
    mat.indptr[i:] -= n
    mat.indptr = mat.indptr[:-1]
    mat._shape = (mat._shape[0]-1, mat._shape[1])

    # col
    mat = mat.tocsc()
    n = mat.indptr[i+1] - mat.indptr[i]
    if n > 0:
        mat.data[mat.indptr[i]:-n] = mat.data[mat.indptr[i+1]:]
        mat.data = mat.data[:-n]
        mat.indices[mat.indptr[i]:-n] = mat.indices[mat.indptr[i+1]:]
        mat.indices = mat.indices[:-n]
    mat.indptr[i:-1] = mat.indptr[i+1:]
    mat.indptr[i:] -= n
    mat.indptr = mat.indptr[:-1]
    mat._shape = (mat._shape[0], mat._shape[1]-1)

    return mat.tocsr()

# Haversine function for stations
def haversine_station(point1, point2):
    # convert decimal degrees to radians
    lat1, lon1 = map(np.radians, point1)
    lat2, lon2 = map(np.radians, point2)

    # Deltas
    delta_lon = lon2 - lon1
    delta_lat = lat2 - lat1

    # haversine formula
    a = np.sin(delta_lat/2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(delta_lon/2)**2
    c = 2 * np.arcsin(np.sqrt(a))
    r = 6371000 # Radius of earth in m
    return c * r

# Load data
df = pd.read_csv('data/processed/VacancySplit.csv', index_col=0, parse_dates = [2]).astype({'time_to_reservation': 'float32', 'park_location_lat': 'float32', 'park_location_long': 'float32', 'leave_location_lat': 'float32', 'leave_location_long': 'float32', 'park_zone': 'int32', 'leave_zone': 'int32', 'park_fuel': 'int8', 'leave_fuel': 'int8', 'moved': 'float32', 'movedTF': 'bool'})

# Time variables
df['weekend'] = df.time.dt.weekday//5

# Time encoding
def circle_transform(col, max_val=86400):
    tot_sec = ((col - col.dt.normalize()) / pd.Timedelta('1 second')).astype(int)
    cos_val = np.cos(2*np.pi*tot_sec/max_val)
    sin_val = np.sin(2*np.pi*tot_sec/max_val)
    return cos_val, sin_val

df['Time_Cos'], df['Time_Sin'] = [x.values for x in circle_transform(df.time)]

# Join weather
df_weather = pd.read_csv('data/processed/weather.csv', index_col=0, parse_dates=[0])
df['timeH'] = df.time.round('H')
df = df.set_index('timeH').join(df_weather).reset_index(drop=True)

# Create init graph
Start_Time = pd.Timestamp('2019-08-31 21:00:00')
End_Time = pd.Timestamp('2019-11-01 12:00:00')
start_df = df[df.time <= Start_Time]
propegate_df = df[(df.time > Start_Time) & (df.time <= End_Time)]

# Start
CarID_dict_start = dict(iter(start_df.groupby('car')))
Start_Garph_data = []

for sub_df in CarID_dict_start.values():
    last_obs = sub_df.iloc[-1]
    if last_obs.action: # True is park
        Start_Garph_data.append(last_obs)

start_df_graph = pd.DataFrame(Start_Garph_data)

# Add dist to station
with open('data/processed/Train_stations.pickle', 'rb') as handle:
    Stations = pickle.load(handle)

start_df_graph['dist_to_station'] = [min({k:haversine_station(v,r[1].values) for k,v in Stations.items()}.values()) for r in start_df_graph[['park_location_lat',	'park_location_long']].iterrows()]
propegate_df['dist_to_station'] = [min({k:haversine_station(v,r[1].values) for k,v in Stations.items()}.values()) for r in tqdm(propegate_df[['park_location_lat',	'park_location_long']].iterrows(), total = len(propegate_df))]

# Adj matrix
A = pd.DataFrame(data = [[haversine_start(start_df_graph, car1, car2) for car1 in start_df_graph.car] for car2 in tqdm(start_df_graph.car)], index = start_df_graph.car, columns=start_df_graph.car, dtype='float16')

# And make it sparse
As = sparse.csr_matrix(A.values)

# Populate
Graph_dict = {pd.Timestamp('2019-08-31 21:00:00'): (start_df_graph ,As)}
node_data = start_df_graph.set_index('car')

positive_time = propegate_df[propegate_df.action].time.diff().shift(-1) > pd.Timedelta(0,'s')
positive_time.iloc[-1] = True

new_day = (propegate_df.time.dt.date.diff().shift(-1) > pd.Timedelta(0,'s'))
new_day.iloc[-1] = True

i = 0
for idx, next_row in tqdm(propegate_df.iterrows(), total = propegate_df.shape[0]):
    if next_row.action: # True is park
        # Get current locs
        locs = [[attr['leave_location_lat'], attr['leave_location_long']] for _, attr in node_data.iterrows()]

        # Add to node data
        node_data = node_data.append(next_row.rename(index = next_row['car']).iloc[1:], verify_integrity = True)

        # Calculate new weights
        new_weights = haversine_add(locs, next_row, max_dist = 1500)

        # Add new weights to adjacency
        As = sparse.hstack([sparse.vstack([As,sparse.csr_matrix(new_weights)]).tocsc(), sparse.csc_matrix(new_weights+[1]).T]).tocsr()

    else:
        # Getindex
        idx_to_drop = np.where(node_data.index == next_row.car)[0][0]

        # Drop it
        As = delete_rc(As, idx_to_drop)

        # Drop from feature-matrix
        node_data.drop(index = next_row.car, inplace=True)

    # Save graph if new time
    if positive_time.get(idx):
        Graph_dict[next_row.time] = (node_data.copy(), As.copy())

    # Save file every day on last obs
    if new_day[idx]:
        f_name = next_row.time.strftime('data/processed/Graphs/%Y%m%d')+'.pickle'
        with open(f_name, 'wb') as handle:
            pickle.dump(Graph_dict, handle, pickle.HIGHEST_PROTOCOL)

        # Clear memory
        Graph_dict = {}


# import necessary libraries
import pandas as pd
from splinter import Browser
from bs4 import BeautifulSoup as Soup
import datetime as dt

def init_browser():

    executable_path = {"executable_path": "C:/chromedriver.exe"}

    return Browser("chrome", **executable_path, headless=False)

def scrape():

    browser = init_browser()
    title, paragraph = news(browser)

    # Run scraping and store the results in a dictionary

    mars_dict = {
        "news_title": title,
        "news_paragraph": paragraph,
        "featured_image": image(browser),
        "facts": mars_facts(),
        "hemispheres": hemispheres(browser),
        "last_modified": dt.datetime.now()
    }

    # Stop Browser connection
    browser.quit()

    return mars_dict

def news(browser):

    # Visit the mars nasa news site with splinter
    url = 'https://mars.nasa.gov/news/'
    browser.visit(url)

     # Use beautiful soup to turn the browser html to a soup object
    html = browser.html
    news_soup = Soup(html, 'html.parser')

    try:
        slide_elem = news_soup.select_one("ul.item_list li.slide")

        # Use bs to get news title and paragraph info from the html object
        title = slide_elem.find("div", class_="content_title").text
        paragraph = slide_elem.find("div", class_="article_teaser_body").text

    except AttributeError:
        return None, None

    return title, paragraph

def image(browser):

    # Visit the mars nasa news site for images
    url_img = "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"
    browser.visit(url_img)
    # Turn webpage into HTML object
    html_img = browser.html
    soup_img = Soup(html_img, "html.parser")
    # Create a list to collect the link to each image displayed on the page
    img_list = []
    try:
    # Locate the images in this webpage
        images = soup_img.find_all('img')

    # Create a loop through the imaged to append the image link to our img_list
        for img in images:
            img_list.append(img['src'])

    # Select the specific desired image

    except AttributeError:
        return None
    # Select the specific desired image
    img_url = img_list[2]

    # full_url = url_img + img_url

    return img_url

def mars_facts():
    # Mars facts to be scraped
    url_facts = 'https://space-facts.com/mars/'

    try:
        # Use pandas to read the tables available on the html
        read_table = pd.read_html(url_facts)

    except BaseException:
        return None
    # Select the first table of the html
    facts_df = read_table[0]
    facts_df.columns =  ['Facts', 'Mars']

    return facts_df.to_html()

def hemispheres(browser):
    # Create a lists to store the large image links
    hemi_url = "https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars"
    browser.visit(hemi_url)

    hemi_links = []

    try:
        # Create a soup object
        html_hemi = browser.html
        soup_hemi = Soup(html_hemi, 'html.parser')
        # Find the desired data by tag and class
        hemis = soup_hemi.find_all("div", class_="item")
        # loop through image data to find title and url info
        for h in hemis:
            # Identify the image name
            title = h.find("h3").text

            # Collect the direct image address
            img_url = h.a["href"]

            # build the complete link
            url = "https://astrogeology.usgs.gov" + img_url

            # Visit img url with splinter
            browser.visit(url)
            each_pic_html = browser.html

            # Create soup object fo each of the individual pictures we are visiting
            soup = Soup(each_pic_html,"html.parser")

            # Find the url for high resolution picture
            new_url = soup.find("img", class_="wide-image")["src"]

            # Build large picture complete link
            hemi_link = "https://astrogeology.usgs.gov" + new_url
            # Create a dictionary with thetitle and link collected on this loop and append it to our hemi_links list
            hemi_links.append({"title": title, "img_url": hemi_link})

    except AttributeError:
        return None

    return hemi_links

if __name__ == "__main__":

    # If running as script, print scraped data
    print(scrape())


#!/usr/bin/env python3
import hashlib
import json
import sys
from datetime import datetime
from urllib.request import urlopen

def md5sum(filename):
    with open(filename, 'rb') as f:
        return hashlib.md5(f.read()).hexdigest()

def generate(filename, version):
    return {
        'checksum': md5sum(filename),
        'changelog': 'Auto Released by Actions',
        'targetAbi': '10.8.0.0',
        'sourceUrl': 'https://github.com/metatube-community/jellyfin-plugin-metatube/releases/download/'
                     f'v{version}/Jellyfin.MetaTube@v{version}.zip',
        'timestamp': datetime.now().strftime('%Y-%m-%dT%H:%M:%SZ'),
        'version': version
    }

def main():
    filename = sys.argv[1]
    version = filename.split('@', maxsplit=1)[1] \
        .removeprefix('v') \
        .removesuffix('.zip')

    with urlopen('https://raw.githubusercontent.com/metatube-community/jellyfin-plugin-metatube/dist/manifest.json') as f:
        manifest = json.load(f)

    manifest[0]['versions'].insert(0, generate(filename, version))

    with open('manifest.json', 'w') as f:
        json.dump(manifest, f, indent=2)

if __name__ == '__main__':
    main()


from typing import List, Dict, Any
import json

def write_binary(out_path: str, machine_code: List[int]) -> None:
    "writes the given machine code to the given output file"
    if not len(out_path):
        raise ValueError("write_binary: out_path is empty")
    if not len(machine_code):
        raise ValueError("write_binary: no machine_code provided")

    try:
        with open(out_path, "w") as fhandle:
            for code in machine_code:
                fhandle.write(f"{hex(code)} ")
    except Exception as ex:
        raise RuntimeError(str(ex))

def write_ast(out_path: str, ast: List[Dict[str, Any]]) -> None:
    "writes the given AST to the given output file as JSON"
    if not len(out_path):
        raise ValueError("write_ast: out_path is empty")
    if not len(ast):
        raise ValueError("write_ast: no ast provided")

    with open(out_path, "w", encoding="utf-8") as fhandle:
        json.dump(ast, fhandle, ensure_ascii=False)


import json

#?#################################################
#?	parse functions
#?#################################################

def file_to_json(fname):
	struct = []
	for line in open(fname, 'r'):
		struct.append(json.loads(line))
	return struct

#?#################################################
#?	sql functions
#?#################################################

def zappos_to_sql(f):
	zappos = file_to_json('zappos.json')
	f.write("--*\n--* zappos\n--*\nINSERT INTO product_apis (id, provider, status) VALUES (DEFAULT, 'zappos', 0);\n\n")

	# products
	for x in zappos:
		f.write("INSERT INTO products (id, api_id, product_id, name, price, rating, brand, link, description, status)" +
		" VALUES (DEFAULT, 1, '" + str(x['id']) + "', '"  + x['name'] + "', " + str(x['price']) + ", " +
		str(x['rating']) + ", '" + x['brand'] + "', '" +x['productUrl'] + "', " + "NULL" + ", " + "0"+ ");\n")
	f.write('\n')

	# photos
	i = 1
	for x in zappos:
		f.write("INSERT INTO product_images (id, product_id, image_link)" +
		" VALUES (DEFAULT, " + str(i) + ", '" + x['thumbnailImageUrl'] + "');\n")
		i+=1
	f.write('\n')

def makeup_to_sql(f):
	makeup = file_to_json('makeup.json')[0:20]
	f.write("--*\n--* makeup\n--*\nINSERT INTO product_apis (id, provider, status) VALUES (DEFAULT, 'makeup', 0);\n\n")
	count = 0

	# products
	for x in makeup:
		# print(x['product_link'])
		# break
		if x.get('rating', 0) is None:
			x['rating'] = 'NULL'
		if x.get('price', 0) is None:
			x['price'] = 'NULL'
		if x.get('brand', 0) is None:
			x['brand'] = 'NULL'
		if x.get('product_link', 0) is None:
			x['product_link'] = 'NULL'

		x['name'] = x['name'].replace("'","''")
		x['brand'] = x['brand'].replace("'","''")
		x['description'] = x['description'].replace("'","''")
		f.write("INSERT INTO products (id, api_id, product_id, name, price, rating, brand, link, description, status)" +
		" VALUES (DEFAULT, 2, '" + str(x['id']) + "', '"  + x['name'] + "', " + str(x['price']) + ", " +
		str(x.get('rating', 0)) + ", '" + x['brand'] + "', '" + x['product_link'] + "', '" + x['description'] + "', " + "0"+ ");\n")
		count+=1
	f.write('\n')

	# photos
	i = 21
	for x in makeup:
		f.write("INSERT INTO product_images (id, product_id, image_link)" +
		" VALUES (DEFAULT, " + str(i) + ", '" + x['image_link'] + "');\n")
		i+=1
	f.write('\n')

#?#################################################
#?	main
#?#################################################
if __name__ == '__main__':

	f = open('products.sql', 'w')
	f.close()

	f = open('products.sql', 'a+')
	f.write("\c alpha\n\n")
	zappos_to_sql(f)
	makeup_to_sql(f)
	f.close()