import asyncio
import os
import warnings
import json
from types import SimpleNamespace
import jwt
from base64 import b64decode

from bson import ObjectId
from django.utils.decorators import sync_and_async_middleware
from django import http
from django.utils.cache import patch_vary_headers
from django.utils.deprecation import RemovedInDjango40Warning

from urllib.parse import urlparse

from base.aes import AES
from base.rf.response import Response
from base.rf import status

from .cors_deafults import *

AES_KEY = bytes(os.getenv('AES_KEY').encode('utf-8'))
JWT_PUBLIC = os.getenv('JWT_PUBLIC').replace('\\n', '\n').strip().replace('"', '')
JWT_ALGORITHM = os.getenv('JWT_ALGORITHM')

ACCESS_CONTROL_ALLOW_ORIGIN = 'Access-Control-Allow-Origin'
ACCESS_CONTROL_EXPOSE_HEADERS = 'Access-Control-Expose-Headers'
ACCESS_CONTROL_ALLOW_CREDENTIALS = 'Access-Control-Allow-Credentials'
ACCESS_CONTROL_ALLOW_HEADERS = 'Access-Control-Allow-Headers'
ACCESS_CONTROL_ALLOW_METHODS = 'Access-Control-Allow-Methods'
ACCESS_CONTROL_MAX_AGE = 'Access-Control-Max-Age'

def get_data(auth):
    if auth:
        auth = auth.strip()
        auth = auth.split(' ')[-1]
        jwt_token = auth
        payload = jwt.decode(jwt_token, JWT_PUBLIC,
                             algorithms=[JWT_ALGORITHM])
        data = b64decode(payload['data'])
        iv = b64decode(payload['identifier'])

        dt = AES(AES_KEY).decrypt_ctr(data, iv)
        dt = dt.replace("'", '"').replace("None", "null")
        data1 = json.loads(dt, object_hook=lambda e: SimpleNamespace(**e))
        return data1
    return None

def authorize(request):
    access_token = request.headers.get('Authorization', None)
    if access_token:
        try:
            user_data = get_data(access_token)
        except jwt.exceptions.InvalidSignatureError:
            return Response(messages=[{'message': "Invalid Signature Error"}],
                            status_code=401
                            , status=status.HTTP_401_UNAUTHORIZED)
        except jwt.exceptions.ExpiredSignatureError:
            return Response(messages=[{'message': 'Signature is Expired'}], status_code=401,
                            status=status.HTTP_401_UNAUTHORIZED)
        except Exception:
            return Response(messages=[{'message': 'Invalid Token'}], status_code=401,
                            status=status.HTTP_401_UNAUTHORIZED)
        request._user = user_data
        request._user.is_authenticated = True
        request._user.id = ObjectId(user_data._id)
    else:
        request._user = None

@sync_and_async_middleware
def auth_middleware(get_response):
    """
        check token exists and decode token and get data for authorization
    """
    if asyncio.iscoroutinefunction(get_response):
        async def middleware(request):
            resp = authorize(request)
            if isinstance(resp, Response):
                return resp
            response = await get_response(request)
            return response

    else:

        def middleware(request):
            resp = authorize(request)
            if isinstance(resp, Response):
                return resp
            response = get_response(request)
            return response

    return middleware

AuthMiddleware = auth_middleware

class CORSMiddleWare:
    sync_capable = True
    async_capable = True

    # RemovedInDjango40Warning: when the deprecation ends, replace with:
    #   def __init__(self, get_response):
    def __init__(self, get_response=None):
        self._get_response_none_deprecation(get_response)
        self.get_response = get_response
        self._async_check()
        super().__init__()

    def _async_check(self):
        """
        If get_response is a coroutine function, turns us into async mode so
        a thread is not consumed during a whole request.
        """
        if asyncio.iscoroutinefunction(self.get_response):
            # Mark the class as async-capable, but do the actual switch
            # inside __call__ to avoid swapping out dunder methods
            self._is_coroutine = asyncio.coroutines._is_coroutine

    def __call__(self, request):
        # Exit out to async mode, if needed
        if asyncio.iscoroutinefunction(self.get_response):
            return self.__acall__(request)
        response = None
        if hasattr(self, 'process_request'):
            response = self.process_request(request)
        response = response or self.get_response(request)
        if hasattr(self, 'process_response'):
            response = self.process_response(request, response)
        return response

    async def __acall__(self, request):
        """
        Async version of __call__ that is swapped in when an async request
        is running.
        """
        response = None
        if hasattr(self, 'async_process_request'):
            response = await self.async_process_request(request)
        response = response or await self.get_response(request)
        if hasattr(self, 'async_process_response'):
            response = await self.async_process_response(request, response)
        return response

    def _get_response_none_deprecation(self, get_response):
        if get_response is None:
            warnings.warn(
                'Passing None for the middleware get_response argument is '
                'deprecated.',
                RemovedInDjango40Warning, stacklevel=3,
            )

    def _https_referer_replace(self, request):
        """
        When https is enabled, django CSRF checking includes referer checking
        which breaks when using CORS. This function updates the HTTP_REFERER
        header to make sure it matches HTTP_HOST, provided that our cors logic
        succeeds
        """
        origin = request.META.get('HTTP_ORIGIN')

        if (request.is_secure() and origin and
                'ORIGINAL_HTTP_REFERER' not in request.META):
            url = urlparse(origin)
            if (not CORS_ORIGIN_ALLOW_ALL and
                    self.origin_not_found_in_white_lists(origin, url)):
                return

            try:
                http_referer = request.META['HTTP_REFERER']
                http_host = "https://%s/" % request.META['HTTP_HOST']
                request.META = request.META.copy()
                request.META['ORIGINAL_HTTP_REFERER'] = http_referer
                request.META['HTTP_REFERER'] = http_host
            except KeyError:
                pass

    def process_request(self, request):
        """
        If CORS preflight header, then create an
        empty body response (200 OK) and return it
        Django won't bother calling any other request
        view/exception middleware along with the requested view;
        it will call any response middlewares
        """
        if CORS_REPLACE_HTTPS_REFERER:
            self._https_referer_replace(request)

        if (request.method == 'OPTIONS' and
                "HTTP_ACCESS_CONTROL_REQUEST_METHOD" in request.META):
            response = http.HttpResponse()
            return response
        return None

    # def process_view(self, request, callback, callback_args, callback_kwargs):
    #     """
    #     Do the referer replacement here as well
    #     """
    #     if CORS_REPLACE_HTTPS_REFERER:
    #         self._https_referer_replace(request)
    #     return None

    def process_response(self, request, response):
        """
        Add the respective CORS headers
        """
        origin = request.META.get('HTTP_ORIGIN')
        if origin:
            # todo: check hostname from db instead
            url = urlparse(origin)

            if CORS_MODEL is not None:
                response[ACCESS_CONTROL_ALLOW_ORIGIN] = origin

            if (not CORS_ORIGIN_ALLOW_ALL and
                    self.origin_not_found_in_white_lists(origin, url)):
                return response

            if CORS_ORIGIN_ALLOW_ALL and not CORS_ALLOW_CREDENTIALS:
                response[ACCESS_CONTROL_ALLOW_ORIGIN] = "*"
            else:
                response[ACCESS_CONTROL_ALLOW_ORIGIN] = origin
                patch_vary_headers(response, ['Origin'])

            if len(CORS_EXPOSE_HEADERS):
                response[ACCESS_CONTROL_EXPOSE_HEADERS] = ', '.join(
                    CORS_EXPOSE_HEADERS)

            if CORS_ALLOW_CREDENTIALS:
                response[ACCESS_CONTROL_ALLOW_CREDENTIALS] = 'true'

            if request.method == 'OPTIONS':
                response[ACCESS_CONTROL_ALLOW_HEADERS] = ', '.join(
                    CORS_ALLOW_HEADERS)
                response[ACCESS_CONTROL_ALLOW_METHODS] = ', '.join(
                    CORS_ALLOW_METHODS)
                if CORS_PREFLIGHT_MAX_AGE:
                    response[ACCESS_CONTROL_MAX_AGE] = \
                        CORS_PREFLIGHT_MAX_AGE
        return response

    async def async_process_response(self, request, response):
        """
        Add the respective CORS headers
        """
        origin = request.META.get('HTTP_ORIGIN')
        if origin:
            # todo: check hostname from db instead
            response[ACCESS_CONTROL_ALLOW_ORIGIN] = origin

            if (not CORS_ORIGIN_ALLOW_ALL and
                    self.origin_not_found_in_white_lists(origin, origin)):
                return response

            if CORS_ORIGIN_ALLOW_ALL and not CORS_ALLOW_CREDENTIALS:
                response[ACCESS_CONTROL_ALLOW_ORIGIN] = "*"
            else:
                response[ACCESS_CONTROL_ALLOW_ORIGIN] = origin
                patch_vary_headers(response, ['Origin'])

            if len(CORS_EXPOSE_HEADERS):
                response[ACCESS_CONTROL_EXPOSE_HEADERS] = ', '.join(
                    CORS_EXPOSE_HEADERS)

            if CORS_ALLOW_CREDENTIALS:
                response[ACCESS_CONTROL_ALLOW_CREDENTIALS] = 'true'

            if request.method == 'OPTIONS':
                response[ACCESS_CONTROL_ALLOW_HEADERS] = ', '.join(
                    CORS_ALLOW_HEADERS)
                response[ACCESS_CONTROL_ALLOW_METHODS] = ', '.join(
                    CORS_ALLOW_METHODS)
                if CORS_PREFLIGHT_MAX_AGE:
                    response[ACCESS_CONTROL_MAX_AGE] = \
                        CORS_PREFLIGHT_MAX_AGE
        return response

    def origin_not_found_in_white_lists(self, origin, url):
        return url.netloc not in CORS_ORIGIN_WHITELIST

    async def async_process_request(self, request):
        """
                If CORS preflight header, then create an
                empty body response (200 OK) and return it
                Django won't bother calling any other request
                view/exception middleware along with the requested view;
                it will call any response middlewares
                """
        if CORS_REPLACE_HTTPS_REFERER:
            self._https_referer_replace(request)

        if (request.method == 'OPTIONS' and
                "HTTP_ACCESS_CONTROL_REQUEST_METHOD" in request.META):
            response = http.HttpResponse()
            return response
        return None


#!/usr/bin/env python3

import re
import sys
from datetime import datetime
from collections import namedtuple

import dateparser
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import matplotlib.lines
from scipy.interpolate import interp1d as interp
import numpy as np
import yaml

DATA_FILE = "data.yaml"
Datum = namedtuple("Datum", ["date", "age", "weight", "height", "bmi"])

UNIT_PATTERN = re.compile(r"(\d+(?:\.\d*)?|\.\d+)\s*(\w*)")
FT_IN_PATTERN = re.compile(r"(\d+)\s*(?:f|ft|foot|feet)\s*(\d+)\s*(?:i|in|inch|inches|inchs)?", re.I)

def parse_unit(rec):
    m = UNIT_PATTERN.match(rec)
    if not m:
        raise ValueError
    return m.groups()

def parse_feet_inches(rec):
    m = FT_IN_PATTERN.match(rec)
    if not m:
        raise ValueError
    return m.groups()

def parse_samples(samples, dob):
    data = []
    for sample in samples:
        print(sample)
        dateStr = sample['date']
        weightStr = sample['weight']
        heightStr = sample['height']

        d = dateparser.parse(dateStr, settings={'DATE_ORDER': 'DMY'})
        try:
            weight, wu = parse_unit(weightStr)
            if wu == 'kg':
                weight *= 0.453592
            else:
                print("warning: bad weight unit, expected kg or lb")
        except ValueError:
            print("error: bad weight record '%s'" % weightStr)
            continue

        try:
            height, hu = parse_unit(heightStr)
            if hu != 'cm':
                print("warning: bad height unit, expected cm or in")
        except ValueError:
            try:
                ft, inc = parse_feet_inches(heightStr)
                height = ft * 0.3048 + inc * 0.0254
            except ValueError:
                print("error: bad height record '%s'" % weightStr)
                continue

        delta = (d - dob)
        w = float(weight)
        h = float(height)
        bmi = w / (h/100)**2
        age = delta.total_seconds() / (365.2425 * 24 * 3600)
        age = int(age * 10) / 10.0
        data.append(Datum(d, age, w, h, bmi))
    return data

def main():
    with open(DATA_FILE, "r") as stream:
        root = yaml.safe_load(stream)
        dob = dateparser.parse(root['DOB'])
        data = parse_samples(root['samples'], dob)

    after = dob
    if len(sys.argv) > 1 and sys.argv[1].lower() == "after":
        after = dateparser.parse(' '.join(sys.argv[2:]))

    data = list(filter(lambda datum: datum.date >= after, data))

    #ax = plt.twiny()
    #TODO: interpolate dates
    fig = plt.figure()
    ax = fig.add_subplot(1, 1, 1)

    maxy = int(max([max(d.weight, d.height, d.bmi) for d in data]) + 10)
    ax.set_yticks(list(range(0, maxy, 10)))
    ax.set_yticks(list(range(0, maxy, 5)), minor=True)

    interpolate = True
    if interpolate:
        ords = [x.date.toordinal() for x in data]
        fine = list(range(min(ords), max(ords)+1, 1))
        kind = "linear"
        #kind = "quadratic"
        height_interp = interp(ords, [x.height for x in data], kind=kind, fill_value="extrapolate")
        weight_interp = interp(ords, [x.weight for x in data], kind=kind, fill_value="extrapolate")
        wfit = np.polyfit(ords, [x.weight for x in data], 1)
        bfit = np.polyfit(ords, [x.weight / (x.height/100)**2 for x in data], 1)
        print("wfit:", wfit, "ords:", ords)
        #weight_interp = np.poly1d(wfit)

        hs = height_interp(fine)
        ws = weight_interp(fine)
        bms = ws / ((hs/100) ** 2)
        dws = np.diff(ws)
        dws = np.append(dws, dws[-1])
        dws = -dws * 100

        dates = [datetime.fromordinal(f) for f in fine]
        ax.plot_date(dates, hs, 'b-', label="Height in cm")
        ax.plot_date(dates, ws, 'r-', label="Weight in kg")
        ax.plot_date(dates, bms, 'g-', label="BMI")
        ax.plot_date(dates, dws, '-', label="-dW/d(decagrams per day)")

        #best fit 1d
        ords.append(ords[-1] + 100)
        ax.plot(ords, np.poly1d(wfit)(ords))
        ax.plot(ords, np.poly1d(bfit)(ords))

    else:
        #age
        xs = list(range(50))
        xt = [datetime(1995+i, 5, 24) for i in xs]
        #ax.set_xticks(xt, map(str, xs))

        #ax_age, ax_date = twiny();
        #x.set_xlabel("Age")

    #plots
    dates = [d.date for d in data]
    ax.plot_date(dates, [d.height for d in data], 'b.')
    ax.plot_date(dates, [d.weight for d in data], 'r.')
    ax.plot_date(dates, [d.bmi for d in data], 'g.')

    #yticks
    ax.grid(True, which="major", axis="y", ls="-", lw=1.0)
    ax.grid(True, which="minor", axis="y", ls="--", lw=0.5)

    #xticks
    years = mdates.YearLocator()
    months = mdates.MonthLocator()
    #ym_fmt = mdates.DateFormatter('%Y')
    month_fmt = mdates.DateFormatter('%b/%Y')
    ax.xaxis.set_major_locator(years)
    ax.xaxis.set_major_formatter(month_fmt)
    ax.xaxis.set_minor_locator(months)
    #ax.xaxis.set_minor_formatter(month_fmt)
    ax.grid(True, which="major", axis="x", ls="-", c="black", lw=1.5)
    ax.grid(True, which="minor", axis="x", ls="--", lw=0.5)

    #today
    today_x = matplotlib.dates.date2num(datetime.now())
    today = matplotlib.lines.Line2D([today_x, today_x], [-10, maxy], c="purple")
    ax.add_line(today)

    #bmi ranges
    bounds = [15, 18.5, 25, 30, 40, 50]
    cols = ["purple", "green", "yellow", "orange", "red"]
    for s, e, c in zip(bounds, bounds[1:], cols):
        ax.axhspan(s, e, color=c, alpha=0.4)

    ax.legend()
    plt.show()

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
import scrapy
from scrapy.selector import Selector
from scrapy.spiders import CrawlSpider, Rule
from datetime import datetime
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
import time

class Yes24CrawlSpider(CrawlSpider):
    name = 'yes24_selenium'
    allowed_domains = ['yes24.com']
    start_urls = [
                'http://www.yes24.com/24/Category/Display/001001046001?ParamSortTp=01',
            ]

    def __init__(self):
        scrapy.Spider.__init__(self)
        self.driver = webdriver.Chrome('C:/chromedriver')
        # self.driver = webdriver.PhantomJS("C:/phantomjs-2.1.1-windows/phantomjs-2.1.1-windows/bin/phantomjs")

    def __del__(self):
        self.driver.close()

    def parse(self, response):
        pageNum = 3
        self.driver.get(response.url)
        time.sleep(1)
        for i in range(5):
            n = 1
            for j in range(20):
                try:
                    xpath = '//*[@id="category_layout"]/ul/li[%d]/div/div[1]/div[2]/a[1]' % n
                    elem = self.driver.find_element_by_xpath(xpath).click()
                except NoSuchElementException:
                    xpath = '//*[@id="category_layout"]/ul/li[%d]/div/div[1]/div[1]/a[1]' % n
                    elem = self.driver.find_element_by_xpath(xpath).click()
                n += 1
                time.sleep(3)
                html = self.driver.find_element_by_xpath('//*').get_attribute('outerHTML')
                selector = Selector(text=html)
                item = {}
                item['book_site'] = 'yes24'
                item['book_isbn'] = selector.xpath('//*[@id="infoset_specific"]/div[2]/div/table/tbody/tr[3]/td/text()')[0].extract()
                item["book_cat"] = selector.xpath('//*[@id="yLocation"]/div/div[3]/a/text()')[0].extract().strip()
                item["book_title"] = selector.xpath('//*[@id="yDetailTopWrap"]/div[2]/div[1]/div/h2/text()')[0].extract().strip()
                item["book_price"] = selector.xpath('//*[@id="yDetailTopWrap"]/div[2]/div[2]/div[1]/div[1]/table/tbody/tr[2]/td/span/em/text()')[0].extract()
                item["book_author"] = selector.xpath('//*[@id="yDetailTopWrap"]/div[2]/div[1]/span[2]/span[1]/a/text()')[0].extract().strip()
                item["book_publish"] = selector.xpath('//*[@id="yDetailTopWrap"]/div[2]/div[1]/span[2]/span[2]/a/text()')[0].extract()
                item["book_publish_date"] = selector.xpath('//*[@id="yDetailTopWrap"]/div[2]/div[1]/span[2]/span[3]/text()')[0].extract()
                item["book_img"] = selector.xpath('//*[@id="yDetailTopWrap"]/div[1]/div[1]/span/em/img/@src')[0].extract()
                item["book_url"] = self.driver.current_url
                item['book_img'] = item['book_img'] + ".jpg"
                item["crawl_time"] = datetime.now()
                yield item
                self.driver.execute_script('window.history.go(-1)')
            page = '//*[@id="cateSubWrap"]/div[2]/div[3]/div[2]/span[1]/div/a[%d]' % pageNum
            pageNum += 1
            elem = self.driver.find_element_by_xpath(page).click()


from __future__ import annotations

import re
from typing import Union, Any, Iterable, Optional, List

from stereotype.fields.annotations import AnnotationResolver
from stereotype.fields.base import Field
from stereotype.utils import Missing, ConfigurationError, PathErrorType, ValidationContextType, Validator

class _AtomicField(Field):
    atomic = True

    def init_from_annotation(self, parser: AnnotationResolver):
        if parser.annotation is not self.type:
            raise parser.incorrect_type(self)

class BoolField(_AtomicField):
    """
    Boolean value (annotation ``bool``), accepting boolean values or true/yes/false/no strings.

    :param default: Means the field isn't required, used as default directly or called if callable
    :param hide_none: If the field's value is None, it will be hidden from serialized output
    :param hide_false: If the field's value is False, it will be hidden from serialized output
    :param primitive_name: Changes the key used to represent the field in serialized data - input or output
    :param to_primitive_name: Changes the key used to represent the field in serialized data - output only
    :param validators: Optional list of validator callbacks - they raise ``ValueError`` if the value is invalid
    """

    __slots__ = Field.__slots__
    type = bool
    type_repr = 'bool'
    empty_value = False

    def __init__(self, *, default: Any = Missing, hide_none: bool = False, hide_false: bool = False,
                 primitive_name: Optional[str] = Missing, to_primitive_name: Optional[str] = Missing,
                 validators: Optional[List[Validator]] = None):
        super().__init__(default=default, hide_none=hide_none, hide_empty=hide_false,
                         primitive_name=primitive_name, to_primitive_name=to_primitive_name, validators=validators)

    def convert(self, value: Any) -> Any:
        if value is Missing:
            return self._fill_missing()
        if value is None:
            return None
        if value is False or value is True:
            return value
        if value in {'true', 'True', 'yes', 'Yes'}:
            return True
        if value in {'false', 'False', 'no', 'No'}:
            return False
        raise TypeError('Value must be a boolean or a true/false/yes/no string value')

class _BaseNumberField(_AtomicField):
    __slots__ = Field.__slots__ + ('min_value', 'max_value')
    min_value: Union[int, float, None]
    max_value: Union[int, float, None]

    def _set_min_max_value_validation(self, min_value: Union[int, float, None], max_value: Union[int, float, None]):
        if min_value is not None and max_value is not None:
            self.native_validate = self._validate_min_max_value
        elif min_value is not None:
            self.native_validate = self._validate_min_value
        elif max_value is not None:
            self.native_validate = self._validate_max_value

    def _validate_min_max_value(self, value: Any, _: ValidationContextType) -> Iterable[PathErrorType]:
        if not (self.min_value <= value <= self.max_value):
            yield (), f'Must be between {self.min_value} and {self.max_value}'

    def _validate_min_value(self, value: Any, _: ValidationContextType) -> Iterable[PathErrorType]:
        if value < self.min_value:
            yield (), f'Must be at least {self.min_value}'

    def _validate_max_value(self, value: Any, _: ValidationContextType) -> Iterable[PathErrorType]:
        if value > self.max_value:
            yield (), f'Must be at most {self.max_value}'

class IntField(_BaseNumberField):
    """
    Integer value (annotation ``int``), accepting integer values, whole float values or strings with integer values.

    :param default: Means the field isn't required, used as default directly or called if callable
    :param hide_none: If the field's value is None, it will be hidden from serialized output
    :param hide_zero: If the field's value is 0, it will be hidden from serialized output
    :param primitive_name: Changes the key used to represent the field in serialized data - input or output
    :param to_primitive_name: Changes the key used to represent the field in serialized data - output only
    :param min_value: Validation enforces the number is greater than or equal to this value
    :param max_value: Validation enforces the number is lower than or equal to this value
    :param validators: Optional list of validator callbacks - they raise ``ValueError`` if the value is invalid
    """

    __slots__ = _BaseNumberField.__slots__
    type = int
    type_repr = 'int'
    empty_value = 0

    def __init__(self, *, default: Any = Missing, hide_none: bool = False, hide_zero: bool = False,
                 primitive_name: Optional[str] = Missing, to_primitive_name: Optional[str] = Missing,
                 min_value: Optional[int] = None, max_value: Optional[int] = None,
                 validators: Optional[List[Validator]] = None):
        super().__init__(default=default, hide_none=hide_none, hide_empty=hide_zero,
                         primitive_name=primitive_name, to_primitive_name=to_primitive_name, validators=validators)
        self.min_value = min_value
        self.max_value = max_value
        self._set_min_max_value_validation(min_value, max_value)

    def convert(self, value: Any) -> Any:
        if value is Missing:
            return self._fill_missing()
        if value is None:
            return None
        if isinstance(value, float) and value != int(value):
            raise TypeError(f'Numeric value {value} is not an integer')
        try:
            return int(value)
        except (ValueError, TypeError):
            raise TypeError(f'Value {value!r} is not an integer number')

class FloatField(_BaseNumberField):
    """
    Floating point value (annotation ``float``), accepting float values, integers or strings with float values.

    :param default: Means the field isn't required, used as default directly or called if callable
    :param hide_none: If the field's value is None, it will be hidden from serialized output
    :param hide_zero: If the field's value is 0.0, it will be hidden from serialized output
    :param primitive_name: Changes the key used to represent the field in serialized data - input or output
    :param to_primitive_name: Changes the key used to represent the field in serialized data - output only
    :param min_value: Validation enforces the number is greater than or equal to this value
    :param max_value: Validation enforces the number is lower than or equal to this value
    :param validators: Optional list of validator callbacks - they raise ``ValueError`` if the value is invalid
    """

    __slots__ = _BaseNumberField.__slots__
    type = float
    type_repr = 'float'
    empty_value = 0.

    def __init__(self, *, default: Any = Missing, hide_none: bool = False, hide_zero: bool = False,
                 primitive_name: Optional[str] = Missing, to_primitive_name: Optional[str] = Missing,
                 min_value: Optional[float] = None, max_value: Optional[float] = None,
                 validators: Optional[List[Validator]] = None):
        super().__init__(default=default, hide_none=hide_none, hide_empty=hide_zero,
                         primitive_name=primitive_name, to_primitive_name=to_primitive_name, validators=validators)
        self.min_value = min_value
        self.max_value = max_value
        self._set_min_max_value_validation(min_value, max_value)

class StrField(_AtomicField):
    """
    String value (annotation ``str``), accepting string values, or anything that can be cast to a string.

    :param default: Means the field isn't required, used as default directly or called if callable
    :param hide_none: If the field's value is None, it will be hidden from serialized output
    :param hide_empty: If the field's value is an empty string, it will be hidden from serialized output
    :param primitive_name: Changes the key used to represent the field in serialized data - input or output
    :param to_primitive_name: Changes the key used to represent the field in serialized data - output only
    :param min_length: Validation enforces the string has a minimum number of characters (1 => non-empty)
    :param max_length: Validation enforces the string has a maximum number of characters
    :param choices: Validation enforces the string matches (case-sensitive) one of these choices
    :param regex: Validation enforces the string matches the regular expression
    :param validators: Optional list of validator callbacks - they raise ``ValueError`` if the value is invalid
    """

    __slots__ = _AtomicField.__slots__ + ('min_length', 'max_length', 'choices', 'regex')
    type = str
    type_repr = 'str'
    empty_value = ''

    def __init__(self, *, default: Any = Missing, hide_none: bool = False, hide_empty: bool = False,
                 primitive_name: Optional[str] = Missing, to_primitive_name: Optional[str] = Missing,
                 min_length: int = 0, max_length: Optional[int] = None, choices: Optional[Iterable[str]] = None,
                 regex: str | re.Pattern | None = None, validators: Optional[List[Validator]] = None):
        super().__init__(default=default, hide_none=hide_none, hide_empty=hide_empty,
                         primitive_name=primitive_name, to_primitive_name=to_primitive_name, validators=validators)
        if sum([min_length > 0 or max_length is not None, choices is not None, regex is not None]) > 1:
            raise ConfigurationError('Can only validate length, choices or regex; not combinations of these')
        self.min_length = min_length
        self.max_length = max_length
        self.choices = {choice: None for choice in choices} if choices is not None else None  # Sets are not ordered
        self.regex = re.compile(regex) if isinstance(regex, str) else regex
        if self.choices is not None:
            self.native_validate = self._validate_choices
        elif min_length > 0 and max_length is not None:
            self.native_validate = self._validate_min_max_length
        elif min_length == 1:
            self.native_validate = self._validate_not_empty
        elif min_length > 0:
            self.native_validate = self._validate_min_length
        elif max_length is not None:
            self.native_validate = self._validate_max_length
        elif regex is not None:
            self.native_validate = self._validate_regex

    def _validate_choices(self, value: str, _: ValidationContextType) -> Iterable[PathErrorType]:
        if value not in self.choices:
            yield (), f'Must be one of: {", ".join(self.choices)}'

    def _validate_min_max_length(self, value: str, _: ValidationContextType) -> Iterable[PathErrorType]:
        if not (self.min_length <= len(value) <= self.max_length):
            if self.min_length == self.max_length:
                yield (), f'Must be exactly {self.min_length} character{"s" if self.min_length > 1 else ""} long'
            else:
                yield (), f'Must be {self.min_length} to {self.max_length} characters long'

    # Note: the validation methods that are put in place of native_validate may not be static
    # noinspection PyMethodMayBeStatic
    def _validate_not_empty(self, value: str, _: ValidationContextType) -> Iterable[PathErrorType]:
        if not value:
            yield (), 'This value cannot be empty'

    def _validate_min_length(self, value: str, _: ValidationContextType) -> Iterable[PathErrorType]:
        if len(value) < self.min_length:
            yield (), f'Must be at least {self.min_length} characters long'

    def _validate_max_length(self, value: str, _: ValidationContextType) -> Iterable[PathErrorType]:
        if len(value) > self.max_length:
            yield (), f'Must be at most {self.max_length} character{"s" if self.max_length > 1 else ""} long'

    def _validate_regex(self, value: str, _: ValidationContextType) -> Iterable[PathErrorType]:
        if not self.regex.match(value):
            case = " (case insensitive)" if self.regex.flags & re.I else ""
            yield (), f'Must match regex `{self.regex.pattern}`{case}'

ATOMIC_TYPE_MAPPING = {
    bool: BoolField,
    int: IntField,
    float: FloatField,
    str: StrField,
}


#!/usr/bin/python3

##########################################################################
############      Designed and tested by Alexander Semenov  ##############
############       email: vj0r.awful@gmail.com              ##############
##########################################################################

import subprocess
import configparser
import logging
import time

config = configparser.ConfigParser()
config.read("/etc/process_list/settings.ini")

command = config["systemd"]["daemons"]
log_path = config["log"]["path"]
sleep_interval = int(config["interval"]["check"])

logging.basicConfig(
    level=logging.INFO,
    filename = log_path,
    format = '%(asctime)s %(levelname)s - %(message)s',
    datefmt="%Y-%m-%d %H:%M:%S",
)

mass_command = command.split(',')

while True:
    for i in mass_command:

        mass_chack = ["systemctl", "status"]
        mass_run = ["systemctl", "restart"]
        mass_chack.append(i)

        if subprocess.call(mass_chack, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False):
            mass_run.append(i)
            try:
                if not subprocess.call(mass_run, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False):
                    logging.info("restart daemon %s" % (i))
                else:
                    logging.error("restart daemon %s failed" % (i))
            except:
                logging.error("restart daemon %s failed" % (i))

    time.sleep(sleep_interval)


from django.forms.models import model_to_dict
import requests
from celery import shared_task
import datetime
import time
from django.apps import apps
from .scraper import StockTickerScraper

def_ticker = 'AAPL'
def_service = 'finnhub'

@shared_task
def perform_scrape(ticker=def_ticker, service=def_service):
    client = StockTickerScraper(service=service)
    pc = client.scrape(ticker=ticker)
    print('task', ticker, pc)
    PriceLookupEvent = apps.get_model("mysite", "PriceLookupEvent")
    PriceLookupEvent.objects.create_event(ticker, pc, source=service)

@shared_task
def company_price_scrape_task(instance_id, service=def_service):
    Stock = apps.get_model("mysite", "Stock")
    obj = Stock.objects.get(id=instance_id)
    ticker = obj.ticker
    perform_scrape(ticker=ticker, service=service)

@shared_task
def company_granular_price_scrape_task(instance_id, service=def_service):
    Stock = apps.get_model("mysite", "Stock")
    obj = Stock.objects.get(id=instance_id)
    print("hi")
    ticker = obj.ticker
    perform_scrape(ticker=ticker, service=service)
    if obj.has_granular_scraping:
        now = datetime.datetime.now()
        expires = now + datetime.timedelta(seconds=65)
        for i in range(1, 60):  # 1 - 59
            perform_scrape.apply_async(kwargs={
                "ticker": ticker,
                "service": service
            }, countdown=i, expires=expires)

"""
def get_data(symbol, stocks):
    url = "https://finnhub.io/api/v1/quote?symbol={}&token=sandbox_c0t96iv48v6r4maen2gg"
    r = requests.get(url.format(symbol)).json()
    obj, created = Stocks.object.get_or_create(symbol=symbol)
    obj.name = symbol
    obj.pc = r['pc']
    obj.save()
    new_data = model_to_dict(obj)
    new_data.update({'state': state})
    stocks.append(new_data)

@shared_task
def get_stocks_data():
    for st in AllStocks:
        get_data(event.data[st]['name'], stocks)
    async_to_sync"""


import pygame
import math
class Ball:
    xcoord = 0
    ycoord = 0
    radius = 0
    color = [0, 0, 0]
    xvelo = 0
    yvelo = 0
    bildschirm = pygame.Surface((2 * radius, 2 * radius))
    edge = []

    def __init__(self, x = 0, y = 0, r = 0, col = [0, 0, 0], xv = 0, yv = 0):
        self.xcoord = x
        self.ycoord = y
        self.radius = r
        self.color = col
        self.xvelo = xv
        self.yvelo = yv
        self.bildschirm = pygame.Surface((2 * self.radius, 2 * self.radius))
        for i in range (1001):
            self.edge.append([self.radius * math.cos(2 * math.pi * i / 1000), self.radius * math.sin(2 * math.pi * i / 1000)])

    def draw_ball(self):
        self.bildschirm.fill((255, 255, 255))
        pygame.draw.circle(self.bildschirm, self.color, (self.radius, self.radius), self.radius)

    def collision_player(self, xplayleft = 0, xplayright = 0, yplayup = 0, yplaydown = 0):
        invert = False
        weird = False
        weird = False
        for pair in self.edge:
            if xplayleft + 2 <= pair[0] + self.xcoord + self.radius <= xplayright - 2 and yplayup == pair[1] + self.ycoord + self.radius:
                invert = True
            if xplayleft + 1 >= pair[0] + self.xcoord + self.radius >= xplayleft - 1 and yplayup + 3 >= pair[1] + self.ycoord + self.radius >= yplayup:
                weird = True
                if self.xvelo <= 0:
                    weird = False
                    invert = True
            if xplayright + 1 >= pair[0] + self.xcoord + self.radius >= xplayright - 1 and yplayup + 3 >= pair[1] + self.ycoord + self.radius >= yplayup:
                weird = True
                if self.xvelo >= 0:
                    weird = False
                    invert = True
        if invert:
            self.yvelo = - self.yvelo
        if weird:
            self.yvelo = - self.yvelo
            self.xvelo = - self.xvelo


import glob, os, datetime, logging

def delete_logs(max_logs, log_dir="logs"):
    logger = logging.getLogger("delete_logs()")
    if max_logs != -1:
        logs = glob.glob(os.path.join(log_dir,"*.log"))
        if len(logs) > max_logs:
            logger.debug("Log limit reached!. Deleting logs...")

            try:
                for x in logs:
                    os.remove(x)
            except Exception as e:
                print("Removal of logs has failed!")
                print(e)
        logger.debug("There were %d logs", len(logs))

def init_log(name, log_dir="logs") -> str | None:
    log_name = os.path.join(log_dir, datetime.datetime.today().strftime(f"%d-%m-%Y_%H-%M-%S_{name}.log"))
    logging.basicConfig(filename=log_name,
                filemode='w',
                format='%(asctime)s:%(msecs)d %(name)s %(levelname)s %(message)s',
                datefmt='%H:%M',
                level=logging.DEBUG)
    console = logging.StreamHandler()
    console.setLevel(logging.DEBUG)
    formatter = logging.Formatter('[ %(name)s ][%(levelname)s] %(message)s')
    console.setFormatter(formatter)
    console.setFormatter
    logging.getLogger('').addHandler(console)
    return [
        getattr(logging.getLoggerClass().root.handlers[0], 'baseFilename', "Null"),
        console,
    ]


import pickle, random
import matplotlib.pyplot as plt

### Hog game

# score needed to win
GOAL_SCORE = 100
# max number of dice the player can roll, ignore rolling 1, sides of the dice
K, START, SIDES = 10, 2, 6
INF = float("inf")
# Dynamic programming approach to compute the probability mass function.
dp = [0]*(6*K + 1)
for i in range(START, SIDES + 1):
    dp[i] += 1

pmf_matrix = [dp]
for i in range(K - 1):
    temp = [0]*(6*K + 1)
    for j in range(len(dp) - 6):
        for v in range(START, SIDES + 1):
            temp[j + v] += dp[j]
    dp = temp
    pmf_matrix.append(list(dp))

fig, ax = plt.subplots()

for k in range(1, 11):
    x = range(2*k, 6*k + 1)
    plt.hist(x, bins=4*k + 1, weights=pmf_matrix[k - 1][2*k: 6*k + 1], density=True, label=f"{k}")

ax.legend()
plt.title(f"Probability mass function of rolling $k$ dice")
plt.xlabel('Sum of rolls')
plt.ylabel('Probability')
plt.savefig(f"pmf.png")


from app import db
from datetime import datetime as dt
import sqlalchemy

def create_tables():
    conn = db.connect()
    init_sql = open('init.sql')
    escaped_sql = sqlalchemy.text(init_sql.read())
    conn.execute(escaped_sql)
    conn.close()

def fetch_todo(user_id) -> dict:
    conn = db.connect()
    query_results = conn.execute(f"Select * from tasks WHERE user_id = '{user_id}' ORDER BY priority DESC;").fetchall()
    conn.close()

    todo_list = []

    for result in query_results:
        item = {
            "id": result[0],
            "task": result[1],
            "priority": round(result[2], 2),
            "status": result[3],
            "added_at": result[4],
            "difficulty": result[5],
            "deadline": result[6],
            "importance": result[7]
        }
        todo_list.append(item)

    return todo_list

def update_task(field, task_id: int, text) -> None:
    conn = db.connect()
    content = f"'{text}'" if isinstance(text, str) else int(text)

    query = f"Update tasks set {field} = {content} where id = {task_id};"
    conn.execute(query)
    conn.close()

def insert_new_task(user_id, text: str, difficulty, deadline, importance) ->  int:
    date = dt.today().strftime('%Y-%m-%d')
    priority = 100 - (round((int(difficulty) + (10-int(importance)) + int(deadline)) / 3, 2) * 10)

    conn = db.connect()
    query = "Insert Into tasks (task, priority, status, difficulty, deadline, importance, added_at, user_id)" + \
                    f"VALUES ('{text}', '{priority}', 'Todo', '{difficulty}', '{deadline}', '{importance}', '{date}', '{user_id}');"
    print(query)
    conn.execute(query)
    query_results = conn.execute("SELECT currval(pg_get_serial_sequence('tasks','id'));")
    query_results = [x for x in query_results]
    task_id = query_results[0][0]
    conn.close()

    return task_id

def remove_task_by_id(task_id: int) -> None:
    conn = db.connect()
    query = 'Delete From tasks where id={};'.format(task_id)
    conn.execute(query)
    conn.close()


import discord, motor, Core.utils as util, asyncio
from discord.ext import commands, tasks
from colorama import Fore as f

class pojcog(commands.Cog):
    def __init__(self, bot):
        self.bot = bot
        self.db = self.bot.db["pingonjoin"]
        self.urgecolor = 0xF3DD6C
        self.errorcol = 0xA90F25 # error color
        self.success = discord.Colour.blurple() #theme
        self.checkmoji = "<:blurple_check:921544108252741723>" # success emoji
        self.xmoji = "<:yy_yno:921559254677200957>" # unsuccessful emoji
        self.urgentmoji = "<:n_:921559211366838282>" # exclamation emoji
        self.cache = {}
        self.pings = {}
        self.clearPings.start()

    @tasks.loop(minutes=100)
    async def cachePings(self):
        cache = self.db.find({})
        guilds = []
        channels = []
        for i in await cache.to_list(length=999999):
            guild = i['guild_id']
            chan = i['channel']
            guilds.append(guild)
            channels.append(chan)
        self.cache = {guilds}

    @tasks.loop(seconds=15)
    async def clearPings(self):
        self.pings.clear()

    @clearPings.before_loop
    async def before_clearPings(self):
        await self.bot.wait_until_ready()

    @commands.group(
        aliases = ['poj', 'pingjoin'],
        usage = "Manage_channels",
        description = "Set a channel in which new members will get pinged at upon joining.",
        brief = "subcommand, arg, subarg",
        help = "```Syntax: poj [subcommand] <arg> <subarg>\nExample: poj add #rules``` "
    )
    async def pingonjoin(self, ctx):
        try:
            if ctx.invoked_subcommand is None:
                embed = discord.Embed(title="Command: pingonjoin", description="Set a channel in which new memers will get pinged at upon joining.\n```Syntax: poj [subcommand] <arg> <subarg>\nExample: poj #rules 3```", color = discord.Color.blurple())
                embed.set_author(name="pingonjoin help", icon_url=ctx.me.avatar.url)
                embed.set_footer(text=f"Page 1/{len([sc for sc in self.bot.get_command('pingonjoin').walk_commands()])} ・ pingonjoin")
                return await ctx.reply(embed=embed)
        except Exception as e:
            print(e)

    @pingonjoin.command(
        aliases = ['set'],
        usage = "Manage_channels",
        description = "Set a channel in which new members will get pinged at upon joining.",
        brief = "channel",
        help = "```Syntax: poj add [channel]\nExample: poj add #rules```"
    )
    #async def add(self, ctx, channel: discord.TextChannel, delete:int=None):
    @commands.has_permissions(manage_channels=True)
    async def add(self, ctx, channel: discord.TextChannel):
        try:
            get = await self.db.count_documents({ "guild_id": ctx.guild.id})
            if len(get) > 20:
                return await ctx.reply(embed=discord.Embed(description=f"{self.urgentmoji} {ctx.author.mention} This **guild** has reached it's **maximum limit** of ``20`` ping on joins.", color=self.urgecolor))
        except:
            pass
            data = await self.db.find_one({ "guild_id": ctx.guild.id, "channel": channel.id})
            print("hi")
            if data:
                return await ctx.reply(embed=discord.Embed(description=f"{self.urgentmoji} A pingonjoin already **exists** for that channel", color=self.urgecolor))
            else:
                await self.db.insert_one({
                    "guild_id": ctx.guild.id,
                    "channel": channel.id

                })
                return await ctx.reply(embed=discord.Embed(description=f"<:check:921544057312915498> **Ping on join** created for {channel.mention}", color = 0x43B581))

    @pingonjoin.command(
        aliases = ['del', 'remove'],
        usage = "Manage_channels",
        description = "Remove a current pingonjoin you have for a channel",
        brief = 'channel',
        help = "```Syntax: poj remove [channel]\nExample: poj remove #rules```"
    )
    @commands.has_permissions(manage_channels=True)
    async def delete(self, ctx, channel: discord.TextChannel=None):
        check = await self.db.find_one({ "guild_id": ctx.guild.id})
        if check:
            findchan = await self.db.find_one({ "guild_id": ctx.guild.id, "channel": channel.id})
            if findchan:
                await self.db.delete_one({ "guild_id": ctx.guild.id, "channel": channel.id})
                return await ctx.reply(embed=discord.Embed(description=f"<:check:921544057312915498> **pingonjoin removed** for channel {channel.mention} and will no longer be sent", color = 0x43B581))
            else:
                return await ctx.reply(embed=discord.Embed(description=f"{self.urgentmoji} **No pingonjoin exists for that channel**", color=self.urgecolor))
        else:
            return await ctx.reply(embed=discord.Embed(description=f"{self.urgentmoji} **No pingonjoin's have been created yet**, use ``poj add`` to **create one**", color=self.urgecolor))

    @pingonjoin.command(
        aliases = ['show', 'list'],
        usage = "Manage_channels",
        description = "View the current existing pinginjoin's in your server",
        brief = "None",
        help = "```Example: poj view```"
    )
    @commands.has_permissions(manage_channels=True)
    async def view(self, ctx):
        check = await self.db.find_one({ "guild_id": ctx.guild.id})
        try:
            if check:
                check_existing = self.db.find({ "guild_id": ctx.guild.id})
                channellist = []
                rows = []
                for doc in await check_existing.to_list(length = 25):
                    channels = doc['channel']
                    channellist.append(channels)
                content = discord.Embed(title="Ping On Join's", description="", color=discord.Color.blurple())
                content.set_author(name=ctx.author.display_name, icon_url=ctx.author.display_avatar.url)
                for count, i in enumerate(channellist, start=1):
                    rows.append(f"``{count})`` <#{i}>")
                content.set_footer(text=f"({count}/{count}) Ping On Joins", icon_url=ctx.guild.icon.url)
                await util.send_as_pages(ctx, content, rows)
            else:
                return await ctx.reply(embed=discord.Embed(description=f"{self.urgentmoji} **No pingonjoin's have been created yet**, use ``poj add`` to **create one**", color=self.urgecolor))
        except Exception as e:
            print(e)

    @commands.Cog.listener()
    async def on_member_join(self, member):
        try:
            guild = member.guild
            check = await self.db.find_one({ "guild_id": guild.id})
            try:
                self.pings[member.guild.id]
            except KeyError:
                self.pings[member.guild.id] =1
            guild2 = self.pings[member.guild.id]
            self.pings[member.guild.id] = self.pings[member.guild.id]+1
            if check and not self.pings[member.guild.id] == 5:
                check_existing = self.db.find({ "guild_id": guild.id })
                for channels in await check_existing.to_list(length=25):
                    channel = guild.get_channel(channels["channel"])
                    await channel.send(member.mention, delete_after=0.8)
            else:
                return
        except:
            pass; return

async def setup(bot):
    await bot.add_cog(pojcog(bot))


import pandas as pd
import nni
import pandas_ta as pd_ta
from hep_ml import reweight
import matplotlib.pyplot as plt
from hep_ml.metrics_utils import ks_2samp_weighted
from fetch_alpaca_data import fetch_alpaca_csv
from pandas import DataFrame
from sklearn.metrics import balanced_accuracy_score, accuracy_score, roc_auc_score, f1_score
import warnings
from sklearn.ensemble import RandomForestClassifier
import seaborn as sns
from tqdm import tqdm
from sklearn.model_selection import train_test_split, cross_val_score
from xgboost import XGBClassifier
import pywt
import pandas_market_calendars as mcal
from collections import Counter
import numpy as np
from time import perf_counter
import yfinance as yf
from scipy.stats import ttest_ind
from sklearn.preprocessing import quantile_transform

warnings.filterwarnings('ignore')

# set random seed on numpy
np.random.RandomState(42)
np.random.seed(42)

# performance csv headers
header = ['test_acc', 'live_acc']

params: dict = {
    'years': 3,
    'wavelet': 'bior1.1',
    'window': 70
}
optimized_params = nni.get_next_parameter()
params.update(optimized_params)

# create script parameters (dynamic)
years = params['years']
trend = params['window']
wavelet = params['wavelet']
shuffle_split: bool = True

# find yesterday closing price
ticker = yf.Ticker('SPY')
yesterday_price = ticker.history(interval='1d')['Close'][-2]

# mean absolute error evaluator for wavelet transform
def madev(d, axis=None):
    """ Mean absolute deviation of a signal """
    return np.mean(np.absolute(d - np.mean(d, axis)), axis)

# de-noise original signal with selected wavelet function
def wavelet_denoising(x, wavelet='db4', level=1):
    coeff = pywt.wavedec(x, wavelet, mode="per")
    sigma = (1 / 0.6745) * madev(coeff[-level])
    uthresh = sigma * np.sqrt(2 * np.log(len(x)))
    coeff[1:] = (pywt.threshold(i, value=uthresh, mode='hard') for i in coeff[1:])
    return pywt.waverec(coeff, wavelet, mode='per')

# smooth initial model features in order to extract longer trend pattern
def _wavelet_smooth(data, wavelet):
    """wavelet transformation applied to curve"""
    processed_arr = []
    for col in ['open', 'high', 'low', 'close', 'volume']:
        processed_arr.append(wavelet_denoising(data[col], wavelet=wavelet, level=1))
    return processed_arr

def _get_indicator_data(data_raw):
    # technical analysis indicators
    data_raw['rsi'] = pd_ta.rsi(data_raw['close'])
    data_raw['macd'] = pd_ta.macd(data_raw['close']).iloc[:, -1]
    data_raw['willr'] = pd_ta.willr(data_raw['high'], data_raw['low'], data_raw['close'])
    data_raw['obv'] = pd_ta.obv(data_raw['close'], data_raw['volume'])
    data_raw['proc'] = pd_ta.roc(data_raw['close'])
    data_raw['stoch_k'] = pd_ta.stoch(data_raw['high'], data_raw['low'], data_raw['close']).iloc[:, 0]

    # Remove columns that won't be used as features
    del (data_raw['open'])
    del (data_raw['high'])
    del (data_raw['low'])
    del (data_raw['volume'])
    try:
        del (data_raw['Adj Close'])
    except KeyError:
        pass
    return data_raw

def _produce_prediction_gaps(data, transformed_data, window, from_hardcoded=False, year=None) -> DataFrame:
    """create predicted target values"""
    try:
        if from_hardcoded:
            return pd.read_csv(f'targets/{year}y_{window}w.csv')
    except FileNotFoundError:
        pass
    # create containers and calendars
    nyse = mcal.get_calendar('NYSE')
    calendar = nyse.schedule(start_date=data['day'].iloc[0], end_date=data['day'].iloc[-1])
    pred_days = data['day'].unique()[-(window // 14):]
    dropped_idx: list = []
    missing_days = []
    missing_counter = 0
    value_counts = data['day'].value_counts()
    for key, value in value_counts.items():
        if value != 14:
            missing_days.append(key)
            missing_counter += 1
            print(value, key)
    data = data.loc[~data['day'].isin(missing_days)]
    transformed_data = transformed_data.loc[~transformed_data['day'].isin(missing_days)]

    # custom mapping for trading window at given intraday time
    map: dict = {
        '09:30:00': 13, '10:00:00': 12, '10:30:00': 11, '11:00:00': 10, '11:30:00': 9, "12:00:00": 8,
        '12:30:00': 7, '13:00:00': 6, '13:30:00': 5, '14:00:00': 4, '14:30:00': 3, '15:00:00': 2,
        '15:30:00': 1, '16:00:00': 0
    }
    # create containers
    times, closes, days = data['time'], data['close'], data['day']
    targets = []
    end_dates = []
    compare_closes = []

    for start_idx in tqdm(range(data.shape[0])):
        # get prices to compare for target variable
        current_close = closes.iloc[start_idx]

        if days.iloc[start_idx] in pred_days:
            end_dates.append(np.nan)
            compare_closes.append(np.nan)
            targets.append(np.nan)
            continue

        # get price comparisons
        try:
            compare_close = closes.iloc[start_idx + window + map[times.iloc[start_idx]]]
            end_dates.append(days.iloc[start_idx + window + map[times.iloc[start_idx]]])
            compare_closes.append(compare_close)
        except IndexError:
            compare_close = np.nan
            end_dates.append(np.nan)
            compare_closes.append(np.nan)  # only at the -window span at the end of the dataframe

        # check market day difference
        if not np.isnan(compare_close):
            diff = calendar.loc[days.iloc[start_idx]:days.iloc[start_idx + window + map[times.iloc[start_idx]]]].shape[
                       0] - 1
            if diff != window // 14 or '16:00:00' not in times.iloc[
                start_idx + window + map[times.iloc[start_idx]]] or compare_close == current_close:
                dropped_idx.append(start_idx)
                targets.append('GAP')
                continue
            else:
                if current_close < compare_close:
                    targets.append(1)
                else:
                    targets.append(0)
        else:
            # check if larger or smaller
            targets.append('GAP')
    print(data.shape[0], len(end_dates))
    data['end_timestamps'] = end_dates
    data['compare_closes'] = compare_closes
    data['pred'] = targets

    # remove dropped indices from the dropped_idx array
    data = data.loc[~data.index.isin(dropped_idx)]
    transformed_data = transformed_data.loc[~transformed_data.index.isin(dropped_idx)]
    # reset indices
    data.reset_index(inplace=True, drop=True)
    transformed_data.reset_index(inplace=True, drop=True)
    # count number of GAP target variables
    same_price_idx = data[data['pred'] == 'GAP'].index.values
    data = data.loc[~data.index.isin(same_price_idx)]
    transformed_data = transformed_data.loc[~transformed_data.index.isin(same_price_idx)]
    # insert original target variables into transformed dataframe
    transformed_data['pred'] = data['pred']
    transformed_data['end_timestamps'] = data['end_timestamps']
    transformed_data['compare_closes'] = data['compare_closes']
    return transformed_data

def _produce_prediction(data, window):
    """create predicted target values"""

    # custom mapping for trading window at given intraday time
    map: dict = {
        '09:30:00': 13, '10:00:00': 12, '10:30:00': 11, '11:00:00': 10, '11:30:00': 9, "12:00:00": 8,
        '12:30:00': 7, '13:00:00': 6, '13:30:00': 5, '14:00:00': 4, '14:30:00': 3, '15:00:00': 2,
        '15:30:00': 1, '16:00:00': 0
    }
    # create
    times, closes = data['time'], data['close']
    targets = np.array([])

    for start_idx in range(data.shape[0]):
        # get prices to compare for target variable
        current_close = closes.iloc[start_idx]
        try:
            compare_close = closes.iloc[start_idx + window + map[times.iloc[start_idx]]]
        except IndexError:
            compare_close = np.nan

        # check if larger or smaller
        if np.isnan(compare_close):
            targets = np.append(targets, np.nan)
        else:
            if current_close < compare_close:
                targets = np.append(targets, 1)
            else:
                targets = np.append(targets, 0)
    data['pred'] = targets
    return data['pred']

def test_boundaries_whole(df, ranges, training_data_boundaries):
    boundary_flag: bool = True
    for var, range in zip(['rsi', 'macd', 'willr', 'obv', 'proc', 'stoch_k'], ranges):
        if training_data_boundaries(range[0], range[1], df[var]):
            boundary_flag = False
    return boundary_flag

def live_accuracy(trends, strikes, comp_price):
    """check the live trades accuracy"""

    # create output container
    container: list = []

    for i, (trend, strike) in enumerate(zip(trends, strikes)):
        if i <= 13:
            if trend == 1:
                if strike < comp_price:
                    container.append(True)
                else:
                    container.append(False)
            else:
                if strike > comp_price:
                    container.append(True)
                else:
                    container.append(False)
        else:
            container.append('-')
    return container

features = ['rsi', 'macd', 'willr', 'obv', 'proc', 'stoch_k']
def draw_distributions(original, target, new_original_weights, plot=False):
    ks_vals: list = []
    for id, column in enumerate(features, 1):
        if plot:
            plt.figure(figsize=[15, 7])
            plt.subplot(2, 3, id)
            sns.kdeplot(data=original, x=column, color='red', weights=new_original_weights)
            sns.kdeplot(data=target, x=column, color='blue')
            plt.title(column)
            plt.show()
        ks_vals.append(ks_2samp_weighted(original[column], target[column],
                                         weights1=new_original_weights, weights2=np.ones(len(target), dtype=float)))
    return sum(ks_vals)/len(ks_vals)

ticker = yf.Ticker('SPY')
yesterday_price_backtest = ticker.history(interval='1d', start="2020-01-01", end="2022-01-21")['Close']
nyse = mcal.get_calendar('NYSE')

def evaluate_preds(strike, close, trend):
  """get actual price movements to live data"""
  if strike < close and trend == 1:
    return True
  elif strike > close and trend == 0:
    return True
  else:
    return False

from datetime import timedelta
from datetime import datetime
def get_n_market_days_ahead(n, start_date):
  """get the next market day in an n-day interval"""
  start_date = str(start_date)
  start_date = datetime.strptime(start_date, "%Y-%m-%d")
  end_date = start_date + timedelta(days=15)

  early = nyse.schedule(start_date=start_date, end_date=end_date)
  actual_end = str(early.iloc[:n+1].iloc[-1]['market_open']).split(' ')[0]
  return yesterday_price_backtest[yesterday_price_backtest.index == str(actual_end)].iloc[0]

def backtest_func(live_data, model, threshold):
  """backtesting function"""
  live_data['trend'] = model.predict(live_data[features])
  live_data['day'] = live_data['start_date'].apply(lambda x: str(x).split(' ')[0])
  live_data['close_price'] = live_data['day'].apply(lambda x: get_n_market_days_ahead(params['window']//14, str(x)))

  live_data['actual'] = live_data.apply(lambda x: evaluate_preds(x.strikes, x.close_price, x.trend), axis=1).astype(int)
  live_data.drop('day', axis=1, inplace=True)
  print(live_data[live_data['prob'] > threshold]['actual'].value_counts())

# backtesting mode turn on/off
backtest: bool = True

def create_model(years=params['years'], trend=params['window'], wavelet=params['wavelet'], facade=False):
    # get data from backend
    data = fetch_alpaca_csv(years)
    data.rename(columns={col: col.lower() for col in data.columns}, inplace=True)
    copy_data = data.copy()

    print(data.shape)  # replace with wavelet transformed version
    data['open'], data['high'], data['low'], data['close'], data['volume'] = _wavelet_smooth(data, wavelet)

    # create indicator features
    data = _get_indicator_data(data)
    copy_data = _get_indicator_data(copy_data)
    data.drop('unnamed: 0', axis=1, inplace=True)
    copy_data.drop('unnamed: 0', axis=1, inplace=True)
    if data.isna().sum().sum() != copy_data.isna().sum().sum():
        print('NaN Error')
        return
    data.dropna(inplace=True)
    copy_data.dropna(inplace=True)

    # remove nan values and reset indices on both dataframes
    data.reset_index(inplace=True)
    copy_data.reset_index(inplace=True)
    data.drop(['index'], axis=1, inplace=True)
    copy_data.drop(['index'], axis=1, inplace=True)
    start = perf_counter()
    data['pred'] = _produce_prediction(copy_data, window=trend)
    print("Target Variable Creation Runtime", perf_counter() - start)

    # create live data samples from equivalent training distribution
    live_data = data[-trend:].drop('pred', axis=1)[['rsi', 'macd', 'willr', 'obv', 'proc', 'stoch_k']]
    strike_prices = copy_data[-trend:]['close']
    live_dates = data[-trend:]['timestamp']

    # resolve formatting and nan issues
    del (data['close'])
    data.dropna(inplace=True)
    print(data.shape, 'tracking data shape 4')
    #data = data.sample(frac=1).reset_index(drop=True)
    features = ['rsi', 'macd', 'willr', 'obv', 'proc', 'stoch_k']
    target = 'pred'

    # train-test data splits
    test_ratio = 0.30
    #X_train, X_test, y_train, y_test = train_test_split(data[features], data[target],
    #                                                    test_size=test_ratio, random_state=42)
    X_train, X_test = data[features][:int(0.8*data.shape[0])], data[features][int(0.8*data.shape[0]):]
    y_train, y_test = data[target][:int(0.8*data.shape[0])], data[target][int(0.8*data.shape[0]):]

    # transform distributions into the normal distribution to standardize training-testing-live samples
    live_data = live_data[features]
    transform: bool = False
    if transform:
        output_dist = 'uniform'
        train_trans = quantile_transform(X_train, output_distribution=output_dist, n_quantiles=len(X_train))
        test_trans = quantile_transform(X_test, output_distribution=output_dist, n_quantiles=len(X_train))
        live_trans = quantile_transform(live_data, output_distribution=output_dist, n_quantiles=len(X_train))
        for i, col in enumerate(X_train.columns):
            X_train[col] = train_trans[:, i]
            X_test[col] = test_trans[:, i]
            live_data[col] = live_trans[:, i]

    reweight_check = False
    opt_weights = np.ones(len(X_train))
    if reweight_check and not transform:
        opt_weights, distance = None, 10000
        for estimator in tqdm([150, 200, 250]):
            for lr in [0.05, 0.1]:
                for depth in [3, 4]:
                    for leaf in [1000, 2000, 3000]:
                        reweighter = reweight.GBReweighter(n_estimators=estimator, learning_rate=lr, max_depth=depth,
                                                           min_samples_leaf=leaf)
                        reweighter.fit(X_train, live_data[features])
                        gb_weights_test = reweighter.predict_weights(X_train)
                        dist_distance = draw_distributions(X_train, live_data, gb_weights_test)
                        if dist_distance < distance:
                            print(dist_distance, distance)
                            opt_weights = gb_weights_test
                            distance = dist_distance

    print(draw_distributions(X_train, live_data, opt_weights, plot=False)) ### DISABLE PLOTTING WHEN TUNING
    ratio = Counter(y_train)
    scale_pos_weight = max(ratio.values()) / min(ratio.values())
    model = XGBClassifier(scale_pos_weight=scale_pos_weight).fit(X_train, y_train, sample_weight=opt_weights)
    #model = RandomForestClassifier(class_weight='balanced').fit(X_train, y_train)

    stat, p_value = ttest_ind(X_train, live_data)
    print("T-test P-value: ", p_value)

    # determine accuracy and append to results
    rf_prediction = model.predict(X_test)
    print("Raw Accuracy: ", accuracy_score(list(y_test), list(rf_prediction)))
    print("ROC AUC: ", roc_auc_score(y_test, rf_prediction))
    print('F1 Score: ', f1_score(y_test, rf_prediction, average='macro'))
    rf_accuracy = balanced_accuracy_score(list(y_test), list(rf_prediction))
    print('Test Accuracy: ', rf_accuracy)
    print(years, trend, wavelet)

    # make live predictions
    live_predictions = model.predict(live_data)
    live_prob_predictions = model.predict_proba(live_data)
    live_data['pred'] = live_predictions
    live_data['strikes'] = strike_prices
    live_data['prob'] = [i[np.argmax(i)] for i in live_prob_predictions]
    live_data['date'] = live_dates
    live_data['window'] = [trend // 14 for _ in range(live_data.shape[0])]
    live_data['day'] = live_data['date'].apply(lambda x: str(x).split(' ')[0])
    live_data = live_data[live_data['day'].isin(live_data['day'].unique()[-3:])].drop('day', axis=1)
    live_data.rename(columns={'pred': 'trend', 'date': 'start_date'}, inplace=True)

    # measure accuracy across different confidence filters
    correct, incorrect, skipped = 0, 0, 0
    threshold = 0.9
    try:
        for pred, prob, actual in zip(rf_prediction, model.predict_proba(X_test), y_test):
            if prob[int(pred)] >= threshold:
                if pred == actual:
                    correct += 1
                else:
                    incorrect += 1
            else:
                skipped += 1
        print((f"Acc: {correct / (correct + incorrect)}"), f'Skip Ratio: {skipped / len(y_test)}')
    except ZeroDivisionError:
        pass

    # create a backtest
    if backtest:
        backtest_func(live_data, model, 0.9)

    if not facade:
        live_data.to_csv('live_data_backtest.csv')
        print(live_data[['window', 'trend', 'start_date', 'prob', 'strikes']][live_data['prob'] > 0.9])

    # report the NNI console
    nni.report_final_result(rf_accuracy)
    #nni.report_intermediate_result(cross_val_acc)

    if not facade:
        return model, rf_accuracy, live_data[['trend', 'prob', 'strikes', 'start_date']], live_data[features], \
               X_train, X_test, y_train, y_test, data
    else:
        return model, accuracy_score(list(y_test), list(rf_prediction)), \
               live_data[['window', 'trend', 'start_date', 'prob', 'strikes']][live_data['prob'] > 0.9]

create_model()
#create_model(years=3, trend=42, wavelet='coif2')


print("Hello Word")
a = 10.36

import matplotlib.pyplot as plt

plt.plot((5,1),'x')

import numpy as np

X = np.arange(0,10)
Y = X + a

Y1 = a - X


#############################################################################################################
# This file contains a Python implementation and adaptation of the Matlab algorithm used by Granata et al.
# (Granata et al. BMC Bioinformatics 2019, 20(Suppl 4):162 https://doi.org/10.1186/s12859-019-2685-9)
# The paper itself uses the Lee-12 algorithm (Matlab) for data integration and model simulation
# (Lee et al. BMC Systems Biology 2012, 6:73 https://doi.org/10.1186/1752-0509-6-73)
#
# COBRApy, the openCOBRA project (https://opencobra.github.io/cobrapy/)
# COBRA Toolbox (https://opencobra.github.io/cobratoolbox/stable/; https://doi.org/10.1038/s41596-018-0098-2)
# Gurobipy obtained form Gurobi and used with an academic license (https://www.gurobi.com/)
#
# Daniel Ribeiro 2020
#############################################################################################################

# System / Python
import os
import sys
import re
import tempfile
from pprint import pprint
from dataclasses import dataclass
import argparse
# Pandas and numpy
import pandas as pd
import numpy as np
# COBRA - cobrapy
import cobra
# Import extensions to Cobrapy
import cobrapy_extensions
# import 'SMOP' converted and reviewed 'Lee12' script
import lee12py
from libsmop import *

######
#Functions
######

def genesToCobraGenes(gene_names: list, cobra_model: cobra.core.Model, attribute: str) -> tuple:
    '''
    Returns tuple(dict_list, dict_names)
    dict_list: Cobrapy DictList of Cobra Genes mapped from our data into the model)
    dict_names: dict of key(gene names): value(Cobra Gene)

    The same name can be mappted to several Cobra Genes (eg. 31_AT1, 31_AT2, 31_AT3, 31_AT4, 31_AT5).
    The number in the ID string, corresponds to the NCBI Entrez ID. Therefore, they signify the same gene (possibly different isoforms).
    Therefore, if a gene name is found, collect only the 1st instance of the Cobra Gene.
    '''
    result = []
    dict_names = {}
    for gene in gene_names:
        regex = re.compile('^{match}$'.format(match=gene))
        query = cobra_model.genes.query(regex, attribute=attribute)
        if len(query) > 0:
            result.append(query[0])
            dict_names[gene] = query[0]
    dict_list = cobra.core.DictList(result)
    return dict_list, dict_names

######
#CODE
######
pd.options.mode.use_inf_as_na = True
idx=pd.IndexSlice

# Define arguments
parser = argparse.ArgumentParser(description="Perform flux balance analysis (FBA) on RNA-seq data.\n\
                                            ")
parser.add_argument("-f", "--file", required=True, help="path/to/rnaseq/file. Requires a Gene per line, a Mean expression (norm counts) column and a SD column.")
parser.add_argument("-g", "--gene_col", required=True, help="Gene column name.")
parser.add_argument("-e", "--expression_col", required=True, help="Expression column name.")
parser.add_argument("-s", "--sd_col", required=True, help="SD column name.")
parser.add_argument("-F", "--measured_fluxes", required=True, help="path/to/measured/fluxes/file. Format:\n\
                    First column are reaction names in the chosen metabolic model.\n\
                    Second column is the measured flux. \n\
                    By convention, exchange reactions are written as export reactions (e.g. ‘glc[e] <==>’),\n\
                    so import of a metabolite is a negative flux and so export of a metabolite is a positive flux\n")
parser.add_argument("-m", "--model", required=True, help="path/to/metabolic/model/file, in XML format.")
parser.add_argument("-o", "--output_path", help="path/to/output/folder.", default="./")
parser.add_argument("-n", "--name", help="Optional name for the analysis", default="flux")
args = parser.parse_args()

# Import tables
print("\nImport tables")
df_rna = pd.read_csv(args.file, sep='\t', na_filter=False, index_col=0, header=0)
df_fluxes = pd.read_csv(args.measured_fluxes, sep='\t', na_filter=False, index_col=0, header=0)
print("Imported RNA-seq:")
print(df_rna)

# Build experimental fluxes dict
experimental_fluxes = {}
for i in range(len(df_fluxes.index)):
    experimental_fluxes[df_fluxes.index[i]] = (df_fluxes.index[i], df_fluxes.iloc[i, 0])
print("Imported fluxes:")
print(experimental_fluxes)
# Build condition dict
condition = {}
condition = {'condition': args.name,
            'df_rna': df_rna.copy(),
            'mean_col': args.expression_col,
            'sd_col': args.sd_col,
            'gene_name': df_rna.index.values.copy(),
            'model': args.model,
            'output': args.output_path,
            'experimental_fluxes': experimental_fluxes
            }

# Create Cobra configuration
print("\n***Load COBRApy***")
cobra_config = cobra.Configuration()

# Display relevant Cobra configuration
print("InstalledPackage versions:")
cobra.show_versions()
cobra_config.solver = 'gurobi'	#Set solver
print("\n***COBRApy configuration***")
print(cobra_config)

# Load model in Cobra
print(f'''\nLoading model '{condition["model"]}' into COBRApy...''')
model = cobra.core.Model()
model = cobra.io.read_sbml_model(condition['model'])

#Model Properties
print("\n***Model description***")
print("Model summary: ", model)
print("ID: ", model.id)
print("Reactions no.: ", len(model.reactions))
print("Genes no.: ", len(model.genes))
print("Subcellular compartment no.: ", len(model.compartments))
#print("Model solver: ", model.solver) #WARNING: will print a lot of info
print("Model objective expression: ", model.objective.expression)
print("Model objective direction: ", model.objective.direction)
objective_reaction = cobra.util.solver.linear_reaction_coefficients(model)
print("Model objective reaction: ")
for key, value in objective_reaction.items():
    print(f"\t{key}", value)
    print("\tName: ", key.name)
    print("\tID: ", key.id)

print("\n***Experimental fluxes to constrain ***")
for key, val in condition['experimental_fluxes'].items():
    measured_flux = val[0] # eg. 'D-glucose exchange' (extracellular consumpion  of glucose)
    measured_flux = model.reactions.get_by_id(measured_flux)
    print("Reaction name: ", measured_flux.name)
    print("Bounds (default): ", measured_flux.bounds)
    print("Reversibility (default): ", measured_flux.reversibility)
#Prepare our data for Lee12 algorithm
print("\nPrepare data for Lee12 algorithm...")
print(f"Using condition: {condition['condition']}")
print(f"Using model: {str(model)}")
print("Clean gene names of data...")
clean_name = [name for name in condition['gene_name'] if name]
print("Lenght of gene names: ", len(clean_name))

print("Convert gene names into model Gene objects...")
gene_dictlist, gene_name_dict = genesToCobraGenes(clean_name, model, attribute='name')
print("Select Mean and SD of genes existing in the model")
boolean = condition['df_rna'].index.isin(gene_dictlist.list_attr('name'))
df_filter = condition['df_rna'].loc[idx[boolean], :].copy()
print("Select the corresponding Gene objects")
gene_name, *_ = genesToCobraGenes(df_filter.index.values, model, attribute='name')
condition['gene_name'] = gene_name
gene_mean = df_filter[condition['mean_col']].values
gene_sd = df_filter[condition['sd_col']].values
print("Filtered genes: ", len(df_filter.index))
##Uncomment to verify gene order
#for i in range(len(df_filter.index)):
#    print("Index: ", df_filter.iloc[i, [2, 3]],
#          "Cobra gene: ", gene_name.list_attr('name')[i])
print(f"\n***Execute Lee12 algorithm with: {condition['condition']} - WIP***")
#Run the gene expression constraint FBA, and get the resulting flux distribution
print("Run the gene expression constraint FBA, and get the resulting flux distribution")
fluxes = lee12py.call_Lee12(model=model,
                            gene_names=condition['gene_name'],
                            gene_exp=gene_mean,
                            gene_exp_sd=gene_sd,
                            #gene_names_cobra=gene_name_dict,
                            #gene_scale_rxn=gene_to_scale,
                            #flux_scale_rxn=flux_to_scale,
                            flux=condition['experimental_fluxes'],
                            model_id=condition['condition'])
#Match Reaction to Flux
#DEBUG
print(f"len(fluxes) ==  len(model.reactions): {fluxes.size == len(model.reactions)}")

print("Save predicted reaction flux table...")
flux_table = {'Reaction.Name': [],
              'Reaction.Id': [],
              'Flux': []
              }
for k in range(len(model.reactions)):
    flux_table['Reaction.Name'].append(model.reactions[k].name)
    flux_table['Reaction.Id'].append(model.reactions[k].id)
    flux_table['Flux'].append(fluxes[k])
flux_table = pd.DataFrame.from_dict(flux_table)
flux_table.to_csv(f"./{condition['output']}/flux_table_{condition['condition']}.txt", sep='\t', index=False)
print("Save the model as COBRApy model in SMBL and JSON formats...")
cobra.io.write_sbml_model(model, f"./{condition['output']}/{condition['condition']}.xml")
cobra.io.save_json_model(model, f"./{condition['output']}/{condition['condition']}.json", pretty=True)

print("******", "\nDONE", "\n******")


from tqdm import tqdm
from typing import Dict, Optional
from lantern.metric import Metric

def ProgressBar(data_loader, name, metrics: Optional[Dict[str, Metric]] = None):
    """Simple progress bar with metrics"""
    if metrics is None:
        for item in tqdm(data_loader, desc=name, leave=False):
            yield item
    else:
        with tqdm(data_loader, desc=name, leave=False) as tqdm_:
            for item in tqdm_:
                yield item

                log_dict = dict()
                for metric_name, metric in metrics.items():
                    metric_value = metric.compute()
                    if isinstance(metric_value, dict):
                        log_dict.update(**metric_value)
                    else:
                        log_dict[metric_name] = metric_value
                tqdm_.set_postfix(log_dict)


import numpy as np
from sklearn.base import ClassifierMixin, MetaEstimatorMixin, BaseEstimator
from scipy.optimize import linear_sum_assignment
import warnings
import pandas as pd

class WhoIsWhoClassifier(BaseEstimator, ClassifierMixin, MetaEstimatorMixin):

    def __init__(self, base_estimator, method="hungarian", conflict_weighted=True):
        """
        Who is Who Classifier
        Kuncheva, L. I., Rodriguez, J. J., & Jackson, A. S. (2017).
        Restricted set classification: Who is there?. <i>Pattern Recognition</i>, 63, 158-170.

        Parameters
        ----------
        base_estimator : ClassifierMixin
            The base estimator to be used for training.
        method : str, optional
            The method to use to assing class, it can be `greedy` to first-look or `hungarian` to use the Hungarian algorithm, by default "hungarian"
        conflict_weighted : bool, default=True
            Whether to weighted the confusion rate by the number of instances with the same group.
        """
        allowed_methods = ["greedy", "hungarian"]
        self.base_estimator = base_estimator
        self.method = method
        if method not in allowed_methods:
            raise ValueError(f"method {self.method} not supported, use one of {allowed_methods}")
        self.conflict_weighted = conflict_weighted

    def fit(self, X, y, instance_group=None, **kwards):
        """Fit the model according to the given training data.
        Parameters
        ----------
        X : {array-like, sparse matrix} of shape (n_samples, n_features)
            The input samples.
        y : array-like of shape (n_samples,)
            The target values.
        instance_group : array-like of shape (n_samples)
            The group. Two instances with the same label are not allowed to be in the same group. If None, group restriction will not be used in training.
        Returns
        -------
        self : object
            Returns self.
        """
        self.base_estimator = self.base_estimator.fit(X, y, **kwards)
        self.classes_ = self.base_estimator.classes_
        if instance_group is not None:
            self.conflict_in_train = conflict_rate(self.base_estimator.predict(X), instance_group, self.conflict_weighted)
        else:
            self.conflict_in_train = None
        return self

    def conflict_rate(self, X, instance_group):
        """Calculate the conflict rate of the model.
        Parameters
        ----------
        X : {array-like, sparse matrix} of shape (n_samples, n_features)
            The input samples.
        instance_group : array-like of shape (n_samples)
            The group. Two instances with the same label are not allowed to be in the same group.
        Returns
        -------
        float
            The conflict rate.
        """
        y_pred = self.base_estimator.predict(X)
        return conflict_rate(y_pred, instance_group, self.conflict_weighted)

    def predict(self, X, instance_group):
        """Predict class for X.
        Parameters
        ----------
        X : {array-like, sparse matrix} of shape (n_samples, n_features)
            The input samples.
        **kwards : array-like of shape (n_samples)
            The group. Two instances with the same label are not allowed to be in the same group.
        Returns
        -------
        array-like of shape (n_samples, n_classes)
            The class probabilities of the input samples.
        """

        y_prob = self.predict_proba(X)

        y_predicted = combine_predictions(y_prob, instance_group, len(self.classes_), self.method)

        return self.classes_.take(y_predicted)

    def predict_proba(self, X):
        """Predict class probabilities for X.
        Parameters
        ----------
        X : {array-like, sparse matrix} of shape (n_samples, n_features)
            The input samples.
        Returns
        -------
        array-like of shape (n_samples, n_classes)
            The class probabilities of the input samples.
        """
        return self.base_estimator.predict_proba(X)

def conflict_rate(y_pred, restrictions, weighted=True):
    """
    Computes the conflict rate of a prediction, given a set of restrictions.
    Parameters
    ----------
    y_pred : array-like of shape (n_samples,)
        Predicted target values.
    restrictions : array-like of shape (n_samples,)
        Restrictions for each sample. If two samples have the same restriction, they cannot have the same y.
    weighted : bool, default=True
        Whether to weighted the confusion rate by the number of instances with the same group.
    Returns
    -------
    conflict rate : float
        The conflict rate.
    """

    # Check that y_pred and restrictions have the same length
    if len(y_pred) != len(restrictions):
        raise ValueError("y_pred and restrictions must have the same length.")

    restricted_df = pd.DataFrame({'y_pred': y_pred, 'restrictions': restrictions})

    conflicted = restricted_df.groupby('restrictions').agg({'y_pred': lambda x: np.unique(x, return_counts=True)[1][np.unique(x, return_counts=True)[1]>1].sum()})
    if weighted:
        return conflicted.sum().y_pred / len(y_pred)
    else:
        rcount = restricted_df.groupby('restrictions').count()
        return (conflicted.y_pred / rcount.y_pred).sum()

def combine_predictions(y_probas, instance_group, class_number, method="hungarian"):
    y_predicted = []
    for group in np.unique(instance_group):

        mask = instance_group == group
        probas_matrix = y_probas[mask]

        preds = list(np.argmax(probas_matrix, axis=1))

        if len(preds) == len(set(preds)) or probas_matrix.shape[0] > class_number:
            y_predicted.extend(preds)
            if probas_matrix.shape[0] > class_number:
                warnings.warn("That the number of instances in the group is greater than the number of classes.", UserWarning)
            continue

        if method == "greedy":
            y = _greedy(probas_matrix)
        elif method == "hungarian":
            y = _hungarian(probas_matrix)

        y_predicted.extend(y)
    return y_predicted

def _greedy(probas_matrix):

    probas = probas_matrix.reshape(probas_matrix.size,)
    order = probas.argsort()[::-1]

    y_pred_group = [None for i in range(probas_matrix.shape[0])]

    instance_to_predict = {i for i in range(probas_matrix.shape[0])}
    class_predicted = set()
    for item in order:
        class_ = item % probas_matrix.shape[0]
        instance = item // probas_matrix.shape[0]
        if instance in instance_to_predict and class_ not in class_predicted:
            y_pred_group[instance] = class_
            instance_to_predict.remove(instance)
            class_predicted.add(class_)

    return y_pred_group

def _hungarian(probas_matrix):

    costs = np.log(probas_matrix)
    costs[costs == -np.inf] = 0  # if proba is 0, then the cost is 0
    _, col_ind = linear_sum_assignment(costs, maximize=True)
    col_ind = list(col_ind)

    return col_ind


#!/usr/bin/env python3

from collections import defaultdict

def nucleo_stats(dna_seq):
    stats = defaultdict(int)
    for nuclio in dna_seq:
        stats[nuclio] += 1
    order_seq = ['A', 'C', 'G', 'T']
    return [stats[nuclio] for nuclio in order_seq]

def main():
    in_data = ''
    current_dir = pathlib.Path(__file__).parent
    with open(f'{current_dir}/input.txt') as fp:
        in_data = fp.read()
    print(' '.join(map(str, nucleo_stats(in_data))))

if __name__ == "__main__":
    main()


# Databricks notebook source
# MAGIC %md
# MAGIC Union de valores clima, altitud, pendiente, ocurrencia de incendio, alto riesgo,  peligro y vulnerabilidad con datos de movilidad urbana

# COMMAND ----------

import pyspark.sql.functions as F

# COMMAND ----------

movilidadDF = spark.read.parquet('/mnt/IncendiosForestalesCAT/prep/mitma/movilidad/')
df = spark.read.parquet(f"/mnt/IncendiosForestalesCAT/prep/riesgo_incendios_catalunya_all")
estacionesMunicipioDF = spark.read.parquet(f"/mnt/IncendiosForestalesCAT/prep/meteocat/estaciones_cercanas_municipio/")

# COMMAND ----------

display(movilidadDF)

# COMMAND ----------

display(df)

# COMMAND ----------

display(estacionesMunicipioDF)

# COMMAND ----------

estacionesMunicipioDF = estacionesMunicipioDF.withColumn("codigo_municipio_m",F.expr("substring(codigo_municipio, 1, length(codigo_municipio)-1)"))

# COMMAND ----------

df.count()

# COMMAND ----------

movilidadDF = movilidadDF.alias("df1").join(estacionesMunicipioDF.alias("df2"), F.col("df1.cod_mun_ine")==F.col("df2.codigo_municipio_m"),"inner").select(["df1.fecha","df2.indicativo","df1.total_viajes","df1.total_viajes_km"])

# COMMAND ----------

movilidadDF.count()

# COMMAND ----------

df_all = df.alias("df1").join(movilidadDF.alias("df2"), (F.col("df1.indicativo")== F.col("df2.indicativo")) & (F.col("df1.fecha")== F.col("df2.fecha")), "left").select(["df1.*","df2.total_viajes","df2.total_viajes_km"])
df_all.count()

# COMMAND ----------

display(df_all)

# COMMAND ----------

df_all.coalesce(1).write.mode("overwrite").option("header",True).csv(f"/mnt/IncendiosForestalesCAT/prep/riesgo_incendios_catalunya_all_movilidad/")

2. TRANSFORMACION/8. All_mas_movilidad/8. Tranformacion_all_movilidad.py


import pyaudio
p=pyaudio.PyAudio()
FRAMES_PERBUFF = 2048 #   no of FRAMES_PERBUFFER
FORMAT = pyaudio.paInt16
CHANNELS = 1
FRAME_RATE = 44100
stream = p.open(format = FORMAT, channels = CHANNELS, rate = FRAME_RATE, input = True, frames_per_buffer = FRAMES_PERBUFF)
frames = []
print("start")

RECORD_SECONDS = 5
nchunks = int(RECORD_SECONDS * FRAME_RATE / FRAMES_PERBUFF)
for i in range(0, nchunks):
    data = stream.read(FRAMES_PERBUFF)
    frames.append(data) # 2 bytes(16 bits) per channel
print("* done recording")
stream.stop_stream()
stream.close()
p.terminate()

print ("done")
import wave
import random
x =random.randint(1,1000000)
wf = wave.open(str(x)+'.wav', 'wb')
wf.setnchannels(CHANNELS)
wf.setsampwidth(p.get_sample_size(FORMAT))
wf.setframerate(FRAME_RATE)
wf.writeframes(b''.join(frames))
wf.close()


from docx import Document
from docx.shared import Pt

documentAnswerSheet = Document()
documentQuestionSheet = Document()
language1 = []
language2 = []

print('What should be the file name:')
fileName = input()

print('How many Vocabularies will there be:')
numVocab = input()

for x in range(int(numVocab)):
    print('Vocab in Language 1:')
    language1.append(input())
    print('Vocab in Language 2:')
    language2.append(input())

for x in range(int(numVocab)):
    run1 = documentAnswerSheet.add_paragraph().add_run(language1[x] + ' = ' + language2[x])
    run2 = documentQuestionSheet.add_paragraph().add_run(language1[x] + ' = ')
    run1.font.size = Pt(24)
    run2.font.size = Pt(24)

documentQuestionSheet.save(fileName + ' - Question Sheet.docx')
documentAnswerSheet.save(fileName + ' - Answer Sheet.docx')