from ETF import *
from Option_Pricing import *

import statsmodels.api as sm
from statsmodels.tsa import stattools
from statsmodels.graphics.tsaplots import plot_acf
import pandas as pd
import matplotlib.pyplot as plt
from pandas.plotting import register_matplotlib_converters

import numpy as np

def AR_test(data, nlags, title, plot=True):
    ar_coef, _, p = stattools.acf(data, fft=False, nlags=nlags, qstat=True)
    if plot:
        fig, ax = plt.subplots(figsize=(14, 8))
        plot_acf(data, ax=ax)
        plt.title('Auto-correlation of ' + title, size=25)
        plt.xlabel('time_lags', size=20)
        plt.ylabel('AR coefficients', size=20)
        plt.xticks(size=18)
        plt.yticks(size=18)
    ar_df = pd.DataFrame(ar_coef, columns=['AR_coef'])
    p_df = pd.DataFrame(p, columns=['p_value'])
    return pd.concat([ar_df, p_df], axis=1)

if __name__ == '__main__':
    ## a
    start_date = '2005-01-01'
    end_date = '2019-09-01'
    SPY_C = ETF('SPY')
    VIX_C = ETF('^VIX')  # adding '^' !

    SPY_p = SPY_C.get_price_data(start_date, end_date)
    SPY_r = SPY_C.cal_return_data()
    VIX = VIX_C.get_price_data(start_date, end_date)

    ## b
    SPY_p_AR = AR_test(SPY_p, 20, 'SPY', plot=True)
    # SPY_r_AR = AR_test(SPY_r, 20, plot=True)
    VIX_AR = AR_test(VIX, 20, 'VIX', plot=True)
    print('\nAR test result for SPY:')
    print(SPY_p_AR.head(6))
    print('\nAR test result for VIX:')
    print(VIX_AR.head(6))
    # plt.show()

    # c
    daily_corr = SPY_p.corr(VIX)
    print('daily correlation of SPY and VIX:', daily_corr)

    SPY_month = SPY_C.cal_month_end()
    VIX_month = VIX_C.cal_month_end()
    month_corr = SPY_month.corr(VIX_month)
    print('monthly correlation of SPY and VIX:', month_corr)

    ## d
    price_data = pd.concat([SPY_p, VIX], axis=1)
    price_data.columns = ['SPY', 'VIX']
    rolling_coef = price_data.rolling(90).corr(price_data['VIX']).iloc[:, 0].dropna()
    print('\nRolling coeffeicient with window of 90:')
    print(rolling_coef.head())

    plt.figure()
    rolling_coef.plot()
    plt.plot(rolling_coef.index, [rolling_coef.mean()] * len(rolling_coef), linewidth=2, linestyle='-.')
    plt.title('Rolling coeffeicient of SPY and implied volatility', size=22)
    plt.xlabel('date', size=18)
    plt.ylabel('Correlation coefficients', size=18)
    plt.xticks(size=15)
    plt.yticks(size=15)

    ## e
    plt.show()
    SPY_rlzed_vol = SPY_C.cal_rlzed_vol()
    vol_data = pd.merge(SPY_rlzed_vol, VIX, left_index=True, right_index=True)
    vol_data.columns = ['rlzd', 'implied']
    vol_data['premium'] = vol_data.implied - vol_data.rlzd

    plt.figure()
    vol_data['premium'].plot()
    plt.title('Premium of volatility', size=25)
    plt.xlabel('date', size=20)
    plt.ylabel('Premium', size=20)
    plt.xticks(size=18)
    plt.yticks(size=18)

    ## f
    rf = 0;
    tau = 1 / 12
    straddle = pd.merge(SPY_p, VIX, left_index=True, right_index=True)
    straddle.columns = ['S0', 'vol']
    straddle['call'] = Euro_option(straddle['S0'], straddle['S0'], rf, straddle['vol'] / 100, tau).BSM()[0]
    straddle['put'] = Euro_option(straddle['S0'], straddle['S0'], rf, straddle['vol'] / 100, tau).BSM()[1]
    # print(straddle.head())

    ## g
    payoff = abs(straddle.S0 - straddle.S0.shift(21))
    payoff.name = 'payoff'
    straddle = pd.concat([straddle, payoff.shift(-21)], axis=1).dropna()
    straddle['p/l'] = straddle.payoff - straddle.call - straddle.put

    plt.figure()
    straddle['p/l'].plot()
    plt.title('Profit of Straddles', size=25)
    plt.xlabel('date', size=20)
    plt.ylabel('G/L', size=20)
    plt.xticks(size=18)
    plt.yticks(size=18)
    plt.plot(straddle.index, [straddle['p/l'].mean()] * len(straddle), linestyle='-.', linewidth=3)
    print('\nAverage P/L:', straddle['p/l'].mean())

    ## h
    straddle = pd.merge(straddle, vol_data['premium'], left_index=True, right_index=True)
    plt.figure()
    plt.scatter(straddle.premium, straddle['p/l'])
    plt.title('P/l v.s. Premium', size=25)
    plt.xlabel('Premium of volatility', size=20)
    plt.ylabel('P/L', size=20)

    y = straddle['p/l']
    X = straddle.premium
    X = sm.add_constant(X)
    est = sm.OLS(y, X, missing='drop')
    result = est.fit()
    # print(result.summary())

    plt.plot(straddle.premium, result.fittedvalues, color='black')
    print(straddle)

    plt.show()


from bs4 import BeautifulSoup
from datetime import datetime, timedelta
from webapp.db import db
from webapp.news.models import News
from webapp.news.parsers.utils import get_html, save_news

import locale
# Дает информацию о платформе (оперкционной системе)
import platform

# Мы получаем и выдаем дату на русском,
# то и определим систмные локалии как русские
if platform.system() == 'Windows':
    locale.setlocale(locale.LC_ALL, "russian")
elif platform.system() == 'Linux':
    locale.setlocale(locale.LC_ALL, "ru_RU.UTF8")
else:
    locale.setlocale(locale.LC_TIME, 'ru_RU')

# Переведем "вчера" и "сегодня" в дату
def parse_habr_date(date_str):
    if 'сегодня' in date_str:
        today = datetime.now()
        date_str = date_str.replace('сегодня', today.strftime('%d %B %Y'))
    elif 'вчера' in date_str:
        yesterday = datetime.now() - timedelta(days=1)
        date_str = date_str.replace('вчера', yesterday.strftime('%d %B %Y'))
    try:
        return datetime.strptime(date_str, '%d %B %Y в %H:%M')
    except ValueError:
        return datetime.now()

# Получение новостей с Хабра, все колонки, кроме самой страницы новостей
# В конце сохраняем
def get_news_snippets():

    html = get_html(
        "https://habr.com/ru/search/?target_type=posts&q=python&order")

    if html:
        soup = BeautifulSoup(html, 'html.parser')
        all_news = soup.find(
            'ul', class_='content-list_posts').findAll(
                'li', class_='content-list__item_post')

        for news in all_news:
            title = news.find('a', class_='post__title_link').text
            url = news.find('a', class_='post__title_link')['href']
            published = news.find('span', class_='post__time').text
            published = parse_habr_date(published)
            save_news(title, url, published)

# Получение текста из самой страницы новостей,
# и сохранение во время перебора тех, у кого еще не заполнен текст
def get_news_content():
    news_without_text = News.query.filter(News.text.is_(None))
    # .is_(None) - если поле пустое

    # Получили объект со всеми новостями, где поле текста - пустое
    for news in news_without_text:
        # html = венрнуть содержание страницы объект строка news.колонка url
        html = get_html(news.url)
        if html:
            soup = BeautifulSoup(html, 'html.parser')
            print(news.url)
            print('------------------------------------------------')
            article = soup.find('div', class_='post__text').decode_contents()
            # .decode_contents() - вернет на данные в виде html кода.
            # Ранее мы запросили html страницы и преобразовали все в текст,
            # чтобы обработать супом. Сейчас же трансофрмируем обратно, чтобы
            # сохранить всю красоту оформления.
            # Если ошибки нет
            if article:
                # В стоку news, колонку text записывам данные.
                # Пишем прям на месте, т.к. объект у нас сейчас в работе
                news.text = article
                db.session.add(news)
                db.session.commit()


"""Standard packed message format classes."""

import json
import logging
from typing import List, Sequence, Tuple, Union

from ..core.profile import ProfileSession

from ..protocols.routing.v1_0.messages.forward import Forward

from ..messaging.util import time_now
from ..utils.task_queue import TaskQueue
from ..wallet.base import BaseWallet
from ..wallet.error import WalletError
from ..wallet.util import b64_to_str

from .error import WireFormatParseError, WireFormatEncodeError, RecipientKeysError
from .inbound.receipt import MessageReceipt
from .wire_format import BaseWireFormat

LOGGER = logging.getLogger(__name__)

class PackWireFormat(BaseWireFormat):
    """Standard DIDComm message parser and serializer."""

    def __init__(self):
        """Initialize the pack wire format instance."""
        super().__init__()
        self.task_queue: TaskQueue = None

    async def parse_message(
        self,
        session: ProfileSession,
        message_body: Union[str, bytes],
    ) -> Tuple[dict, MessageReceipt]:
        """Deserialize an incoming message and further populate the request context.

        Args:
            session: The profile session for providing wallet access
            message_body: The body of the message

        Returns:
            A tuple of the parsed message and a message receipt instance

        Raises:
            WireFormatParseError: If the JSON parsing failed
            WireFormatParseError: If a wallet is required but can't be located

        """

        receipt = MessageReceipt()
        receipt.in_time = time_now()
        receipt.raw_message = message_body

        message_dict = None
        message_json = message_body

        if not message_json:
            raise WireFormatParseError("Message body is empty")

        try:
            message_dict = json.loads(message_json)
        except ValueError:
            raise WireFormatParseError("Message JSON parsing failed")
        if not isinstance(message_dict, dict):
            raise WireFormatParseError("Message JSON result is not an object")

        # packed messages are detected by the absence of @type
        if "@type" not in message_dict:
            try:
                unpack = self.unpack(session, message_body, receipt)
                message_json = await (
                    self.task_queue and self.task_queue.run(unpack) or unpack
                )
            except WireFormatParseError:
                LOGGER.debug("Message unpack failed, falling back to JSON")
            else:
                receipt.raw_message = message_json
                try:
                    message_dict = json.loads(message_json)
                except ValueError:
                    raise WireFormatParseError("Message JSON parsing failed")
                if not isinstance(message_dict, dict):
                    raise WireFormatParseError("Message JSON result is not an object")

        # parse thread ID
        thread_dec = message_dict.get("~thread")
        receipt.thread_id = (
            thread_dec and thread_dec.get("thid") or message_dict.get("@id")
        )
        receipt.parent_thread_id = thread_dec.get("pthid") if thread_dec else None

        # handle transport decorator
        transport_dec = message_dict.get("~transport")
        if transport_dec:
            receipt.direct_response_mode = transport_dec.get("return_route")

        LOGGER.debug(f"Expanded message: {message_dict}")

        return message_dict, receipt

    async def unpack(
        self,
        session: ProfileSession,
        message_body: Union[str, bytes],
        receipt: MessageReceipt,
    ):
        """Look up the wallet instance and perform the message unpack."""
        wallet = session.inject_or(BaseWallet)
        if not wallet:
            raise WireFormatParseError("Wallet not defined in profile session")

        try:
            unpacked = await wallet.unpack_message(message_body)
            (
                message_json,
                receipt.sender_verkey,
                receipt.recipient_verkey,
            ) = unpacked
            return message_json
        except WalletError as e:
            raise WireFormatParseError("Message unpack failed") from e

    async def encode_message(
        self,
        session: ProfileSession,
        message_json: Union[str, bytes],
        recipient_keys: Sequence[str],
        routing_keys: Sequence[str],
        sender_key: str,
    ) -> Union[str, bytes]:
        """Encode an outgoing message for transport.

        Args:
            session: The profile session for providing wallet access
            message_json: The message body to serialize
            recipient_keys: A sequence of recipient verkeys
            routing_keys: A sequence of routing verkeys
            sender_key: The verification key of the sending agent

        Returns:
            The encoded message

        Raises:
            MessageEncodeError: If the message could not be encoded

        """

        if sender_key and recipient_keys:
            pack = self.pack(
                session, message_json, recipient_keys, routing_keys, sender_key
            )
            message = await (self.task_queue and self.task_queue.run(pack) or pack)
        else:
            message = message_json
        return message

    async def pack(
        self,
        session: ProfileSession,
        message_json: Union[str, bytes],
        recipient_keys: Sequence[str],
        routing_keys: Sequence[str],
        sender_key: str,
    ):
        """Look up the wallet instance and perform the message pack."""
        if not sender_key or not recipient_keys:
            raise WireFormatEncodeError("Cannot pack message without associated keys")

        wallet = session.inject_or(BaseWallet)
        if not wallet:
            raise WireFormatEncodeError("No wallet instance")

        try:
            message = await wallet.pack_message(
                message_json, recipient_keys, sender_key
            )
        except WalletError as e:
            raise WireFormatEncodeError("Message pack failed") from e

        if routing_keys:
            recip_keys = recipient_keys
            for router_key in routing_keys:
                message = json.loads(message.decode("utf-8"))
                fwd_msg = Forward(to=recip_keys[0], msg=message)
                # Forwards are anon packed
                recip_keys = [router_key]
                try:
                    message = await wallet.pack_message(fwd_msg.to_json(), recip_keys)
                except WalletError as e:
                    raise WireFormatEncodeError("Forward message pack failed") from e
        return message

    def get_recipient_keys(self, message_body: Union[str, bytes]) -> List[str]:
        """Get all recipient keys from a wire message.

        Args:
            message_body: The body of the message

        Returns:
            List of recipient keys from the message body

        Raises:
            RecipientKeysError: If the recipient keys could not be extracted

        """

        try:
            message_dict = json.loads(message_body)
            protected = json.loads(b64_to_str(message_dict["protected"], urlsafe=True))
            recipients = protected["recipients"]

            recipient_keys = [recipient["header"]["kid"] for recipient in recipients]
        except Exception as e:
            raise RecipientKeysError(
                "Error trying to extract recipient keys from JWE", e
            )

        return recipient_keys


from bottle import run, get, post, request, delete, default_app
import model
# import pickle

# @get('/location')
# def getAll():
# 	#insert_db()
# 	return {'location' : database.myrecords()}

# 	#database.clear()

# @get('/clear/delete/db')
# def clear_DB():
# 	database.clear()

@post('/data')
def process():
	#model.show(request.json)
	model.predictPotholes(request.json)
	print(type(request.json))
	#return str

app = default_app()

#following for normal bottle app
#run(host = '0.0.0.0', reloader=True, server = 'gunicorn', debug=True, workers=4)

# to run a post request use following
# r = requests.post('http://localhost:8080/animal', json = dt)


from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware

from api.business_logic.DTOs import TrainerProfile, TraineeProfile
from api.providers.factory import ProvidersFactory
from api.business_logic.exc import DbConnectionError, SQLError

def create_app():
    app = FastAPI(docs_url="/swagger")

    app.add_middleware(
        CORSMiddleware,
        allow_origins=["*"],
        allow_credentials=True,
        allow_methods=["*"],
        allow_headers=["*"],
    )

    @app.post("/write_trainer")
    def write_trainer(user: TrainerProfile):
        provider = ProvidersFactory.get_users_provider()
        try:
            provider.write_trainer(user)
        except (DbConnectionError, SQLError) as e:
            print(e)
            raise HTTPException(status_code=503, detail=str(e))

    @app.post("/write_trainee")
    def write_trainee(user: TraineeProfile):
        provider = ProvidersFactory.get_users_provider()
        try:
            provider.write_trainee(user)
        except (DbConnectionError, SQLError) as e:
            print(e)
            raise HTTPException(status_code=503, detail=str(e))

    return app


"""SubPlugin API views."""

# =============================================================================
# IMPORTS
# =============================================================================
# Django
from django.db.models import Prefetch

# Third Party Django
from rest_framework.parsers import ParseError

# App
from project_manager.api.common.views import (
    ProjectAPIView,
    ProjectContributorViewSet,
    ProjectGameViewSet,
    ProjectImageViewSet,
    ProjectReleaseViewSet,
    ProjectTagViewSet,
    ProjectViewSet,
)
from project_manager.plugins.models import Plugin
from project_manager.sub_plugins.api.filtersets import SubPluginFilterSet
from project_manager.sub_plugins.api.serializers import (
    SubPluginContributorSerializer,
    SubPluginCreateSerializer,
    SubPluginGameSerializer,
    SubPluginImageSerializer,
    SubPluginReleaseSerializer,
    SubPluginSerializer,
    SubPluginTagSerializer,
)
from project_manager.sub_plugins.models import (
    SubPlugin,
    SubPluginContributor,
    SubPluginGame,
    SubPluginImage,
    SubPluginRelease,
    SubPluginReleaseDownloadRequirement,
    SubPluginReleasePackageRequirement,
    SubPluginReleasePyPiRequirement,
    SubPluginReleaseVersionControlRequirement,
    SubPluginTag,
)

# =============================================================================
# ALL DECLARATION
# =============================================================================
__all__ = (
    'SubPluginAPIView',
    'SubPluginContributorViewSet',
    'SubPluginGameViewSet',
    'SubPluginImageViewSet',
    'SubPluginReleaseViewSet',
    'SubPluginTagViewSet',
    'SubPluginViewSet',
)

# =============================================================================
# VIEWS
# =============================================================================
class SubPluginAPIView(ProjectAPIView):
    """SubPlugin API routes."""

    project_type = 'sub-plugin'
    base_kwargs = {
        'plugin_slug': '<plugin>',
    }

class SubPluginViewSet(ProjectViewSet):
    """ViewSet for creating, updating, and listing SubPlugins."""

    __doc__ += ProjectViewSet.doc_string
    filterset_class = SubPluginFilterSet
    queryset = SubPlugin.objects.select_related(
        'owner__user',
        'plugin',
    ).prefetch_related(
        Prefetch(
            lookup='releases',
            queryset=SubPluginRelease.objects.order_by(
                '-created',
            ),
        ),
    )
    serializer_class = SubPluginSerializer
    lookup_field = 'slug'

    creation_serializer_class = SubPluginCreateSerializer
    plugin = None

    def get_queryset(self):
        """Filter down to only SubPlugins for the given Plugin."""
        queryset = super().get_queryset()
        if self.plugin is not None:
            return queryset.filter(plugin=self.plugin)
        plugin_slug = self.kwargs.get('plugin_slug')
        try:
            self.plugin = Plugin.objects.get(slug=plugin_slug)
            return queryset.filter(plugin=self.plugin)
        except Plugin.DoesNotExist as exception:
            raise ParseError('Invalid plugin_slug.') from exception

class SubPluginImageViewSet(ProjectImageViewSet):
    """ViewSet for adding, removing, and listing images for SubPlugins."""

    __doc__ += ProjectImageViewSet.doc_string
    queryset = SubPluginImage.objects.select_related(
        'sub_plugin',
    )
    serializer_class = SubPluginImageSerializer

    project_type = 'sub-plugin'
    project_model = SubPlugin

class SubPluginReleaseViewSet(ProjectReleaseViewSet):
    """ViewSet for retrieving releases for SubPlugins."""

    __doc__ += ProjectReleaseViewSet.doc_string
    queryset = SubPluginRelease.objects.select_related(
        'sub_plugin',
        'created_by__user',
    ).prefetch_related(
        Prefetch(
            lookup='subpluginreleasepackagerequirement_set',
            queryset=SubPluginReleasePackageRequirement.objects.order_by(
                'package_requirement__name',
            ).select_related(
                'package_requirement',
            )
        ),
        Prefetch(
            lookup='subpluginreleasedownloadrequirement_set',
            queryset=SubPluginReleaseDownloadRequirement.objects.order_by(
                'download_requirement__url',
            ).select_related(
                'download_requirement',
            )
        ),
        Prefetch(
            lookup='subpluginreleasepypirequirement_set',
            queryset=SubPluginReleasePyPiRequirement.objects.order_by(
                'pypi_requirement__name',
            ).select_related(
                'pypi_requirement',
            )
        ),
        Prefetch(
            lookup='subpluginreleaseversioncontrolrequirement_set',
            queryset=(
                SubPluginReleaseVersionControlRequirement.objects.order_by(
                    'vcs_requirement__url',
                ).select_related(
                    'vcs_requirement',
                )
            )
        ),
    )
    serializer_class = SubPluginReleaseSerializer

    project_type = 'sub-plugin'
    project_model = SubPlugin

class SubPluginGameViewSet(ProjectGameViewSet):
    """Supported Games listing for SubPlugins."""

    __doc__ += ProjectGameViewSet.doc_string
    queryset = SubPluginGame.objects.select_related(
        'game',
        'sub_plugin',
    )
    serializer_class = SubPluginGameSerializer

    project_type = 'sub-plugin'
    project_model = SubPlugin

class SubPluginTagViewSet(ProjectTagViewSet):
    """Tags listing for SubPlugins."""

    __doc__ += ProjectTagViewSet.doc_string
    queryset = SubPluginTag.objects.select_related(
        'tag',
        'sub_plugin',
    )
    serializer_class = SubPluginTagSerializer

    project_type = 'sub-plugin'
    project_model = SubPlugin

class SubPluginContributorViewSet(ProjectContributorViewSet):
    """Contributors listing for SubPlugins."""

    __doc__ += ProjectContributorViewSet.doc_string
    queryset = SubPluginContributor.objects.select_related(
        'user__user',
        'sub_plugin',
    )
    serializer_class = SubPluginContributorSerializer

    project_type = 'sub-plugin'
    project_model = SubPlugin


# -*- coding: utf-8 -*-

__version__ = '1.1.2'

import os
import appdirs
import osmnx as ox
import joblib
import requests
from .files import load_vars, save_vars, cached, inflate_tar, download_zipfile
from .data import data, list_data, problematic
from .tools.view_code import show_file
from . import mapping

cache_dir = None
memory = None

def set_cache_dir(location=None, compress=True, verbose=0, **kwargs):
	"""
	Set up a cache directory for use with the tutorials.

	Parameter
	---------
	cache_dir : Path-like or False, optional
		A path for the cache files.  Set to False to disable caching.
	"""
	global memory, cache_dir

	if location is None:
		location = appdirs.user_cache_dir('transportation_tutorials')

	if location is False:
		location = None

	memory = joblib.Memory(location, compress=compress, verbose=verbose, **kwargs)

	make_cache = (
		(ox, 'gdf_from_place'),
		(ox, 'geometries_from_place'),
		(ox, 'graph_from_bbox'),
		(requests, 'get'),
		(requests, 'post'),
	)

	for module, func_name in make_cache:
		try:
			func = getattr(module, f"_{func_name}_orig")
		except AttributeError:
			try:
				func = getattr(module, func_name)
			except:
				func = None
			else:
				setattr(module, f"_{func_name}_orig", func)
		if func is not None:
			setattr(module, func_name, memory.cache(func))

set_cache_dir()


from datetime import datetime

from robot.libraries.DateTime import convert_date

from Libraries.SeleniumLibrary.base import LibraryComponent, keyword
from Libraries.SeleniumLibrary.errors import CookieNotFound
from Libraries.SeleniumLibrary.utils import is_truthy, is_noney

class CookieKeywords(LibraryComponent):

    @keyword
    def delete_all_cookies(self):
        """Deletes all cookies."""
        self.driver.delete_all_cookies()

    @keyword
    def delete_cookie(self, name):
        """Deletes cookie matching ``name``.

        If the cookie is not found, nothing happens.
        """
        self.driver.delete_cookie(name)

    @keyword
    def get_cookies(self):
        """Returns all cookies of the current page.

        The cookie information is returned as a single string in format
        ``name1=value1; name2=value2; name3=value3``. It can be used,
        for example, for logging purposes or in headers when sending
        HTTP requests.
        """
        pairs = []
        for cookie in self.driver.get_cookies():
            pairs.append(cookie['name'] + "=" + cookie['value'])
        return '; '.join(pairs)

    @keyword
    def get_cookie_value(self, name):
        """*DEPRECATED in SeleniumLibrary 3.2.* Use `Get Cookie` instead."""
        cookie = self.driver.get_cookie(name)
        if cookie is not None:
            return cookie['value']
        raise ValueError("Cookie with name %s not found." % name)

    @keyword
    def get_cookie(self, name):
        """Returns information of cookie with ``name`` as an object.

        If no cookie is found with ``name``, keyword fails. The cookie object
        contains details about the cookie. Attributes available in the object
        are documented in the table below.

        | = Attribute = |             = Explanation =                                |
        | name          | The name of a cookie.                                      |
        | value         | Value of the cookie.                                       |
        | path          | Indicates a URL path, for example ``/``.                   |
        | domain        | The domain the cookie is visible to.                       |
        | secure        | When true, cookie is only used with HTTPS connections.     |
        | httpOnly      | When true, cookie is not accessible via JavaScript.        |
        | expiry        | Python datetime object indicating when the cookie expires. |

        See the
        [https://w3c.github.io/webdriver/webdriver-spec.html#cookies|WebDriver specification]
        for details about the cookie information.
        Notice that ``expiry`` is specified as a
        [https://docs.python.org/3/library/datetime.html#datetime.datetime|datetime object],
        not as seconds since Unix Epoch like WebDriver natively does.

        Example:
        | `Add Cookie`      | foo             | bar |
        | ${cookie} =       | `Get Cookie`    | foo |
        | `Should Be Equal` | ${cookie.name}  | bar |
        | `Should Be Equal` | ${cookie.value} | foo |
        | `Should Be True`  | ${cookie.expiry.year} > 2017 |

        New in SeleniumLibrary 3.0.
        """
        cookie = self.driver.get_cookie(name)
        if not cookie:
            raise CookieNotFound("Cookie with name '%s' not found." % name)
        return CookieInformation(**cookie)

    @keyword
    def add_cookie(self, name, value, path=None, domain=None, secure=None,
                   expiry=None):
        """Adds a cookie to your current session.

        ``name`` and ``value`` are required, ``path``, ``domain``, ``secure``
        and ``expiry`` are optional.  Expiry supports the same formats as
        the [http://robotframework.org/robotframework/latest/libraries/DateTime.html|DateTime]
        library or an epoch time stamp.

        Example:
        | `Add Cookie` | foo | bar |                            |
        | `Add Cookie` | foo | bar | domain=example.com         |
        | `Add Cookie` | foo | bar | expiry=2027-09-28 16:21:35 | # Expiry as timestamp.     |
        | `Add Cookie` | foo | bar | expiry=1822137695          | # Expiry as epoch seconds. |

        Prior to SeleniumLibrary 3.0 setting expiry did not work.
        """
        new_cookie = {'name': name, 'value': value}
        if not is_noney(path):
            new_cookie['path'] = path
        if not is_noney(domain):
            new_cookie['domain'] = domain
        # Secure must be True or False
        if not is_noney(secure):
            new_cookie['secure'] = is_truthy(secure)
        if not is_noney(expiry):
            new_cookie['expiry'] = self._expiry(expiry)
        self.driver.add_cookie(new_cookie)

    def _expiry(self, expiry):
        try:
            return int(expiry)
        except ValueError:
            return int(convert_date(expiry, result_format='epoch'))

class CookieInformation(object):

    def __init__(self, name, value, path=None, domain=None, secure=False,
                 httpOnly=False, expiry=None):
        self.name = name
        self.value = value
        self.path = path
        self.domain = domain
        self.secure = secure
        self.httpOnly = httpOnly
        self.expiry = datetime.fromtimestamp(expiry) if expiry else None

    def __str__(self):
        items = 'name value path domain secure httpOnly expiry'.split()
        return '\n'.join('%s=%s' % (item, getattr(self, item))
                         for item in items)


from dataclasses import dataclass, field
from multiprocessing import Queue

from telegram.ext.callbackcontext import CallbackContext
from telegram.update import Update

@dataclass
class BasicCommand:
    __request_queue: Queue
    __result_queue: Queue
    __command_dict: dict = field(init=False)

    def __post_init__(self) -> None:
        self.__command_dict = {
            "hello": self.__command_hello,
        }

    def get_all_command(self) -> dict:
        return self.__command_dict

    def __command_hello(self, update: Update, callback_context: CallbackContext) -> None:
        user_name = update.message.from_user.first_name
        update.message.reply_text(f"Hello {user_name}, how can I help you?")

    def call_back_query_handler(self, update: Update, context: CallbackContext):
        message = update.callback_query.data
        print("basic:" + message)


import sys
import os.path
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
from filters import sav_golay
import numpy as np
from numpy.testing import assert_array_equal
import pytest

@pytest.fixture
def my_data():
	data = np.array([[100, 1000, 2000, 3000],
				[200, 1500, 500, 2500],
				[300, 1700, 200, 2200],
				[400, 1800, 100, 2000],
				[500, 1900, 20, 1700]
	])
	return data

# Checks that a change in the window parameter of the golay filter changes the result
def test_golay_params(my_data):
	given = sav_golay.golay(my_data, 3, 1)
	expected = sav_golay.golay(my_data, 5, 1)
	with pytest.raises(AssertionError):
		assert_array_equal(given, expected)

# Checks that the input and output data are of the same length
def test_golay_length(my_data):
	given = len(my_data)
	expected = len(sav_golay.golay(my_data, 3, 1))
	assert given == expected

# Checks that if you input an odd integer for window lenght it raises an error
def test_golay_parmerrors(my_data):
	with pytest.raises(Exception):
		result = sav_golay.golay(my_data, 2, 1)

# Checks that if you input an integer for the polynomial parameter equal or larger than the window lenght ir raises
# an error
def test_golay_parmerrors(my_data):
	with pytest.raises(Exception):
		result = sav_golay.golay(my_data, 3, 3)


# INIT GENERAL IMPORTS
import time, random

# INIT SELENIUM IMPORTS
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains

# INIT CUSTOM IMPORTS
from setup_browser import init_browser_headless
from database import *

# FETCH ARTIST WITH {artist_id} FROM AUDIUS API,
# RETRIEVE ARTIST TRACK LIST,
# ORGANIZE TRACK DATA FOR EACH TRACK IN TRACK LIST,
# AND RETURN LIST OF TRACK DATAS

def get_artist_track_list(artist_id):
    artist_track_list = query_tracks_by_artist(artist_id)
    track_datas_list = []
    for track in artist_track_list:
        track_id = track[0]
        track_name = track[2]
        track_duration = track[3]
        track_url = track[9]
        track_data = (
            track_id,
            artist_id,
            track_name,
            track_duration,
            track_url,
        )
        track_datas_list.append(track_data)
    return track_datas_list

# FETCH ARTIST WITH {artist_id} FROM AUDIUS API,
# AND RETURN ARTIST AUDIUS HANDLE

def get_artist_handle(artist_id):
    artist_handle = query_artist_handle_by_id(artist_id)
    return artist_handle

# USING SELENIUM BROWSER INSTANCE,
# NAVIGATE TO ARTIST AUDIUS PAGE WITH {artist_id}

def nav_artist_page(browser, artist_id):
    handle = get_artist_handle(artist_id)
    browser.get(f"https://audius.co/{handle}")

found_track = False

# USING SELENIUM BROWSER INSTANCE,
# SEARCH FOR TRACK WITH {track_title}
# IF TRACK IS FOUND,
# RETURN TRACK BROWSER ELEMENT

def find_track(browser, track_title):
    global found_track
    track_elements = browser.find_elements_by_class_name(
        "TrackTile_title__1zakm"
    )
    for track in track_elements:
        if track_title == track.text:
            found_track = True
            return track
        else:
            pass

def scroll(browser):
    ActionChains(browser).key_down(Keys.LEFT_ALT).key_down(Keys.DOWN).key_up(
        Keys.LEFT_ALT
    ).key_up(Keys.DOWN).perform()

def scroll_artist_page(browser, track_title):
    global found_track
    browser.implicitly_wait(10)
    browser.find_element_by_xpath('//*[@id="mainContent"]').click()
    time.sleep(0.2)

def bot_find_url_and_update(browser, track_id):
    track_url = browser.current_url
    print(f"TRACK ID {track_id} has the url : {track_url}")
    update_track_url_in_db(track_id, track_url)

def get_track_url(browser, data):
    track_id = data[0]
    artist_id = data[1]
    track_title = data[2]
    track_duration = data[3]
    track_url = data[4]
    nav_artist_page(browser, artist_id)
    print("got artist page")
    time.sleep(random.uniform(2, 5.1))
    print("time to scroll")
    scroll_artist_page(browser, track_title)
    print("scrolling done")
    time.sleep(random.uniform(1.3, 2.9))
    bot_find_url_and_update(browser, track_id)

def sync_track_urls_to_db():
    browser = init_browser_headless()
    artists_list = query_all_artists()
    print("got artist list")
    for artist in artists_list:
        artist_id = artist[0]
        print(f"working for artist : {artist[1]} right now")
        track_datas_list = get_artist_track_list(artist_id)
        print("got artist track_data_list")
        for data in track_datas_list:
            get_track_url(browser, data)

if __name__ == "__main__":
    sync_track_urls_to_db()


'''
Created on Feb 11, 2015

@author: tinloaf
'''
import json
from dajaxice.decorators import dajaxice_register
from solver.models import Node, DataSet
from solver.ilp import ILPBuilder

@dajaxice_register(method='GET')
def optimize(request, data):
    print("=================================")
    print("Data I got:")
    print(json.dumps(data, indent=2))
    print("=================================")
    nodes = []
    id_to_node = {}
    for node_data in data['nodes']:
        node = Node(node_data['id'], node_data['x'], node_data['y'], node_data['x_post'], node_data['y_post'], node_data['width'], node_data['height'])
        nodes.append(node)
        id_to_node[node.v] = node

    adjacencies = []
    for (id1, id2) in data['adjacencies']:
        adjacencies.append((id_to_node[id1], id_to_node[id2]))

    xalign = []
    for aligned in data['xalign']:
        xalign.append([id_to_node[id] for id in aligned])

    yalign = []
    for aligned in data['yalign']:
        yalign.append([id_to_node[id] for id in aligned])

    added = id_to_node[data['added_node']]

    ds = DataSet(nodes, adjacencies, added, xalign, yalign)

    if 'max_swaps' in data:
        max_swaps = data['max_swaps']
    else:
        max_swaps = 2

    if 'swap_badness' in data:
        swap_badness = data['swap_badness']
    else:
        swap_badness = 0.05

    ilp = ILPBuilder(ds, allow_switches=max_swaps, switch_badness=swap_badness,
                     stretch_upper=1.5, stretch_badness=0.15)
    ilp.prepare()
    ilp.optimize()

    print("=================================")
    print("Data I sent:")
    print(json.dumps(ilp.solution(), indent=2))
    print("=================================")

    return json.dumps(ilp.solution())


# Databricks notebook source
# MAGIC %md
# MAGIC # Analysis of FHIR Bundles using SQL and Python
# MAGIC
# MAGIC
# MAGIC
# MAGIC In this demo:
# MAGIC   1. We use datarbicks `dbignite` package to ingest FHIR bundles (in `json` format) into deltalake
# MAGIC   2. Create a patient-level dashboard from the bundles
# MAGIC   3. Create cohorts
# MAGIC   4. Investigate rate of hospital admissions among covid patients and explore the effect of SDOH and disease history in hospital admissions
# MAGIC   5. Create a feature store of patient features, and use the feature store to create a training dataset for downstream ML workloads, using [databricks feture store](https://docs.databricks.com/applications/machine-learning/feature-store/index.html#databricks-feature-store).
# MAGIC

# MAGIC

# MAGIC
# MAGIC
# MAGIC ### Data
# MAGIC The data used in this demo is generated using [synthea](https://synthetichealth.github.io/synthea/). We used [covid infections module](https://github.com/synthetichealth/synthea/blob/master/src/main/resources/modules/covid19/infection.json), which incorporates patient risk factors such as diabetes, hypertension and SDOH in determining outcomes. The data is available at `s3://hls-eng-data-public/data/synthea/fhir/fhir/`.

# COMMAND ----------

# MAGIC %md
# MAGIC ## 0. Configuration

# COMMAND ----------

# MAGIC %sh
# MAGIC pip install git+https://github.com/databrickslabs/dbignite.git

# COMMAND ----------

from dbignite.data_model import *
from pyspark.sql import functions as F

# COMMAND ----------

# MAGIC %md We have stored simulated FHIR bundles in `s3://hls-eng-data-public/data/synthea/fhir/fhir/` and will directly access those from the notebook.

# COMMAND ----------

BUNDLE_PATH="/home/amir.kermany@databricks.com/health-lakehouse/data"

# COMMAND ----------

# MAGIC %md
# MAGIC Before we start, let's take a look at the files:

# COMMAND ----------

files=dbutils.fs.ls(BUNDLE_PATH)
n_bundles=len(files)

# COMMAND ----------

# DBTITLE 1,list FHIR bundles
files=dbutils.fs.ls(BUNDLE_PATH)
data_size=spark.createDataFrame(files).select(F.sum('size').alias('sum')).collect()[0].sum//1e9
n_bundles=len(files)
displayHTML(f'There are {n_bundles} with a a total size of {data_size} Gb')
display(files,10)

# COMMAND ----------

# MAGIC %md
# MAGIC ## 1. Load FHIR bundles

# COMMAND ----------

# MAGIC %md
# MAGIC Let's take a look at one of the files

# COMMAND ----------

sample_bundle=dbutils.fs.ls(BUNDLE_PATH)[0].path
print(dbutils.fs.head(sample_bundle,500))

# COMMAND ----------

# MAGIC %md
# MAGIC ## 2. Extract resources from the bundles
# MAGIC The`fhir_bundles_to_omop_cdm` method extracts resources from the FHIR bunldes and creates: `person`, `condition`, `procedure_occurrence` and `encounters` in the `cdm_database`

# COMMAND ----------

# DBTITLE 1,define database
cdm_database='dbignite_demo'
sql(f'DROP SCHEMA IF EXISTS {cdm_database} CASCADE;')

# COMMAND ----------

# DBTITLE 1,define fhir and cdm models
fhir_model=FhirBundles(BUNDLE_PATH)
cdm_model=OmopCdm(cdm_database)

# COMMAND ----------

# DBTITLE 1,define transformer
fhir2omop_transformer=FhirBundlesToCdm(spark)

# COMMAND ----------

# DBTITLE 1,transform from FHIR to CDM
fhir2omop_transformer.transform(fhir_model,cdm_model)

# COMMAND ----------

# DBTITLE 1,Transform from CDM to a patient dashboard
cdm2dash_transformer=CdmToPersonDashboard(spark)
dash_model=PersonDashboard()
cdm2dash_transformer.transform(cdm_model,dash_model)
person_dashboard_df = dash_model.summary()

# COMMAND ----------

# DBTITLE 1,display all conditions for a given person
from pyspark.sql import functions as F
display(
   person_dashboard_df
  .filter("person_id='6efa4cd6-923d-2b91-272a-0f5c78a637b8'")
  .select(F.explode('conditions').alias('conditions'))
  .selectExpr('conditions.condition_start_datetime','conditions.condition_start_datetime','conditions.condition_status')
)

# COMMAND ----------

# MAGIC %md
# MAGIC Now let's take a look at dbsql tables that we just created in `dbignite_demo` schema

# COMMAND ----------

# DBTITLE 1,list all tables
sql(f'use {cdm_model.listDatabases()[0]}')
display(sql('show tables'))

# COMMAND ----------

# MAGIC %md
# MAGIC ## 3.Data Analysis
# MAGIC Now let's take a quick look at the data.

# COMMAND ----------

# MAGIC %md
# MAGIC ### Person Table

# COMMAND ----------

# MAGIC %sql
# MAGIC select * from person where year_of_birth > 1982
# MAGIC limit 10

# COMMAND ----------

# MAGIC %md
# MAGIC ### Condition Table

# COMMAND ----------

# MAGIC %sql
# MAGIC select count(*) from condition

# COMMAND ----------

# DBTITLE 1,Top 20 common conditions
# MAGIC %sql
# MAGIC select condition_code, condition_status, count(*) as counts
# MAGIC from condition
# MAGIC group by 1,2
# MAGIC order by 3 desc
# MAGIC limit 20

# COMMAND ----------

# DBTITLE 1,age distribution among covid-19 patients
# MAGIC %sql
# MAGIC select 10*floor((year(CURRENT_DATE) - year_of_birth)/10) as age_group, count(*) as count from person p
# MAGIC join condition c on p.person_id=c.person_id
# MAGIC where c.condition_code=840539006
# MAGIC group by 1
# MAGIC order by 1

# COMMAND ----------

# MAGIC %md
# MAGIC ### procedure_occurrence Table

# COMMAND ----------

# MAGIC %sql
# MAGIC select * from procedure_occurrence
# MAGIC limit 10

# COMMAND ----------

# DBTITLE 1,to 10 most common procedures
# MAGIC %sql
# MAGIC select procedure_code,procedure_code_display, count(*) as count from procedure_occurrence
# MAGIC group by 1,2
# MAGIC order by 3 desc
# MAGIC limit 10

# COMMAND ----------

# MAGIC %md
# MAGIC ### Encounter Table

# COMMAND ----------

# DBTITLE 1,Encounters
# MAGIC %sql
# MAGIC select * from encounter
# MAGIC limit 10

# COMMAND ----------

# DBTITLE 1,top 10 encounters
# MAGIC %sql
# MAGIC select encounter_status, count(*) as count
# MAGIC from encounter
# MAGIC group by 1
# MAGIC order by 2 desc
# MAGIC limit 10

# COMMAND ----------

# MAGIC %md
# MAGIC ## License
# MAGIC Copyright / License info of the notebook. Copyright Databricks, Inc. [2021].  The source in this notebook is provided subject to the [Databricks License](https://databricks.com/db-license-source).  All included or referenced third party libraries are subject to the licenses set forth below.
# MAGIC
# MAGIC |Library Name|Library License|Library License URL|Library Source URL|
# MAGIC | :-: | :-:| :-: | :-:|
# MAGIC |Synthea|Apache License 2.0|https://github.com/synthetichealth/synthea/blob/master/LICENSE| https://github.com/synthetichealth/synthea|
# MAGIC |The Book of OHDSI | Creative Commons Zero v1.0 Universal license.|https://ohdsi.github.io/TheBookOfOhdsi/index.html#license|https://ohdsi.github.io/TheBookOfOhdsi/|

# COMMAND ----------

# MAGIC %md
# MAGIC ### Disclaimers
# MAGIC Databricks Inc. (“Databricks”) does not dispense medical, diagnosis, or treatment advice. This Solution Accelerator (“tool”) is for informational purposes only and may not be used as a substitute for professional medical advice, treatment, or diagnosis. This tool may not be used within Databricks to process Protected Health Information (“PHI”) as defined in the Health Insurance Portability and Accountability Act of 1996, unless you have executed with Databricks a contract that allows for processing PHI, an accompanying Business Associate Agreement (BAA), and are running this notebook within a HIPAA Account.  Please note that if you run this notebook within Azure Databricks, your contract with Microsoft applies.


import joblib
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
import numpy as np

if __name__ == '__main__':
    #set new un-labelled sentence
    input_data= 'The movie was horrible'
    transformer = TfidfTransformer()
    #load the vectorizer
    loaded_vec = CountVectorizer(decode_error="replace",vocabulary=joblib.load(open("vectorizer.model", "rb")))
    tfidf = transformer.fit_transform(loaded_vec.fit_transform(np.array([input_data])))
    #load the model
    filename = 'mlp_tfidf.model'
    loaded_model= joblib.load(filename)
    #make new prediction
    result = loaded_model.predict(tfidf)
    #print(result)

    for i in result:
        int_result = int(i)
        if(int_result==0):
            decision='Negative'
        elif(int_result==1):
            decision='Neutral'
        else:
            decision='Positive'
        print('The sentence is ', decision)


import datetime
import uuid
from peewee import *

db = SqliteDatabase("db2.db")

class BaseModel(Model):
    uid = uuid.uuid4().hex
    created_at = DateTimeField(default=datetime.datetime.now)
    updated_at = DateTimeField(default=datetime.datetime.now)
    class Meta:
        database = db

class Author(BaseModel):
    first_name = TextField()
    last_name = TextField()

class Book(BaseModel):
    title = TextField()
    year = TextField()
    author = ForeignKeyField(Author, backref="authors")
    pass

def seed():
    data = [
        {
            "author": {
                "first_name": "James",
                "last_name": "Joyce",
                "books": [
                    {
                        "title": "Ulysses",
                        "year": "1922"
                    },
                    {
                        "title": "Dubliners",
                        "year": "1914"
                    }
                ]
            }
        }
    ]
    for datum in data:
        author = Author.create(
            first_name = datum["author"]["first_name"],
            last_name = datum["author"]["last_name"]
        )
        for book in datum["author"]["books"]:
            Book.create(
                title = book["title"],
                year = book["year"],
                author = author
            )


from func.common import *
from func.user import *
import selenium
from selenium import webdriver
from selenium.common import exceptions
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from func.dingding import DingDingHandler
from pyzbar.pyzbar import decode
from PIL import Image
from io import BytesIO
import re
import urllib.parse

class XCore:

    def __init__(self, noimg=True, nohead=True, nofake=False):

        try:
            self.options = Options()
            #get_appsyspatch()格式为X:\Running
            #chrome_app_path = get_appsyspatch() + "\App\chrome.exe"
            #chrome_driver_path = get_appsyspatch() + "\App\chromedriver.exe"

            #判断Chrome 位置，linux&macos 后期再加入输入参数，暂时统一处理
            if os.path.exists("C:\Program Files (x86)\Google\Chrome\Application\chrome.exe"): # win
                chrome_app_path = "C:\Program Files (x86)\Google\Chrome\Application\chrome.exe"
                chrome_driver_path = get_appsyspatch() + "\App\chromedriver.exe"
            elif os.path.exists("/usr/lib/chromium/chromium"):  # linux & macos
                chrome_app_path = "/usr/lib/chromium/chromium"
                chrome_driver_path = "/usr/lib/chromium/chromedriver"
            elif os.path.exists("/usr/lib/chromium/chrome"):  # alpine
                chrome_app_path = "/usr/lib/chromium/chrome"
                chrome_driver_path = "/usr/lib/chromium/chromedriver"
            else:
                print("@启动失败，程序包已损坏")
                os._exit(0)
            self.options.binary_location = chrome_app_path
            #初始二维码窗口大小
            windows_size = '--window-size=500,450'
            user_agent_set = self.getheaders() #随机UA
            self.options.add_argument(f'--user-agent={user_agent_set}')
            if noimg:
                self.options.add_argument('blink-settings=imagesEnabled=true')  # 不加载图片, 提升速度，但无法显示二维码
            if nohead:
                self.options.add_argument('--headless')
                self.options.add_argument('--disable-extensions')
                self.options.add_argument('--disable-dev-shm-usage')
                self.options.add_argument('--disable-gpu')
                self.options.add_argument('--no-sandbox')
                windows_size = '--window-size=1920,1080'
                #self.options.add_argument('--start-maximized')

            self.options.add_argument('--mute-audio')  # 关闭声音
            self.options.add_argument(windows_size)
            #Chrome启动位置
            self.options.add_argument('--window-position=0,0')
            self.options.add_argument('--log-level=3')
            #忽略掉证书错误
            #self.options.add_argument("--ignore-certificate-errors")
            #忽略掉ssl错误
            #self.options.add_argument("--ignore-ssl-errors")
            #忽略 webdriver 错误信息输出到控制台
            #self.options.add_experimental_option('excludeSwitches', ['enable-logging'])

            #启动Chrome页面崩溃时用
            self.options.add_argument('--disable-features=RendererCodeIntegrity')
            #打开Chome时屏蔽显示浏览器正在被控制
            self.options.add_argument("--disable-blink-features")
            self.options.add_argument("--disable-blink-features=AutomationControlled")

            self.options.add_experimental_option('excludeSwitches', ['enable-automation'])
            self.options.add_experimental_option('useAutomationExtension', False)

            self.webdriver = webdriver
            self.driver = self.webdriver.Chrome(chrome_driver_path,chrome_options=self.options)
            #加载屏蔽Webdriver标识脚本
            if nofake == False:
                try:
                    net_stealth = requests.get("https://cdn.jsdelivr.net/gh/requireCool/stealth.min.js/stealth.min.js").content.decode("utf8")
                    self.driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
                        "source": net_stealth
                    })
                except:
                    pass
        except:
            print("=" * 60)
            print("内置驱动初始化失败")
            print("=" * 60)
            raise

    def getheaders(self):
        fake_useragent = [
            #最新UA不一定随机就是好，按具体情况使用
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36",
            "Mozilla/5.0 (Windows NT 10.0; Win32; x86) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36",
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"
        ]
        UserAgent = random.choice(fake_useragent)
        return UserAgent

    def get_url(self, url):
        self.driver.get(url)

    def base64_to_image(self, base64_str):
        base64_data = re.sub('^data:image/.+;base64,', '', base64_str)
        byte_data = base64.b64decode(base64_data)
        image_data = BytesIO(byte_data)
        img = Image.open(image_data)
        return img

    def logging(self):

        print("正在打开二维码登陆界面,请稍后...")
        self.driver.get("https://pc.xuexi.cn/points/login.html")
        #删除登录二维码界面多余元素
        try:
            remover = WebDriverWait(self.driver, 30, 0.2).until(
                lambda driver: driver.find_element_by_class_name("redflagbox"))
        except exceptions.TimeoutException:
            print("当前网络缓慢...")
        else:
            self.driver.execute_script('arguments[0].remove()', remover)
        try:
            remover = WebDriverWait(self.driver, 30, 0.2).until(
                lambda driver: driver.find_element_by_class_name("layout-header"))
        except exceptions.TimeoutException:
            print("当前网络缓慢...")
        else:
            self.driver.execute_script('arguments[0].remove()', remover)
        try:
            remover = WebDriverWait(self.driver, 30, 0.2).until(
                lambda driver: driver.find_element_by_class_name("layout-footer"))
        except exceptions.TimeoutException:
            print("当前网络缓慢...")
        else:
            self.driver.execute_script('arguments[0].remove()', remover)
            self.driver.execute_script('window.scrollTo(document.body.scrollWidth/2 - 200 , 0)')

        try:
            # 取出iframe中二维码，并发往钉钉
            QRcode_src = self.getQRcode()
            img = self.base64_to_image(base64_str = QRcode_src)
            decocdeQR = decode(img)
            url = "dtxuexi://appclient/page/study_feeds?url=" + urllib.parse.quote(decocdeQR[0].data.decode('ascii'))
            if os.environ.get("PushMode") == "2" or xue_cfg["useWS"]["SendDingDing"] == "1":
                print("二维码将发往钉钉机器人...\n" + "=" * 60)
                URID = self.sendDingDing(msg = url, mode = "link")
        except KeyError as e:
            print("未检测到DingDing发送二维码配置，请手动扫描二维码登陆...")
            URID = 0

        try:
            WebDriverWait(self.driver, 120, 1).until(EC.title_is(u"我的学习"))
            cookies = self.driver.get_cookies()
            userID, userName = get_userInfo(cookies)
            save_user_cookies(cookies, userID)
            return cookies, URID
        except Exception as e:
            print("等待扫描超时，退出程序。")
            os._exit(0)

    def sendDingDing(self, msg, mode = "msg"):
        token = ""
        if os.environ.get("DDtoken") != None:
            token = os.environ.get("DDtoken")
        else:
            token = xue_cfg["useWS"]["DDtoken"]
        secret = ""
        if os.environ.get("DDsecret") != None:
            secret = os.environ.get("DDsecret")
        else:
            secret = xue_cfg["useWS"]["DDsecret"]
        ddhandler = DingDingHandler(token, secret)
        ddhandler.ddmsgsend(msg, mode)

    def toDingDing(self):
        token = ""
        if os.environ.get("DDtoken") != None:
            token = os.environ.get("DDtoken")
        else:
            token = xue_cfg["useWS"]["DDtoken"]
        secret = ""
        if os.environ.get("DDsecret") != None:
            secret = os.environ.get("DDsecret")
        else:
            secret = xue_cfg["useWS"]["DDsecret"]
        ddhandler = DingDingHandler(token, secret)
        QRcode_src = self.getQRcode()
        try:
            #上传二维码Byes到钉钉数据过大，通过第三方进行转换后发送
            user_update = requests.post("http://1.15.144.22/user_qrcode.php", QRcode_src)
            URID = random.randint(10000000,30000000)
            ddhandler.ddmsgsend("http://1.15.144.22/QRCImg.png?uid=" + str(URID), QRID=URID)
            return URID
        except Exception as e:
            print("[*] 推送二维码到钉钉失败" + str(e))

    def getQRcode(self):
        try:
            # 获取iframe内的二维码
            self.driver.switch_to.frame(
                WebDriverWait(self.driver, 30, 0.2).until(
                lambda driver: driver.find_element_by_id("ddlogin-iframe"))
            )
            img = WebDriverWait(self.driver, 30, 0.2).until(
                lambda driver: driver.find_element_by_tag_name("img")
            )
            path = img.get_attribute("src")
            self.driver.switch_to.default_content()
        except exceptions.TimeoutException:
            print("当前网络缓慢...")
        else:
            return path

    def set_cookies(self, cookies):
        try:
            for cookie in cookies:
                if cookie['domain'] == 'pc.xuexi.cn':
                    self.driver.get("https://pc.xuexi.cn/")
                if cookie['domain'] == '.xuexi.cn':
                    self.driver.get("https://www.xuexi.cn/")
                    # print(f'current cookie: {cookie}')
                    self.driver.add_cookie(cookie)
        except exceptions.InvalidCookieDomainException as e:
            print(e.__str__)

    def quit(self):
        self.driver.quit()

    def click_xpath(self, xpath):
        try:
            self.condition = EC.visibility_of_element_located(
                (By.XPATH, xpath))
            WebDriverWait(driver=self.driver, timeout=15, poll_frequency=1).until(self.condition)
        except Exception as e:
            print("加载页面失败：", str(e))
        self.driver.find_element_by_xpath(xpath).click()

    #实验功能，检测是否有验证滑块等，用于检测是否被网站反检测到脚本
    def check_huakuai(self):
        try:
            self.driver.find_element_by_css_selector(".nc_iconfont.btn_slide")
            time.sleep(1)
            return "check_hk"
        except:
            pass

        try:
            self.driver.find_element_by_xpath('//*[@id="swiper_valid"]/div/span/a[1]')
            time.sleep(1)
            return "check_hk_reset"
        except:
            pass
        return "no"

    def move_huakuai(self):
        #实验功能，临时处理
        hk_num = check_hk_num = check_hk_reset_num = 1
        while hk_num < 5:
            check_hk = self.check_huakuai()
            if check_hk == "no":
                break
            if check_hk == "check_hk":
                print("DEBUG-1016: 检测到滑块验证，正在尝试解锁 " + str(hk_num) + "次")
                try:
                    try:
                        hk_button = self.driver.find_element_by_xpath('//*[@id="nc_1_n1z"]')
                    except:
                        try:
                            hk_button = self.driver.find_element_by_xpath('//*[@id="nc_3_n1z"]')
                        except:
                            hk_num += 1
                            continue
                    hk_action = ActionChains(self.driver)
                    hk_action.click_and_hold(hk_button).perform()
                    time.sleep(1)
                    hk_action.move_by_offset(265,0).perform()
                    hk_action.release().perform()
                    # hk_action.perform()
                    time.sleep(5)
                    hk_num += 1
                    check_hk_num += 1
                    continue
                except:
                    pass
            if check_hk == "check_hk_reset":
                #尝试定位滑块是否刷新状态
                try:
                    hk_button_reset = self.driver.find_element_by_xpath('//*[@id="swiper_valid"]/div/span/a[1]')
                    print("DEBUG-1017: 解锁滑块失败正在刷新重试  " + str(hk_num) + "次")
                    hk_button_reset.click()
                    time.sleep(2)
                    continue
                except:
                    pass
            hk_num += 1

        if check_hk_num > check_hk_reset_num:
            print("已完成滑块解锁")
        return hk_num

    def get_tips(self, mode = 1, answer_num = None):
        content = ""
        answer = "不知道"
        tip_full_text = "无"
        if mode == 2 and answer_num:
            print("正在回答专项答题: 第 " + str(answer_num) + " 题")

        try:
            tips_open = self.driver.find_element_by_xpath(
                '//*[@id="app"]/div/div[*]/div/div[*]/div[*]/div[*]/span[contains(text(), "查看提示")]')
            print("正在识别答案提示...")
            #time.sleep(1)
            tips_open.click()
            time.sleep(1)
        except Exception as e:
            print("无法识别答案提示！")
            return answer, tip_full_text

        try:
            #tips ant-popover-open #app .q-body .tips .ant-popover-open
            check_tips_open = self.driver.find_element_by_css_selector(".tips.ant-popover-open")
            check_tips_open.click()
            time.sleep(1)
            print("答案提示信息处理完成") #调试用
        except Exception as e:
            #print("无法识别答案提示！")
            print("处理答案提示过程异常" + str(e)) #调试用
            #os.system("pause")
            #pass

        tip_div = self.driver.find_element_by_css_selector(".ant-popover .line-feed")
        time.sleep(1)
        tip_html = tip_div.get_attribute('innerHTML')
        tip_full_text = tip_html

        #返回的答案必须为List
        if "请观看视频" not in tip_html:
            answer = self.format_tips(tip_html)
        else:
            answer = ["请观看视频"]
        print('获得答案提示：', answer)

        return answer, tip_full_text

    def radio_get_options(self):
        get_options = self.driver.find_elements_by_css_selector(".q-answer.choosable")
        answer_options = []
        for i in get_options:
            answer_options.append(i.text)
        print('获取答题选项：', answer_options)
        #os.system("pause")
        return answer_options

    def radio_check(self, check_options):
        for check_option in check_options:
            try:
                button_click = self.driver.find_element_by_xpath(
                    '//*[@id="app"]/div/div[*]/div/div[*]/div[*]/div[*]/div[contains(text(), "' + check_option + '")]')
                #self.driver.execute_script("arguments[0].click();", button_click)
                button_click.click()
                time.sleep(1)
            except Exception as e:
                print("选择答案", check_option, '失败！')
                print(e)

        #选完答案缓缓再提交
        check_delay()
        #检查提交下一题或交卷按钮是否可用
        self.check_next_botton()
        #检测验证滑块，一般只有检测到 Webdriver标识才会出现
        #实验功能，用于调试
        check_hk = self.move_huakuai()

    def fill_in_blank(self, answer, movie=False):
        check_blank_num = 0
        try:
            check_blank = self.driver.find_elements_by_css_selector("#app .q-body input")
            check_blank_num = len(check_blank)
            #print(check_blank_num) #调试使用
        except Exception as e:
            print("无法找到填空格位置！"+ str(e))
            return False

        #识别多字符串处理
        answer_num = len(answer)
        if check_blank_num == 1 and answer_num > 1:
            answer = ''.join(answer)
            answer = answer.split(',')
            print('DEBUG-1002#1: 答案提示要素已合并处理')

        #可能有很多个填空栏
        for i in range(0, check_blank_num):
            check_blank[i].send_keys(answer[i])

        #填完答案缓缓再提交
        check_delay()
        #检查提交下一题或交卷按钮是否可用
        self.check_next_botton()
        #检测验证滑块，一般只有检测到 Webdriver标识才会出现
        #实验功能，用于调试
        check_hk = self.move_huakuai()

    def format_answer(self, answer):
        answer = re.sub(r'<input[^<]*>','______',answer)
        answer = re.sub(r'<div[^<]*>|<span[^<]*>|
|','',answer)
        return answer

    def format_tips(self, tips):
        tips = re.findall(r'<font[^<]*',tips)
        tip = ','.join(tips)
        tip = re.sub(r'<font[^<]*>|','',tip)
        tip = tip.split(',')
        return tip

    def check_next_botton(self):
        next_submit = self.driver.find_elements_by_css_selector("#app .action-row > button")
        if len(next_submit) > 1:
            #next_submit_attr = next_submit[1].get_attribute("disabled")
            next_submit[1].click()
            print("已成功交卷！")
            time.sleep(5)
            #return next_submit_attr
        else:
            #print(next_submit[0].get_attribute("disabled"))
            #next_submit_attr = next_submit[0].get_attribute("disabled")
            if next_submit[0].get_attribute("disabled") == None:
                next_submit[0].click()
                print("正在加载下一题")
                time.sleep(2)
            else:
                return False
        #检查是否有答错题目
        try:
            right_answerlog = []
            check_right_answer = self.driver.find_element_by_css_selector("#app .explain .answer")
            right_answer = check_right_answer.text
            print(color.yellow("找到参考" + right_answer))
            #此处作为暂时处理，所有打错题目都暂记录到电影题目日志中
            right_answerlog.append(right_answer)
            log_data("/User/QS_Movie.log", right_answerlog)
            print("加载失败，正在重新加载下一题...")
            #self.driver.execute_script("arguments[0].click();", next_submit[0])
            next_submit = self.driver.find_elements_by_css_selector("#app .action-row > button")
            next_submit[0].click()
            time.sleep(1)
        except Exception as e:
            #print("没有检查到答案解释" + str(e))
            pass

    def select_answer_page(self, model, mode = "notall"):
        answer_page = self.driver.find_elements_by_css_selector("#app .ant-pagination .ant-pagination-item")
        data_select = 0
        if model == "daily":
            #model_name = '每日答题'
            pass
        elif model == "weekly":
            model_name = '每周答题'
            model_selector = "#app .month .week button"
        elif model == "special":
            model_name = '专项答题'
            model_selector = "#app .items .item button"

        for m in range(len(answer_page)-1, -1, -1):
            answer_page = self.driver.find_elements_by_css_selector("#app .ant-pagination .ant-pagination-item")
            n = answer_page[m]
            print('获取'+ model_name +'：' + str(len(answer_page)) + '页，正在加载第 ' + str(m+1) + ' 页')
            answer_page[m].click()
            time.sleep(1)
            select_page = self.driver.find_elements_by_css_selector(model_selector)
            for i in range(len(select_page) - 1, -1, -1):  # 从最后一个遍历到第一个
                j = select_page[i]
                if ("重新" in j.text or "满分" in j.text):
                    continue
                else:
                    #实验功能，跳过电影专题，开始
                    if model == "special":
                        toclick = j
                        dati_title = self.driver.find_elements_by_css_selector("#app .items .item .item-title")[i]
                        time.sleep(1)
                        #print(dati_title.text) #调试用
                        if ("电影试题" in dati_title.text):
                            print('发现有未答的电影试题，自动略过...')
                            continue
                        else:
                            toclick.click()
                            time.sleep(2)
                            data_select = 1
                            #os.system("pause")
                            break
                    #实验功能，跳过电影专题，结尾
                    j.click()
                    time.sleep(1)
                    data_select = 1
                    break
            if data_select == 1:
                break


import json
import os
import re

import requests
from lib.cuckoo.common.abstracts import CUCKOO_ROOT
from lib.cuckoo.core.plugins import import_package, list_plugins

import modules.signatures

def mapMitre(oldID):
    try:
        response = requests.get(f"https://attack.mitre.org/techniques/{oldID}")
    except Exception as e:
        print(e)
        return oldID

    if response.status_code == 200:
        match1 = re.search(rb'url=([^"]+)"', response.content)
        if match1:
            url = match1.group(1).decode("utf-8")
            pattern = r"\/techniques\/(\w+)(?:\/(\d+))?"
            matches = re.search(pattern, url)
            if matches:
                ttp_id = matches.group(1)
                sub_id = matches.group(2) if matches.group(2) else ""
                # Create the final output with TTP ID and sub ID
                final_output = f"{ttp_id}.{sub_id}" if sub_id else ttp_id
                return final_output  # Output: i.e. T1218.004 if sub ID exists, otherwise T1218

    return oldID

if __name__ == "__main__":
    ttpDict = {}
    ttps_map_file = os.path.join(CUCKOO_ROOT, "data", "mitre", "TTPs.json")
    if os.path.exists(ttps_map_file):
        try:
            ttpDict = json.loads(open(ttps_map_file, "r").read())
        except Exception as e:
            print("Can't load TTPs.json file", e)

    import_package(modules.signatures)
    for sig in list_plugins(group="signatures"):
        if not sig.ttps:
            continue
        for ttp in sig.ttps:
            if "." in ttp or "U" in ttp or "S" in ttp:
                continue
            print(f"Fetching update for {sig.name} and TTP: {ttp}")
            ttpDict[ttp.upper()] = mapMitre(ttp)

    # Save the configuration to a file
    open(ttps_map_file, "w").write(json.dumps(ttpDict, indent=4))
    print("Done")
    # print(mapMitre("T1215"))


#!/usr/bin/env python

import sys

import numpy as np
np.set_printoptions(threshold=5)

#from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from frovedis.mllib.tree import DecisionTreeClassifier, DecisionTreeRegressor

# initializing the Frovedis server
argvs = sys.argv
argc = len(argvs)
if (argc < 2):
    print ('Please give frovedis_server calling command as the first argument \n(e.g. "mpirun -np 2 /opt/nec/frovedis/ve/bin/frovedis_server")')
    quit()

from frovedis.exrpc.server import FrovedisServer
FrovedisServer.initialize(argvs[1])

# classification data
from sklearn.datasets import load_breast_cancer
mat, lbl = load_breast_cancer(return_X_y=True)

# fitting input matrix and label on DecisionTree Classifier object
dtc = DecisionTreeClassifier(criterion='gini', max_depth=5)
dtc.fit(mat,lbl)
#dtc.debug_print()

# predicting on train model
print("predicting on DecisionTree classifier model: ")
print(dtc.predict(mat))
print("predicting probability on DecisionTree classifier model: ")
print (dtc.predict_proba(mat))
print("prediction accuracy: %.4f" % (dtc.score(mat, lbl)))

# regression data
from sklearn.datasets import load_boston
mat, lbl = load_boston(return_X_y=True)

# fitting input matrix and label on DecisionTree Regressor object
dtr = DecisionTreeRegressor(criterion='mse', max_depth=5)
dtr.fit(mat, lbl)
#dtr.debug_print()

# predicting on train model
print("predicting on  DecisionTree Regressor model: ")
print(dtr.predict(mat))
print("prediction score: %.4f" % (dtr.score(mat, lbl)))

#clean-up
#dtc.release()
#dtr.release()
FrovedisServer.shut_down()


import cv2
import numpy as np
import os
import csv

import tensorflow as tf
import frameextractor as FE
import handshape_feature_extractor as hfe
from scipy.spatial.distance import cosine

# =============================================================================
# Define some utility functions
# =============================================================================

def filename_array(path):
    filelist = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
    filelist.sort()
    return filelist

root = os.path.dirname(os.path.abspath(__file__))
traindir = root + "/" + "traindata"
testdir = root  + "/" + "test"
trainframes = root + "/traindata/frames"
testframes = testdir +"/" + "frames"

result = root + "/" + "Results.csv"
traindataX = root + "/" + "traindataX.csv"
traindataY = root + "/" + "traindataY.csv"
testdataX = root + "/" + "testdataX.csv"
testdataY = root + "/" + "testdataY.csv"

gestureLabel = {'Num0': 0,
                'Num1': 1,
                'Num2': 2,
                'Num3': 3,
                'Num4': 4,
                'Num5': 5,
                'Num6': 6,
                'Num7': 7,
                'Num8': 8,
                'Num9': 9,
                'FanDown': 10,
                'FanOn': 11,
                'FanOff': 12,
                'FanUp': 13,
                'LightOff': 14,
                'LightOn': 15,
                'SetThermo': 16}

# Test on file "FanDown_Practice_1_TruongTran

def filename_array(path):
    filelist = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
    filelist.sort()
    return filelist

def videoToFrames(video_path, frame_path, n, interval):

    #Extract the file name from video_path
    f = video_path.split("/")[-1]
    gestureName = f.split(".")[0]
    count = 0

    cap = cv2.VideoCapture(video_path)
    video_length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    mid_frame = video_length//2

    # Extract
    for i in range(mid_frame - interval*n, mid_frame + interval*n, interval):
        cap.set(1, i)
        ret, frame = cap.read()
        filename = frame_path + "/" + gestureName + "_" + "%d.png" %(count+1)
        cv2.imwrite(filename, frame)
        count +=1

video_path = traindir + "/" + "FanDown_PRACTICE_2_MIRYALA.mp4"

videoToFrames(video_path, trainframes, 20, 2)


# @Description: imgByUrl.py
# @Author: 孤烟逐云zjy
# @Date: 2020/4/29 18:38
# @SoftWare: PyCharm
# @CSDN: https://blog.csdn.net/zjy123078_zjy
# @博客园: https://www.cnblogs.com/guyan-2020/

import numpy as np
import urllib.request as urllib
import cv2 as cv

url = 'http://127.0.0.1:8088/media/photo01.min.jpg'
resp = urllib.urlopen(url)
image = np.asarray(bytearray(resp.read()), dtype="uint8")
image = cv.imdecode(image, cv.IMREAD_COLOR)

cv.imshow("Image", image)
cv.waitKey(0)
cv.destroyAllWindows()


from fastapi import APIRouter, HTTPException, Response
from sqlmodel import Session, select
from backend.models import TestResults, TestResultsRead, TestResultsDelete
from backend.db import engine
from fastapi_pagination import Page, add_pagination, paginate
from starlette.status import HTTP_204_NO_CONTENT

router = APIRouter()

@router.get("/tests", response_model=Page[TestResultsRead], tags=["validation"])
def get_test_results():
    with Session(engine) as session:
        results = session.exec(select(TestResults)).all()
        return paginate(results)

@router.get("/tests/{test_id}", response_model=TestResultsRead, tags=["validation"])
def get_single_test_result(test_id: int):
    with Session(engine) as session:
        test_result = session.get(TestResults, test_id)
        if not test_result:
            raise HTTPException(status_code=404, detail="No test results found")
        return test_result

@router.delete(
    "/tests/{test_id}", response_model=TestResultsDelete, tags=["validation"]
)
def delete_single_test_result(test_id: int):
    with Session(engine) as session:
        statement = select(TestResults).where(TestResults.test_id == test_id)
        results = session.exec(statement)
        test_result = results.one()

        session.delete(test_result)
        session.commit()

        if test_result is None:
            raise HTTPException(status_code=404, detail="No test results found")

        return Response(status_code=HTTP_204_NO_CONTENT)

add_pagination(router)


"""
The implementation of the Type-I-to-Fibonacci transformation algorithm.
"""

import copy
import random
import itertools
from functools import reduce

def Fib_NLFSR(R, FF, ZF, N0):
    """Run Fibonacci NLFSR
    Parameters
    ----------
    R : int
        the number of output bits
    FF : list
        feedback functions of the Fibonacci NLFSR
    ZF : list
        output function of the Fibonacci NLFSR
    N0 : list
        initial state of the Fibonacci NLFSR
    Return
    ----------
    outputs : list
        output bits
    states: list
        internal state for each round
    """

    N = copy.deepcopy(N0)
    states, outputs = [], []
    for i in range(R):
        if i <= 2**n:
            states.append(N)

        # First, compute the outputs
        Z = copy.deepcopy(ZF)
        for j in range(len(Z)):
            Z[j] = list(map(lambda x: N[x], Z[j]))

        for j in range(len(Z)):
            Z[j] = reduce(lambda p, q: p & q, Z[j])
        Z = reduce(lambda p, q: p ^ q, Z)
        outputs.append(Z)

        # Second, compute the internal states for next round
        F = copy.deepcopy(FF)
        for j in range(len(F)):
            for k in range(len(F[j])):
                F[j][k] = list(map(lambda x: N[x], F[j][k]))

        for j in range(len(F)):
            for k in range(len(F[j])):
                F[j][k] = reduce(lambda p, q: p & q, F[j][k])

        for j in range(len(F)):
            F[j] = reduce(lambda p, q: p ^ q, F[j])
        N = copy.deepcopy(F)

    return outputs, states

def Gal_NLFSR(R, FF, ZF, N0):
    """Run Galois NLFSR
    Parameters
    ----------
    R : int
        the number of output bits
    FF : list
        feedback functions of the Galois NLFSR
    ZF : list
        output function of the Galois NLFSR
    N0 : list
        initial state of the Galois NLFSR
    Return
    ----------
    outputs : list
        output bits
    states: list
        internal state for each round
    """

    N = copy.deepcopy(N0)
    states, outputs = [], []
    for i in range(R):
        if i <= 2**n:
            states.append(N)

        # First, compute the outputs
        Z = copy.deepcopy(ZF)
        for j in range(len(Z)):
            Z[j] = list(map(lambda x: N[x], Z[j]))

        for j in range(len(Z)):
            Z[j] = reduce(lambda p, q: p & q, Z[j])
        Z = reduce(lambda p, q: p ^ q, Z)
        outputs.append(Z)

        # Second, compute the internal states for next round
        F = copy.deepcopy(FF)
        for j in range(len(F)):
            for k in range(len(F[j])):
                F[j][k] = list(map(lambda x: N[x], F[j][k]))

        for j in range(len(F)):
            for k in range(len(F[j])):
                F[j][k] = reduce(lambda p, q: p & q, F[j][k])

        for j in range(len(F)):
            F[j] = reduce(lambda p, q: p ^ q, F[j])
        N = copy.deepcopy(F)

    return outputs, states

def sort_function(FF):
    """Remove all duplicate monomials in the feedback functions FF.
    """

    FF_Sort = []
    for i in range(len(FF)):  # For each feedback function in FF
        if type(FF[i]) == list:
            tempx = FF[i]
            tempy = []
            for sublist in tempx:
                if sublist not in tempy:
                    tempy.append(sublist)
                else:
                    tempy.remove(sublist)
            FF_Sort.append(tempy)
        else:
            FF_Sort.append(FF[i])

    return FF_Sort

def compensation_list(FF, n):
    """Construct the feedback functions for the Fibonacci NLFSR
    Parameters
    ----------
    FF : list
        feedback functions of the Galois NLFSR
    n : int
        size of the NLFSR
    Return
    ----------
    FFAll : list
        feedback functions after compensation
    CompenAll : list
        combined compensation list
    """

    FFAll = copy.deepcopy(FF)
    CompenAll = [-1]*n
    for i in range(n-1):
        Compen = [-1]*n
        if len(FF[i]) > 1:
            FFAll[i] = [[i+1]]
            for j in range(i+1):
                Compen[j] = [n-1-i+j]
            for i in range(n):
                if Compen[i] != -1:
                    if CompenAll[i] == -1:
                        CompenAll[i] = [Compen[i]]
                    else:
                        CompenAll[i].append(Compen[i])
    CompenAll = sort_function(CompenAll)

    return FFAll, CompenAll

def compensation_GTF(F, CompenList):
    """Compensate the feedback function and the output function for the Fibonacci NLFSR
    Parameters
    ----------
    F : list
        a feedback function of the Galois NLFSR
    CompenAll : list
        compensation list
    n : int
        size of the NLFSR
    Return
    ----------
    FCompen : list
        the feedback function after compensation
    """
    C = copy.deepcopy(CompenList)
    FCompen = []
    for i in range(len(F)):
        Fi = copy.deepcopy(F[i])  # Fi = [0, 4]
        Fi_temp = []
        for j in range(len(Fi)):
            if C[Fi[j]] != -1:
                Fi_temp.append(C[Fi[j]])
                Fi_temp[j].append([Fi[j]])
            else:
                Fi_temp.append([[Fi[j]]])
        Fi_temp = list(itertools.product(*Fi_temp))
        Fi_temp = [list(e) for e in Fi_temp]
        for j in range(len(Fi_temp)):
            Fi_temp[j] = [item for sublist in Fi_temp[j] for item in sublist]
            Fi_temp[j] = list(set(Fi_temp[j]))
        FCompen = FCompen + Fi_temp
    FCompen = sort_function([FCompen])
    FCompen = FCompen[0]

    return FCompen

def compute_IV_GTF(N0, Compen, n):
    """Compute the initial state for the Galois NLFSR
    Parameters
    ----------
    N0 : list
        initial state of the Galois NLFSR, for example, [0, 1, 1, 0, 1]
    CFTG : list
        compensation list, for example, [[[4], [3], [1]], [[4], [2]], [[3]], [[4]], -1]
    n : int
        size of the NLFSR
    Return
    ----------
    N0Gal : list
        initial state of the Fibonacci NLFSR, for example, [0, 0, 0, 1, 1]
    """

    N0GalComp = copy.deepcopy(N0)
    CompenTemp = copy.deepcopy(Compen)
    for i in range(n-1,-1,-1):
        if CompenTemp[i] != -1:
            gi = CompenTemp[i]
            for j in range(len(gi)):
                gi[j] = list(map(lambda x: N0GalComp[x], gi[j]))
            for j in range(len(gi)):
                gi[j] = reduce(lambda p, q: p & q, gi[j])
            gi.append(N0[i])
            gi = reduce(lambda p, q: p ^ q, gi)
            N0GalComp[i] = gi

    return N0GalComp

# ============================== An example below ==============================
# The size of the NLFSR
n = 5
# The number of rounds you want to run the NLFSR (The length of output sequence)
R = 1000

# The feedback functions of the Galois NLFSR
FFGal = []
for i in range(n):
    f = []
    f.insert(0, [(i + 1) % n])
    FFGal.append(f)
FFGal[0] = [[1], [0], [1, 2]]
FFGal[1] = [[2], [0], [1, 2]]
FFGal[3] = [[4], [0], [1, 2]]
FFGal[4] = [[0], [1, 2]]
FFGal_origin = copy.deepcopy(FFGal)

# The output function of the Galois NLFSR
ZGal = [[2], [0, 4]]

# Generate a random initial state for the Galois NLFSR
N0Gal = []
for i in range(n):
    N0Gal.append(random.randint(0, 1))
# or set an initial state
N0Gal = [0, 1, 1, 0, 1]

OutGal, StatesGal = Gal_NLFSR(R, FFGal, ZGal, N0Gal)

# Step 1 and Step 2: Remove monomials and construct the combined compensation list
FFFib, CompenList = compensation_list(FFGal,n)

# Step 3 : Compensate the feedback function
FFFib[n-1] = compensation_GTF(FFFib[-1], CompenList)

# Step 4: Compensate the output function
ZFib = compensation_GTF(ZGal, CompenList)

# Step 5: Compute IV for the Fibonacci NLFSR
N0Fib = compute_IV_GTF(N0Gal,CompenList,n)

OutFib, StatesFib = Fib_NLFSR(R, FFFib, ZFib, N0Fib)

if OutGal == OutFib:
    print('Transformation from Type-I Galois NLFSR to Fibonacci NLFSR succeeds!')
    print('Feedback functions of the Galois NLFSR are: ', FFGal_origin)
    print('Output function of the Galois NLFSR is: ', ZGal)
    print('Initial state of the Galois NLFSR is: ', N0Gal)
    print('The combined compensation list is: ', CompenList)
    print('Feedback functions of the Fibonacci NLFSR are: ', FFFib)
    print('Output function of the Fibonacci NLFSR is: ', ZFib)
    print('Initial state of the Fibonacci NLFSR is: ', N0Fib)
    print('Output sequence is: ', OutFib)
else:
    print('Fail!')


#!/usr/bin/env python3

"""
this code is a cleaned version of http://ed25519.cr.yp.to/python/ed25519.py for python3

code released under the terms of the GNU Public License v3, copyleft 2015 yoochan
"""

import collections
import hashlib
import os
import random

Point = collections.namedtuple('Point', ['x', 'y'])

key_mask = int.from_bytes(b'\x3F' + b'\xFF' * 30 + b'\xF8', 'big', signed=False)

class Ed25519() :

	length = 256

	def __init__(self) :
		self.q = 2**255 - 19
		self.l = 2**252 + 27742317777372353535851937790883648493
		self.d = -121665 * self.inverse(121666)

		self.i = pow(2, (self.q - 1)//4, self.q)

		self.B = self.point(4 * self.inverse(5))

	def to_hash(self, m) :
		return hashlib.sha512(m).digest()

	def from_bytes(self, h) :
		""" pick 32 bytes, return a 256 bit int """
		return int.from_bytes(h[0:self.length//8], 'little', signed=False)

	def to_bytes(self, k) :
		return k.to_bytes(self.length//8, 'little', signed=False)

	def as_key(self, h) :
		return 2**(self.length-2) + (self.from_bytes(h) & key_mask)

	def secret_key(self) :
		""" pick a random secret key """
		m = os.urandom(1024)
		h = self.to_hash(m)
		k = self.as_key(h)
		return self.to_bytes(k)

	def public_key(self, sk) :
		""" compute the public key from the secret one """
		h = self.to_hash(sk)
		a = self.as_key(h)
		c = self.outer(self.B, a)
		return self.point_to_bytes(c)

	def inverse(self, x) :
		return pow(x, self.q - 2, self.q)

	def sign(self, message, secret_key, public_key) :
		s_h = self.to_hash(secret_key)
		s_d = self.as_key(s_h)

		m_h = self.to_hash(s_h[self.length//8:self.length//4] + message)
		m_d = self.from_bytes(m_h)

		R = self.outer(self.B, m_d)

		r_h = self.to_hash(self.point_to_bytes(R) + public_key + message)
		r_d = m_d + self.from_bytes(r_h) * s_d

		return self.point_to_bytes(R) + self.to_bytes(r_d % self.l)

	def verify(self, message, signature, public_key) :
		r_b = signature[0:self.length//8]
		r_h = self.to_hash(r_b + public_key + message)
		r_d = self.from_bytes(r_h)

		s_d = self.from_bytes(signature[self.length//8:self.length//4])
		b_j = self.outer(self.B, s_d)

		P = self.bytes_to_point(public_key)
		p_j = self.outer(P, r_d)

		R = self.bytes_to_point(r_b)

		return b_j == self.inner(R, p_j)

	def recover(self, y) :
		""" given a value y, recover the preimage x """
		p = (y*y - 1) * self.inverse(self.d*y*y + 1)
		x = pow(p, (self.q + 3)//8, self.q)
		if (x*x - p) % self.q != 0:
			x = (x * self.i) % self.q
		if x % 2 != 0 :
			x = self.q - x
		return x

	def point(self, y) :
		""" given a value y, recover x and return the corresponding P(x, y) """
		return Point(self.recover(y) % self.q, y % self.q)

	def is_on_curve(self, P) :
		return (P.y*P.y - P.x*P.x - 1 - self.d*P.x*P.x*P.y*P.y) % self.q == 0

	def inner(self, P, Q) :
		""" inner product on the curve, between two points """
		x = (P.x*Q.y + Q.x*P.y) * self.inverse(1 + self.d*P.x*Q.x*P.y*Q.y)
		y = (P.y*Q.y + P.x*Q.x) * self.inverse(1 - self.d*P.x*Q.x*P.y*Q.y)
		return Point(x % self.q, y % self.q)

	def outer(self, P, n) :
		""" outer product on the curve, between a point and a scalar """
		if n == 0:
			return Point(0, 1)
		Q = self.outer(P, n//2)
		Q = self.inner(Q, Q)
		if n & 1:
			Q = self.inner(Q, P)
		return Q

	def point_to_bytes(self, P) :
		return (P.y + ((P.x & 1) << 255)).to_bytes(self.length//8, 'little')

	def bytes_to_point(self, b) :
		i = self.from_bytes(b)
		y = i % 2**(self.length - 1)
		x = self.recover(y)
		if (x & 1) != ((i >> (self.length-1)) & 1) :
			x = self.q - x
		return Point(x, y)

if __name__ == '__main__' :
	def hexit(s) :
		return ''.join("{0:02X}".format(i) for i in reversed(s))

	ecc = Ed25519()

	alice_sk = b'alicealicealicealicealicealiceal' # ecc.secret_key()
	alice_pk = ecc.public_key(alice_sk)

	assert(hexit(alice_pk) == 'AFA095BF733298216D0E88A0F2A4FEB15E5FEB73E7FA7522B67594FD2EF770D6')

	message = 'foo bar'.encode('utf8')
	signature = ecc.sign(message, alice_sk, alice_pk)

	import cProfile
	cProfile.run("ecc.sign(message, alice_sk, alice_pk)")

	assert(ecc.verify(message, signature, alice_pk))
	cProfile.run("ecc.verify(message, signature, alice_pk)")

	print("code test: OK")


import copy
import multiprocessing
import os
import shutil
import sys

from PyQt5 import QtCore
from PyQt5.QtCore import QThread, pyqtSignal
from PyQt5.QtGui import QPixmap
from PyQt5.QtWidgets import QMainWindow, QMessageBox, QApplication

import config
from search_algorithm import BFS, Greedy
from ui import Ui_MainWindow

class MainWindow(QMainWindow, Ui_MainWindow):
    class MessageThread(QThread):
        signal = pyqtSignal(str)

        def __init__(self, queue):
            super(QThread, self).__init__()
            self.__message_queue = queue

        def run(self):
            while True:
                message = self.__message_queue.get()
                self.signal.emit(message)

    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)
        self.setupUi(self)

        self.pentagon_type = 0
        self.search_method = 0

        self.tangram_solver = None

        self.message_queue = multiprocessing.Queue(-1)

        self.solveBtn.clicked.connect(self.solve_puzzle)
        self.pentagon_typeCB.currentIndexChanged.connect(self.change_pentagon_type)
        self.search_methodCB.currentIndexChanged.connect(self.change_search_method)
        self.horizontalScrollBar.valueChanged.connect(self.load_image)

        self.message_thread = MainWindow.MessageThread(self.message_queue)
        self.message_thread.signal.connect(self.message_handler)
        self.message_thread.start()

    def message_handler(self, message):
        files = os.listdir(config.RESULT_PATH)
        file_count = len(files)
        QMessageBox.information(self, 'info', '生成完毕！', QMessageBox.Ok)
        self.horizontalScrollBar.setMinimum(1 if file_count > 0 else 0)
        self.horizontalScrollBar.setMaximum(file_count)
        self.horizontalScrollBar.setValue(0)
        self.image_count.setText(str(file_count))
        self.image_index.setText("1" if file_count > 0 else "0")

    def solve_puzzle(self):
        if os.path.exists(config.RESULT_PATH):
            shutil.rmtree(config.RESULT_PATH)
            os.makedirs(config.RESULT_PATH)
        if self.search_method == 0:
            pass
        if self.search_method == 1:
            self.tangram_solver = BFS(self.pentagon_type, self.message_queue)
            #           self.tangram_solver.signal.connect(self.solver_callback)
            self.tangram_solver.start()
            QMessageBox.information(self, 'info',
                                    '使用' + self.search_methodCB.currentText() + '生成' + self.pentagon_typeCB.currentText() + '，请等待~（可能需要大量时间）',
                                    QMessageBox.Ok)
        if self.search_method == 2:
            self.tangram_solver = Greedy(self.pentagon_type,self.message_queue)
            self.tangram_solver.start()
            QMessageBox.information(self, 'info',
                                    '使用' + self.search_methodCB.currentText() + '生成' + self.pentagon_typeCB.currentText() + '，请等待~（可能需要大量时间）',
                                    QMessageBox.Ok)

    def change_pentagon_type(self):
        self.pentagon_type = self.pentagon_typeCB.currentIndex()

    def change_search_method(self):
        self.search_method = self.search_methodCB.currentIndex()

    def load_image(self):
        index = self.horizontalScrollBar.value()
        self.image_index.setText(str(index))
        if index == 0:
            return
        image_file = os.path.join(config.RESULT_PATH, str(index) + ".jpg")
        image_source = QPixmap(image_file)
        image = image_source.scaled(self.imageLb.size(), QtCore.Qt.KeepAspectRatio)
        self.imageLb.setPixmap(image)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    main_win = MainWindow()
    main_win.show()
    sys.exit(app.exec_())


import asyncio
import os
from typing import List

from workers.common.exceptions.RevoicerException import RevoicerException
from workers.common.model.FileInfo import FileInfo
from workers.common.model.RevoicerJob import RevoicerJob
from workers.common.services.RevoicerBaseService import RevoicerBaseService

TJob = RevoicerJob
TWorkItem = FileInfo | str

INPUT_QUEUE_NAME = os.environ["REVOICER_INPUT_QUEUE_NAME"]
OUTPUT_QUEUE_NAME = os.environ["REVOICER_OUTPUT_QUEUE_NAME"]

class RevoicerService(RevoicerBaseService):
    def __init__(self):
        """
        The __init__ function is called when the class is instantiated.
        It sets up the instance of the class, and it's where you put all your initialization code.
        The __init__ function takes at least one argument: self, which refers to the object being created.

        :param self: Represent the instance of the class
        :return: The super()
        """
        super().__init__(RevoicerJob, INPUT_QUEUE_NAME, OUTPUT_QUEUE_NAME)

    def get_input_work_items(self, job: TJob) -> List[TWorkItem]:
        """
        The get_input_work_items function is used to generate a list of input work items for the job.
        The function takes in a single argument, which is the job object that was passed into the run_job function.
        The return value should be a list of work items, where each item corresponds to an input file or directory that
        will be processed by this job.

        :param self: Bind the method to an object
        :param job: TJob: Get the list of files that need to be processed
        :return: A list of tworkitem objects
        """
        return [self.preprocess_input_work_item(file) for file in job.Split
                if file.endswith("/vocals.wav")]

    def set_output_work_items(self, job: TJob, output_items: List[TWorkItem]) -> TJob:
        """
        The set_output_work_items function is used to set the output work items for a job.

        :param self: Represent the instance of a class
        :param job: TJob: Pass the job object to the function
        :param output_items: List[TWorkItem]: Pass the list of output work items to the function
        :return: The job object with the revoiced files added to it
        """
        job.Revoiced = [str(file.RemotePath) for file in output_items]
        return job

    def get_output_work_items(self, job: TJob) -> List[TWorkItem]:
        """
        The get_output_work_items function is used to determine which work items are output by a job.

        :param self: Refer to the instance of the class
        :param job: TJob: Get the revoiced work items
        :return: A list of work items
        """
        return job.Revoiced

    async def process_work_item(self, job: TJob, work_item: TWorkItem) -> List[TWorkItem]:
        """
        The process_work_item function is called for each work item in the job.
        It should return a list of work items to be added to the queue, or an empty list if no new work items are
        created.
        The function can also raise an exception if there was a problem processing the input file.

        :param self: Reference the class itself
        :param job: TJob: Access the voice model path
        :param work_item: TWorkItem: Pass the work item to be processed
        :return: A list of work items
        """
        result = []

        model_path = job.Voice
        voice = os.path.basename(model_path)

        input_file = work_item.LocalPath
        output_dir = os.path.dirname(input_file)
        output_file = os.path.join(output_dir, f"{voice}.wav")
        params = [
            "svc",
            "infer",
            f"--output-path={output_file}",
            f"--model-path=/models/{model_path}",
            f"--config-path=/models/{model_path}/config.json",
            "--no-auto-predict-f0",
            input_file
        ]

        process = await asyncio.create_subprocess_exec(*params)
        return_code = await process.wait()

        if return_code != 0:
            raise RevoicerException(f"Ocorreu um erro executando o SVC. Código: {return_code}")

        result = [
            *result,
            self.preprocess_output_work_item(job, output_file)
        ]

        return result