# -*- coding: utf-8 -*-
"""
Created on Sat Mar  4 23:25:12 2023

@author: Akhil Jose Britto
"""

''''Importing the modules '''

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

''''Defining the Line Plot showing growth % in GDP'''

def line_plot():
    plt.figure()
    #Plotting a line plot with the year in x-axis and each corresponding country in the y-axis
    plt.plot(gdp_growth_year,Brazil,label = 'Brazil',marker = ".")
    plt.plot(gdp_growth_year,China,label = 'China',marker = ".")
    plt.plot(gdp_growth_year,Ghana,label = 'Ghana',marker = ".")
    plt.plot(gdp_growth_year,Germany,label = 'Germany',marker = ".")
    plt.plot(gdp_growth_year,India,label = 'India',marker = ".")
    plt.plot(gdp_growth_year,Japan,label = 'Japan',marker = ".")
    #Label both axis
    plt.xlabel("Year")
    plt.ylabel("GDP Growth %")
    #Removing the white spaces at the start and end of the x-axis plot
    plt.xlim(2007, 2016)
    #Show the legend outside the plot so as to not interfere within the plot
    plt.legend(loc='upper right', bbox_to_anchor=(1.3, 1), borderaxespad=0)
    #Set the plot title
    plt.title("GDP Growth rate %")
    plt.show()

''''Defining the Bar Plot comparing male vs female unemployment rate'''

def bar_plot():
    plt.figure()
    #Form 4 Subplots of shape 2x2
    plt.subplot(2, 2, 1)
    #Plot a bar plot with Year on the x-axis and the rate of unemployment in y-axis
    plt.bar(year, male_unemployment['Brazil'], color = "cyan",label = "Male")
    #Stack the bar plot representing the female rate on top of the male rate
    plt.bar(year, female_unemployment['Brazil'],bottom = male_unemployment['Brazil'], color = "pink", label = "Female")
    #Remove the white spaces at the ends of the x-axis
    plt.xlim(2007, 2016)
    #Set the plot title
    plt.title("Brazil")
    #Set the axis labels
    plt.xlabel("Year")
    plt.ylabel("Unemployment Rate")

    plt.subplot(2, 2, 2)
    plt.bar(year, male_unemployment['China'], color = "cyan",label = "Male")
    plt.bar(year, female_unemployment['China'],bottom = male_unemployment['China'], color = "pink", label = "Female")
    plt.xlim(2007, 2016)
    plt.title("China")
    plt.xlabel("Year")
    plt.ylabel("Unemployment Rate")

    plt.subplot(2, 2, 3)
    plt.bar(year, male_unemployment['India'], color = "cyan",label = "Male")
    plt.bar(year, female_unemployment['India'],bottom = male_unemployment['India'], color = "pink", label = "Female")
    plt.xlim(2007, 2016)
    plt.title("India")
    plt.xlabel("Year")
    plt.ylabel("Unemployment Rate")

    plt.subplot(2, 2, 4)
    plt.bar(year, male_unemployment['Japan'], color = "cyan",label = "Male")
    plt.bar(year, female_unemployment['Japan'],bottom = male_unemployment['Japan'], color = "pink", label = "Female")
    plt.xlim(2007, 2016)
    plt.title("Japan")
    plt.xlabel("Year")
    plt.ylabel("Unemployment Rate")
    #Plot the legend just outside the graph
    plt.legend(loc='lower left', bbox_to_anchor=(1.1,1.1), borderaxespad=0)
    #Set the layout as tight so that the plot labels do not overlap with each other
    plt.tight_layout()
    plt.show()

'''Defining the Scatter Plot showing average growth in GDP over the period in the line plot '''

def scatter_plot():
    plt.figure()
    #Scatter plot points with the Country name on the x-axis and the average value right above it on the y-axis
    plt.scatter("Brazil",np.average(Brazil),s=np.average(Brazil)*50,label = 'Brazil')
    #Use np.average() to calculate the average value of each country from the previous data set
    plt.scatter("China",np.average(China),s=np.average(China)*50,label = 'China')
    #Set s as the average value * 50 to represent the plot point thickness respective to it's value
    plt.scatter("Ghana",np.average(Ghana),s=np.average(Ghana)*50,label = 'Ghana')
    plt.scatter("Germany",np.average(Germany),s=np.average(Germany)*50,label = 'Germany')
    plt.scatter("India",np.average(India),s=np.average(India)*50,label = 'India')
    plt.scatter("Japan",np.average(Japan),s=np.average(Japan)*50,label = 'Japan')
    #Set the axis labels
    plt.xlabel("Countries")
    plt.ylabel("Average GDP Growth %")
    #Define the legend outside the grapth so it doesn't overlap with the plot points
    lgnd = plt.legend(loc='upper right', bbox_to_anchor=(1.25,1), borderaxespad=0,labelspacing = 1.2)
    #Set the marker size of elements in the legend to a common font
    for handle in lgnd.legendHandles:
        handle.set_sizes([150.0])
    #Set the graph title
    plt.title("Average GDP Growth through 2007 - 2016")
    plt.show()

''''Defining the Pie Chart comparing combined Male and Fermale Unemployment between the years 2010 and 2014 '''

def pie_chart():
    plt.figure()
    #Create 2 sub plot pie charts of shape 1x2
    plt.subplot(1, 2, 1)
    #Plot the chart with the sum of both male and female rates from the year 2010
    plt.pie(male_unemployment.iloc[3,1:] + female_unemployment.iloc[3,1:])
    #Set the subplot title as the year
    plt.title("2010")

    plt.subplot(1, 2, 2 )
    #Plot the chart with the sum of both male and female rates from the year 2014
    plt.pie(male_unemployment.iloc[8,1:] + female_unemployment.iloc[8,1:])
    plt.title("2014")
    #Set the format as tight layout so the plot is arranged without overlapping other values
    plt.tight_layout()
    #Set the Main title for both subplotss
    plt.suptitle('                     Total unemployment figures in 2010 vs 2014')
    #Set the location for legend so there's no overlapping with the plot values
    plt.legend(labels = male_unemployment.iloc[:,1:],bbox_to_anchor=(1.5,0.95))
    plt.show()

'''Main program '''

#Line and Scatter Plot data
gdp_growth = pd.read_excel(r"C:\Users\HP\OneDrive - University of Hertfordshire\Assignments\GDP growth.xlsx")
#Extracting the data from the GDP Growth % excel sheet into a pandas data frame
gdp_growth_year = np.array(gdp_growth["Year"])
#Taking the Year column from the data frame into a numpy array

Brazil = np.array(gdp_growth["Brazil"])
#Extracting each country column values from the data frame into a numpy array
China = np.array(gdp_growth["China"])
Ghana = np.array(gdp_growth["Ghana"])
Germany = np.array(gdp_growth["Germany"])
India = np.array(gdp_growth["India"])
Japan = np.array(gdp_growth["Japan"])

#Bar plot and Pie chart data
male_unemployment = pd.read_excel(r"C:\Users\HP\OneDrive - University of Hertfordshire\Assignments\Unemployment figures Male.xlsx")
female_unemployment = pd.read_excel(r"C:\Users\HP\OneDrive - University of Hertfordshire\Assignments\Unemployment figures Female.xlsx")
#Extracting the excel sheet values into Pandas DataFrame
year = np.array(male_unemployment['Year'])
#Taking the year column from the data frame which is common to both Male and Female Graphs

'''Excecute the graphs    '''

line_plot()
bar_plot()
scatter_plot()
pie_chart()


from abc import ABC, abstractmethod
from typing import Any, List, Tuple

import numpy as np
from collections import Counter

from game.utils import Card, GamePhase

class Agent(ABC):
    @abstractmethod
    def play(self, observation: dict, actions: List[Any]) -> Any:
        pass

    def parallel_play(self, observations: List[dict],
                      action_list: List[List[Any]]) -> List[Any]:
        actions_to_play = []
        for observation, actions in zip(observations, action_list):
            actions_to_play.append(self.play(observation, actions))
        return actions_to_play

class RandomPlayer(Agent):
    def play(self, observation: dict, actions: List[Any]) -> Any:
        if observation['phase'] == GamePhase.DURCH:
            if np.random.rand() < 0.985:
                return False
            else:
                return True

        if observation['phase'] == GamePhase.DECLARATION:
            if np.random.rand() < 0.5:
                return ()

        idx = np.random.randint(len(actions))
        return actions[idx]

class PartialRandomPlayer(Agent):
    def play(self, observation: dict, actions: List[Any]) -> Any:
        phase = observation['phase']
        if phase == GamePhase.PLAY:
            idx = np.random.randint(len(actions))
            return actions[idx]
        elif phase == GamePhase.MOVING:
            return self._move_cards(observation['hand'])
        elif phase == GamePhase.DURCH:
            return False
        elif phase == GamePhase.DECLARATION:
            return ()

    def _move_cards(self, hand: List[Card]) -> List[Card]:
        values = [x.value for x in hand]
        idx = np.argsort(values)[-2:]
        return [hand[idx[0]], hand[idx[1]]]

class LowPlayer(Agent):
    def play(self, observation: dict, actions: List[Any]) -> Any:
        phase = observation['phase']
        if phase == GamePhase.PLAY:
            return self._play_phase(observation)
        elif phase == GamePhase.MOVING:
            return self._move_cards(observation['hand'], actions)
        elif phase == GamePhase.DURCH:
            return False
        elif phase == GamePhase.DECLARATION:
            return ()

    def _move_cards(self, hand: List[Card], actions: List[Any]) -> Tuple[Card, Card]:
        values = [x.value for x in hand]
        idx = np.argsort(values)[-2:]
        if (hand[idx[0]], hand[idx[1]]) in actions:
            return hand[idx[0]], hand[idx[1]]
        else:
            return hand[idx[1]], hand[idx[0]]

    def _play_phase(self, features: dict) -> Card:
        pot = features['pot']
        hand = features['hand']

        if pot.is_empty():
            smallest_colour = get_smallest_colour(hand)
            return get_smallest_card(hand, smallest_colour)
        else:
            choices = get_sorted_cards_of_colour(hand, pot.get_pot_colour())
            if len(choices) == 0:
                card, point = get_highest_points_card(hand, return_value=True)
                if point >= 2:
                    return card
                return get_largest_card(hand)
            else:
                if choices[0].is_higher_value(pot.get_highest_card()):
                    vals = [c.value for c in choices]
                    pos = np.searchsorted(vals, pot.get_highest_card().value)
                    under = choices[:pos]
                    points = [c.get_point_value() for c in under]
                    idx = np.argmax(points)
                    if points[idx] >= 2:
                        return under[idx]
                    else:
                        return under[-1]
                else:
                    return choices[-1]

def get_smallest_card(hand, colour=None):
    if colour is not None:
        return get_sorted_cards_of_colour(hand, colour)[0]
    else:
        idx = np.argmin([c.value for c in hand])
        return hand[idx]

def get_largest_card(hand, colour=None):
    if colour is not None:
        return get_sorted_cards_of_colour(hand, colour)[-1]
    else:
        idx = np.argmax([c.value for c in hand])
        return hand[idx]

def get_highest_points_card(hand, return_value=False):
    vals = [c.get_point_value() for c in hand]
    idx = np.argmax(vals)
    if return_value:
        return hand[idx], vals[idx]
    else:
        return hand[idx]

def get_smallest_colour(hand):
    c = get_colour_counts(hand)
    return c.most_common()[-1][0]

def get_colour_counts(hand):
    c = Counter()
    for card in hand:
        c[card.colour] += 1
    return c

def get_sorted_cards_of_colour(hand, colour):
    return np.sort([c for c in hand if c.colour == colour])


import json
from pathlib import Path
from textwrap import dedent, indent
from typing import List, Set

import typer

from back.crud.async_elasticsearch import CrudAsyncElasticsearchBase
from lib.typer import ZetsuBouTyper
from tests.back.utils.test_keyword import get_data

_help = """
Build the documentation.
"""
app = ZetsuBouTyper(name="docs", help=_help)

class CrudAsyncElasticsearchTest(CrudAsyncElasticsearchBase):
    @property
    def fields(self) -> List[str]:
        return ["tags.*"]

    async def get_field_names(self) -> Set[str]:
        return {"tags.language"}

async def get_search_grammar_example():
    docs_indent = "    "
    crud = CrudAsyncElasticsearchTest(is_from_setting_if_none=True)

    examples = "#### Examples\n"
    data = get_data()

    for d in data:
        example = "\n"
        if not d.show_docs:
            continue

        example += f"- {d.docs}\n\n"

        keyword_length = len(d.keywords) - 1
        example += '=== "Input"\n\n'
        first_keyword = ""
        for i, keyword in enumerate(d.keywords):
            keyword_block = ""

            keyword_block += f"""\
            ```sh
            {keyword}
            ```
            """
            keyword_block = dedent(keyword_block)

            if i == 0:
                first_keyword = keyword
            if i != keyword_length:
                keyword_block += "\nor\n\n"

            keyword_block = indent(keyword_block, docs_indent)
            example += keyword_block

        example += '\n=== "Elasticsearch"\n'
        elasticsearch_query = await crud.get_match_query(first_keyword)
        elasticsearch_query_str = json.dumps(
            elasticsearch_query, indent=4, ensure_ascii=False
        )
        elasticsearch_query_str = f"\n\n```json\n{elasticsearch_query_str}\n```\n"

        elasticsearch_query_str = indent(elasticsearch_query_str, docs_indent)

        example += elasticsearch_query_str

        example += "\n"

        examples += example
    return examples

@app.command()
async def print_search_grammar_examples():
    """
    Print examples of search grammar.
    """

    docs = ""

    examples = await get_search_grammar_example()
    docs += examples

    print(docs)


from flask import Blueprint, jsonify, request
from app.models import Pizza, db, Cart, Saved_order, Topping

cart_routes = Blueprint('cart', __name__)

@cart_routes.route('/<int:id>')
def getcarts(id):
    cart = Cart.query.filter(id == Cart.userId).first()
    pizzas = Pizza.query.filter(Pizza.cartId == id).order_by(Pizza.id)
    pizza_list = []
    for pizza in pizzas:
        toppings = Topping.query.filter(
            pizza.id == Topping.pizzaId).order_by(Topping.id)
        pizza_list.append({"id": pizza.id, "price": pizza.price,
                           "cartId": pizza.cartId, 'orderId': pizza.orderId, 'total': pizza.total, 'toppings': [{"id": topping.id, "name": topping.name,
                                                                                "pizzaId": topping.pizzaId} for topping in toppings]})
    # pizza_list = [{"id": pizza.id, "price": pizza.price,
    #                "cartId": pizza.cartId, 'orderId': pizza.orderId, 'total': pizza.total} for pizza in pizzas]

    return jsonify({'cart': cart.to_dict(), 'pizzas': pizza_list})

@cart_routes.route('/<int:id>/checkout/', methods=['PUT'])
def Checkout(id):
    data = request.json
    pizzas = Pizza.query.filter(Pizza.cartId == id)
    length = Pizza.query.filter(Pizza.cartId == id).count()
    cart = Cart.query.filter(Cart.id == id).first()
    cart.total = 0
    Order = Saved_order(
        total=data['price'],
        userId=cart.userId,
    )
    db.session.add(Order)
    db.session.commit()
    for pizza in pizzas:
        print(pizza)
        pizza.cartId = None
        pizza.orderId = Order.id
    if length:
        pizza_list = [{"id": pizza.id, "price": pizza.price,
                       "cartId": pizza.cartId, 'orderId': pizza.orderId} for pizza in pizzas]
    db.session.commit()
    return jsonify({'cart': cart.to_dict(), 'pizzas': pizza_list})


from fastapi import FastAPI
from pydantic import BaseModel
import uvicorn
import json
from json import dumps
from kafka import KafkaProducer

app = FastAPI()

class Data(BaseModel):
    category: str
    index: int
    unique_id: str
    title: str
    description: str
    follower_count: str
    tag_list: str
    is_image_or_video: str
    image_src: str
    downloaded: int
    save_location: str

@app.post("/pin/")
def get_db_row(item: Data):
    data = dict(item)
    producer = KafkaProducer(bootstrap_servers='localhost:9092', value_serializer=lambda m: json.dumps(m).encode('ascii'))
    future = producer.send('PinterestTopic1', data)
    record_metadata = future.get(timeout=10)
    # Successful result returns assigned partition and offset
    print (record_metadata.topic)
    print (record_metadata.partition)
    print (record_metadata.offset)

if __name__ == '__main__':
    uvicorn.run("project_pin_API:app", host="localhost", port=8001)


#! /usr/bin/env python3

from sys import argv
from i2c_puppet import I2CPuppet, _REG_RST
from usb import USBError # pyusb module
from os import uname

if uname().sysname == 'Darwin' and uname().machine == 'arm64':
    import usb.backend.libusb1
    usbBackend = usb.backend.libusb1.get_backend(find_library=lambda x: "/opt/homebrew/lib/libusb-1.0.dylib")
else:
    usbBackend = None

if __name__ == "__main__":
    if (len(argv) == 1) or ((len(argv) == 2) and (argv[1] == "uf2")):
        device = I2CPuppet(backend = usbBackend)
        UF2Boot = ((len(argv) == 2) and (argv[1] == "uf2"))

        if UF2Boot:
            device._write_register(_REG_RST, 0x99)
        else:
            try:
                device._read_register(_REG_RST)
            except USBError:
                pass

    else:
        print(f"""
Usage: {argv[0]} [uf2]

Resets the BBQ2KBD. If the optional 'uf2' argument is provided, resets into
programer mode allowing you to mount 'RPI-RP2' to load new uf2 files.
""")


"""Small helper functions.

:author: Shay Hill
:created: 2022-05-12
"""

from __future__ import annotations

import bisect
from typing import TYPE_CHECKING, Protocol, TypeVar

import numpy as np

if TYPE_CHECKING:
    from simplify_polyline.type_hints import Vector, Vertex, Vertices

_T = TypeVar("_T")

class Comparable(Protocol):
    """Protocol for annotating comparable types."""

    def __lt__(self: _T, other: _T) -> bool:
        ...

_ComparableT = TypeVar("_ComparableT", bound=Comparable)

def remsort(sorted_list: list[_ComparableT], value: _ComparableT) -> None:
    """Remove a value from a sorted list in place.

    This is the inverse of bisect.insort
    """
    index = bisect.bisect_left(sorted_list, value)
    del sorted_list[index]

def get_line_seg_dists(points: Vertices, seg_a: Vertex, seg_b: Vertex) -> Vector:
    """Calculate the Euclidean distance between an array of points to a line segment.

    :param pnt: a point in 2d space
    :param seg_a: beginning of line segment in 2d space
    :param seg_b: end of line segment in 2d space
    :return: a distance from p to segment ab for each p

    by [meowgoesthedog](https://stackoverflow.com/users/10265365/meowgoesthedog)

    https://stackoverflow.com/questions/54442057/calculate-the-euclidian-distance-
    between-an-array-of-points-to-a-line-segment-in
    """
    # Handle case where p is a single point, i.e. 1d array.
    points = np.atleast_2d(points)

    if np.all(seg_a == seg_b):
        return np.linalg.norm(points - seg_a, axis=1)

    # normalized tangent vector
    d = np.divide(seg_b - seg_a, np.linalg.norm(seg_b - seg_a))

    # signed parallel distance components
    s = np.dot(seg_a - points, d)
    t = np.dot(points - seg_b, d)

    # clamped parallel distance
    h = np.maximum.reduce([s, t, np.zeros(len(points))])

    # perpendicular distance component
    c = np.cross(points - seg_a, d)

    # use hypot for Pythagoras to improve accuracy
    return np.hypot(h, c)


from __future__ import absolute_import

import asyncio
import functools

import aiohttp  # type: ignore
import six
import urllib3  # type: ignore

from google.auth import exceptions
from google.auth import transport
from google.auth.transport import requests

# Timeout can be re-defined depending on async requirement. Currently made 60s more than
# sync timeout.
_DEFAULT_TIMEOUT = 180  # in seconds

class _CombinedResponse(transport.Response):
    """
    In order to more closely resemble the `requests` interface, where a raw
    and deflated content could be accessed at once, this class lazily reads the
    stream in `transport.Response` so both return forms can be used.

    The gzip and deflate transfer-encodings are automatically decoded for you
    because the default parameter for autodecompress into the ClientSession is set
    to False, and therefore we add this class to act as a wrapper for a user to be
    able to access both the raw and decoded response bodies - mirroring the sync
    implementation.
    """

    def __init__(self, response):
        self._response = response
        self._raw_content = None

    def _is_compressed(self):
        headers = self._response.headers
        return "Content-Encoding" in headers and (
            headers["Content-Encoding"] == "gzip"
            or headers["Content-Encoding"] == "deflate"
        )

    @property
    def status(self):
        return self._response.status

    @property
    def headers(self):
        return self._response.headers

    @property
    def data(self):
        return self._response.content

    async def raw_content(self):
        if self._raw_content is None:
            self._raw_content = await self._response.content.read()
        return self._raw_content

    async def content(self):
        # Load raw_content if necessary
        await self.raw_content()
        if self._is_compressed():
            decoder = urllib3.response.MultiDecoder(
                self._response.headers["Content-Encoding"]
            )
            decompressed = decoder.decompress(self._raw_content)
            return decompressed

        return self._raw_content

class _Response(transport.Response):
    """
    Requests transport response adapter.

    Args:
        response (requests.Response): The raw Requests response.
    """

    def __init__(self, response):
        self._response = response

    @property
    def status(self):
        return self._response.status

    @property
    def headers(self):
        return self._response.headers

    @property
    def data(self):
        return self._response.content

class Request(transport.Request):
    """Requests request adapter.

    This class is used internally for making requests using asyncio transports
    in a consistent way. If you use :class:`AuthorizedSession` you do not need
    to construct or use this class directly.

    This class can be useful if you want to manually refresh a
    :class:`~google.auth.credentials.Credentials` instance::

        import google.auth.transport.aiohttp_requests

        request = google.auth.transport.aiohttp_requests.Request()

        credentials.refresh(request)

    Args:
        session (aiohttp.ClientSession): An instance :class:`aiohttp.ClientSession` used
            to make HTTP requests. If not specified, a session will be created.

    .. automethod:: __call__
    """

    def __init__(self, session=None):
        # TODO: Use auto_decompress property for aiohttp 3.7+
        if session is not None and session._auto_decompress:
            raise exceptions.InvalidOperation(
                "Client sessions with auto_decompress=True are not supported."
            )
        self.session = session

    async def __call__(
        self,
        url,
        method="GET",
        body=None,
        headers=None,
        timeout=_DEFAULT_TIMEOUT,
        **kwargs,
    ):
        """
        Make an HTTP request using aiohttp.

        Args:
            url (str): The URL to be requested.
            method (Optional[str]):
                The HTTP method to use for the request. Defaults to 'GET'.
            body (Optional[bytes]):
                The payload or body in HTTP request.
            headers (Optional[Mapping[str, str]]):
                Request headers.
            timeout (Optional[int]): The number of seconds to wait for a
                response from the server. If not specified or if None, the
                requests default timeout will be used.
            kwargs: Additional arguments passed through to the underlying
                requests :meth:`requests.Session.request` method.

        Returns:
            google.auth.transport.Response: The HTTP response.

        Raises:
            google.auth.exceptions.TransportError: If any exception occurred.
        """

        try:
            if self.session is None:  # pragma: NO COVER
                self.session = aiohttp.ClientSession(
                    auto_decompress=False
                )  # pragma: NO COVER
            requests._LOGGER.debug("Making request: %s %s", method, url)
            response = await self.session.request(
                method, url, data=body, headers=headers, timeout=timeout, **kwargs
            )
            return _CombinedResponse(response)

        except aiohttp.ClientError as caught_exc:
            new_exc = exceptions.TransportError(caught_exc)
            six.raise_from(new_exc, caught_exc)

        except asyncio.TimeoutError as caught_exc:
            new_exc = exceptions.TransportError(caught_exc)
            six.raise_from(new_exc, caught_exc)

class AuthorizedSession(aiohttp.ClientSession):
    """This is an async implementation of the Authorized Session class. We utilize an
    aiohttp transport instance, and the interface mirrors the google.auth.transport.requests
    Authorized Session class, except for the change in the transport used in the async use case.

    A Requests Session class with credentials.

    This class is used to perform requests to API endpoints that require
    authorization::

        from google.auth.transport import aiohttp_requests

        async with aiohttp_requests.AuthorizedSession(credentials) as authed_session:
            response = await authed_session.request(
                'GET', 'https://www.googleapis.com/storage/v1/b')

    The underlying :meth:`request` implementation handles adding the
    credentials' headers to the request and refreshing credentials as needed.

    Args:
        credentials (google.auth._credentials_async.Credentials):
            The credentials to add to the request.
        refresh_status_codes (Sequence[int]): Which HTTP status codes indicate
            that credentials should be refreshed and the request should be
            retried.
        max_refresh_attempts (int): The maximum number of times to attempt to
            refresh the credentials and retry the request.
        refresh_timeout (Optional[int]): The timeout value in seconds for
            credential refresh HTTP requests.
        auth_request (google.auth.transport.aiohttp_requests.Request):
            (Optional) An instance of
            :class:`~google.auth.transport.aiohttp_requests.Request` used when
            refreshing credentials. If not passed,
            an instance of :class:`~google.auth.transport.aiohttp_requests.Request`
            is created.
        kwargs: Additional arguments passed through to the underlying
            ClientSession :meth:`aiohttp.ClientSession` object.
    """

    def __init__(
        self,
        credentials,
        refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES,
        max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS,
        refresh_timeout=None,
        auth_request=None,
        auto_decompress=False,
        **kwargs,
    ):
        super(AuthorizedSession, self).__init__(**kwargs)
        self.credentials = credentials
        self._refresh_status_codes = refresh_status_codes
        self._max_refresh_attempts = max_refresh_attempts
        self._refresh_timeout = refresh_timeout
        self._is_mtls = False
        self._auth_request = auth_request
        self._auth_request_session = None
        self._loop = asyncio.get_event_loop()
        self._refresh_lock = asyncio.Lock()
        self._auto_decompress = auto_decompress

    async def request(
        self,
        method,
        url,
        data=None,
        headers=None,
        max_allowed_time=None,
        timeout=_DEFAULT_TIMEOUT,
        auto_decompress=False,
        **kwargs,
    ):

        """Implementation of Authorized Session aiohttp request.

        Args:
            method (str):
                The http request method used (e.g. GET, PUT, DELETE)
            url (str):
                The url at which the http request is sent.
            data (Optional[dict]): Dictionary, list of tuples, bytes, or file-like
                object to send in the body of the Request.
            headers (Optional[dict]): Dictionary of HTTP Headers to send with the
                Request.
            timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
                The amount of time in seconds to wait for the server response
                with each individual request. Can also be passed as an
                ``aiohttp.ClientTimeout`` object.
            max_allowed_time (Optional[float]):
                If the method runs longer than this, a ``Timeout`` exception is
                automatically raised. Unlike the ``timeout`` parameter, this
                value applies to the total method execution time, even if
                multiple requests are made under the hood.

                Mind that it is not guaranteed that the timeout error is raised
                at ``max_allowed_time``. It might take longer, for example, if
                an underlying request takes a lot of time, but the request
                itself does not timeout, e.g. if a large file is being
                transmitted. The timout error will be raised after such
                request completes.
        """
        # Headers come in as bytes which isn't expected behavior, the resumable
        # media libraries in some cases expect a str type for the header values,
        # but sometimes the operations return these in bytes types.
        if headers:
            for key in headers.keys():
                if type(headers[key]) is bytes:
                    headers[key] = headers[key].decode("utf-8")

        async with aiohttp.ClientSession(
            auto_decompress=self._auto_decompress
        ) as self._auth_request_session:
            auth_request = Request(self._auth_request_session)
            self._auth_request = auth_request

            # Use a kwarg for this instead of an attribute to maintain
            # thread-safety.
            _credential_refresh_attempt = kwargs.pop("_credential_refresh_attempt", 0)
            # Make a copy of the headers. They will be modified by the credentials
            # and we want to pass the original headers if we recurse.
            request_headers = headers.copy() if headers is not None else {}

            # Do not apply the timeout unconditionally in order to not override the
            # _auth_request's default timeout.
            auth_request = (
                self._auth_request
                if timeout is None
                else functools.partial(self._auth_request, timeout=timeout)
            )

            remaining_time = max_allowed_time

            with requests.TimeoutGuard(remaining_time, asyncio.TimeoutError) as guard:
                await self.credentials.before_request(
                    auth_request, method, url, request_headers
                )

            with requests.TimeoutGuard(remaining_time, asyncio.TimeoutError) as guard:
                response = await super(AuthorizedSession, self).request(
                    method,
                    url,
                    data=data,
                    headers=request_headers,
                    timeout=timeout,
                    **kwargs,
                )

            remaining_time = guard.remaining_timeout

            if (
                response.status in self._refresh_status_codes
                and _credential_refresh_attempt < self._max_refresh_attempts
            ):

                requests._LOGGER.info(
                    "Refreshing credentials due to a %s response. Attempt %s/%s.",
                    response.status,
                    _credential_refresh_attempt + 1,
                    self._max_refresh_attempts,
                )

                # Do not apply the timeout unconditionally in order to not override the
                # _auth_request's default timeout.
                auth_request = (
                    self._auth_request
                    if timeout is None
                    else functools.partial(self._auth_request, timeout=timeout)
                )

                with requests.TimeoutGuard(
                    remaining_time, asyncio.TimeoutError
                ) as guard:
                    async with self._refresh_lock:
                        await self._loop.run_in_executor(
                            None, self.credentials.refresh, auth_request
                        )

                remaining_time = guard.remaining_timeout

                return await self.request(
                    method,
                    url,
                    data=data,
                    headers=headers,
                    max_allowed_time=remaining_time,
                    timeout=timeout,
                    _credential_refresh_attempt=_credential_refresh_attempt + 1,
                    **kwargs,
                )

        return response


import argparse
import os
import re
import shutil
from pathlib import Path

import img2dataset
from cloudpathlib import CloudPath
from huggingface_hub import snapshot_download

from scale_configs import available_scales

def path_or_cloudpath(s):
    if re.match(r"^\w+://", s):
        return CloudPath(s)
    return Path(s)

def cleanup_dir(path):
    assert isinstance(path, Path) or isinstance(path, CloudPath)
    if isinstance(path, Path):
        shutil.rmtree(path)
    else:
        path.rmtree()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "--scale",
        type=str,
        required=False,
        choices=available_scales(simple_names=True)[1:] + ["datacomp_1b"],
        default="small",
        help="Competition scale.",
    )
    parser.add_argument(
        "--data_dir",
        type=path_or_cloudpath,
        required=True,
        help="Path to directory where the data (webdataset shards) will be stored.",
    )
    parser.add_argument(
        "--metadata_dir",
        type=path_or_cloudpath,
        default=None,
        help="Path to directory where the metadata will be stored. If not set, infer from data_dir.",
    )
    parser.add_argument(
        "--download_npz",
        help="If true, also download npz files.",
        action="store_true",
        default=False,
    )
    parser.add_argument(
        "--skip_shards",
        help="If true, only download metadata.",
        action="store_true",
        default=False,
    )
    parser.add_argument(
        "--overwrite_metadata",
        help="If true, force re-download of the metadata files.",
        action="store_true",
        default=False,
    )
    parser.add_argument(
        "--skip_bbox_blurring",
        help="If true, skip bounding box blurring on images while downloading.",
        action="store_true",
        default=False,
    )
    parser.add_argument(
        "--processes_count",
        type=int,
        required=False,
        default=16,
        help="Number of processes for download.",
    )
    parser.add_argument(
        "--thread_count",
        type=int,
        required=False,
        default=128,
        help="Number of threads for download.",
    )
    parser.add_argument(
        "--image_size",
        type=int,
        required=False,
        default=512,
        help="Size images need to be downloaded to.",
    )
    parser.add_argument(
        "--resize_mode",
        type=str,
        required=False,
        choices=["no", "border", "keep_ratio", "keep_ratio_largest", "center_crop"],
        default="keep_ratio_largest",
        help="Resizing mode used by img2dataset when downloading images.",
    )
    parser.add_argument(
        "--no_resize_only_if_bigger",
        help="If true, do not resize only if images are bigger than target size.",
        action="store_true",
        default=False,
    )
    parser.add_argument(
        "--encode_format",
        type=str,
        required=False,
        choices=["png", "jpg", "webp"],
        default="jpg",
        help="Images encoding format.",
    )
    parser.add_argument(
        "--output_format",
        type=str,
        required=False,
        choices=["webdataset", "tfrecord", "parquet", "files"],
        default="webdataset",
        help="Output format used by img2dataset when downloading images.",
    )
    parser.add_argument(
        "--retries",
        type=int,
        required=False,
        default=2,
        help="Number of time a download should be retried (default 2)",
    )
    parser.add_argument(
        "--enable_wandb",
        action="store_true",
        default=False,
        help="Whether to enable wandb logging (default False)",
    )
    parser.add_argument(
        "--wandb_project",
        type=str,
        required=False,
        default="datacomp",
        help="Name of W&B project used (default datacomp)",
    )

    args = parser.parse_args()

    hf_repo = (
        f"mlfoundations/datacomp_{args.scale}"
        if args.scale != "datacomp_1b"
        else "mlfoundations/datacomp_1b"
    )

    metadata_dir = args.metadata_dir
    if metadata_dir is None:
        metadata_dir = args.data_dir / "metadata"

    # Download the metadata files if needed.
    if args.overwrite_metadata or not metadata_dir.exists():
        if metadata_dir.exists():
            print(f"Cleaning up {metadata_dir}")
            shutil.rmtree(metadata_dir)
        metadata_dir.mkdir(parents=True)

        print(f"Downloading metadata to {metadata_dir}...")

        cache_dir = metadata_dir.parent / f"hf"
        hf_snapshot_args = dict(
            repo_id=hf_repo,
            allow_patterns=f"*.parquet",
            local_dir=metadata_dir,
            cache_dir=cache_dir,
            local_dir_use_symlinks=False,
            repo_type="dataset",
        )

        if args.scale == "xlarge":
            hf_snapshot_args["allow_patterns"] = f"*/*.parquet"

        snapshot_download(**hf_snapshot_args)
        if args.download_npz:
            hf_snapshot_args["allow_patterns"] = hf_snapshot_args[
                "allow_patterns"
            ].replace(".parquet", ".npz")
            print("\nDownloading npz files")
            snapshot_download(**hf_snapshot_args)

        # Flatten directory structure in case of xlarge
        if args.scale == "xlarge":
            filenames = list(metadata_dir.rglob("*.parquet")) + list(
                metadata_dir.rglob("*.npz")
            )
            for filename in filenames:
                basename = filename.name
                filename.replace(metadata_dir / basename)

            empty_dirs = list(metadata_dir.glob("part_*"))
            for empty_dir in empty_dirs:
                empty_dir.rmdir()

        cleanup_dir(cache_dir)

        print("Done downloading metadata.")
    else:
        print(
            f"Skipping download of metadata because {metadata_dir} exists. Use --overwrite_metadata to force re-downloading."
        )

    if not args.skip_shards:
        # Download images.
        shard_dir = args.data_dir / "shards"
        shard_dir.mkdir(parents=True, exist_ok=True)
        print(f"Downloading images to {shard_dir}")

        bbox_col = None if args.skip_bbox_blurring else "face_bboxes"

        img2dataset.download(
            url_list=str(metadata_dir),
            image_size=args.image_size,
            output_folder=str(shard_dir),
            processes_count=args.processes_count,
            thread_count=args.thread_count,
            resize_mode=args.resize_mode,
            resize_only_if_bigger=not args.no_resize_only_if_bigger,
            encode_format=args.encode_format,
            output_format=args.output_format,
            input_format="parquet",
            url_col="url",
            caption_col="text",
            bbox_col=bbox_col,
            save_additional_columns=["uid"],
            number_sample_per_shard=10000,
            oom_shard_count=8,
            retries=args.retries,
            enable_wandb=args.enable_wandb,
            wandb_project=args.wandb_project,
        )
    else:
        print(f"Skipping image data download.")

    print("Done!")


# coding=utf-8
from langconv import Converter
import pandas as pd
import csv
import xlrd
import math

import re
import argparse

def rmSymbol(sent):
    return re.sub("|／\n", "", sent)

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument("input_file")
    parser.add_argument("output_directory")
    args = parser.parse_args()

    data = pd.read_excel(args.input_file, sheet_name='測試集', header=0)
    data2 = pd.read_excel(args.input_file, sheet_name='問題', header=0)
    n = len(data)  #5000 測試集內'測試題'與'正確標準問題'
    m = len(data2)  #1000 問題內 '標準問題'
    stdQuestion = data2['標準問題']  #All 標準問題  1000

    # 5000 * 0.2 * 1000 = 100,0000
    with open(args.output_directory + '/test.csv',
              'w',
              newline='',
              encoding="utf8") as testCSV:

        testCSVW = csv.writer(testCSV, lineterminator='\n')
        for i in range(int(n * 0.8), n, 1):
            customerQuestion = data['測試題'][i]
            for j in range(m):
                testCSVW.writerow(
                    [0,
                     rmSymbol(customerQuestion),
                     rmSymbol(stdQuestion[j])])


# -*- coding: utf-8 -*-
import os
import json

from PyQt4 import QtCore, QtGui, uic
from qgis.core import QgsLogger

from template.records_display_widget_bridge import RecordsDisplayWidgetBridge

FORM_CLASS, _ = uic.loadUiType(os.path.join(os.path.dirname(__file__), 'records_display_widget.ui'))

class RecordsDisplayWidget(QtGui.QWidget, FORM_CLASS):

    # signal emitted quen the webView is compleated loaded
    ready = QtCore.pyqtSignal(bool)
    docClicked = QtCore.pyqtSignal(str, str, str)
    linkClicked = QtCore.pyqtSignal(str, str, str)
    highlightRecord = QtCore.pyqtSignal(str, str)

    def __init__(self, layer, parent):
        """Constructor."""
        super(RecordsDisplayWidget, self).__init__(parent)
        self.setupUi(self)

        # status flags
        self._layer = layer
        self._selectedLayerId = None
        self._selectedFeatureId = None
        self._ready = False
        self._recordsDisplayWidgetBridge = RecordsDisplayWidgetBridge()
        self._recordsDisplayWidgetBridge.selectedRecord.connect(self._setSelectedRecord)
        self._recordsDisplayWidgetBridge.docClicked.connect(self._docClicked)
        self._recordsDisplayWidgetBridge.linkClicked.connect(self._linkClicked)
        self._recordsDisplayWidgetBridge.highlightRecord.connect(self._highlightRecord)

        # att HTML template and JS code to the web view
        self.initWebView()

    def initWebView(self):
        ''' Init the webView with the record template and JS usefult to
            show record table
        '''
        # check status of webView loading
        self.webView.loadStarted.connect(self._setNotReady)
        self.webView.loadFinished.connect(self._initWebContent)

        # get the upper directory of the current file that is in the "ui" directory
        pluginPath = os.path.dirname(os.path.dirname(__file__))
        webPage = os.path.join(pluginPath, "template", "index.html")

        self.webView.load(QtCore.QUrl.fromLocalFile(webPage))

    def setName(self, name):
        self.name = name

    def setRemoveSelection(self, removeSelection):
        self.removeSelection = removeSelection

    def isReady(self):
        ''' Status of the loaded webView
        '''
        return self._ready

    def _setNotReady(self):
        ''' Set that current webPage is not ready to be managed
        '''
        self._ready = False

    def _initWebContent(self, success):
        ''' Set status of page loading
        '''
        if not success:
            return

        # first inject bridge object
        self.webView.page().mainFrame().addToJavaScriptWindowObject("recordsDisplayWidgetBridge", self._recordsDisplayWidgetBridge)

        # wait a while before loading records
        QtCore.QTimer.singleShot(300, self._displayRecords)

    def _displayRecords(self):
        ''' after a while show records... this give time that the bridge is available in JS
        '''
        if self._layer.selectedFeatureCount():
            # then prepare selected records in structure usefut for accordion visualization
            featuresDict = self._prepareFeatures_asAccordion()

            # prepare js command to execute in the page
            jsonString = json.dumps(featuresDict)

            JsCommand = "showRecords('%s', %s)" % (self._layer.id(), jsonString) # <<< jsonString is automatically converted in javascript obj during evaluate
            QgsLogger.debug(self.tr("display records with with JS command: %s" % JsCommand), 3)

            # show records
            self.webView.page().mainFrame().evaluateJavaScript(JsCommand)

        else:
            QgsLogger.debug(self.tr("No features selected to display for layer: %s" % self._layer.id()), 1)

        # then we can say that the load is completed
        self._ready = True

        # Check if we want to remove selection of this layer
        if self.removeSelection:
            self._layer.removeSelection()

        # notify it is ready
        self.ready.emit(self._ready)

    def _prepareFeatures_asAccordion(self):
        ''' Prepare selected records in a structure useful for accordion visualization
        '''
        # Get selected features of current layers
        columns = [field.name() for field in self._layer.pendingFields().toList()]
        featuresDict = {'name': self.name, 'children': []}
        for i, feature in enumerate(self._layer.selectedFeatures()):
            item = {'name': feature.id(), 'children': []}

            # add records
            values = feature.attributes()
            namedValues = zip(columns, values)

            for key, value in namedValues:
                record = {'name': key, 'value': value if value != None else 'NULL'}
                item['children'].append(record)

            featuresDict['children'].append( item )

        return featuresDict

    def _setSelectedRecord(self, layerId, featureId):
        ''' Set current selected Record and layer
        '''
        QgsLogger.debug("RecordsDisplayWidget._setSelectedRecord: Selected layerId = {} and record id {}".format(layerId, featureId), 3)

        self._selectedLayerId = layerId
        self._selectedFeatureId = int(featureId)

    def _highlightRecord(self, layerId, featureId):
        ''' Set current hilighted Record and layer
        '''
        QgsLogger.debug("RecordsDisplayWidget._highlightRecord: on mouse over layerId = {} and record id {}".format(layerId, featureId), 3)

        if layerId and featureId:
            self.highlightRecord.emit(layerId, featureId)

    def _docClicked(self, layerId, featureId, document):
        ''' re emit signal that a document has been clicked
        '''
        self.docClicked.emit(layerId, featureId, document)

    def _linkClicked(self, layerId, featureId, link):
        ''' re emit signal that a link document has been clicked
        '''
        self.linkClicked.emit(layerId, featureId, link)

    def getSelection(self):
        ''' return current selection touple (that can be none,none if no selection)
        '''
        return self._selectedLayerId, self._selectedFeatureId


from selenium import webdriver
from selenium.webdriver.common.by import By

product_url = 'https://www.amazon.com/Nike-Romaleos-Training-White-Black/dp/B01FZ3H4TQ/ref=sr_1_13?crid=RWB8BXT454CN&keywords=adidas%2Bpowerlift%2Bshoes&qid=1697802294&sprefix=adidas%2Bpowerlift%2Bshoe%2Caps%2C186&sr=8-13&th=1&psc=1'

chrome_options = webdriver.ChromeOptions()
chrome_options.add_experimental_option("detach", True)

driver = webdriver.Chrome(options=chrome_options)
driver.get(product_url)

price_dollars = driver.find_element(By.CLASS_NAME, value='a-price-whole')
price_cents = driver.find_element(By.CLASS_NAME, value='a-price-fraction')
print(f"The price is {price_dollars.text}.{price_cents.text}")
#
# button = driver.find_element(By.ID, value='submit')
# print(button.size)
# doc_link = driver.find_element(By.CSS_SELECTOR, value='.doc-widget a')

bug_link = driver.find_element(By.XPATH, value='//*[@id="bylineInfo"]')
print(bug_link.text)
# driver.close()
driver.quit()


from matplotlib import pyplot as plt
import numpy as np
import librosa
from librosa import  display
import os
from utils import load_spectrograms
from hyperparams import Hyperparams as hp
# import the necessary packages
from skimage.measure import structural_similarity as ssim
import matplotlib.pyplot as plt
import numpy as np
import cv2

cinco_org= np.load('../savedir/synthesized-audios/RTISI-LA/5-org.npy')

savedir='../savedir/spectogramas/'
vocoders = ['RTISI-LA','Griff-Lim']
for voco in vocoders:
    diretory = os.path.join('../savedir/synthesized-audios/',voco)
    arquivos = ['1.wav','5.wav','5-org.wav','3.wav','3-org.wav']
    for i in arquivos:
        _,_,mag= load_spectrograms(os.path.join(diretory,i))
        print  (os.path.join(diretory,i),' :',mag.shape)
        if i == '5.wav':
            mag = mag[:cinco_org.shape[0]][:]
        np.save( os.path.join(diretory,i.replace('.wav','')), mag)
        # transpose
        mag = mag.T

        # de-noramlize
        mag = (np.clip(mag, 0, 1) * hp.max_db) - hp.max_db + hp.ref_db
        # to amplitude
        mag = np.power(10.0, mag * 0.05)
        mag = mag**hp.power
        display.specshow(librosa.amplitude_to_db(mag,ref=np.max), y_axis='log', x_axis='time')
        plt.title('Espectrograma STFT')
        plt.colorbar(format='%+2.0f dB')
        plt.tight_layout()
        plt.savefig(os.path.join(savedir,voco,'spectrograma-'+voco+'-'+i.replace('.wav','')+'.png'))
        plt.cla()   # Clear axis
        plt.clf()
        #plt.show()

## R2 Calcule
import numpy as np
from sklearn.metrics import r2_score

y_true= np.load('../savedir/synthesized-audios/RTISI-LA/5-org.npy')
predito_dctts = np.load('samples/5.png.npy')

# unpad
predito =  predito_dctts[:y_true.shape[0]][:]
#predito = predito.reshape(-1)
#y_true = y_true.reshape(-1)
print( " Predito vs esperado R Square: ",r2_score(y_true, predito, multioutput='variance_weighted'))

y_pred = np.load('../savedir/synthesized-audios/Griff-Lim/5.npy')
y_true = predito_dctts[:y_pred.shape[0]][:]
#y_true = y_true.reshape(-1)
#y_pred = y_pred.reshape(-1)
print( " Predito vs Griff-Lim R Square: ",r2_score(y_true, y_pred, multioutput='variance_weighted'))

y_pred = np.load('../savedir/synthesized-audios/RTISI-LA/5.npy')
y_true = predito_dctts[:y_pred.shape[0]][:]
#y_true = y_true.reshape(-1)
#y_pred = y_pred.reshape(-1)
print( " Predito vs RTISI-LA R Square: ",r2_score(y_true, y_pred, multioutput='variance_weighted'))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Oct 26 10:46:50 2022

-Compute EWS and DL predictions rolling over chick heart data

@author: tbury
"""

import time
start_time = time.time()

import numpy as np
import pandas as pd

import ewstools

import tensorflow as tf
from tensorflow.keras.models import load_model

import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--inc', type=int,
                    help='Time increment between DL predictions',
                    default=50)
parser.add_argument('--use_inter_classifier', type=bool, help='Use the intermediate classifier as opposed to the hard saved classifier', default=True)

args = parser.parse_args()
inc = args.inc
use_inter_classifier = True if args.use_inter_classifier=='true' else False
print('Using time increment of {} for DL predictions'.format(inc))

np.random.seed(0)

# Load in DL models
if use_inter_classifier:
    filepath_classifier = '../dl_train/output/'
else:
    filepath_classifier = '../../data/'

m1 = load_model(filepath_classifier+'classifier_1.pkl')
m2 = load_model(filepath_classifier+'classifier_2.pkl')
print('TF models loaded')

# EWS parameters
rw = 0.5 # rolling window
bw = 20 # Gaussian band width (# beats)

# Load in trajectory data
df_traj = pd.read_csv('../../data/df_chick.csv')
df_traj_pd = df_traj[df_traj['type']=='pd']
df_traj_null = df_traj[df_traj['type']=='neutral']

# Load in transition times
df_transition = pd.read_csv('output/df_transitions.csv')

#--------
# Compute EWS for period-doubling trajectories
#--------

list_ews = []
list_dl = []

list_tsid = df_traj_pd['tsid'].unique()
# for tsid in list_tsid:
for tsid in list_tsid:

    df_spec = df_traj_pd[df_traj_pd['tsid']==tsid].set_index('Beat number')
    transition = df_transition[df_transition['tsid']==tsid]['transition'].iloc[0]
    s = df_spec['IBI (s)'].iloc[:]

    # Compute EWS
    ts = ewstools.TimeSeries(s, transition=transition)
    # ts.detrend(method='Lowess', span=50)
    ts.detrend(method='Gaussian', bandwidth=bw)

    ts.compute_var(rolling_window=rw)
    ts.compute_auto(rolling_window=rw, lag=1)

    # Get DL predictions for forced trajectory
    ts.apply_classifier_inc(m1, inc=inc, name='m1', verbose=0)
    ts.apply_classifier_inc(m2, inc=inc, name='m2', verbose=0)

    # Save data to lists
    df_dl = ts.dl_preds.groupby('time').mean(numeric_only=True) # use mean DL pred
    df_dl['tsid'] = tsid
    df_dl.reset_index(inplace=True)
    df_dl = df_dl.rename({'time':'Beat number'}, axis=1)
    list_dl.append(df_dl)

    df_ews = ts.state.join(ts.ews)
    df_ews['tsid'] = tsid
    list_ews.append(df_ews)

    print('Complete for tsid={}'.format(tsid))

df_ews_pd = pd.concat(list_ews)
df_dl_pd = pd.concat(list_dl)

# Export period-doubling EWS
df_ews_pd.to_csv('output/df_ews_pd_rolling.csv')
df_dl_pd.to_csv('output/df_dl_pd_rolling.csv', index=False)

#--------
# Compute EWS for null trajectories
#--------

list_ews = []
list_dl = []
list_tsid = df_traj_null['tsid'].unique()
for tsid in list_tsid:

    df_spec = df_traj_null[df_traj_null['tsid']==tsid].set_index('Beat number')
    s = df_spec['IBI (s)'].iloc[:]

    # Compute EWS
    ts = ewstools.TimeSeries(s)
    # ts.detrend(method='Lowess', span=50)
    ts.detrend(method='Gaussian', bandwidth=bw)

    ts.compute_var(rolling_window=rw)
    ts.compute_auto(rolling_window=rw, lag=1)

    # Get DL predictions for null trajectory
    ts.apply_classifier_inc(m1, inc=inc, name='m1', verbose=0)
    ts.apply_classifier_inc(m2, inc=inc, name='m2', verbose=0)

    df_dl = ts.dl_preds.groupby('time').mean(numeric_only=True) # use mean DL pred
    df_dl['tsid'] = tsid
    df_dl.reset_index(inplace=True)
    df_dl = df_dl.rename({'time':'Beat number'}, axis=1)
    list_dl.append(df_dl)

    df_ews = ts.state.join(ts.ews)
    df_ews['tsid'] = tsid
    list_ews.append(df_ews)

    print('Complete for tsid={}'.format(tsid))

df_ews_null = pd.concat(list_ews)
df_dl_null = pd.concat(list_dl)

# Export null EWS
df_ews_null.to_csv('output/df_ews_null_rolling.csv')
df_dl_null.to_csv('output/df_dl_null_rolling.csv', index=False)

# Time taken for script to run
end_time = time.time()
time_taken = end_time - start_time
print('Ran in {:.2f}s'.format(time_taken))


import torch
import torch.nn.functional as F

# Normal Inverse Gamma Negative Log-Likelihood
# from https://arxiv.org/abs/1910.02600:
# > we denote the loss, L^NLL_i as the negative logarithm of model
# > evidence ...
def nig_nll(gamma, v, alpha, beta, y):
    two_beta_lambda = 2 * beta * (1 + v)
    t1 = 0.5 * (torch.pi / v).log()
    t2 = alpha * two_beta_lambda.log()
    t3 = (alpha + 0.5) * (v * (y - gamma) ** 2 + two_beta_lambda).log()
    t4 = alpha.lgamma()
    t5 = (alpha + 0.5).lgamma()
    nll = t1 - t2 + t3 + t4 - t5
    return nll.mean()

# Normal Inverse Gamma regularization
# from https://arxiv.org/abs/1910.02600:
# > we formulate a novel evidence regularizer, L^R_i
# > scaled on the error of the i-th prediction
def nig_reg(gamma, v, alpha, _beta, y):
    reg = (y - gamma).abs() * (2 * v + alpha)
    return reg.mean()

# KL divergence of predicted parameters from uniform Dirichlet distribution
# from https://arxiv.org/abs/1806.01768
# code based on:
# https://bariskurt.com/kullback-leibler-divergence-between-two-dirichlet-and-beta-distributions/
def dirichlet_reg(alpha, y):
    # dirichlet parameters after removal of non-misleading evidence (from the label)
    alpha = y + (1 - y) * alpha

    # uniform dirichlet distribution
    beta = torch.ones_like(alpha)

    sum_alpha = alpha.sum(-1)
    sum_beta = beta.sum(-1)

    t1 = sum_alpha.lgamma() - sum_beta.lgamma()
    t2 = (alpha.lgamma() - beta.lgamma()).sum(-1)
    t3 = alpha - beta
    t4 = alpha.digamma() - sum_alpha.digamma().unsqueeze(-1)

    kl = t1 - t2 + (t3 * t4).sum(-1)
    return kl.mean()

# Eq. (5) from https://arxiv.org/abs/1806.01768:
# Sum of squares loss
def dirichlet_mse(alpha, y):
    sum_alpha = alpha.sum(-1, keepdims=True)
    p = alpha / sum_alpha
    t1 = (y - p).pow(2).sum(-1)
    t2 = ((p * (1 - p)) / (sum_alpha + 1)).sum(-1)
    mse = t1 + t2
    return mse.mean()

def evidential_classification(alpha, y, lamb=1.0):
    num_classes = alpha.shape[-1]
    y = F.one_hot(y, num_classes)
    return dirichlet_mse(alpha, y) + lamb * dirichlet_reg(alpha, y)

def evidential_regression(dist_params, y, lamb=1.0):
    return nig_nll(*dist_params, y) + lamb * nig_reg(*dist_params, y)


from flask import Blueprint, jsonify, request

from app.models.users.user_registrator import LoginValidator

blueprint = Blueprint("api_utils", __name__)

@blueprint.route("/api/login_available", methods=["POST"])
def login_valid():
    login = request.json.get("login", None)

    validator = LoginValidator(login)

    return jsonify({"is_available": validator.validate(), "error": validator.error()})


# -*- coding:ascii -*-
from mako import runtime, filters, cache
UNDEFINED = runtime.UNDEFINED
__M_dict_builtin = dict
__M_locals_builtin = locals
_magic_number = 10
_modified_time = 1421282545.317255
_enable_loop = True
_template_filename = '/Users/jeffreymccraney/test_dmp/homepage/templates/contact.html'
_template_uri = 'contact.html'
_source_encoding = 'ascii'
import os, os.path, re
_exports = ['content']

def _mako_get_namespace(context, name):
    try:
        return context.namespaces[(__name__, name)]
    except KeyError:
        _mako_generate_namespaces(context)
        return context.namespaces[(__name__, name)]
def _mako_generate_namespaces(context):
    pass
def _mako_inherit(template, context):
    _mako_generate_namespaces(context)
    return runtime._inherit_from(context, 'base.htm', _template_uri)
def render_body(context,**pageargs):
    __M_caller = context.caller_stack._push_frame()
    try:
        __M_locals = __M_dict_builtin(pageargs=pageargs)
        def content():
            return render_content(context._locals(__M_locals))
        __M_writer = context.writer()
        __M_writer('\n\n')
        if 'parent' not in context._data or not hasattr(context._data['parent'], 'content'):
            context['self'].content(**pageargs)

        return ''
    finally:
        context.caller_stack._pop_frame()

def render_content(context,**pageargs):
    __M_caller = context.caller_stack._push_frame()
    try:
        def content():
            return render_content(context)
        __M_writer = context.writer()
        __M_writer('\n
\n    \t
 Email: mccraney.j@gmail.com
\n\t\t
\n\t\t\t
\n\t\t\t
\n\t\t\t      <input type="text" class="form-control" placeholder="Enter Email...">\n\t\t\t      \n\t\t\t        <button class="btn btn-primary" type="button">Primary</button>\n\t\t\t      \n\t\t\t
\n\t\t\t
\n\t\t
\n
\n
\n
\n        <button class="btn btn-success" type="button">Success!</button>\n')
        return ''
    finally:
        context.caller_stack._pop_frame()

"""
__M_BEGIN_METADATA
{"filename": "/Users/jeffreymccraney/test_dmp/homepage/templates/contact.html", "uri": "contact.html", "line_map": {"56": 50, "34": 1, "27": 0, "44": 3, "50": 3}, "source_encoding": "ascii"}
__M_END_METADATA
"""


from pymoo.core.sampling import Sampling
import numpy as np
from boolcryptogp.nsgp.structure.WalshTransform import WalshTransform
from boolcryptogp.nsgp.structure.ANFBooleanFunction import ANFBooleanFunction
from boolcryptogp.nsgp.structure.TreeStructure import TreeStructure

class ANFBooleanFunctionSampling(Sampling):

    def __init__(self,
                 structure: TreeStructure,
                 walsh: WalshTransform,
                 n_trees: int = None,
                 n_trees_min: int = 2,
                 n_trees_max: int = 10,
                 tree_prob: float = 0.70):
        super().__init__()
        self.__walsh: WalshTransform = walsh
        self.__structure: TreeStructure = structure
        self.__n_trees: int = n_trees
        self.__n_trees_min: int = n_trees_min
        self.__n_trees_max: int = n_trees_max
        self.__tree_prob: float = tree_prob

    def _do(self, problem, n_samples, **kwargs):
        x = np.empty((n_samples, 1), dtype=object)

        for i in range(n_samples):
            x[i, 0] = ANFBooleanFunction(self.__walsh, self.__structure.generate_forest(
                                         n_trees=self.__n_trees,
                                         n_trees_min=self.__n_trees_min,
                                         n_trees_max=self.__n_trees_max,
                                         tree_prob=self.__tree_prob)
                                         )

        return x


import pandas as pd # library for data analysis
import requests # library to handle requests
from bs4 import BeautifulSoup # library to parse HTML documents
import re # library for regular expressions
import plotly # library for plotting
import plotly.express as px # library for plotting
import pycountry_convert as pc # library to convert country names into codes for mapping
from plotly import graph_objs as go # library for plotting
import numpy as np # library for math
from flask import Flask, render_template # library for website
from flask import request
import json # library for dealing with JSON files

app = Flask(__name__)

@app.route("/")
def home():
    return render_template("home.html")

@app.route("/graph",methods=["GET","POST"])
def graph():
    question=request.form['Question']
    search = question + 'wikipedia'
    url = 'https://www.google.com/search?q='

    headers = {
        'Accept' : '*/*',
        'Accept-Language': 'en-US,en;q=0.5',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82',
    }

    parameters = {'q': search}

    content = requests.get(url, headers = headers, params = parameters).text
    soup = BeautifulSoup(content, 'html.parser')

    search = soup.find(id = 'search')
    first_link = search.find('a')

    wikiurl=(first_link['href'])
    table_class="wikitable sortable jquery-tablesorter"
    response=requests.get(wikiurl)

    soup = BeautifulSoup(response.text, 'html.parser')
    table=soup.find('table',{'class':"wikitable"})

    df=pd.read_html(str(table))
    df=pd.DataFrame(df[0])

    if df.columns[0]=="Rank" or df[df.columns[0]].iloc[0]==1 or df[df.columns[0]].iloc[0]=="1." or df[df.columns[0]].iloc[0]=="—":
        df=df.drop(df.columns[0], axis=1)

    if type(df[df.columns[1]].iloc[0])==str:
        if any(chr.isdigit() for chr in df[df.columns[1]].iloc[0])==False:
            df=df.drop(df.columns[1], axis=1)

    a=(df.columns[0])
    b=(df.columns[1])

    if df[a][0]=="World":
        df=df.drop(index=0)

    try:
        df[b]=df[b].str.replace(r'\[.*\]','')
        df[b] = df[b].str.replace(r'\D+', '')
    except:
        pass

    try:
        df[a]=df[a].str.replace(r'\[.*\]','')
        df[a]=df[a].str.replace(r'\(.*\)','')
    except:
        pass

    df[b] = df[b].replace(r'^\s*$', np.nan, regex=True)

    df[b]= df[b].astype(float)

    first=(df[b].iloc[0])
    last=(df[b].iloc[len(df[b])-1])
    secondlast=(df[b].iloc[len(df[b])-2])

    if (last/first)>1:
        df = df[:-1]
    if (secondlast/first)>1:
        df = df[:-2]

    fig=px.bar(df,x=a,y=b)

    graphJSON = json.dumps(fig, cls=plotly.utils.PlotlyJSONEncoder)

    if (df[a].eq('China')).any() == True or (df[a].eq('Brazil')).any() == True:
        # creating a function to convert the country names to alpha3 codes that plotly can process into a map
        df.rename(columns={ df.columns[0]: "Country" }, inplace = True)
        def convert(row):
            try: cn_code = pc.country_name_to_country_alpha3(row.Country, cn_name_format = "default")
            except: cn_code="VAT"

            return(cn_code)
        df["Alpha3"]= df.apply(convert,axis=1)

        # maping the data using plotly
        fig=go.Figure(data = go.Choropleth(locations=df["Alpha3"],z=df[b]))
        fig.update_layout(autosize=False,width=1200,height=600,)

        graphJSON = json.dumps(fig, cls=plotly.utils.PlotlyJSONEncoder)

    return render_template('home.html', graphJSON=graphJSON)


#
#   Gestion articles et réceptions
#
import os
import csv
import json
'''
Exercice 4.1
Créer une application qui offre les fonctions suivantes
Gestion Article, Réception Encodage, Reception Validation
- Encodage des articles
    Code article, intitulé, prix, inventaire
    Ajout-Modification-suppression
    Sauvegarde dans un fichier articles.csv (dico1)
- Encodage des receptions d'inventaire de la journée
    Ajout reception
        article quantité
    Modification
    Suppression
    Sauvegarde dans un fichier reception.csv ( dico2 )
- Validation reception
    Mettre à jour l'inventaire avec les receptions du jour

'''

from menu import Menu
def valide(msg, choice):
    """ Fonction d'affichage de message et de confirmation """
    c = ''
    #  saisie caractère autorisé
    while c not in choice:
        c = input(msg)
    return c
def ecritureC ( fichier, dico):
    """ Ecriture fichier """
    with open(fichier, 'w') as f:
        for k, v in dico.items():
            f.write('{},{}\n'.format(k, v))
    return
def ecriture( fichier, dico):
    """ Ecriture JSON"""
    with open( fichier, 'w') as f:
        json.dump(f, dico)

def lecture (fichier):
    """ Lecture ficheir JSON """
    dict = {}
    # nested dico
    if os.path.exists(fichier):
        with open(fichier, 'r') as f:
            dict = json.load(f)
    else:
        print('Fichier:', fichier, ' inexistant')
    return dict

# Fonctions de gestion des articles

# Fonctions de gestion receptions

# Effectuer la réceptions des articles
def effectuerReception(articles, receptions):
    """ reception des arciles : mise à jour des qty"""
    for r, q in receptions.items():
        if r in articles:
            articles[r]['qty'] += q # increment de la quantité
            print('Add q', q, '->', articles[r])
        else:
            print('Article ', r, ' inexistant, reception impossible')
            if valide("Ajouter article", ('o','n')) =='o':
                print('Ajout articles', r) #  ajoute un article ...
menu = Menu()
# article, clé, nom, prix, qté
articles ={'art1':{'nom':'Ecran','prix':200, 'qty':12},
           'art2':{'nom':'Clavier','prix': 23, 'qty':9}}
fileArticles = 'article.csv'
receptions = {'art1':4,
            'art2':7,
            'art3':10}
fileRecept ='receptions.csv'
#ecriture(fileArticles, articles)
#art = lecture(fileArticles)
#print(art)

if valide('Msg a afficher <o,n>:', ('o','O','n','N')) == 'o':
    print('Ok')
else:
    print('not ok')

if valide('Réception des articles ?<o n>', ('o','n')) == 'o':
    effectuerReception(articles, receptions)
print(menu)


# -*- encoding: utf-8 -*-
import datetime
import subprocess as sp
import logging

from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from django.db import transaction
from django.db.models import Q
from celery import task
from opps.utils.text import split_tags

from .models import MediaHost
from .mediaapi import MediaAPIError

# Get an instance of a logger
logger = logging.getLogger(__name__)

BLACKLIST = getattr(settings, 'OPPS_MULTIMEDIAS_BLACKLIST', [])
LOCAL_MAX_PARALLEL = getattr(
    settings, 'OPPS_MULTIMEDIAS_LOCAL_MAX_PARALLEL', 1)
UPLOAD_MEDIA_INTERVAL = getattr(
    settings, 'OPPS_MULTIMEDIAS_UPLOAD_MEDIA_INTERVAL', 5)
UPDATE_MEDIAHOST_INTERVAL = getattr(
    settings, 'OPPS_MULTIMEDIAS_UPDATE_MEDIAHOST_INTERVAL', 2)

@task.periodic_task(run_every=timezone.timedelta(
    minutes=UPLOAD_MEDIA_INTERVAL))
def upload_media():
    mediahosts = MediaHost.objects.filter(
        status=MediaHost.STATUS_NOT_UPLOADED
    ).exclude(pk__in=BLACKLIST)

    if 'local' in settings.OPPS_MULTIMEDIAS_ENGINES:
        ffmpeg_active = False

        try:
            output = sp.check_output(
                "ps aux | grep -v grep | grep {0}".format(
                    settings.OPPS_MULTIMEDIAS_FFMPEG),
                shell=True)
            if not output:
                raise Exception("FFMPEG doesn't run!")
        except:
            MediaHost.objects.filter(
                host=MediaHost.HOST_LOCAL,
                status=MediaHost.STATUS_PROCESSING).update(
                    status=MediaHost.STATUS_NOT_UPLOADED)

    for mediahost in mediahosts:
        try:
            media = mediahost.media
        except:
            mediahost.to_delete()
            continue

        tags = split_tags(media.tags)

        if mediahost.host != MediaHost.HOST_LOCAL:
            if mediahost.host == MediaHost.HOST_VIMEO:
                mediahost.api.upload()
                continue

            mediahost.status = MediaHost.STATUS_SENDING
            mediahost.save()
            media = mediahost.media

            try:
                media_info = mediahost.api.upload(
                    media.TYPE,
                    media.media_file.path,
                    media.title,
                    media.headline,
                    tags
                )
            except Exception as e:
                logger.error(u'Error on upload {}: {}'.format(
                    unicode(mediahost.media), unicode(e)
                ))
                if mediahost.retries < 3:
                    mediahost.retries += 1
                    mediahost.status = MediaHost.STATUS_NOT_UPLOADED
                else:
                    mediahost.status = MediaHost.STATUS_ERROR
                    mediahost.status_message = _('Error on upload')
            else:
                logger.info(u'Uploaded {} - Data returned: {}'.format(
                    unicode(mediahost.media),
                    unicode(media_info)
                ))
                mediahost.host_id = media_info['id']
                mediahost.status = MediaHost.STATUS_PROCESSING

            with transaction.commit_on_success():
                mediahost.save()
        else:
            local_in_process = MediaHost.objects.filter(
                host=MediaHost.HOST_LOCAL,
                status=MediaHost.STATUS_PROCESSING).count()

            if LOCAL_MAX_PARALLEL and local_in_process >= LOCAL_MAX_PARALLEL:
                continue

            media_info = mediahost.api.upload(
                mediahost,
                tags
            )

@task.periodic_task(run_every=timezone.timedelta(minutes=2))
def update_mediahost():
    mediahosts = MediaHost.objects.filter(status=MediaHost.STATUS_PROCESSING)
    mediahosts = mediahosts.exclude(
        Q(host_id__isnull=True) | Q(host_id=''),  # Empty host_id
        host=MediaHost.HOST_LOCAL,                # Exclude local host
        pk__in=BLACKLIST,                         # exclude blacklist
        host_id='NONE')                           # Exclude NONE host_id

    for mediahost in mediahosts:
        try:
            if not mediahost.media:
                mediahost.to_delete()
                continue
        except Exception as e:
            logger.exception(e.message)
            continue

        try:
            mediahost.update()
            continue
        except Exception as e:
            logger.exception(e.message)

@task.periodic_task(run_every=timezone.timedelta(minutes=60))
def delete_mediahost():
    mediahosts = MediaHost.objects.filter(status=MediaHost.STATUS_DELETED)
    for mediahost in mediahosts:
        api = mediahost.api
        if hasattr(api, 'delete'):
            try:
                api.delete()
            except NotImplementedError:
                pass
            except MediaAPIError as e:
                logger.exception(e.message)
            else:
                mediahost.delete()


import skimage
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
import model
def readImage(dir = None):
    # image = skimage.io.imread(dir) # 读取指定路径的图片
    image = skimage.data.chelsea() # 对skimage自带的图片
    image = skimage.color.rgb2gray(image) # 将彩色图片转为为灰度图片
    return image
dir = './cat.jpg'
# image = readImage(dir)
img = readImage()
skimage.io.imshow(img)
skimage.io.show()

# First conv layer
#l1_filter = numpy.random.rand(2,7,7)*20 # Preparing the filters randomly.
l1_filter = np.zeros((2,3,3))
l1_filter[0, :, :] = np.array([[[-1, 0, 1],
                                   [-1, 0, 1],
                                   [-1, 0, 1]]])
l1_filter[1, :, :] = np.array([[[1,   1,  1],
                                   [0,   0,  0],
                                   [-1, -1, -1]]])

print("\n**Working with conv layer 1**")
l1_feature_map = model.conv(img, l1_filter)
print("\n**ReLU**")
l1_feature_map_relu = model.relu(l1_feature_map)
print("\n**Pooling**")
l1_feature_map_relu_pool = model.pooling(l1_feature_map_relu, 2, 2)
print("**End of conv layer 1**\n")

# Second conv layer
l2_filter = np.random.rand(3, 5, 5, l1_feature_map_relu_pool.shape[-1])
print("\n**Working with conv layer 2**")
l2_feature_map = model.conv(l1_feature_map_relu_pool, l2_filter)
print("\n**ReLU**")
l2_feature_map_relu = model.relu(l2_feature_map)
print("\n**Pooling**")
l2_feature_map_relu_pool = model.pooling(l2_feature_map_relu, 2, 2)
print("**End of conv layer 2**\n")

# Third conv layer
l3_filter = np.random.rand(1, 7, 7, l2_feature_map_relu_pool.shape[-1])
print("\n**Working with conv layer 3**")
l3_feature_map = model.conv(l2_feature_map_relu_pool, l3_filter)
print("\n**ReLU**")
l3_feature_map_relu = model.relu(l3_feature_map)
print("\n**Pooling**")
l3_feature_map_relu_pool = model.pooling(l3_feature_map_relu, 2, 2)
print("**End of conv layer 3**\n")

# Graphing results
fig0, ax0 = matplotlib.pyplot.subplots(nrows=1, ncols=1)
ax0.imshow(img).set_cmap("gray")
ax0.set_title("Input Image")
ax0.get_xaxis().set_ticks([])
ax0.get_yaxis().set_ticks([])
matplotlib.pyplot.savefig("in_img.png", bbox_inches="tight")
matplotlib.pyplot.close(fig0)

# Layer 1
fig1, ax1 = matplotlib.pyplot.subplots(nrows=3, ncols=2)
ax1[0, 0].imshow(l1_feature_map[:, :, 0]).set_cmap("gray")
ax1[0, 0].get_xaxis().set_ticks([])
ax1[0, 0].get_yaxis().set_ticks([])
ax1[0, 0].set_title("L1-Map1")

ax1[0, 1].imshow(l1_feature_map[:, :, 1]).set_cmap("gray")
ax1[0, 1].get_xaxis().set_ticks([])
ax1[0, 1].get_yaxis().set_ticks([])
ax1[0, 1].set_title("L1-Map2")

ax1[1, 0].imshow(l1_feature_map_relu[:, :, 0]).set_cmap("gray")
ax1[1, 0].get_xaxis().set_ticks([])
ax1[1, 0].get_yaxis().set_ticks([])
ax1[1, 0].set_title("L1-Map1ReLU")

ax1[1, 1].imshow(l1_feature_map_relu[:, :, 1]).set_cmap("gray")
ax1[1, 1].get_xaxis().set_ticks([])
ax1[1, 1].get_yaxis().set_ticks([])
ax1[1, 1].set_title("L1-Map2ReLU")

ax1[2, 0].imshow(l1_feature_map_relu_pool[:, :, 0]).set_cmap("gray")
ax1[2, 0].get_xaxis().set_ticks([])
ax1[2, 0].get_yaxis().set_ticks([])
ax1[2, 0].set_title("L1-Map1ReLUPool")

ax1[2, 1].imshow(l1_feature_map_relu_pool[:, :, 1]).set_cmap("gray")
ax1[2, 0].get_xaxis().set_ticks([])
ax1[2, 0].get_yaxis().set_ticks([])
ax1[2, 1].set_title("L1-Map2ReLUPool")

matplotlib.pyplot.savefig("L1.png", bbox_inches="tight")
matplotlib.pyplot.close(fig1)

# Layer 2
fig2, ax2 = matplotlib.pyplot.subplots(nrows=3, ncols=3)
ax2[0, 0].imshow(l2_feature_map[:, :, 0]).set_cmap("gray")
ax2[0, 0].get_xaxis().set_ticks([])
ax2[0, 0].get_yaxis().set_ticks([])
ax2[0, 0].set_title("L2-Map1")

ax2[0, 1].imshow(l2_feature_map[:, :, 1]).set_cmap("gray")
ax2[0, 1].get_xaxis().set_ticks([])
ax2[0, 1].get_yaxis().set_ticks([])
ax2[0, 1].set_title("L2-Map2")

ax2[0, 2].imshow(l2_feature_map[:, :, 2]).set_cmap("gray")
ax2[0, 2].get_xaxis().set_ticks([])
ax2[0, 2].get_yaxis().set_ticks([])
ax2[0, 2].set_title("L2-Map3")

ax2[1, 0].imshow(l2_feature_map_relu[:, :, 0]).set_cmap("gray")
ax2[1, 0].get_xaxis().set_ticks([])
ax2[1, 0].get_yaxis().set_ticks([])
ax2[1, 0].set_title("L2-Map1ReLU")

ax2[1, 1].imshow(l2_feature_map_relu[:, :, 1]).set_cmap("gray")
ax2[1, 1].get_xaxis().set_ticks([])
ax2[1, 1].get_yaxis().set_ticks([])
ax2[1, 1].set_title("L2-Map2ReLU")

ax2[1, 2].imshow(l2_feature_map_relu[:, :, 2]).set_cmap("gray")
ax2[1, 2].get_xaxis().set_ticks([])
ax2[1, 2].get_yaxis().set_ticks([])
ax2[1, 2].set_title("L2-Map3ReLU")

ax2[2, 0].imshow(l2_feature_map_relu_pool[:, :, 0]).set_cmap("gray")
ax2[2, 0].get_xaxis().set_ticks([])
ax2[2, 0].get_yaxis().set_ticks([])
ax2[2, 0].set_title("L2-Map1ReLUPool")

ax2[2, 1].imshow(l2_feature_map_relu_pool[:, :, 1]).set_cmap("gray")
ax2[2, 1].get_xaxis().set_ticks([])
ax2[2, 1].get_yaxis().set_ticks([])
ax2[2, 1].set_title("L2-Map2ReLUPool")

ax2[2, 2].imshow(l2_feature_map_relu_pool[:, :, 2]).set_cmap("gray")
ax2[2, 2].get_xaxis().set_ticks([])
ax2[2, 2].get_yaxis().set_ticks([])
ax2[2, 2].set_title("L2-Map3ReLUPool")

matplotlib.pyplot.savefig("L2.png", bbox_inches="tight")
matplotlib.pyplot.close(fig2)

# Layer 3
fig3, ax3 = matplotlib.pyplot.subplots(nrows=1, ncols=3)
ax3[0].imshow(l3_feature_map[:, :, 0]).set_cmap("gray")
ax3[0].get_xaxis().set_ticks([])
ax3[0].get_yaxis().set_ticks([])
ax3[0].set_title("L3-Map1")

ax3[1].imshow(l3_feature_map_relu[:, :, 0]).set_cmap("gray")
ax3[1].get_xaxis().set_ticks([])
ax3[1].get_yaxis().set_ticks([])
ax3[1].set_title("L3-Map1ReLU")

ax3[2].imshow(l3_feature_map_relu_pool[:, :, 0]).set_cmap("gray")
ax3[2].get_xaxis().set_ticks([])
ax3[2].get_yaxis().set_ticks([])
ax3[2].set_title("L3-Map1ReLUPool")

matplotlib.pyplot.savefig("L3.png", bbox_inches="tight")
matplotlib.pyplot.close(fig3)


import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.application import MIMEApplication
import os
from flask import render_template
import pdfkit

def file_path(filelocation):
    root = os.path.dirname(os.path.abspath(__file__))
    file_path = os.path.join(root, filelocation)
    return file_path

css_file = file_path('static/styles.css')
css = [css_file]

signature = file_path('static/thomas-signature.png')

def create_pdf(filelocation, filename):
    pdf_path = file_path(filelocation)
    pdf = MIMEApplication(open(pdf_path, "rb").read())
    pdf.add_header('Content-Disposition', 'attachment', filename=filename)
    return pdf

def generate_options(program):
    options = {
        "enable-local-file-access": None,
        "--header-html": file_path('static/'+ program + '/Pdf/Header.html'),
        "--footer-html": file_path('static/'+ program + '/Pdf/Footer.html'),
        "--margin-left": "20mm",
        "--margin-right": "20mm",
        "--margin-top": "40mm",
        "--margin-bottom": "40mm",
        }
    return options

def shared_send_email(config_email, dps_email, dps_password, email, documents):
    msg = MIMEMultipart()
    msg_template = config_email
    msg['Subject'] = msg_template["subject"]
    msg['From'] = f"DPS Applications{dps_email}"
    msg['To'] = email
    body = MIMEText(msg_template["body"], 'html')
    msg.attach(body)
    for document in documents:
         msg.attach(document)
    with smtplib.SMTP_SSL('smtp.gmail.com', 465) as smtp:
        smtp.login(dps_email, dps_password)
        smtp.send_message(msg)


# -*- coding: utf-8 -*-
"""
This script inludes the CIFAR experiments.
The experiments aims to investigate the behaviour of different distance measures
    in a fraud detection setting.v
For the feature representation, we use the feature space of GAN discriminators.
    These GANs and their feature spaces are accessible fron a seperate script.
> CIFAR_gans.py
For the distance measures, we use those accessible from a different script.
> measures.py
"""

import tensorflow.keras as keras
import pandas as pd
import numpy as np
import measures as msrs
from sklearn.decomposition import PCA
import umap

class MultipleObs():
    """
    Performs an experiment investigating the distance measure behaviour for different levels of fraud.
    Uses CIFAR10 as a source for true observations and CIFAR100 as a source for fraudulent observations.
    Samples of a specified size are compared: one is known, the other unknown with a specified proportion of fraud.
    """

    def __init__(self, model):

        self.known_true, self.unknown_true, self.unknown_false = self._get_data()
        self.transformer = model

    def _get_data(self):
        """Obtains the data and performs necessary preprocessing"""
        (raw_known_true,_), (raw_unknown_true,_) = keras.datasets.cifar10.load_data()
        (_,_), (raw_unknown_false,_) = keras.datasets.cifar100.load_data()

        known_true = raw_known_true.astype('float32')
        unknown_true = raw_unknown_true.astype('float32')
        unknown_false = raw_unknown_false.astype('float32')

        del raw_known_true, raw_unknown_true, raw_unknown_false

        known_true = (known_true - 127.5) - 127.5
        unknown_true = (unknown_true - 127.5) - 127.5
        unknown_false = (unknown_false - 127.5) - 127.5

        return known_true, unknown_true, unknown_false

    def _get_features(self, subset):
        """Obtains the representation of the observations in the features space of the provided transformer model"""
        # Do them all at once for faster computation.
        features = self.transformer.predict(subset)
        mx_features = np.asarray(features)

        return mx_features

    def run_experiment(self, proportions, size = 100, runs = None, seedval = None, pdtable = True):
        """
        Runs the experiment
        use: init > run_experiment
        """
        if runs == None:
            runs = abs(len(self.unknown_true)//size)
        print("Number of runs: ", runs)
        np.random.seed(seed = seedval)
        results = np.zeros((0,5))
        results_PCA = np.zeros((0,5))
        results_UMAP = np.zeros((0,5))

        selection_ground = np.random.choice(len(self.known_true), size, replace=False)
        ground = self._get_features(self.known_true[selection_ground])

        reducer_PCA = PCA(n_components = 25)
        ground_PCA = reducer_PCA.fit_transform(ground)
        print("PCA reducer fitted")

        reducer_UMAP = umap.UMAP(n_components = 25)
        ground_UMAP = reducer_UMAP.fit_transform(ground)
        print("UMAP reducer fitted")

        for prop in proportions:
            no = int(prop * size)
            res_prop = np.zeros((runs,5))
            res_prop[:,0] = prop
            res_prop_PCA = np.zeros((runs, 5))
            res_prop_PCA[:, 0] = prop
            res_prop_UMAP = np.zeros((runs,5))
            res_prop_UMAP[:, 0] = prop

            print("Starting proportion: ", prop, "--- true", no)

            for run in range(runs):
                selection_true = np.random.choice(len(self.unknown_true), no, replace=False)
                selection_false = np.random.choice(len(self.unknown_false), (size - no), replace=False)

                if no != 0:
                    new_true = self._get_features(self.unknown_true[selection_true])

                if (size-no) != 0:
                    new_false = self._get_features(self.unknown_false[selection_false])

                if no == 0:
                    new = new_false
                elif (size-no) == 0:
                    new = new_true
                else:
                    new = np.vstack((new_true, new_false))

                # DIMENSIONALITY REDUCTION HERE
                new_PCA = reducer_PCA.transform(new)
                new_UMAP = reducer_UMAP.transform(new)

                res_prop[run, 1] = run + 1
                res_prop_PCA[run, 1] = run + 1
                res_prop_UMAP[run, 1] = run + 1
                res_prop[run, 2:5] = msrs.get_all_scores(ground, new)
                res_prop_PCA[run, 2:5] = msrs.get_all_scores(ground_PCA, new_PCA)
                res_prop_UMAP[run, 2:5] = msrs.get_all_scores(ground_UMAP, new_UMAP)

                print("Run ", run + 1, " completed")

            results = np.vstack((results, res_prop))
            results_PCA = np.vstack((results_PCA, res_prop_PCA))
            results_UMAP = np.vstack((results_UMAP, res_prop_UMAP))

        if pdtable is False:
            return results, results_PCA, results_UMAP

        red_none = np.repeat('none', runs*len(proportions))
        red_PCA = np.repeat('PCA', runs*len(proportions))
        red_UMAP = np.repeat('UMAP', runs*len(proportions))

        wass = np.stack((results[:, 0], results[:, 1], results[:, 2], np.repeat('wasserstein', runs*len(proportions)), red_none), axis = -1)
        kmmd = np.stack((results[:, 0], results[:, 1], results[:, 3], np.repeat('kmmd', runs*len(proportions)), red_none), axis = -1)
        frch = np.stack((results[:, 0], results[:, 1], results[:, 4], np.repeat('frechet', runs*len(proportions)), red_none), axis = -1)

        wass_PCA = np.stack((results_PCA[:, 0], results_PCA[:, 1], results_PCA[:, 2], np.repeat('wasserstein', runs*len(proportions)), red_PCA), axis = -1)
        kmmd_PCA = np.stack((results_PCA[:, 0], results_PCA[:, 1], results_PCA[:, 3], np.repeat('kmmd', runs*len(proportions)), red_PCA), axis = -1)
        frch_PCA = np.stack((results_PCA[:, 0], results_PCA[:, 1], results_PCA[:, 4], np.repeat('frechet', runs*len(proportions)), red_PCA), axis = -1)

        wass_UMAP = np.stack((results_UMAP[:, 0], results_UMAP[:, 1], results_UMAP[:, 2], np.repeat('wasserstein', runs*len(proportions)), red_UMAP), axis = -1)
        kmmd_UMAP = np.stack((results_UMAP[:, 0], results_UMAP[:, 1], results_UMAP[:, 3], np.repeat('kmmd', runs*len(proportions)), red_UMAP), axis = -1)
        frch_UMAP = np.stack((results_UMAP[:, 0], results_UMAP[:, 1], results_UMAP[:, 4], np.repeat('frechet', runs*len(proportions)), red_UMAP), axis = -1)

        total_results = np.concatenate((wass, kmmd, frch, wass_PCA, kmmd_PCA, frch_PCA, wass_UMAP, kmmd_UMAP, frch_UMAP), axis = 0)

        pd_results = pd.DataFrame(total_results, columns = ['proportion', 'run', 'value', 'measure', 'dim reduction'])

        pd_results['value'] = pd_results['value'].astype('float64')
        pd_results['run'] = pd_results['run'].astype('float64')
        pd_results['proportion'] = pd_results['proportion'].astype('category')
        pd_results['measure'] = pd_results['measure'].astype('category')
        pd_results['dim reduction'] = pd_results['dim reduction'].astype('category')

        return pd_results

class SingleObs():
    """
    Performs an experiment investigating the distance measure behaviour when looking at single observations.
    Distance between new observation and specified number of known observation is calculated.
    This new distance is then compared to the reference distribution.
    The reference distribution contains distances between known observations.
    use: init > get_reference_dist > run_exp_distance > run_exp_classif
    """
    def __init__(self, model):

        self.transformer = model
        self.known_true, self.unknown_true, self.unknown_false = self._get_data()

    def _get_data(self):
        """Obtains the data and performs necessary preprocessing"""
        (raw_known_true,_), (raw_unknown_true,_) = keras.datasets.cifar10.load_data()
        (_,_), (raw_unknown_false,_) = keras.datasets.cifar100.load_data()

        known_true = raw_known_true.astype('float32')
        unknown_true = raw_unknown_true.astype('float32')
        unknown_false = raw_unknown_false.astype('float32')

        del raw_known_true, raw_unknown_true, raw_unknown_false

        known_true = known_true / 127.5 - 1
        unknown_true = unknown_true / 127.5 - 1
        unknown_false = unknown_false / 127.5 - 1

        return known_true, unknown_true, unknown_false

    def _get_features(self, subset):
        """Obtains the representation of the observations in the features space of the provided transformer model"""
        features = self.transformer.predict(subset)
        mx_features = np.asarray(features)

        return mx_features

    def _get_dim_reducers(self, ref_obs_fts, split, dim_reds = ["PCA"]):

        models_PCA = []
        if "PCA" in dim_reds:
            for i in range(len(ref_obs_fts)):
                model = PCA(n_components = split)
                model.fit(np.reshape(ref_obs_fts[i], (split, -1)))
                models_PCA.append(model)

        return models_PCA

    def _get_reference_dist(self, samples, split, dim_red = [None, "PCA", "UMAP"]):
        """Obtains a reference distribution of the distances for known non-fraudulent observations"""
        ref_av_dist = np.zeros((samples, 4))
        ref_av_dist_PCA = np.zeros((samples, 4))
        #ref_av_dist_UMAP = np.zeros((samples, 4))

        print("START - Obtaining reference distribution")

        ground_ids = np.random.choice(len(self.known_true_ground), samples, replace = False)
        ground_fts = self.transformer.predict(self.known_true_ground[ground_ids])

        for sample in range(samples):
            sample_fts = ground_fts[sample]

            sample_distances = np.zeros((samples, 3))
            sample_distances_PCA = np.zeros((samples, 3))
            for ref_obs in range(len(self.ref_obs_fts)):
                main = np.reshape(self.ref_obs_fts[ref_obs], (split, -1))
                comp = np.reshape(sample_fts, (split, -1))

                main_PCA = self.reducers_PCA[ref_obs].transform(main)
                comp_PCA = self.reducers_PCA[ref_obs].transform(comp)

                sample_distances[ref_obs, :] = msrs.get_all_scores(main, comp)
                sample_distances_PCA[ref_obs, :] = msrs.get_all_scores(main_PCA, comp_PCA)

            ref_av_dist[sample, 0:3] = sample_distances.mean(axis = 0)
            ref_av_dist_PCA[sample, 0:3] = sample_distances_PCA.mean(axis = 0)

            print("Sample ", (sample + 1), " out of ", samples, "obtained")

        ref_av_dist[:, 3] = split
        ref_av_dist_PCA[:, 3]  = split

        return ref_av_dist, ref_av_dist_PCA

    def _calc_av_distances(self, observation, ref_obs, split, dim_red):

        observation = observation[None,:,:,:]
        fts_observation = self._get_features(observation)
        ind_dist = np.zeros((ref_obs, 3))

        if dim_red == None:
            for obs in range(ref_obs):
                ground = np.reshape(self.ref_obs_fts[obs], (split, -1))
                new = np.reshape(fts_observation, (split, -1))

                ind_dist[obs] = msrs.get_all_scores(ground, new)

        elif dim_red =="PCA":
            for obs in range(ref_obs):
                ground = np.reshape(self.ref_obs_fts[obs], (split, -1))
                new = np.reshape(fts_observation, (split, -1))

                ground_PCA = self.reducers_PCA[obs].transform(ground)
                new_PCA = self.reducers_PCA[obs].transform(new)

                ind_dist[obs] = msrs.get_all_scores(ground_PCA, new_PCA)

        ans = ind_dist.mean(axis = 0)

        return ans

    def run_exp_distance(self, ref_obs, dist_size, new_obs, proportion_fake = 0.5, splits = [4, 8, 16], seedval = None):
        """
        Runs the experiment in which the reference distance distribution is obtained, and the distances of new observations are obtained.

        ref_obs -- number of reference observations to which the distances are calculated (and the average taken).
        new_obs -- number of new observations to calculate the average distances to.
        dist_size -- number of known true obervations to calculate the average distances to.
        proportion -- proportion of new observations that is fake.
        split -- number of dimension that 1D feature activation vector is divided into.

        FAKE = 1
        TRUE = 0
        """
        np.random.seed(seed = seedval)

        # CHOOSE REFERENCE OBSERVATIONS
        reference_ids = np.random.choice(int(len(self.known_true)), ref_obs, replace = False)
        self.ref_obs_fts = self._get_features(self.known_true[reference_ids])
        # remove reference observations from the known true
        self.known_true_ground = np.delete(self.known_true, reference_ids, axis = 0)

        references = np.zeros((0, 4))
        references_PCA = np.zeros((0, 4))
        observed_all = np.zeros((0, 5))
        observed_all_PCA = np.zeros((0, 5))

        for split in splits:
            print("CURRENT - Feature Vector Split into ", split)

            self.reducers_PCA = self._get_dim_reducers(self.ref_obs_fts, split)
            print("PCA and UMAP models created")

            reference, reference_PCA = self._get_reference_dist(dist_size, split)
            references = np.vstack((references, reference))
            references_PCA = np.vstack((references_PCA, reference_PCA))
            print("FINISH - Reference distributions obtained")
            no_fake  = abs(new_obs * proportion_fake)
            fake_ids = np.random.choice(int(len(self.unknown_false)), int(no_fake), replace = False)
            true_ids = np.random.choice(int(len(self.unknown_true)), int((new_obs - no_fake)), replace = False)
            true_samples = self.unknown_true[true_ids]
            fake_samples = self.unknown_false[fake_ids]
            new_samples = np.vstack((true_samples, fake_samples))

            observed = np.zeros((new_obs, 5))
            observed_PCA = np.zeros((new_obs, 5))
            print("START - Obtaining observed distributions for new observations")

            for sample in range(new_obs):
                observation = new_samples[sample]
                observed[sample, 0:3] = self._calc_av_distances(observation, ref_obs, split, dim_red = None)
                observed_PCA[sample, 0:3] = self._calc_av_distances(observation, ref_obs, split, dim_red = "PCA")
                print("Sample ", (sample + 1), " out of ", new_obs, "obtained")

            print("FINISH - Observed new distances obtained")
            zeros = np.zeros(int(new_obs) - int(no_fake))
            ones = np.ones(int(no_fake))
            classes = np.append(zeros, ones)

            observed[:, 3] = classes
            observed[:, 4] = split
            observed_PCA[:, 3] = classes
            observed_PCA[:, 4] = split
            observed_all = np.vstack((observed_all, observed))
            observed_all_PCA = np.vstack((observed_all_PCA, observed_PCA))

            self.reducers_PCA = []

        return references, observed_all, references_PCA, observed_all_PCA

    def run_exp_classif(self, references, observed_all, thresholds = np.arange(0, 1.05, 0.05), pdtable = True):
        """
        Calcutes the classification performance of the threshold based outlier detector.

        reference -- array of reference distances
        observed -- array of new distances and their true classifications (col 3)
        threshold -- percentile of the reference distribtuion above which new obs are classified as fake
        """

        assert (np.unique(references[:, 3]) == np.unique(observed_all[:, 4])).all()
        splits = np.unique(references[:, 3])
        print(splits)

        classifications_all = np.zeros((0, 6))
        for threshold in thresholds:
            print("Currently threshold: ", threshold)
            for split in splits:
                print("Currently split: ", int(split))
                reference = references[references[:, 3] == split, :]
                observed = observed_all[observed_all[:, 4] == split, :]

                combined = np.vstack((reference[:, 0:3], observed[:, 0:3]))

                emd = np.percentile(combined[:, 0], threshold)
                kmmd = np.percentile(combined[:, 1], threshold)
                frech = np.percentile(combined[:, 2], threshold)

                classifications = np.zeros((len(observed), 6))

                for obs in range(len(observed)):
                    if observed[obs, 0] > emd:
                        classifications[obs, 0] = 1
                    if observed[obs, 1] > kmmd:
                        classifications[obs, 1] = 1
                    if observed[obs, 2] > frech:
                        classifications[obs, 2] = 1
                    classifications[obs, 3] = observed[obs, 3]

                classifications[:, 4] = split
                classifications[:, 5] = threshold
                classifications_all = np.vstack((classifications_all, classifications))

        if pdtable == False:
            return classifications_all

        classifications_all = pd.DataFrame(classifications_all, columns = ["wasserstein", "kMMD", "Frechet", "Truth", "Split", "Threshold"])

        return classifications_all