import torch.nn as nn
import torch
import torch.nn.functional as F
from KGEModel.KGEModel import KGEModel

class TransH(KGEModel):
    def __init__(self, args):
        super(TransH, self).__init__(args)
        self.args = args
        self.entity_embedding = None
        self.relation_embedding = None
        self.norm_flag = args.norm_flag
        self.init_emb()

    def init_emb(self):
        self.epsilon = 2.0
        self.margin = nn.Parameter(torch.Tensor([self.args.margin]), requires_grad=False)
        self.embedding_range = nn.Parameter(torch.Tensor([(self.margin.item() + self.epsilon) / self.args.emb_dim]),
                                            requires_grad=False,)

        self.entity_embedding = nn.Embedding(self.args.nentity, self.args.emb_dim)
        self.relation_embedding = nn.Embedding(self.args.nrelation, self.args.emb_dim)
        self.norm_vector = nn.Embedding(self.args.nrelation, self.args.emb_dim)
        nn.init.uniform_(tensor=self.entity_embedding.weight.data, a=-self.embedding_range.item(), b=self.embedding_range.item())
        nn.init.uniform_(tensor=self.relation_embedding.weight.data, a=-self.embedding_range.item(), b=self.embedding_range.item())
        nn.init.uniform_(tensor=self.norm_vector.weight.data, a=-self.embedding_range.item(), b=self.embedding_range.item())

    def score_func(self, head_emb, relation_emb, tail_emb, mode):
        if self.norm_flag:
            head_emb = F.normalize(head_emb, 2, -1)
            relation_emb = F.normalize(relation_emb, 2, -1)
            tail_emb = F.normalize(tail_emb, 2, -1)
        if mode == "head-batch":
            score = head_emb + (relation_emb - tail_emb)
        else:
            score = (head_emb + relation_emb) - tail_emb
        score = self.margin.item() - torch.norm(score, p=1, dim=-1)
        return score

    def forward(self, triples, mode="single"):
        if mode == "single":
            head_emb = self.entity_embedding(triples[:, 0]).unsqueeze(1)  # [bs, 1, dim]
            relation_emb = self.relation_embedding(triples[:, 1]).unsqueeze(1)  # [bs, 1, dim]
            tail_emb = self.entity_embedding(triples[:, 2]).unsqueeze(1)  # [bs, 1, dim]
            norm_vector = self.norm_vector(triples[:, 1]).unsqueeze(dim=1)  # shape:[bs, 1, dim]

        elif mode == "head-batch":
            tail_part, head_part = triples
            batch_size, negative_sample_size = head_part.size(0), head_part.size(1)

            head_emb = torch.index_select(self.entity_embedding.weight.data, dim=0, index=head_part.view(-1)).view(
                batch_size, negative_sample_size, -1)  # [bs, num_neg, dim]
            relation_emb = self.relation_embedding(tail_part[:, 1]).unsqueeze(1)  # [bs, 1, dim]
            tail_emb = self.entity_embedding(tail_part[:, 2]).unsqueeze(1)  # [bs, 1, dim]
            norm_vector = self.norm_vector(tail_part[:, 1]).unsqueeze(dim=1)  # shape:[bs, 1, dim]

        elif mode == "tail-batch":
            head_part, tail_part = triples  # head是positive samples，tail是negative tail_id
            batch_size, negative_sample_size = tail_part.size(0), tail_part.size(1)

            head_emb = self.entity_embedding(head_part[:, 0]).unsqueeze(1)  # [bs, 1, dim]
            relation_emb = self.relation_embedding(head_part[:, 1]).unsqueeze(1)  # [bs, 1, dim]
            tail_emb = torch.index_select(self.entity_embedding.weight.data, dim=0, index=tail_part.view(-1)).view(
                batch_size, negative_sample_size, -1)  # [bs, num_neg, dim]
            norm_vector = self.norm_vector(head_part[:, 1]).unsqueeze(dim=1)  # shape:[bs, 1, dim]

        head_emb = self._transfer(head_emb, norm_vector)
        tail_emb = self._transfer(tail_emb, norm_vector)
        score = self.score_func(head_emb, relation_emb, tail_emb, mode)

        return score

    def _transfer(self, emb, norm_vector):
        if self.norm_flag:
            norm_vector = F.normalize(norm_vector, p=2, dim=-1)
        return emb - torch.sum(emb * norm_vector, -1, True) * norm_vector

    def loss(self, pos_score, neg_score, subsampling_weight=None):
        neg_score = F.logsigmoid(-neg_score)  # shape:[bs]
        pos_score = F.logsigmoid(pos_score)  # shape:[bs, 1]
        if self.args.uni_weight:
            positive_sample_loss = - pos_score.mean()
            negative_sample_loss = - neg_score.mean()
        else:
            positive_sample_loss = - (subsampling_weight * pos_score).sum() / subsampling_weight.sum()
            negative_sample_loss = - (subsampling_weight * neg_score).sum() / subsampling_weight.sum()

        loss = (positive_sample_loss + negative_sample_loss) / 2
        return loss


from __future__ import print_function

import argparse
import pprint
import sys
sys.path.append("../")
import grpc
from protobuf import dlserver_pb2
from protobuf import dlserver_pb2_grpc
from protobuf import pb2_Feature, pb2_Features, pb2_Int64List, pb2_Example, pb2_BytesList
from conf import config

import cv2
import numpy as np

def create_examples(images):

    def getImageByteString(imgInput):
        try:
            img_encode = cv2.imencode('.jpg', imgInput)[1]
            data_encode = np.array(img_encode)
            imageByteString = data_encode.tostring()
            return imageByteString
        except Exception:
            return None

    def getExample(imageName):
        try:
            # image_reader = ImageReader()
            # with tf.Session('') as sess:
            image = cv2.imread(imageName)  # 读入图片W
            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)  # 将图片转换为RGB格式，因为cv2读入为BGRW
            height = 299
            width = 299
            image = cv2.resize(image, (height, width))
            imageByteString = getImageByteString(image)

            #多标签W
            # imageName = getImageName(imageName)  # 获取图像对应的名字，用于预测W
            example = pb2_Example(features=pb2_Features(feature={
                'image/encoded': pb2_Feature(bytes_list=pb2_BytesList(value=[imageByteString])),
                'image/format': pb2_Feature(bytes_list=pb2_BytesList(value=[b'jpg'])),
                'image/class/labelPP': pb2_Feature(int64_list=pb2_Int64List(value=[0])),
                'image/class/labelFood': pb2_Feature(int64_list=pb2_Int64List(value=[0])),
                'image/class/labelScene': pb2_Feature(int64_list=pb2_Int64List(value=[0]))
            }))

            return example
        except Exception as e:
            pass
        return None

    examples = list(filter(lambda x: x != None, map(getExample, images))) #将lines中的每一行转为TFRecordW

    print("request examples len: {}".format(len(examples)))
    example_list = dlserver_pb2.ExampleList(examples=examples) #生成一个list请求W

    return example_list

def getScores(stub, dl_request):
    feature = stub.GetOutputs(dl_request)
    return feature

# host_port = '10.126.106.160:50051' if config.debug_mode else '10.126.106.210:50051'
host_port = '10.126.106.210:50051'
channel = grpc.insecure_channel(host_port)
stub = dlserver_pb2_grpc.ExamplesDLServerStub(channel)

def requestDLServerLabels(images):
    print("-------------- GetScores --------------")
    example_list = create_examples(images) #自定义解码W
    dl_request = dlserver_pb2.DLRequest(task_id="803", example_list=example_list) #服务idW
    return getScores(stub, dl_request)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.register("type", "bool", lambda v: v.lower() == "true")
    parser.add_argument(
        "--debug_mode",
        type="bool",
        default=True,
        help="use debug mode")
    parser.add_argument(
        "--use_hdfs",
        type=bool,
        default=False,
        help="use hdfs file")
    parser.add_argument(
        "--partition_num",
        type=int,
        default=64,
        help="the number of the repartition num")

    FLAGS, unparsed = parser.parse_known_args()
    pp = pprint.PrettyPrinter().pprint
    pp(FLAGS)
    pp(unparsed)

    images = ['./tmp/image/WechatIMG{}.jpeg'.format(i) for i in range(17, 21)]

    for i in range(1):
        result = requestDLServerLabels(images)
        print(result)


import abc
import collections
import itertools
from typing import Any

import gymnasium
import numpy as np
from gymnasium import spaces
from jdrones.data_models import State
from jdrones.data_models import States
from jdrones.data_models import URDFModel
from jdrones.envs.base.basecontrolledenv import BaseControlledEnv
from jdrones.envs.dronemodels import DronePlus
from jdrones.envs.lqr import LQRDroneEnv
from jdrones.trajectory import BasePolynomialTrajectory
from jdrones.trajectory import FifthOrderPolynomialTrajectory
from jdrones.trajectory import FirstOrderPolynomialTrajectory
from jdrones.types import DType
from jdrones.types import PositionAction
from jdrones.types import PositionVelocityAction
from jdrones.types import VEC3

class BasePositionDroneEnv(gymnasium.Env, abc.ABC):
    """
    Baseclass for other position drone environments. These are ones where step takes
    a :math:`(x,y,z)` argument and makes a drone fly from its current position to there.
    """

    env: BaseControlledEnv

    dt: float

    model: URDFModel

    def __init__(
        self,
        model: URDFModel = DronePlus,
        initial_state: State = None,
        dt: float = 1 / 240,
        env: LQRDroneEnv = None,
    ):
        if env is None:
            env = LQRDroneEnv(model=model, initial_state=initial_state, dt=dt)
        self.env = env
        self.dt = dt
        self.model = model
        self.observation_space = spaces.Sequence(self.env.observation_space, stack=True)
        act_bounds = np.array([[0, 10], [0, 10], [1, 10]], dtype=DType)
        self.action_space = spaces.Box(
            low=act_bounds[:, 0], high=act_bounds[:, 1], dtype=DType
        )

    @staticmethod
    def get_reward(states: States) -> float:
        """
        Calculate the cost of the segment with

        .. math::
            \\begin{gather}
            J = \\sum_{i=0}^{T/dt} Q \\frac{\\vec x_i}{dt}
            \\\\
            Q =
            \\left[
                1000,1000,1000,0,0,0,0,10,10,10,1,1,1,1,1,1,0,0,0,0
            \\right]
            \\\\
            \\vec x =
            \\left[
                x,y,z,q_x,q_y,q_z,q_w,\\phi,\\theta,\\psi,\\dot x,\\dot y,\\dot z,
                p,q,r,P_1,P_2,P_3,P_4
            \\right]
            \\end{gather}

        Where :math:`\\vec x_i` is the state matrix at time-step :math:`i`, and
        :math:`Q` is a cost matrix prioritizing error in position and angle.

        Parameters
        ----------
        states : jdrones.data_models.States
            Iterable containing the observed states at regular intervals :math:`dt`

        Returns
        -------
        float
            The calculated cost
        """
        df = states.to_df(tag="temp")
        df_sums = (
            df.sort_values("t")
            .groupby(df.variable)
            .apply(lambda r: np.trapz(r.value.abs(), x=r.t))
        )

        df_sums[["P0", "P1", "P2", "P3"]] = 0
        df_sums[["qw", "qx", "qy", "qz"]] = 0
        df_sums[["x", "y", "z"]] *= 1000
        df_sums[["phi", "theta", "psi"]] *= 10

        return df_sums.sum()

class PolynomialPositionBaseDronEnv(BasePositionDroneEnv):
    @staticmethod
    def calc_traj(cur: State, tgt: State, max_vel: float = 1):
        raise NotImplementedError

    @staticmethod
    def update_u_from_traj(u: State, traj: BasePolynomialTrajectory, t: float):
        if hasattr(traj, "position"):
            u.pos = traj.position(t)
        if hasattr(traj, "velocity"):
            u.vel = traj.velocity(t)
        return u

    @staticmethod
    def _validate_action_input(
        action: PositionAction | PositionVelocityAction,
    ) -> State:
        """
        Validate the action input. Has to either be only position, or position and
        velocity

        Parameters
        ----------
        action : VEC3 | tuple[VEC3, VEC3]
            Target coordinates :math:`(x,y,z)` or target coordinates and velocity (
            :math:`(v_x,v_y,v_z)`)

        Returns
        -------
        State
            The valid and converted action as a :class:`~jdrones.data_models.State`

        """
        action_as_state = State()
        action_np = np.asarray(action).squeeze()
        if action_np.shape == (3,):
            action_as_state.pos = action_np
        elif action_np.shape == (2, 3):
            action_as_state.pos = action_np[0]
            action_as_state.vel = action_np[1]
        elif action_np.shape == (6,):
            action_as_state.pos = action_np[:3]
            action_as_state.vel = action_np[3:]
        else:
            raise ValueError(
                f"Incorrect shape {action_np.shape}. Expected either " f"(3,) or (2,3)"
            )
        return action_as_state

    def reset(
        self,
        *,
        seed: int | None = None,
        options: dict[str, Any] | None = None,
    ) -> tuple[States, dict[str, Any]]:
        super().reset(seed=seed, options=options)

        obs, _ = self.env.reset(seed=seed, options=options)

        return States([np.copy(obs)]), {}

    def step(
        self, action: PositionAction | PositionVelocityAction
    ) -> tuple[States, float, bool, bool, dict[str, Any]]:
        """
        A step from the viewpoint of a
        :class:`~jdrones.envs.position.BasePositionDroneEnv` is making the drone
        fly from its current position :math:`A` to the target position :math:`B`.

        Parameters
        ----------
        action : VEC3 | tuple[VEC3, VEC3]
            Target coordinates :math:`(x,y,z)` or target coordinates and velocity (
            :math:`(v_x,v_y,v_z)`)

        Returns
        -------
            states: jdrones.data_models.States
            reward: float
            term: bool
            trunc: bool
            info: dict
        """
        action_as_state = self._validate_action_input(action)

        term, trunc, info = False, False, {}
        if np.allclose(action, self.env.state.pos):
            # Avoid a singularity since the trajectory is scaled with distance
            term = True
            traj = None
        else:
            traj = self.calc_traj(
                self.env.state, action_as_state, self.model.max_vel_ms
            )

        observations = collections.deque()
        observations.append(self.env.state.copy())

        u: State = action_as_state.copy()

        counter = itertools.count(-1)
        while not (term or trunc):
            t = next(counter) * self.dt
            if t > traj.T:
                u.pos = action_as_state.pos
                u.vel = (0, 0, 0)
            else:
                u = self.update_u_from_traj(u, traj, t)

            obs, _, term, trunc, info = self.env.step(u.to_x())

            observations.append(obs.copy())

            dist = np.linalg.norm(self.env.state.pos - action_as_state.pos)
            if np.any(np.isnan(dist)):
                trunc = True

            if dist < 0.01:
                term = True
                info["error"] = dist

        states = States(observations)
        return states, self.get_reward(states), term, trunc, info

class FifthOrderPolyPositionDroneEnv(PolynomialPositionBaseDronEnv):
    """
    Uses :class:`jdrones.trajectory.FifthOrderPolynomialTrajectory` to give target
    position and velocity commands at every time point until the target is reached.
    If the time taken exceeds :math:`T`, the original target position is given as a raw
    input. However, if this were to happen, the distance is small enough to ensure
    stability.

    >>> import jdrones
    >>> import gymnasium
    >>> gymnasium.make("FifthOrderPolyPositionDroneEnv-v0")
    <OrderEnforcing<PassiveEnvChecker<FifthOrderPolyPositionDroneEnv<FifthOrderPolyPositionDroneEnv-v0>>>>

    """

    @staticmethod
    def calc_traj(
        cur: State, tgt: State, max_vel: float = 1
    ) -> FifthOrderPolynomialTrajectory:
        """
        Calculate the trajectory for the drone to traverse.

        Total time to traverse the polynomial is defined as

        .. math::
            T = \\frac{||x_{t=T}-x_{t=0}||}{v_\\max}

        to ensure dynamic compatibility.

        Parameters
        ----------
        cur : jdrones.data_models.State
            Current state
        tgt : jdrones.data_models.State
            Target state
        max_vel : float
            Maximum vehicle velocity

        Returns
        -------
        jdrones.trajectory.FifthOrderPolynomialTrajectory
            The solved trajectory
        """
        dist = np.linalg.norm(tgt.pos - cur.pos)
        T = dist / max_vel

        t = FifthOrderPolynomialTrajectory(
            start_pos=cur.pos,
            start_vel=cur.vel,
            start_acc=(0, 0, 0),
            dest_pos=tgt.pos,
            dest_vel=tgt.vel,
            dest_acc=(0, 0, 0),
            T=T,
        )
        return t

class FirstOrderPolyPositionDroneEnv(PolynomialPositionBaseDronEnv):
    """
    Uses :class:`jdrones.trajectory.FirstOrderPolynomialTrajectory` to give target
    position commands at every time point until the target is reached.
    If the time taken exceeds :math:`T`, the original target position is given as a raw
    input. However, if this were to happen, the distance is small enough to ensure
    stability.

    >>> import jdrones
    >>> import gymnasium
    >>> gymnasium.make("FirstOrderPolyPositionDroneEnv-v0")
    <OrderEnforcing<PassiveEnvChecker<FirstOrderPolyPositionDroneEnv<FirstOrderPolyPositionDroneEnv-v0>>>>
    """

    @staticmethod
    def calc_traj(
        cur: State, tgt: State, max_vel: float = 1
    ) -> FirstOrderPolynomialTrajectory:
        """
        Calculate the trajectory for the drone to traverse.

        Total time to traverse the polynomial is defined as

        .. math::
            T = \\frac{||x_{t=T}-x_{t=0}||}{v_\\max}

        to ensure dynamic compatibility.

        Parameters
        ----------
        cur : jdrones.data_models.State
            Current state
        tgt : jdrones.data_models.State
            Target state
        max_vel : float
            Maximum vehicle velocity

        Returns
        -------
        jdrones.trajectory.FirstOrderPolynomialTrajectory
            The solved trajectory
        """
        dist = np.linalg.norm(tgt.pos - cur.pos)
        T = dist / max_vel

        t = FirstOrderPolynomialTrajectory(
            start_pos=cur.pos,
            dest_pos=tgt.pos,
            T=T,
        )
        return t

class FifthOrderPolyPositionWithLookAheadDroneEnv(BasePositionDroneEnv):
    """
    Supplements the :class:`jdrones.env.FifthOrderPolyPositionDroneEnv` by including
    the next waypoints position in the trajectory generation calculation.

    >>> import jdrones
    >>> import gymnasium
    >>> gymnasium.make("FifthOrderPolyPositionWithLookAheadDroneEnv-v0")
    <OrderEnforcing<PassiveEnvChecker<FifthOrderPolyPositionWithLookAheadDroneEnv<FifthOrderPolyPositionWithLookAheadDroneEnv-v0>>>>
    """

    env: FifthOrderPolyPositionDroneEnv

    def __init__(
        self,
        model: URDFModel = DronePlus,
        initial_state: State = None,
        dt: float = 1 / 240,
        env: FifthOrderPolyPositionDroneEnv = None,
    ):
        if env is None:
            env = FifthOrderPolyPositionDroneEnv(
                model=model, initial_state=initial_state, dt=dt, env=env
            )
        super().__init__(model=model, initial_state=initial_state, dt=dt, env=env)

        self.action_space = spaces.Box(
            low=np.array((self.env.action_space.low,) * 2, dtype=DType),
            high=np.array((self.env.action_space.high,) * 2, dtype=DType),
            shape=(2, 3),
            dtype=DType,
        )

    def reset(
        self,
        *,
        seed: int | None = None,
        options: dict[str, Any] | None = None,
    ) -> tuple[States, dict[str, Any]]:
        super().reset(seed=seed, options=options)

        obs, _ = self.env.reset(seed=seed, options=options)

        return obs, {}

    @staticmethod
    def calc_v_at_B(A: VEC3, B: VEC3, C: VEC3, *, V: float, N: float = 3):
        """
        For the velocity vector at :math:`B`, it is ideal to create a smooth
        transition between the segments. One solution is to reduce velocity to 0.
        However, this is very inefficient and not representative of real-world flight,
        especially for the case where there are multiple waypoints in a straight line.
        The preferred, albeit slightly more computationally expensive, solution is to
        use the waypoints before and after to calculate a smooth velocity at :math:`B`.
        This is calculated by

        .. math::
            \\begin{gather}
                \\theta =
                    \\frac{(-\\vec r_{AB}) \\cdot \\vec r_{BC}}
                    {|\\vec r_{AB}| |\\vec r_{BC}|}, \\\\
                \\chi = \\frac{(1-\\theta)}{2}, \\\\
                \\vec v_{B,-} = \\vec v_{B,+} = V \\cdot
                    \\frac{\\vec r_{AC}}
                    {| \\vec r_{AC} |}
                    \\cdot \\chi^N,
            \\end{gather}

        Parameters
        ----------
        A : VEC3
            Current position
        B : VEC3
            Target position
        C : VEC3
            Position after going through :math:`B`
        V : float
            Scaling factor for the magnitude of :math:`\\vec v_B`
        N : float
            Scaling factor for the magnitude of :math:`\\chi`.
            :math:`N = 3` for an aggressive reduction in velocity for non-straight line
            waypoints to closely follow the path. :math:`N = 1` would be more suitable
            for cases where following the path is not as important.
            (Default = 3)

        Returns
        -------
        VEC3
            The velocity at :math:`B`
        """

        rAC = C - A
        rBA = A - B
        rBC = C - B
        rAC_unit = rAC / np.clip(np.linalg.norm(rAC), 1e-6, np.inf)
        theta = rBA.dot(rBC) / (np.linalg.norm(rBA) * np.linalg.norm(rBC))
        chi = np.power((1 - theta) / 2, N)
        return V * rAC_unit * chi

    def step(
        self, action: tuple[PositionAction, PositionAction]
    ) -> tuple[States, float, bool, bool, dict[str, Any]]:
        """
        A step from the viewpoint of a
        :class:`~jdrones.envs.position.
        FifthOrderPolyPositionWithLookAheadDroneEnv` is making the drone
        fly from its current position :math:`A` to the target position :math:`C` via
        :math:`B`.

        .. seealso::
            :meth:`~jdrones.envs.position.
            FifthOrderPolyPositionDroneEnvWithLookAheadDroneEnv.calc_v_at_B`

        Parameters
        ----------
        action : tuple[VEC3, VEC3]
            Target coordinates :math:`B=(x_1,y_1,z_1)` and next target coordinates
            :math:`C=(x_2,y_2,z_2)`

        Returns
        -------
            states: jdrones.data_models.States
            reward: float
            term: bool
            trunc: bool
            info: dict
        """
        A, B, C = np.array([self.env.env.state.pos, *action])
        if np.allclose(B, C):
            v_at_B = (0, 0, 0)
        else:
            v_at_B = self.calc_v_at_B(A, B, C, V=self.model.max_vel_ms * 0.3)
        tgt_pos = B
        tgt_vel = v_at_B
        return self.env.step((tgt_pos, tgt_vel))


import copy
from pyaudio import PyAudio, paInt16
from threading import Thread
import numpy as np
import sys

class AudioAnalyzer(Thread):
    SAMPLING_RATE = 48000
    CHUNK_SIZE = 1024
    BUFFER_TIMES = 50
    ZERO_PADDING = 3
    NUM_HPS = 3

    NOTE_NAMES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B']

    def __init__(self, queue, *args, **kwargs):
        Thread.__init__(self, *args, **kwargs)

        self.queue = queue
        self.buffer = np.zeros(self.CHUNK_SIZE * self.BUFFER_TIMES)
        self.hanning_window = np.hanning(len(self.buffer))
        self.running = False

        try:
            self.audio_object = PyAudio()
            self.stream = self.audio_object.open(format=paInt16,
                                                 channels=1,
                                                 rate=self.SAMPLING_RATE,
                                                 input=True,
                                                 output=False,
                                                 frames_per_buffer=self.CHUNK_SIZE)
        except Exception as e:
            sys.stderr.write('Error: Line {} {} {}\n'.format(sys.exc_info()[-1].tb_lineno, type(e).__name__, e))
            return

    @staticmethod
    def frequency_to_number(freq, a4_freq):

        if freq == 0:
            sys.stderr.write("Error: No frequency data. Program has potentially no access to microphone\n")
            return 0

        return 12 * np.log2(freq / a4_freq) + 69

    @staticmethod
    def number_to_frequency(number, a4_freq):
        return a4_freq * 2.0**((number - 69) / 12.0)

    @staticmethod
    def number_to_note_name(number):
        return AudioAnalyzer.NOTE_NAMES[int(round(number) % 12)]

    @staticmethod
    def frequency_to_note_name(frequency, a4_freq):
        """ converts frequency to note name (for example: 440 returns 'A') """

        number = AudioAnalyzer.frequency_to_number(frequency, a4_freq)
        note_name = AudioAnalyzer.number_to_note_name(number)
        return note_name

    def run(self):
        """ Main function where the microphone buffer gets read and
            the fourier transformation gets applied """

        self.running = True

        while self.running:
            try:
                data = self.stream.read(self.CHUNK_SIZE, exception_on_overflow=False)
                data = np.frombuffer(data, dtype=np.int16)

                self.buffer[:-self.CHUNK_SIZE] = self.buffer[self.CHUNK_SIZE:]
                self.buffer[-self.CHUNK_SIZE:] = data

                magnitude_data = abs(np.fft.fft(np.pad(self.buffer * self.hanning_window,
                                                       (0, len(self.buffer) * self.ZERO_PADDING),
                                                       "constant")))
                magnitude_data = magnitude_data[:int(len(magnitude_data) / 2)]

                magnitude_data_orig = copy.deepcopy(magnitude_data)
                for i in range(2, self.NUM_HPS+1, 1):
                    hps_len = int(np.ceil(len(magnitude_data) / i))
                    magnitude_data[:hps_len] *= magnitude_data_orig[::i]  # multiply every i element

                frequencies = np.fft.fftfreq(int((len(magnitude_data) * 2) / 1),
                                             1. / self.SAMPLING_RATE)

                for i, freq in enumerate(frequencies):
                    if freq > 60:
                        magnitude_data[:i - 1] = 0
                        break

                self.queue.put(round(frequencies[np.argmax(magnitude_data)], 2))

            except Exception as e:
                sys.stderr.write('Error: Line {} {} {}\n'.format(sys.exc_info()[-1].tb_lineno, type(e).__name__, e))

        self.stream.stop_stream()
        self.stream.close()
        self.audio_object.terminate()


import sys
import numpy as np
import os
import glob
import cv2
import tensorflow as tf
from PyQt5 import QtWidgets, uic
from PyQt5.QtGui import QPixmap
from skimage import io
from skimage.transform import resize
from sklearn.model_selection import train_test_split
from tensorflow.keras.utils import to_categorical

class DataLoader():
	def CountFiles(somefolderpath):
		return len([name for name in os.listdir(somefolderpath) if os.path.isfile(os.path.join(somefolderpath, name))])

	def LoadAllData(allimagescount, datasetpath, rastersize, maxarraysize_in_GB):
		array_approximatesize = allimagescount * rastersize * rastersize * 3 * 1
		if (array_approximatesize < maxarraysize_in_GB * 1024 * 1024 * 1024):
			i = 0       			 # - ID растра
			j = 0		 			 # - ID класса растения

			nparray_rasters = np.zeros((allimagescount, rastersize, rastersize, 3), dtype=np.uint8)
			nparray_labels = np.zeros((allimagescount), dtype = np.uint16)
			subfolders_list = list([f.path for f in os.scandir(datasetpath) if f.is_dir()])

			# Вспомогательные переменные, чтобы отслеживать прогресс загрузки.
			temp_uploading_onepercent = int(allimagescount / 100)
			temp_uploading_step = 5

			for subf in subfolders_list:
				subf = os.path.join(subf,'*g')
				subf_rasters = glob.glob(subf)

				for r in subf_rasters:
					# Подготовка растра:
					# 1. Загрузка изображения из папки:
					image_loaded = cv2.imread(r)

					# 2. Обработка изображения - приведение к необходимому размеру с таким сохранением отношения сторон, насколько это возможно
					image_resized = resize(image_loaded, (rastersize, rastersize))

					# 3. Перевод обработанного изображения в растр (массив NxNx3, где третяя размерность хранит RGB-координаты пикселя)
					# Для того, чтобы точки, выделенные другим оттенком цвета (напр. граничные) не обнулились после занесения в массив типа uint8,
					# используется функция sign, превращающая малые значения RGB в белый цвет, напр: [0.1, 0.4, 0.3] -> [1, 1, 1]
					# raster = np.asarray(image_resized, dtype=np.float32)
					raster = np.sign(np.asarray(image_resized, dtype=np.float64)).astype('uint8')

					# Полученный растр необходимо загрузить в массив растров, каждая новая папка subf в цикле - новое значение j.
					nparray_rasters[i] = raster
					nparray_labels[i] = j

					if (i % (temp_uploading_onepercent*temp_uploading_step) == 0):
						print("Обработано растров: " + str(i) + "  (" + str(int(100*i/allimagescount)) + "%)")

					i += 1
				j += 1
		else:
			print("Ошибка: массив будет иметь размер более, чем: " + str(maxarraysize_in_GB) + "Гб")
		return nparray_rasters, nparray_labels

	def LoadSingleImageAsRaster(someimagepath, rastersize):
		image_loaded = cv2.imread(someimagepath)
		image_resized = resize(image_loaded, (rastersize, rastersize))
		return np.asarray(image_resized, dtype=np.float64)

	def LoadArray(arrayname):
		return np.load(arrayname)

	def LoadModel(modelname):
		return tf.keras.models.load_model(modelname)

	def CountAllImages(folderpath):
		count = 0
		subfolders_list = list([f.path for f in os.scandir(folderpath) if f.is_dir()])

		for subf in subfolders_list:
			count += len([name for name in os.listdir(subf) if os.path.isfile(os.path.join(subf, name))])
		return count

	def GetAllClassesNamesList(somefolderpath):
		# Названия классов определяются названиями папок в библиотеке LeafData
		subfolders_path_list = list([f.path for f in os.scandir(somefolderpath) if f.is_dir()])
		classesnames_list = []

		for subf_path in subfolders_path_list:
			l = len(subf_path)
			for i in range(0,l-1):
				if subf_path[l-1-i] == "\\" and i >= 1:
					classesnames_list.append(subf_path[l-i : l])
					break

		return classesnames_list

class DataSaver():
	def SaveArray(arrayname, numpyarray):
		np.save(arrayname,numpyarray)

	def SaveModel(modelname, tensorflowmodel):
		tensorflowmodel.save(modelname)

class MainWindow(QtWidgets.QMainWindow):
	def __init__(self,tensorflowmodel,rastersize,predictionstopN,classesnames,somedatasetpath,window_width,window_height,image_width,image_height,txtlabel_leftrightmargin,txtlabel_height):
		# Загрузка параметров окна:
		super(MainWindow, self).__init__()
		uic.loadUi('mainwindow_design.ui', self)

		# Настройка окна:
		self.setFixedSize(window_width, window_height)
		self.imagewidth = image_width
		self.imageheight = image_height
		self.frombottomtxtboxmargin = window_height-min([self.findChild(QtWidgets.QPushButton, 'goLeft').y(),self.findChild(QtWidgets.QPushButton, 'goRight').y()])+5 # расположение панели кнопок определяется файлом интерфейса .ui, поэтому я учитываю расположение кнопок для последующей верстки

		# Загрузка модели и переменных, связанных с моделью:
		self.tfmodel = tensorflowmodel
		self.rsize = rastersize
		self.predtopN = predictionstopN
		self.clnames = classesnames

		# Добавление методов-обработчиков на кнопки:
		self.button_goLeft = self.findChild(QtWidgets.QPushButton, 'goLeft')
		self.button_goRight = self.findChild(QtWidgets.QPushButton, 'goRight')
		self.button_goLeft.clicked.connect(self.ButtonClick_goLeft)
		self.button_goRight.clicked.connect(self.ButtonClick_goRight)

		# Занесение списка путей изображений из ValidationSet в поле класса, подсчет изображений:
		self.allimagespath = glob.glob(os.path.join(somedatasetpath,'*g'))
		self.allimagesamount = len(self.allimagespath)

		# Подготовка окна для отображения текста:
		self.txtlabel = QtWidgets.QTextEdit(self)
		self.txtlabelleftrightmargin = txtlabel_leftrightmargin
		self.txtlabelheight = txtlabel_height
		self.txtlabel.setGeometry(txtlabel_leftrightmargin, window_height-txtlabel_height-self.frombottomtxtboxmargin, window_width-txtlabel_leftrightmargin*2, txtlabel_height)

		# Подготовка окна для отображения изображения:
		self.imglabel = QtWidgets.QLabel(self)
		self.imglabel.setGeometry(int((window_width-image_width)/2), int((window_height-image_height-txtlabel_height-self.frombottomtxtboxmargin)/2), image_width, image_height)

		# Сначала показывается первое изображение в выбранной папке:
		self.currentimagecounter = 0;
		self.ImageLoad(self.allimagespath[0], image_width, image_height)
		self.TextLoad(tensorflowmodel,classesnames,rastersize,predictionstopN)
		self.show()

	def ImageLoad(self, imgpath, imgwidth, imgheight):
		self.imglabel.clear()
		self.imglabel.setPixmap((QPixmap(imgpath)).scaled(imgwidth, imgheight))
		self.show()

	def TextLoad(self, TFmodel, clnames, rsize, n):
		currentraster = np.zeros((1,rsize,rsize,3), dtype=np.float64)
		currentraster[0] = DataLoader.LoadSingleImageAsRaster(self.allimagespath[self.currentimagecounter], rsize)
		topNlabels, topNresults = WorkWithTensorFlowModel.MakePrediction(TFmodel,clnames,currentraster,n)

		predictiontext = "Нейросеть думает, что это:" + '\n'
		for i in range(0,n):
			predictiontext = predictiontext + topNlabels[i] + "  ----  " + str(100*topNresults[i]) + "%" + '\n'
		self.txtlabel.setText(predictiontext)

	# Обработчики нажатия на кнопки:
	def ButtonClick_goLeft(self):
		if self.currentimagecounter == 0:
			self.currentimagecounter = self.allimagesamount - 1 		# - к последнему изображению:
		else:
			self.currentimagecounter = self.currentimagecounter - 1

		self.ImageLoad(self.allimagespath[self.currentimagecounter],self.imagewidth,self.imageheight)
		self.TextLoad(self.tfmodel,self.clnames,self.rsize,self.predtopN)

	def ButtonClick_goRight(self):
		if self.currentimagecounter == self.allimagesamount - 1:
			self.currentimagecounter = 0								# - к первому изображению.
		else:
			self.currentimagecounter = self.currentimagecounter + 1

		self.ImageLoad(self.allimagespath[self.currentimagecounter],self.imagewidth,self.imageheight)
		self.TextLoad(self.tfmodel,self.clnames,self.rsize,self.predtopN)

class WorkWithTensorFlowModel():
	def MakePrediction(model, labelsnames, raster, topN):
		predictions = (model.predict(raster))[0]
		labelsnames_copy = labelsnames.copy()
		n = predictions.shape[0]

		if n < topN:
			print("Нельзя выделить " + str(topN) + " классов, так как это больше, чем общее число классов: " + str(allclasses_count))
		else:
			for i in range(0, n):
				for j in range(i+1, n):
					if predictions[i] < predictions[j]:
						predictions[i], predictions[j] = predictions[j], predictions[i]
						labelsnames_copy[i], labelsnames_copy[j] = labelsnames_copy[j], labelsnames_copy[i]

		return labelsnames_copy[:topN], predictions[:topN]

def main(traindatapath,validationdatapath,rastersarray_save_name,labelssarray_save_name,model_save_name,rastersize,maxarraysizeGB,testsize,predictions_topN,mode1="new",mode2="new"):
	# Подсчет общего количества фотографий в библиотеке:
	count = DataLoader.CountAllImages(traindatapath)
	print("Обнаружено фотографий: " + str(count))

	# Загрузка всех данных - растров, меток и список названий всех классов:
	if mode1 == "new":
		# Путь первый - извлечение и обработка исходных данных, преобразование их в массив, сохранение массивов:
		rasters, labels = DataLoader.LoadAllData(count, traindatapath, rastersize, maxarraysizeGB)
		DataSaver.SaveArray(rastersarray_save_name, rasters)
		DataSaver.SaveArray(labelssarray_save_name, labels)

	elif mode1 == "load":
		# Путь второй - загрузка уже заранее обработанных данных из
		rasters, labels = DataLoader.LoadArray(rastersarray_save_name), DataLoader.LoadArray(labelssarray_save_name)

	# Загрузка всех названий классов:
	labelsnames = DataLoader.GetAllClassesNamesList(traindatapath)
	amountofspecies = len(labelsnames)

	# Разбиение выборки:
	rasters_train, rasters_test, labels_train, labels_test = train_test_split(rasters, labels, test_size=testsize, random_state=42)
	print("Выборка разделена в отношении: ")
	print(" - Train: " + str(rasters_train.shape[0]))
	print(" - Test: "+ str(rasters_test.shape[0]))

	# Модель:
	if mode2 == "load":
		# Путь первый: загрузка готовой модели:
		model = DataLoader.LoadModel(model_save_name)

	elif mode2 == "new":
		# Путь второй: создание новой модели:
		# Программирование архитектуры модели при помощи пакетов Tensorflow/Keras:
		model = tf.keras.models.Sequential()
		model.add(tf.keras.layers.Conv2D(32,(3,3),padding='same', input_shape=(rastersize,rastersize,3), activation='relu'))
		model.add(tf.keras.layers.Conv2D(32,(3,3),activation='relu'))
		model.add(tf.keras.layers.MaxPooling2D(pool_size=(2,2)))
		model.add(tf.keras.layers.Dropout(0.25))
		model.add(tf.keras.layers.Conv2D(64,(3,3),padding='same',activation='relu'))
		model.add(tf.keras.layers.Conv2D(64,(3,3),activation='relu'))
		model.add(tf.keras.layers.MaxPooling2D(pool_size=(2,2)))
		model.add(tf.keras.layers.Dropout(0.25))
		model.add(tf.keras.layers.Flatten())
		model.add(tf.keras.layers.Dense(512,activation='relu'))
		model.add(tf.keras.layers.Dropout(0.5))
		model.add(tf.keras.layers.Dense(amountofspecies,activation='softmax'))

		# Настройка оптимизатора SGD к модели:
		sgd = tf.keras.optimizers.SGD(lr = 0.01, decay = 1e-6, momentum=0.9, nesterov=True)
		model.compile(loss='categorical_crossentropy', optimizer=sgd,metrics=['accuracy'])

		# Обучение модели:
		model.fit(rasters_train, to_categorical(labels_train), batch_size=50, nb_epoch=20, validation_split=0.0, shuffle=False)

		# Сохранение обученной модели в папку с названием model_save_name:
		DataSaver.SaveModel(model_save_name,model)

	# Проверка модели на тестовых данных:
	scores = model.evaluate(rasters_test, to_categorical(labels_test), verbose=0)
	print("Точность на тестовых данных: "+str(100*scores[1]))

	# Запуск окна:
	app = QtWidgets.QApplication(sys.argv)    																			 # Создание приложения
	window = MainWindow(model,rastersize,predictions_topN,labelsnames,validationdatapath,440, 560, 375, 375, 32, 120)    # Создание окна
	window.show()																										 # Отображение окна
	app.exec_()

if __name__ == '__main__':
	# Входные параметры:
	traindata_path = "C:\\Users\\VG\\Desktop\\LeafProject\\LeafDataSet\\Segmented\\field"
	validationdata_path = "C:\\Users\\VG\\Desktop\\LeafProject\\Program\\ValidationSet"
	rastersarray_save_name = 'rasters.npy'
	labelsarray_save_name = 'labels.npy'
	model_save_name = '40percents'
	raster_size = 32
	maxarraysize_in_Gigabytes = 6
	test_size = 0.2
	predictions_topN = 5
	arraymode = "load"
	modelmode = "load"

	main(traindata_path, validationdata_path, rastersarray_save_name, labelsarray_save_name, model_save_name, raster_size, maxarraysize_in_Gigabytes, test_size, predictions_topN, mode1=arraymode, mode2=modelmode)


from sage.all import *
from polyrank import *

import unittest
import coverage

class TestPolyRankMMatrix(unittest.TestCase):
    def test_matrix_addition(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2], [3, 4]]))
        m_2 = mmatrix.MMatrix(matrix([[5, 6], [7, 8]]))

        result = m_1 + m_2
        expected = matrix([[6, 8], [10, 12]])

        self.assertEqual(result.matrix, expected)

    def test_matrix_subtraction(self):
        m_1 = mmatrix.MMatrix(matrix([[5, 6], [7, 8]]))
        m_2 = mmatrix.MMatrix(matrix([[1, 2], [3, 4]]))

        result = m_1 - m_2
        expected = matrix([[4, 4], [4, 4]])

        self.assertEqual(result.matrix, expected)

    def test_matrix_multiplication(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2], [3, 4]]))
        m_2 = mmatrix.MMatrix(matrix([[5, 6], [7, 8]]))

        result = m_1 * m_2
        expected = matrix([[19, 22], [43, 50]])

        self.assertEqual(result.matrix, expected)

    def test_matrix_get_item(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2], [3, 4]]))

        row = list(m_1[0])
        expected = [1, 2]

        self.assertEqual(row, expected)

    def test_matrix_set_item(self):
        m_1 = mmatrix.MMatrix(matrix([[0, 0], [3, 4]]))
        m_1[0] = [1, 2]

        expected = matrix([[1, 2], [3, 4]])

        self.assertEqual(m_1.matrix, expected)

    def test_matrix_row_size(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2, 3], [4, 5, 6]]))

        row_size = m_1.row_size()
        expected = 3

        self.assertEqual(row_size, expected)

    def test_matrix_column_size(self):
        m_1 = mmatrix.MMatrix(matrix([[1], [2], [3]]))

        column_size = m_1.column_size()
        expected = 3

        self.assertEqual(column_size, expected)

    def test_matrix_row_reduce(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2], [3, 4]]))

        result = m_1.row_reduce()[0]
        expected = matrix([[1, 0], [0, -2]])

        self.assertEqual(result.matrix, expected)

    def test_matrix_scalar_mult(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2], [3, 4]]))
        m_1.scalar_mult(2)

        expected = matrix([[2, 4], [6, 8]])

        self.assertEqual(m_1.matrix, expected)

    def test_matrix_swap_rows(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2], [3, 4]]))
        m_1.swap_rows(0, 1)

        expected = matrix([[3, 4], [1, 2]])

        self.assertEqual(m_1.matrix, expected)

    def test_matrix_subtract_row(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2], [3, 4]]))
        m_1.subtract_row(2, 0, 1)

        expected = matrix([[1, 2], [1, 0]])

        self.assertEqual(m_1.matrix, expected)

    def test_matrix_get_pivots(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2], [3, 4]]))
        reduced = m_1.row_reduce()[0]

        pivots = reduced.get_pivots()
        expected = [1, -2]

        self.assertEqual(pivots, expected)

    def test_matrix_is_reduced(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2], [3, 4]]))
        self.assertFalse(m_1._is_reduced())

        reduced = m_1.row_reduce()[0]
        self.assertTrue(reduced._is_reduced())

    def test_matrix_max_pivots(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2], [3, 4], [5, 6]]))

        max_pivots = m_1.max_pivots()
        expected = 2

        self.assertEqual(max_pivots, expected)

    def test_matrix_apply_fn(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2], [3, 4]]))

        result = m_1.apply_fn(lambda x: x + 1)
        expected = matrix([[2, 3], [4, 5]])

        self.assertEqual(result.matrix, expected)

    def test_matrix_update_entry(self):
        m_1 = mmatrix.MMatrix(matrix([[1, 2], [3, 4]]))
        m_1.update_entry(0, 0, 0)

        expected = matrix([[0, 2], [3, 4]])

        self.assertEqual(m_1.matrix, expected)

if __name__ == "__main__":
    cov = coverage.coverage(source=["polyrank"])
    cov.start()

    unittest.main(exit=False)

    cov.stop()
    cov.save()


from datetime import date, datetime
from sqlalchemy import func
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.appointment.models import Appointment
from app.status.models.status import Status

class AppointmentRepository:

    def __init__(self, db: Session):
        """
        This function is used to initialize the database session

        :param db: Session = Depends(get_db)
        :type db: Session
        """
        self.db = db

    def create_appointment(self, appointment_datetime, client_id, employee_id, pending_status_id):
        """
        It creates an appointment object, adds it to the database, commits the changes, and then
        refreshes the appointment object

        :param appointment_datetime: datetime.datetime(2020, 5, 1, 10, 0)
        :param client_id: 1
        :param employee_id: 1
        :param pending_status_id: 1
        :return: The appointment object
        """
        try:
            appointment = Appointment(appointment_datetime, client_id, employee_id, pending_status_id)
            self.db.add(appointment)
            self.db.commit()
            self.db.refresh(appointment)
            return appointment
        except IntegrityError as e:
            raise e

    def get_appointments_by_client_id(self, client_id: str):
        """
        It returns all appointments for a given client_id

        :param client_id: str
        :type client_id: str
        :return: A list of Appointment objects
        """
        return self.db.query(Appointment).filter(Appointment.client_id == client_id).all()

    def get_all_appointments_by_employee_id_and_status(self, employee_id : str, status_id: str):
        """
        Get all appointments by employee id and status

        :param employee_id: str
        :type employee_id: str
        :param status_id: 1 = Pending, 2 = Confirmed, 3 = Cancelled
        :type status_id: str
        :return: A list of Appointment objects
        """
        return self.db.query(Appointment).filter(Appointment.employee_id == employee_id, Appointment.status_id == status_id).all()

    def get_all_appointments_by_employee_id_by_status_for_date(self, employee_id : str, status_id: str, appointment_datetime: str = None):
        """
        Get all appointments for a given employee, status, and date

        :param employee_id: str
        :type employee_id: str
        :param status_id: str = '1'
        :type status_id: str
        :param appointment_datetime: str = None
        :type appointment_datetime: str
        :return: A list of Appointment objects
        """
        return self.db.query(Appointment).filter(Appointment.employee_id == employee_id, Appointment.status_id == status_id, func.date(Appointment.appointment_datetime) == appointment_datetime).all()

    def get_appointment_by_id(self, appointment_id: str):
        """
        It returns the first appointment in the database that matches the appointment_id

        :param appointment_id: str
        :type appointment_id: str
        :return: A single appointment object
        """
        appointment = self.db.query(Appointment).filter(Appointment.id == appointment_id).first()
        return appointment

    def get_all_appointments(self):
        """
        It returns all the appointments in the database
        :return: A list of all the appointments in the database.
        """
        appointments = self.db.query(Appointment).all()
        return appointments

    def delete_appointment_by_id(self, appointment_id:str):
        """
        It deletes an appointment from the database by its id

        :param appointment_id: The id of the appointment to be deleted
        :type appointment_id: str
        :return: The return value is a boolean value.
        """
        try:
            appointment = self.db.query(Appointment).filter(Appointment.id == appointment_id).first()
            self.db.delete(appointment)
            self.db.commit()
            return True
        except Exception as e:
            raise e

    def cancel_appointement(self, appointment_id: str):
        """
        It takes an appointment_id and sets the status of the appointment to 'rejected'

        :param appointment_id: str
        :type appointment_id: str
        :return: The appointment object
        """
        try:
            rejected_status = self.db.query(Status).filter_by(status = 'rejected').one()
            appointment = self.db.query(Appointment).filter(Appointment.id == appointment_id).first()
            appointment.status_id = rejected_status.id
            self.db.add(appointment)
            self.db.commit()
            self.db.refresh(appointment)
            return appointment
        except IntegrityError as e:
            raise e

    def accept_appointment(self, appointment_id: str):
        """
        It takes an appointment_id, finds the appointment in the database, changes the status_id to the
        id of the accepted status, and returns the appointment and sets status to accepted.

        :param appointment_id: str
        :type appointment_id: str
        :return: The appointment object
        """
        try:
            accepted_status = self.db.query(Status).filter_by(status = 'accepted').one()
            appointment = self.db.query(Appointment).filter(Appointment.id == appointment_id).first()
            appointment.status_id = accepted_status.id
            self.db.add(appointment)
            self.db.commit()
            self.db.refresh(appointment)
            return appointment
        except IntegrityError as e:
            raise e

    def update_appointment_by_id(self, appointment_id, appointment_datetime, client_id, employee_id, status_id):
        """
        It takes in an appointment_id, appointment_datetime, client_id, employee_id, and status_id, and
        updates the appointment with the given appointment_id with the given values

        :param appointment_id: 1
        :param appointment_datetime: datetime.datetime(2019, 1, 1, 0, 0)
        :param client_id: 1
        :param employee_id: 1
        :param status_id: 1
        :return: The appointment object
        """
        try:
            appointment = self.db.query(Appointment).filter(Appointment.id == appointment_id).first()
            appointment.appointment_datetime = appointment_datetime
            appointment.client_id = client_id
            appointment.employee_id = employee_id
            appointment.status_id = status_id
            self.db.add(appointment)
            self.db.commit()
            self.db.refresh(appointment)
            return appointment
        except IntegrityError as e:
            raise e


import re
from textblob import TextBlob
import pandas as pd
import numpy as np
from Data_collection import tweet_collect
import tweepy
from tweepy import OAuthHandler
from textblob import TextBlob
from tweepy import API
from tweepy import Cursor
from tweepy.streaming import StreamListener
from tweepy import Stream

class TweetAnalyzer():
    """
    Preprocessing and analyaing the tweets with textblob.
    """

    def clean_tweet(self, tweet):
        return ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ", tweet).split())

    def analyze_sentiment(self, tweet):
        analysis = TextBlob(self.clean_tweet(tweet))

        if analysis.sentiment.polarity > 0:
            return 1
        elif analysis.sentiment.polarity == 0:
            return 0
        else:
            return -1

    def tweets_to_data_frame(self, tweets):
        df = pd.DataFrame(data=[tweet.text[:150] for tweet in tweets], columns=['tweets'])

        df['id'] = np.array([tweet.id for tweet in tweets])
        df['date'] = np.array([tweet.created_at for tweet in tweets])

        return df

"""
DOING SENTIMENT ANALYSIS WITH TEXTBLOB
"""

def predicting_Sentiment(tweets,query):
	df = tweet_analyzer.tweets_to_data_frame(tweets)
	print(len(df))
	df['sentiment'] = np.array([tweet_analyzer.analyze_sentiment(tweet) for tweet in df['tweets']])
	positive=100*len(df['sentiment'][df['sentiment']==1])/len(df)
	negative=100*len(df['sentiment'][df['sentiment']==-1])/len(df)
	t_score=positive+negative
	score=(positive)/negative

	print('{} positive {}%'.format(query,100*len(df['sentiment'][df['sentiment']==1])/len(df)))
	print('{} negative {}%'.format(query,100*len(df['sentiment'][df['sentiment']==-1])/len(df)))
	print('{} neutral {}%'.format(query,00*len(df['sentiment'][df['sentiment']==0])/len(df)))

	file=str(query)+'.csv'
	df.to_csv(file)
	return score

"""
Calculating voting percentage among different parties.
"""

def voting_percentage(aap_score,bjp_score,cong_score):

	print('AAP Adjusted Voting Percentage  {}'.format(100*(aap_score )/((aap_score)+bjp_score+cong_score)))
	print('BJP Adjusted Voting Percentage  {}'.format(100*(bjp_score)/(aap_score+(bjp_score)+cong_score)))
	print('cong Adjusted Voting Percentage  {}'.format(100*(cong_score)/(aap_score+bjp_score+(cong_score))))

def main():

	with open('API_key.json') as json_file:
		data = json.load(json_file)
		for Key in API['Key']:
			API_Key=Key['API_KEY']
			API_SECRET=Key['API_SECRET']

	auth = tweepy.AppAuthHandler(API_KEY, API_SECRET)

	api = tweepy.API(auth,wait_on_rate_limit=True,wait_on_rate_limit_notify=True)

	parser=argparse.ArgumnetParser()

	parser.add_argument('query_keyword',help='Name of party which tweets you want to extract')
	parser.add_argument('MaxTweets',help='Number of Tweets you want to get')

	query= parser.query_keyword
	maxTweets=parser.MaxTweets

	tweets=[]

	"""
	Collecting Data on basis of Date i have collected data from 1 feb 20 to 8 feb 20 So there should be not any bias due to final results.

	"""
	for i in range(1,9):
		until='2020-2-'+str(i)
		tweet_on=tweet_collect(query=query,maxTweets=maxTweets,until=until)
		tweets=tweets+tweet_on

	score=predicting_Sentiment(tweets,query)

	"""
	After getting scores for all three parties AAP,BJP and CONGRESS use
	voting_percentage function to get the ouput result.
	"""

if __name__ == '__main__':
	main()


from base64 import b64encode
from struct import calcsize, pack, unpack
from random import randint

class SizeTag:
    SIZE_TAG_SIZE = 8

    def __init__(self, data_size):
        self.data_size = data_size

    def serialize(self):
        return pack("i", self.data_size)

    def deserialize(serialized_size_tag):
        i = calcsize("i")
        (data_size,) = unpack("i", serialized_size_tag[0:i])
        return SizeTag(data_size=data_size)

class Marker:
    MARKER_SIZE = 12

    def __init__(self, end_of_payload=0, next_marker=0):
        self.end_of_payload = end_of_payload
        self.next_marker = next_marker

    def serialize(self):
        return pack("ii", self.end_of_payload, self.next_marker)

    def deserialize(serialized_marker):
        size = calcsize("ii")
        (end_of_payload, next_marker) = unpack("ii", serialized_marker[0:size])
        return Marker(end_of_payload=end_of_payload, next_marker=next_marker)

class SubChunk:
    MIN_SUBCHUNK_SIZE = 24

    def __init__(self, size, marker=None, payload=None, dummy=None):
        if size >= self.MIN_SUBCHUNK_SIZE:
            self.size = size
        else:
            raise ValueError(
                f"SubChunk muss mindestens {self.MIN_SUBCHUNK_SIZE} Zeichen groß sein!"
            )
        self.marker = marker
        self.payload = payload
        self.dummy = dummy

    def serialize(self):
        return b64encode(self.marker.serialize()) + self.payload + self.dummy

class Chunk:
    MIN_CHUNK_SIZE = 100 - SizeTag.SIZE_TAG_SIZE

    def __init__(self, size, subchunks_to_set=None):
        if size >= self.MIN_CHUNK_SIZE:
            self.size = size
        else:
            raise ValueError(
                f"Chunk muss mindestns {self.MIN_CHUNK_SIZE} Zeichen groß sein!"
            )
        self.subchunks = subchunks_to_set

    def serialize(self):
        byte = b""
        for subchunk in self.subchunks:
            byte = byte + subchunk.serialize()
        return byte

    def sub_chunking(self, payload_percentage, message):
        if len(message) == 0:
            raise ValueError(f"'message' darf nicht 0 sein!")
        # setze Methoden Variable
        rest_chunk_size = self.size

        # SUBCHUNKING-START
        while rest_chunk_size > 0:
            subchunk_size = randint(SubChunk.MIN_SUBCHUNK_SIZE, rest_chunk_size)
            # bleibt genug übrig um danach einen weiteren SubChunk zu bilden?
            if (
                rest_chunk_size - subchunk_size < SubChunk.MIN_SUBCHUNK_SIZE
                or len(message) == 0
            ):
                subchunk_size = rest_chunk_size

            rest_chunk_size = rest_chunk_size - subchunk_size

            subchunk = SubChunk(size=subchunk_size)
            subchunk_size = subchunk_size - Marker.MARKER_SIZE
            if len(message) == 0:
                subchunk.dummy = DummyGenerator.get_dummy_data(subchunk_size)
                subchunk.marker = Marker(next_marker=subchunk.size)
                self.subchunks.append(
                    SubChunk(
                        size=subchunk.size,
                        marker=subchunk.marker,
                        payload=b"",
                        dummy=subchunk.dummy,
                    )
                )
                break
            if len(message) < int(
                (subchunk_size - Marker.MARKER_SIZE) * payload_percentage
            ):
                # letzter SubChunk
                subchunk.payload = message
                message = ""
            else:
                subchunk.payload = message[: int(subchunk_size * payload_percentage)]
                message = message[int(subchunk_size * payload_percentage) :]
            subchunk.dummy = DummyGenerator.get_dummy_data(
                subchunk_size - len(subchunk.payload)
            )
            subchunk.marker = Marker(
                end_of_payload=len(subchunk.payload),
                next_marker=subchunk.size - Marker.MARKER_SIZE,
            )
            if self.subchunks is None:
                self.subchunks = []

            self.subchunks.append(
                SubChunk(
                    size=subchunk.size,
                    marker=subchunk.marker,
                    payload=subchunk.payload,
                    dummy=subchunk.dummy,
                )
            )
            del subchunk
        # SUBCHUNKING-ENDE

        # gebe Rest der Message zurück
        return message

class TransferObject:
    def __init__(self, message, size_tag=None, chunks=None):
        self.message = message
        self.size_tag = size_tag
        self.chunks = chunks

    def serialize(self):
        array = []
        for chunk in self.chunks:
            if len(array) == 0:
                byte = b64encode(self.size_tag.serialize()) + chunk.serialize()
            else:
                byte = chunk.serialize()
            array.append(byte)
        return array

    def chunking(self, payload_percentage, chunk_size_range):
        # setze Methoden Variable
        temp_message = self.message
        self.chunks = []
        first = True
        while len(temp_message) > 0:
            if first:
                if (
                    chunk_size_range.stop - SizeTag.SIZE_TAG_SIZE
                    >= chunk_size_range.start
                ):
                    chunk = Chunk(
                        size=randint(
                            chunk_size_range.start,
                            chunk_size_range.stop - SizeTag.SIZE_TAG_SIZE,
                        )
                    )
                else:
                    chunk = Chunk(size=chunk_size_range.stop - SizeTag.SIZE_TAG_SIZE)
                first = False
            else:
                chunk = Chunk(
                    size=randint(chunk_size_range.start, chunk_size_range.stop)
                )
            temp_message = chunk.sub_chunking(
                payload_percentage=payload_percentage, message=temp_message
            )
            self.chunks.append(Chunk(size=chunk.size, subchunks_to_set=chunk.subchunks))
            for subchunk in chunk.subchunks:
                del subchunk
            del chunk.subchunks
            del chunk

class DummyGenerator:
    WORDS = [
        "Hallo",
        "Welt",
        "Wie",
        "Gehts",
        "Schoen",
        "Haesslich",
        "Leben",
        "Ist",
        "Butter",
        "Birne",
        "Abfall",
        "Kopf",
        "Rofl",
        "Jumks",
        "Und",
        "Auch",
    ]

    MARKS = [".", ",", "!", "?"]

    def get_dummy_data(size):
        i = 0
        random_string = ""
        index = 0
        while len(random_string) < size:
            i = randint(0, len(DummyGenerator.WORDS) - 1)
            random_string = f"{random_string} {DummyGenerator.WORDS[i]}"
            if index % 4 == 0:
                i = randint(0, len(DummyGenerator.MARKS) - 1)
                random_string = random_string + DummyGenerator.MARKS[i]
            index = index + 1

        string = random_string[:size]
        if string[len(string)-1] == " ":
            string = string[:len(string)-1]
            string = f"{string}a"
        return string.encode()


#!/usr/bin/env python
import argparse
import json
import os
import re
import subprocess
import sys
from collections import defaultdict
import htcondor
import logging

#TODO USE ENVIRONMENTAL VARIABLES
#TODO USE CONFIG FILE ALSO

class CondorUtils:

    def __init__(self):
        self.collector = self.get_condor_collector()

    def get_queue_stats(self):
        print(get_queue_stats())
        return {'Queue_stats' : '1'}

    def get_user_prio(self):
        return {'User_prio' : '1'}

    @staticmethod
    def get_condor_collector():
        hostname = CondorUtils.get_condor_hostname()
        collector = htcondor.Collector(hostname)
        collectors = collector.query(htcondor.AdTypes.Collector, "true", ["Name"])
        numCollectors = len(collectors)
        negotiators = collector.query(htcondor.AdTypes.Negotiator, "true", ["Name"])
        numNegotiators = len(negotiators)
        print(numCollectors)
        print(numNegotiators)
        return collector

    @staticmethod
    def get_condor_hostname():
        hostname = "ci.kbase.us:9618"
        CondorUtils.setup_condor()
        return hostname

def get_queue_stats(file=None):
    """

    :param file:
    :return:
    """
    if file is not None:
        condor_data = load_condor_q_from_file(file)
    else:
        condor_data = load_condor_q()

    jobs_by_status = get_jobs_by_status(condor_data)

    cg = defaultdict(lambda: defaultdict(int))

    for status in jobs_by_status.keys():
        jobs = jobs_by_status[status]

        for job in jobs:
            if 'Requirements' in job:
                client_group = get_client_group(job)
                cg[client_group][status] += 1

    return cg

#TODO DECIDE IF YOU SHOULD CHANGE job status here or in JS.. Probably Here
#TODO UNCOMMENT FILTER USERNAME
#TODO ADD COLUMNS WITH TIMES
def get_jobs_status(username, file=None):
    """
    Get filtered job status rows, filtered job status, and filtered by username
    :param username: filter user jobs
    :param file: Optional - use filename
    :return:
    """
    if file is not None:
        condor_data = load_condor_q_from_file(file)
    else:
        condor_data = load_condor_q()

    rows = []

    summary_keys = ['AcctGroup', 'ClusterId', 'JobBatchName',
                    'kb_app_id', 'QDate', 'JobStatus', 'kb_function_name', 'kb_module_name']

    for job in condor_data:
        job_summary = {}

        for key in summary_keys:
            if key == 'JobStatus':
                job_summary[key] = job_status[job[key]]
            else:
                job_summary[key] = job[key]

        job_summary['client_group'] = get_client_group(job)

        # if job_summary['AcctGroup'] != username:
        #     del job_summary['AcctGroup']

        rows.append(job_summary)

    return rows

job_status = {
    5: 'Held',
    4: 'Completed',
    3: 'Removed',
    2: 'Running',
    1: 'Idle',
    'U': 'Unexpanded',
    'E': 'Submission_error'
}

def separator(f):
    sep = "=" * 160
    def wrap(*args, **kwargs):
        print(sep)
        return f(*args, **kwargs)
    return wrap

# TODO: Cache this
def get_jobs_by_status(data):
    jobs = defaultdict(list)
    for item in data:
        jobstatus = job_status[item['JobStatus']]
        jobs[jobstatus].append(item)
    return jobs

def print_general_queue_status(jobs_by_status):
    for status in jobs_by_status:
        print("{}:{}".format(status, len(jobs_by_status[status])))

def get_client_group(job):
    try:
        return re.search('CLIENTGROUP == (".+?")', job['Requirements']).group(1).replace('"', '')
    except Exception:
        return 'unknown'

@separator
def print_job_status_by_status():
    print("Job Stats by Status")
    jobs_by_status = get_jobs_by_status(json_data)

    for status in jobs_by_status.keys():
        jobs = jobs_by_status[status]

        print("{}:{}".format(status, len(jobs_by_status[status])))

        cg = defaultdict(int)

        for job in jobs:
            if 'Requirements' in job:
                client_group = get_client_group(job)
                cg[client_group] += 1

        for c in cg:
            print("\t{}:{}".format(c, cg[c]))

@separator
def print_job_status_by_client_group():
    print("Job Stats by ClientGroup")
    jobs_by_status = get_jobs_by_status(json_data)

    cg = defaultdict(lambda: defaultdict(int))

    for status in jobs_by_status.keys():
        jobs = jobs_by_status[status]

        for job in jobs:
            if 'Requirements' in job:
                client_group = get_client_group(job)
                cg[client_group][status] += 1

    for c in cg:
        print("{}:{}".format(str(c), dict(cg[c])))

def load_condor_q_from_file(file):
    with open(file) as f:
        return json.load(f)

def look_for_condor_container():
    cmd = "docker ps | grep kbase/condor | cut -f1 -d' '"
    lines = subprocess.check_output(cmd, shell=True).decode().split("\n")

    names = []
    for line in lines:
        if len(line) > 2:
            names.append(line)

    if len(names) > 1:
        raise Exception("Found too many container names" + str(names))
    elif len(names) == 0 :
        raise Exception("Couldn't find any docker containers for condor")
    else:
        return str(names[0])

def load_condor_q():
    if cmd_exists('condor_q'):
        comm = ['condor_q', '-json']
        output = subprocess.check_output(comm, shell=True).decode()
        return json.loads(output)
    else:
        container_id = look_for_condor_container()
        comm = " ".join(['docker', 'exec', '-it', '-u', '0', container_id, 'condor_q', '-json'])
        output = subprocess.check_output(comm, shell=True).decode()
        return json.loads(output)

def cmd_exists(cmd):
    return any(
        os.access(os.path.join(path, cmd), os.X_OK)
        for path in os.environ["PATH"].split(os.pathsep)
    )

def print_table(table):
    longest_cols = [
        (max([len(str(row[i])) for row in table]) + 3)
        for i in range(len(table[0]))
    ]
    row_format = "".join(["{:>" + str(longest_col) + "}" for longest_col in longest_cols])
    for row in table:
        print(row_format.format(*row))

@separator
def display_job_info(status='Idle', fields=None):
    jobs_by_status = get_jobs_by_status(json_data)

    if fields is None:
        fields = ['JobBatchName', 'ClusterId', 'AcctGroup', 'RemoteHost', 'kb_app_id',
                  'LastRejMatchReason']

    print_lines = []
    for job in jobs_by_status[status]:
        line = []

        for field in fields:
            if field in job:
                line.append("{}".format(job[field]))
            else:
                line.append("")

        line.append(get_client_group(job))
        print_lines.append(line)

    fields.append("ClientGroup")

    print_lines.insert(0, fields)
    print(print_table(print_lines))

def display_idle_jobs():
    display_job_info('Idle')

def display_running_jobs():
    display_job_info('Running')

def display_removed_jobs():
    display_job_info('Removed')

def display_held_jobs():
    display_job_info('Held')

def display_completed_jobs():
    display_job_info('Completed')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='See info extracted from condor_q')
    parser.add_argument('--file', help='Specify output file from condor_q -json')
    parser.add_argument('--status', action='store_true', help='Get status of all jobs')
    parser.add_argument('--status_client_group', action='store_true',
                        help='Get status of all queues')
    parser.add_argument('--idle', action='store_true', help='Get status of all idle jobs')
    parser.add_argument('--running', action='store_true', help='Get status of all running jobs')
    parser.add_argument('--removed', action='store_true', help='Get status of all removed jobs')
    parser.add_argument('--held', action='store_true', help='Get status of all held jobs')
    parser.add_argument('--completed', action='store_true', help='Get status of all completed jobs')

    args = parser.parse_args()

    if args.file:
        json_data = load_condor_q_from_file(args.file)
    else:
        json_data = load_condor_q()

    if args.status:
        print_job_status_by_status()
    if args.status_client_group:
        print_job_status_by_client_group()
    if args.idle:
        display_idle_jobs()
    if args.running:
        display_running_jobs()
    if args.removed:
        display_removed_jobs()
    if args.held:
        display_held_jobs()
    if args.completed:
        display_completed_jobs()

#TODO Cache call to condor_q
#TODO Figure out why None is being printed


import numpy as np
from typing import TypeVar

P = TypeVar('P', bound='Perceptron')
class Perceptron:
    b = 0
    eta = 1
    modifed = True
    def __init__(self, eta = 1) -> None:
        self.eta = eta

    def _gram_matrix(self) -> None:
        # 生成Gram矩阵，方便之后直接取数据无需过多计算
        self.gram = np.zeros((self.n, self.n))
        for i in range(self.n):
            for j in range(self.n):
                self.gram[i][j] = np.inner(self.X[i], self.X[j])

    def _omega(self):
        # 计算omega的累加值
        temp = np.zeros(self.X.shape[1])
        for i in range(self.n):
            temp += self.alpha[i] * self.y[i] * self.X[i]
        return temp

    def _fit_once(self) -> None:
        for i in range(self.n):
            # 样本点到超平面距离不为正，则处于错误的数据集中
            if self.y[i] * (np.inner(self._omega(), self.X[i]) + self.b) <= 0:
                # 误分类时调整平面的方向和截距 使超平面向该误分类的一侧移动
                self.modifed = True
                self.alpha[i] += self.eta
                self.b += self.eta * self.y[i]

    def fit(self, X, y) -> P:
        self.n = X.shape[0]
        self.X = X
        self.y = y
        self.alpha = np.zeros(self.n)
        self._gram_matrix()

        # 如果有修改，则继续进行训练，并把该轮次训练状态设置为还未修改
        while self.modifed:
            self.modifed = False
            self._fit_once()

        self.w = np.zeros(len(X[0]))
        for i in range(self.n):
            self.w += self.alpha[i] * y[i] * np.array(X[i])

        return self

    def predict(self, X_test) -> list:
        res = []
        for x in X_test:
            res.append(np.sign(np.dot(x, self.w) + self.b))
        return np.array(res)

    def score(self, X_test, y_test):
        right = 0
        for i, y in enumerate(self.predict(X_test)):
            if (y == y_test[i]): right += 1
        return right / len(X_test)

from sklearn.datasets import load_iris
iris_dataset = load_iris()
target = []
for y in iris_dataset['target']:
    if y == 0: target.append(-1)
    else: target.append(y)

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(
    iris_dataset['data'][:100], target[:100], random_state=0
)

p = Perceptron()
p.fit(X_train, y_train)
print(p.score(X_test, y_test))


import requests
from Question import Question

BASE_URL = 'https://opentdb.com'
NUM_QUESTIONS = 5
ENCODING = 'urlLegacy'

def main():
	token = get_token()
	result = get_questions(token, NUM_QUESTIONS, None)
	if result is None:
		reset_token(token)
		result = get_questions(token, NUM_QUESTIONS, None)
	return result

def get_token():
	url = BASE_URL + '/api_token.php?command=request'
	response = requests.get(url).json()
	code = response['response_code']
	if code is 0:
		return response['token']
	else:
		print('Error getting session token')
		exit(1)

def reset_token(token):
	url = BASE_URL + 'api_token.php?command=reset&token=' + token
	response = requests.get(url).json()
	code = response['response_code']
	if code is not 0:
		print('Error resetting token')
		exit(1)

def get_questions(token, num, category):
	json = req_questions(token, num, category)
	return json
	'''code = json['response_code']
	if code is 0:
		questions = []
		for i in range(0, len(json['results'])):
			questions.append(Question(json['results'][i]))
		return questions
	elif code is 1:
		return None'''

def req_questions(token, num, category):
	url = BASE_URL + '/api.php?amount=' + str(num) \
			+ '&token=' + token + '&encode=' + ENCODING
	if category is not None:
		url += '&category=' + str(category)
	response = requests.get(url)
	return response#.json()

main()


import torch
import torch.nn.functional as F
from torch_geometric.nn import GCNConv
from torch_geometric.datasets import Planetoid

# Load the CORA dataset (a benchmark dataset for node classification in a graph)
dataset = Planetoid(root='/tmp/Cora', name='Cora')

class GCN(torch.nn.Module):
    def __init__(self):
        super(GCN, self).__init__()
        # Define GCN layers
        self.conv1 = GCNConv(dataset.num_node_features, 16)
        self.conv2 = GCNConv(16, dataset.num_classes)

    def forward(self, data):
        x, edge_index = data.x, data.edge_index

        # First GCN layer with ReLU activation
        x = self.conv1(x, edge_index)
        x = F.relu(x)
        x = F.dropout(x, training=self.training)

        # Second GCN layer
        x = self.conv2(x, edge_index)

        return F.log_softmax(x, dim=1)

# Instantiate the model and optimizer
model = GCN()
optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)

# Training loop
def train():
    model.train()
    optimizer.zero_grad()
    # Use the first graph in the dataset as data
    data = dataset[0]
    # Get the output from the model
    out = model(data)
    # Compute the loss (only for labeled nodes, to simulate semi-supervised setting)
    loss = F.nll_loss(out[data.train_mask], data.y[data.train_mask])
    # Backpropagate
    loss.backward()
    optimizer.step()
    return loss.item()

# Train for 200 epochs
for epoch in range(200):
    loss = train()
    print(f"Epoch {epoch+1}, Loss: {loss:.4f}")


from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
from active_learning.core import ActiveLearner, MAL1, MismatchFirstFarthestTraversal, LargestNeighborhood

def fully_supervised(X_train, y_train, X_test, y_test):
    ceiling_classifier = LogisticRegression()
    ceiling_classifier.fit(X_train, y_train)
    y_test_pred = ceiling_classifier.predict(X_test)
    f1 = metrics.f1_score(y_test, y_test_pred, average='macro')
    print ("With all the training dataset labeled, \
    the classification accuracy is  {0}.".format(f1))

def random_sampling(X_train, y_train, X_test, y_test):
    learner = ActiveLearner(X_train, initial_batch_size=200, batch_size=200)
    n_batch = 50
    print("Query strategy: Random sampling...")
    for i in range(n_batch):
        print("Batch {0}:".format(i + 1))
        batch = learner.draw_next_batch()
        learner.annotate_batch(batch, y_train[batch])
        print("Annotated instances: {0}".format(len(learner.L)))

        print("Training starts.")
        learner.train()
        print("Training is done.")

        y_test_pred = learner.classifier.predict(X_test)

        f1 = metrics.f1_score(y_test, y_test_pred, average='macro')
        print("The average F1 score is ", f1)

def largest_neighborhood(X_train, y_train, X_test, y_test):
    learner = LargestNeighborhood(X_train, batch_size=200, initial_batch_size=1000)
    n_batch = 50

    print("Query strategy: Mismatch first largest neigbourhood...")

    for i in range(n_batch):
        print("Batch {0}:".format(i + 1))
        batch = learner.draw_next_batch()
        learner.annotate_batch(batch, y_train[batch])
        print("Annotated instances: {0}".format(len(learner.L)))

        print("Training starts.")
        learner.train_with_propagated_labels()
        print("Training is done.")

        y_test_pred = learner.classifier.predict(X_test)

        f1 = metrics.f1_score(y_test, y_test_pred, average='macro')
        print("The average F1 score is ", f1)

def mismatch_first_farthest_traversal(X_train, y_train, X_test, y_test):
    learner = MismatchFirstFarthestTraversal(X_train, initial_batch_size=2000, batch_size=200)
    learner.K = 2000
    n_batch = 50
    print("Query strategy: Mismatch-first farthest-traversal...")
    for i in range(n_batch):
        print("Batch {0}:".format(i + 1))
        if learner.n_batch == 0:
            learner.batch_size = 2000
        else:
            learner.batch_size = 200

        batch = learner.draw_next_batch()
        learner.annotate_batch(batch, y_train[batch])
        print("Annotated instances: {0}".format(len(learner.L)))

        print("Training starts.")
        learner.train_with_propagated_labels()
        print("Training is done.")

        y_test_pred = learner.classifier.predict(X_test)

        f1 = metrics.f1_score(y_test, y_test_pred, average='macro')
        print("The average F1 score is ", f1)

if __name__ == '__main__':
    newsgroups_train = fetch_20newsgroups(subset='train')
    newsgroups_test = fetch_20newsgroups(subset='test')
    vectorizer = TfidfVectorizer()
    classifier = LogisticRegression()

    # The vectors are normalized, thus inner product is equivalent to cosine similarity
    X_train = vectorizer.fit_transform(newsgroups_train.data)
    y_train = newsgroups_train.target
    X_test = vectorizer.transform(newsgroups_test.data)
    y_test = newsgroups_test.target

    # fully_supervised(X_train, y_train, X_test, y_test)
    # The accuracy with all the data labeled should be aroung 81%-82%

    random_sampling(X_train, y_train, X_test, y_test)
    largest_neighborhood(X_train, y_train, X_test, y_test)
    mismatch_first_farthest_traversal(X_train, y_train, X_test, y_test)


# -*- coding: utf-8 -*-
__license__ = \
    """Copyright 2019 West University of Timisoara

       Licensed under the Apache License, Version 2.0 (the "License");
       you may not use this file except in compliance with the License.
       You may obtain a copy of the License at

           http://www.apache.org/licenses/LICENSE-2.0

       Unless required by applicable law or agreed to in writing, software
       distributed under the License is distributed on an "AS IS" BASIS,
       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
       See the License for the specific language governing permissions and
       limitations under the License.
    """

import logging
from collections import OrderedDict

import os
import random
import re

log = logging.getLogger(__name__)

class DatasetScanner(object):

    def __init__(self,
                 template=None,
                 validation_source=None,
                 randomise=False,
                 randomise_training=False,
                 randomise_validation=False,
                 mapping=None,
                 type_format="{type}",
                 id_format="{name}-{idx}",
                 custom_attributes={},
                 filter=lambda dataset_id, match_components, dataset: True,
                 validation_percent=0,
                 prepend_path=""):

        if template:
            self._template = template
        if template:
            self._re = re.compile(template)
        self._validation_source = validation_source
        self._prepend_path = prepend_path
        self._id_format = id_format
        self._type_format = type_format

        self._randomise = randomise
        self._randomise_training = randomise_training
        self._randomise_validation = randomise_validation
        self._validation_percent = validation_percent
        self._mapping = mapping
        self._custom_attributes = custom_attributes
        self._datasets = OrderedDict()
        self._validation_datasets = OrderedDict()

        self._filter = filter
        self.update_datasets()
        if self._validation_source:
            self.update_datasets(directory=self._validation_source, datasets=self._validation_datasets)
        self.__update_split()

    def __len__(self):
        return len(self._datasets)

    def __update_split(self):
        _datasets = list(self._datasets.items())
        if self._randomise:
            random.shuffle(_datasets)

        _validation_datasets = list(self._validation_datasets.items())
        if len(_validation_datasets) > 0:
            # num_evaluation = int(len(_validation_datasets) * self._validation_percent)
            if self._randomise_training:
                random.shuffle(_datasets)
            if self._randomise_validation:
                random.shuffle(_validation_datasets)
            self._train_list = tuple(_datasets)
            self._evaluation_list = tuple(_validation_datasets[:])
        else:
            num_evaluation = int(len(_datasets) * self._validation_percent)
            self._evaluation_list, self._train_list = tuple(_datasets[:num_evaluation]), tuple(
                _datasets[num_evaluation:])

    def get_full_datasets(self):
        return self._datasets

    def get_training_datasets(self):
        return self._train_list

    def get_validation_datasets(self):
        return self._evaluation_list

    def scan_datasets(self):
        raise NotImplementedError()

    def get_dataset_id(self, components):
        return self._id_format.format(**components)

    def get_dataset_by_id(self, dataset_id, dataset=None):
        if dataset is None:
            dataset = self._datasets
        return dataset[dataset_id]

    def remove_dataset_by_id(self, dataset_id, dataset=None):
        if dataset is None:
            dataset = self._datasets
        return dataset.pop(dataset_id)

    def update_dataset(self, dataset=None, dataset_id=None, match_components={}, dataset_path=None):
        """"""

        # print("dataset_id={} match_components={} dataset_path={}".format(dataset_id, match_components,dataset_path))

        if dataset is None:
            dataset = self._datasets

        if self._custom_attributes:
            for k, v in self._custom_attributes.items():
                match_components[k] = v(**match_components)
        if dataset_id is None and not match_components.get("__id_generated", False):
            dataset_id = self.get_dataset_id(match_components)
            match_components["__id_generated"] = True

        if dataset_id not in dataset:
            dataset[dataset_id] = {}
            components = dataset[dataset_id]
        else:
            components = dataset[dataset_id]
        dataset_type = self._type_format.format(**match_components)

        if dataset_type in components:
            raise KeyError("Already registered: %s %s" % (dataset_type, dataset_path))
        components[dataset_type] = dataset_path
        return dataset_id

class FileScanner(DatasetScanner):
    def __init__(self, source_file, *args, **kw):
        self._source_file = source_file
        DatasetScanner.__init__(self, *args, **kw)

    def update_datasets(self, filter=None):
        if filter is None:
            filter = self._filter

        file_list = []
        log.info("Updateing datasets from file list: %s", self._source_file)
        if self._source_file.startswith("gs://"):
            log.info("Using tensorflow for IO")
            from tensorflow.python.lib.io.file_io import FileIO
            input_file = FileIO(self._source_file, "r")
            log.info("Tensorflow reported size: %d", input_file.size())
        else:
            input_file = open(self._source_file)

        lines = input_file.readlines()
        for line in lines:
            fpath = line.strip()
            parts = fpath.split("/")
            file_name = parts[-1]
            directory_name = "/".join(parts[:-1])
            match = self._re.match(file_name)
            if not match:
                continue
            match_components = match.groupdict()
            dataset_path = self._prepend_path + fpath
            dataset_id = self.update_dataset(match_components=match_components, dataset_path=dataset_path)
            dataset = self.get_dataset_by_id(dataset_id)
            if not filter(dataset_id, match_components, dataset):
                self.remove_dataset_by_id(dataset_id)
        input_file.close()

class FilesystemScanner(DatasetScanner):

    def __init__(self, directory, *args, **kw):
        self._directory = directory
        DatasetScanner.__init__(self, *args, **kw)

    def update_datasets(self, directory=None, datasets=None, filter=None):
        if directory is None:
            directory = self._directory
        if datasets is None:
            datasets = self.get_full_datasets()

        if filter is None:
            filter = self._filter
        for directory_entry in os.walk(directory, followlinks=True):
            directory_name = directory_entry[0]
            directory_members = directory_entry[2]
            for file_name in directory_members:
                match = self._re.match(file_name)
                fpath = os.path.join(directory_name, file_name)
                if not match:
                    continue

                match_components = match.groupdict()
                dataset_path = self._prepend_path + fpath
                dataset_id = self.update_dataset(dataset=datasets, match_components=match_components,
                                                 dataset_path=dataset_path)
                dataset = self.get_dataset_by_id(dataset_id, dataset=datasets)

                if not filter(dataset_id, match_components, dataset):
                    self.remove_dataset_by_id(dataset_id)


import re
from datetime import date, datetime
from enum import Enum
from typing import Optional

from pydantic import BaseModel, PositiveFloat

class AppwriteDocumentPermissions(BaseModel):
    """Appwrite document permissions."""

    read: list[str]
    write: list[str]

class AppwriteDocument(BaseModel):
    """Basic fields for an Appwrite document."""

    id: str
    collection: str
    permissions: AppwriteDocumentPermissions

    def __init__(self, **kwargs) -> None:
        for k in list(kwargs.keys()):
            if isinstance(k, str):
                _k = re.sub("^\\$", "", k)
                kwargs[_k] = kwargs.pop(k)
        super().__init__(**kwargs)

class CoffeeBeanRoast(Enum):
    """Types of roasts."""

    LIGHT = "LIGHT"
    LIGHT_MEDIUM = "LIGHT_MEDIUM"
    MEDIUM = "MEDIUM"
    MEDIUM_DARK = "MEDIUM_DARK"
    DARK = "DARK"
    FRENCH = "FRENCH"
    ESPRESSO = "ESPRESSO"

class CoffeeBag(BaseModel):
    """Coffee bag data."""

    brand: str
    name: str
    mass: PositiveFloat
    start_date: Optional[date] = None
    end_date: Optional[date] = None
    active: bool
    roast: Optional[CoffeeBeanRoast] = None

    def __init__(self, **data) -> None:
        for k in ["start_date", "end_date", "roast"]:
            if data.get(k, None) == "":
                data[k] = None
        super().__init__(**data)

class CoffeeBagDocument(CoffeeBag, AppwriteDocument):
    """Coffee Bag Document."""

class CoffeeCup(BaseModel):
    """Coffee cup data."""

    bag_id: str
    datetime: datetime

class CoffeeCupDocument(CoffeeCup, AppwriteDocument):
    """Coffee Cup Document."""


from datetime import datetime
import os
import mne
import json
from db_connections import save_preprocessed_file_location_in_db, save_epochs_file_location_in_db, query_for_index

datatype = 'eeg'
extension = '.fif'
timestamp = datetime.now()
timestamp_str = timestamp.strftime("%d-%b-%Y (%H:%M:%S.%f)")

def set_file_name(subject, level, ctr=None):
    if ctr is not None:
        filename = f'sub_{subject}_{level}_{ctr}_raw.fif'
    else:
        filename = f'sub_{subject}_{level}_raw.fif'
    return filename

def get_path(directory_name):
    path = f'{os.getcwd() }/{directory_name}'
    if not os.path.exists(path):
        # Path does not exist yet, create it
        os.makedirs(path)
    return path

def save_preprocessed_data_offline(raw, subject):
    level = 'preprocessed'
    path = get_path('preprocessed_data')
    if not os.path.exists(path):
        # Path does not exist yet, create it
        os.makedirs(path)
    raw.save(os.path.join(path, set_file_name(subject, level)), fmt='single', overwrite=True)

async def save_preprocessed_data_online(raw, subject, channel_num):
    level = 'preprocessed'
    path = get_path('preprocessed_data')
    if not os.path.exists(path):
        # Path does not exist yet, create it
        os.makedirs(path)
    index = await query_for_index() + 1
    raw.save(os.path.join(path, set_file_name(subject, level, index)), fmt='single', overwrite=False)
    await save_preprocessed_file_location_in_db(index, subject, f'{path}/{set_file_name(subject, level, index)}')

def read_raw_fif(filepath):
    return mne.io.read_raw_fif(filepath, preload=True)

def read_fif_epochs(filepath):
    return mne.read_epochs(filepath, preload=True)

def get_event_dict(subject_id) -> dict:
    file_name = f"{get_path('offline_module_data')}/subject_{subject_id}.json"
    file = open(file_name)
    data = json.load(file)
    return data['Event_Dictionary']

def set_montage():
    montage_path = get_path('montage_folder') + '/AC-64.bvef'
    montage = mne.channels.read_custom_montage(montage_path, head_size=0.085)
    return montage


import random
import urllib.request
from bs4 import BeautifulSoup
import math
import sys
import sqlite3
import os

def get_db():
    con = sqlite3.connect(os.path.join(os.path.dirname(__file__), "ubuntu.db"))
    con.execute("create table if not exists bugs (id text, isok bool, link text, title text)")
    con.commit()
    return con

def db_fetch(bug_number):
    db = get_db()
    crs = db.cursor()
    crs.execute("select id, isok, link, title from bugs where id = ?", [str(bug_number)])
    rows = crs.fetchall()
    db.close()
    if len(rows) == 0:
        return None
    else:
        return rows[0]

def db_store(bug_number, isok, link="", title=""):
    db = get_db()
    db.execute("insert into bugs (id, isok, link, title) values (?, ?, ?, ?)", [str(bug_number), isok, link, title])
    db.commit()
    db.close()

def get_individual(bug_number, known_good=False):
    if known_good:
        # get a bug from our known good list
        db = get_db()
        crs = db.cursor()
        crs.execute("select id, link, title from bugs where isok = 1 order by random() limit 1")
        rows = crs.fetchall()
        id, link, title = rows[0]
        return True, {"title": title, "link": link}
        db.close()
    else:
        val = db_fetch(bug_number)
        if val is None:
            pass  # if we know nothing of this bug, carry on with it
        else:
            id, isok, link, title = val
            if isok == 0:
                print("Bug", bug_number, "known bad")
                return False, None  # bail if we know this one to be a dud
            else:
                # this bug is known to be ok, then return it
                return True, {"title": title, "link": link}
    bug_page = "https://pad.lv/{}".format(bug_number)
    print("trying", bug_page)
    try:
        fp = urllib.request.urlopen(bug_page)
    except:
        db_store(bug_number, False)
        return False, None
    soup = BeautifulSoup(fp.read(), "lxml")
    fp.close()
    project = soup.select("div#watermark a")[0]["href"].split("/")[-1]
    if project != "ubuntu":
        print("Not Ubuntu project:", project)
        db_store(bug_number, False)
        return False, None
    dupe = soup.select("div#bug-is-duplicate .bug-duplicate-details a")
    if dupe:
        override_result = dupe[0]["href"].split("/")[-1]
        print("Duplicate of", override_result)
        db_store(bug_number, False)
        return False, None
    try:
        status = soup.select("div.status-content")[0].text.strip()
    except IndexError:
        print("Bug unreadable")
        db_store(bug_number, False)
        return False, None
    if status in ("New", "Incomplete", "Confirmed"):
        title = soup.select("h1#edit-title")[0].text.strip()
        db_store(bug_number, True, bug_page, title)
        return True, {
            "title": title,
            "link": bug_page
        }
    else:
        print("Not seeking contributions: status", status)
        db_store(bug_number, False)
        return False, None

def get(max_bug, override_result=None, known_good=False):
    # get a random int up to max_bug, but biased towards the more recent
    # https://gamedev.stackexchange.com/a/116875 with min = 0
    while True:
        if override_result:
            result = override_result
            override_result = None
        else:
            result = math.floor(max_bug * pow(random.random(), 0.8))
        isok, bug = get_individual(result, known_good)
        if not isok:
            if max_bug == 0:
                max_bug = get_max_bug_number()
            continue
        return bug

def get_max_bug_number():
    fp = urllib.request.urlopen(
        "https://bugs.launchpad.net/ubuntu/+bugs?orderby=-id&start=0")
    soup = BeautifulSoup(fp.read(), "lxml")
    return int(soup.select("span.bugnumber")[0].text[1:])

if __name__ == "__main__":
    # for testing, run as python3 projects/ubuntu_project.py
    # or python3 projects/ubuntu_project.py 434733
    # or to get a definitely known good bug, python3 projects/ubuntu_project.py -1
    if len(sys.argv) > 1 and sys.argv[1] == "-1":
        print(get(0, None, True))
    elif len(sys.argv) > 1:
        print(get(0, sys.argv[1]))
    else:
        max_bug = get_max_bug_number()
        print(get(max_bug))


import difflib
import sys
import argparse

def get_binary(ground_truth, actual_string):
    diffs = difflib.SequenceMatcher(None, actual_string, ground_truth)
    binary_out = [0] * len(actual_string)
    for tag, i1, i2, j1, j2 in diffs.get_opcodes():
        if tag == "delete":
            binary_out[i1:i2] = [1]*(i2-i1)
    return binary_out

def main():
    with open(args.output_file, "w+", encoding='utf-8') as out_file:
        with open(args.truth, encoding='utf-8') as gt , \
                open(args.actual, encoding='utf-8') as og:
            for line_gt, line_og in zip(gt, og):
                out_file.write("".join(map(str, get_binary(line_gt, line_og)))+"\n")

if __name__ == '__main__':
    # Read files
    # Get binary out
    # Append to output file
    parser = argparse.ArgumentParser(description=(
        "Convert FCE data into required input format for Reformer"))
    parser.add_argument('output_file',
                        help='Output file path')
    parser.add_argument('--truth',
                        help='Path to the ground truth file')
    parser.add_argument('--actual',
                        help='Path to the actual file to be compared')
    args = parser.parse_args()
    main()


"""Site Representation."""
# Standard Library
from typing import Tuple
from xml.etree import ElementTree as ET

# Math
from decimal import Decimal

Coordinates = Tuple[Decimal, Decimal]

class Point:
    """Point representation."""

    x: Decimal
    y: Decimal

    def __init__(self, x: Decimal, y: Decimal) -> None:
        """Construct point."""
        self.x = x
        self.y = y

    def __str__(self) -> str:
        """Get String representation."""
        return f"{'{0:.4f}'.format(self.x)}, {'{0:.4f}'.format(self.y)}"

    def __repr__(self) -> str:
        """Get object representation."""
        return self.__str__()

    def __eq__(self, point: "Point") -> str:
        """Get equality between points."""
        return self.x == point.x and self.y == point.y

    def get_tuple(self) -> Coordinates:
        """Get tuple of coordinates (x, y)."""
        return (self.x, self.y)

    def get_xml(self, label, exp) -> str:
        """Get xml representation."""
        point_expression = ET.Element("expression")
        point_expression.set(
            "label", label,
        )
        point_expression.set("exp", exp)
        point_expression.set("type", "point")

        """
        <expression label="G" exp="(p_{x}, p_{y})" type="point" />
        <element type="point" label="G">
            <show object="true" label="true"/>
            <objColor r="0" g="0" b="0" alpha="0"/>
            <layer val="0"/>
            <labelMode val="0"/>
            <pointSize val="5"/>
            <pointStyle val="0"/>
        </element>
        """
        point_element = ET.Element("element")
        point_element.set("type", "point")
        point_element.set("label", label)
        point_show = ET.SubElement(point_element, "show")
        point_show.set("object", "true")
        point_show.set("label", "true")
        point_obj_color = ET.SubElement(point_element, "objColor")
        point_obj_color.set("r", "0")
        point_obj_color.set("g", "0")
        point_obj_color.set("b", "0")
        point_obj_color.set("alpha", "0")
        point_layer = ET.SubElement(point_element, "layer")
        point_layer.set("val", "0")
        point_label_mode = ET.SubElement(point_element, "labelMode")
        point_label_mode.set("val", "0")
        point_size = ET.SubElement(point_element, "pointSize")
        point_size.set("val", "5")
        point_style = ET.SubElement(point_element, "pointStyle")
        point_style.set("val", "0")

        expression_xml = (
            ET.tostring(point_expression, encoding="unicode", method="xml")
            + "\n"
            + ET.tostring(point_element, encoding="unicode", method="xml")
            + "\n"
        )
        return expression_xml