import numpy as np
from jax import numpy as jnp
from flax import struct
from flax.traverse_util import flatten_dict, unflatten_dict
from flax.core import Scope, lift, freeze, unfreeze
from commplax import comm, xcomm, xop, adaptive_filter as af
from commplax.util import wrapped_partial as wpartial
from typing import Any, NamedTuple, Iterable, Callable, Optional

Array = Any

# related: https://github.com/google/jax/issues/6853
@struct.dataclass
class SigTime:
  start: int = struct.field(pytree_node=False)
  stop: int = struct.field(pytree_node=False)
  sps: int = struct.field(pytree_node=False)

class Signal(NamedTuple):
    val: Array
    t: Any = SigTime(0, 0, 2)

    def taxis(self):
        return self.t[0].shape[0], -self.t[0].shape[1]

    def __mul__(self, other):
        Signal._check_type(other)
        return Signal(self.val * other, self.t)

    def __add__(self, other):
        Signal._check_type(other)
        return Signal(self.val + other, self.t)

    def __sub__(self, other):
        Signal._check_type(other)
        return Signal(self.val - other, self.t)

    def __truediv__(self, other):
        Signal._check_type(other)
        return Signal(self.val / other, self.t)

    def __floordiv__(self, other):
        Signal._check_type(other)
        return Signal(self.val // other, self.t)

    def __imul__(self, other):
        return self * other

    def __iadd__(self, other):
        return self + other

    def __isub__(self, other):
        return self - other

    def __itruediv__(self, other):
        return self / other

    def __ifloordiv__(self, other):
        return self // other

    @classmethod
    def _check_type(cls, other):
        assert not isinstance(other, cls), 'not implemented'

def zeros(key, shape, dtype=jnp.float32): return jnp.zeros(shape, dtype)
def ones(key, shape, dtype=jnp.float32): return jnp.ones(shape, dtype)
def delta(key, shape, dtype=jnp.float32):
    k1d = comm.delta(shape[0], dtype=dtype)
    return jnp.tile(np.expand_dims(k1d, axis=list(range(1, len(shape)))), (1,) + shape[1:])
def gauss(key, shape, dtype=jnp.float32):
    taps = shape[0]
    k1d = comm.gauss(comm.gauss_minbw(taps), taps=taps, dtype=dtype)
    return jnp.tile(np.expand_dims(k1d, axis=list(range(1, len(shape)))), (1,) + shape[1:])

def dict_replace(col, target, leaf_only=True):
    col_flat = flatten_dict(unfreeze(col))
    diff = {}
    for keys_flat in col_flat.keys():
        for tar_key, tar_val in target.items():
            if (keys_flat[-1] == tar_key if leaf_only else (tar_key in keys_flat)):
                diff[keys_flat] = tar_val
    col_flat.update(diff)
    col = unflatten_dict(col_flat)
    return col

def update_varcolbykey(var, col_name, target, leaf_only=True):
    wocol, col = var.pop(col_name)
    col = dict_replace(col, target, leaf_only=leaf_only)
    del var
    return freeze({**wocol, col_name: col})

def update_aux(var, tar):
    return update_varcolbykey(var, 'aux_inputs', tar, leaf_only=True)

def conv1d_t(t, taps, rtap, stride, mode):
    assert t.sps >= stride, f'sps of input SigTime must be >= stride: {stride}, got {t.sps} instead'
    if rtap is None:
        rtap = (taps - 1) // 2
    delay = -(-(rtap + 1) // stride) - 1
    if mode == 'full':
        tslice = (-delay * stride, taps - stride * (rtap + 1)) #TODO: think more about this
    elif mode == 'same':
        tslice = (0, 0)
    elif mode == 'valid':
        tslice = (delay * stride, (delay + 1) * stride - taps)
    else:
        raise ValueError('invalid mode {}'.format(mode))
    return SigTime((t.start + tslice[0]) // stride, (t.stop + tslice[1]) // stride, t.sps // stride)

def conv1d_slicer(taps, rtap=None, stride=1, mode='valid'):
    def slicer(signal):
        x, xt = signal
        yt = conv1d_t(xt, taps, rtap, stride, mode)
        D = xt.sps // yt.sps
        zt = SigTime(yt.start * D, yt.stop * D, xt.sps)
        x = x[zt.start - xt.start: x.shape[0] + zt.stop - xt.stop]
        return Signal(x, zt)
    return slicer

def fullsigval(inputs: Signal, fill_value=1):
    x, t = inputs
    full_shape = (x.shape[0] + t.start - t.stop,) + x.shape[1:]
    return jnp.full(full_shape, fill_value, dtype=x.dtype)

def vmap(f,
         variable_axes={
             'params': -1,
             'const': None
         },
         split_rngs={
             'params': True,
         },
         in_axes=(Signal(-1, None),), out_axes=Signal(-1, None)):
    # in_axes needs to be wrapped by a tuple, see Flax's lifted vmap implemetation:
    # https://github.com/google/flax/blob/82e9798274c927286878c4600b4b09650d1e7935/flax/core/lift.py#L395
    vf = lift.vmap(f,
                   variable_axes=variable_axes, split_rngs=split_rngs,
                   in_axes=in_axes, out_axes=out_axes)
    vf.__name__ = 'vmapped_' + f.__name__ # [Workaround]: lifted transformation does not keep the original name
    return vf

def scan(f, in_axes=0, out_axes=0):
    sf = lift.scan(f, in_axes=in_axes, out_axes=out_axes)
    sf.__name__ = 'scanned' + f.__name__
    return sf

def simplefn(scope, signal, fn=None, aux_inputs=None):
    assert fn is not None, 'simple function cannot be None'
    aux = ()
    if aux_inputs is not None:
        aux_name, aux_init = aux_inputs
        aux += scope.variable('aux_inputs', aux_name, aux_init, signal).value,
    return fn(signal, *aux)

def batchpowernorm(scope, signal, momentum=0.999, mode='train'):
    running_mean = scope.variable('norm', 'running_mean',
                                  lambda *_: 0. + jnp.ones(signal.val.shape[-1]), ())
    if mode == 'train':
        mean = jnp.mean(jnp.abs(signal.val)**2, axis=0)
        running_mean.value = momentum * running_mean.value + (1 - momentum) * mean
    else:
        mean = running_mean.value
    return signal / jnp.sqrt(mean)

def conv1d(
    scope: Scope,
    signal,
    taps=31,
    rtap=None,
    mode='valid',
    kernel_init=delta,
    conv_fn = xop.convolve):

    x, t = signal
    t = scope.variable('const', 't', conv1d_t, t, taps, rtap, 1, mode).value
    h = scope.param('kernel',
                     kernel_init,
                     (taps,), np.complex64)
    x = conv_fn(x, h, mode=mode)

    return Signal(x, t)

def mimoconv1d(
    scope: Scope,
    signal,
    taps=31,
    rtap=None,
    dims=2,
    mode='valid',
    kernel_init=zeros,
    conv_fn=xop.convolve):

    x, t = signal
    t = scope.variable('const', 't', conv1d_t, t, taps, rtap, 1, mode).value
    h = scope.param('kernel', kernel_init, (taps, dims, dims), np.float32)
    y = xcomm.mimoconv(x, h, mode=mode, conv=conv_fn)
    return Signal(y, t)

def mimofoeaf(scope: Scope,
              signal,
              framesize=100,
              w0=0,
              train=False,
              preslicer=lambda x: x,
              foekwargs={},
              mimofn=af.rde,
              mimokwargs={},
              mimoinitargs={}):

    sps = 2
    dims = 2
    tx = signal.t
    # MIMO
    slisig = preslicer(signal)
    auxsig = scope.child(mimoaf,
                         mimofn=mimofn,
                         train=train,
                         mimokwargs=mimokwargs,
                         mimoinitargs=mimoinitargs,
                         name='MIMO4FOE')(slisig)
    y, ty = auxsig # assume y is continuous in time
    yf = xop.frame(y, framesize, framesize)

    foe_init, foe_update, _ = af.array(af.frame_cpr_kf, dims)(**foekwargs)
    state = scope.variable('af_state', 'framefoeaf',
                           lambda *_: (0., 0, foe_init(w0)), ())
    phi, af_step, af_stats = state.value

    af_step, (af_stats, (wf, _)) = af.iterate(foe_update, af_step, af_stats, yf)
    wp = wf.reshape((-1, dims)).mean(axis=-1)
    w = jnp.interp(jnp.arange(y.shape[0] * sps) / sps,
                   jnp.arange(wp.shape[0]) * framesize + (framesize - 1) / 2, wp) / sps
    psi = phi + jnp.cumsum(w)
    state.value = (psi[-1], af_step, af_stats)

    # apply FOE to original input signal via linear extrapolation
    psi_ext = jnp.concatenate([w[0] * jnp.arange(tx.start - ty.start * sps, 0) + phi,
                               psi,
                               w[-1] * jnp.arange(tx.stop - ty.stop * sps) + psi[-1]])

    signal = signal * jnp.exp(-1j * psi_ext)[:, None]
    return signal

def mimoaf(
    scope: Scope,
    signal,
    taps=32,
    rtap=None,
    dims=2,
    sps=2,
    train=False,
    mimofn=af.ddlms,
    mimokwargs={},
    mimoinitargs={}):

    x, t = signal
    t = scope.variable('const', 't', conv1d_t, t, taps, rtap, 2, 'valid').value
    x = xop.frame(x, taps, sps)
    mimo_init, mimo_update, mimo_apply = mimofn(train=train, **mimokwargs)
    state = scope.variable('af_state', 'mimoaf',
                           lambda *_: (0, mimo_init(dims=dims, taps=taps, **mimoinitargs)), ())
    truth_var = scope.variable('aux_inputs', 'truth',
                               lambda *_: None, ())
    truth = truth_var.value
    if truth is not None:
        truth = truth[t.start: truth.shape[0] + t.stop]
    af_step, af_stats = state.value
    af_step, (af_stats, (af_weights, _)) = af.iterate(mimo_update, af_step, af_stats, x, truth)
    y = mimo_apply(af_weights, x)
    state.value = (af_step, af_stats)
    return Signal(y, t)

def fdbp(
    scope: Scope,
    signal,
    steps=3,
    dtaps=261,
    ntaps=41,
    sps=2,
    d_init=delta,
    n_init=gauss):

    x, t = signal
    dconv = vmap(wpartial(conv1d, taps=dtaps, kernel_init=d_init))

    for i in range(steps):
        x, td = scope.child(dconv, name='DConv_%d' % i)(Signal(x, t))
        c, t = scope.child(mimoconv1d, name='NConv_%d' % i)(Signal(jnp.abs(x)**2, td),
                                                            taps=ntaps,
                                                            kernel_init=n_init)
        x = jnp.exp(1j * c) * x[t.start - td.start: t.stop - td.stop + x.shape[0]]

    return Signal(x, t)

def identity(scope, inputs):
    return inputs

def fanout(scope, inputs, num):
    return (inputs,) * num

# compositors

def serial(*fs):
    def _serial(scope, inputs, **kwargs):
        for f in fs:
            if isinstance(f, tuple) or isinstance(f, list):
                name, f = f
            else:
                name = None
            inputs = scope.child(f, name=name)(inputs, **kwargs)
        return inputs
    return _serial

def parallel(*fs):
    def _parallel(scope, inputs, **kwargs):
        outputs = []
        for f, inp in zip(fs, inputs):
            if isinstance(f, tuple) or isinstance(f, list):
                name, f = f
            else:
                name = None
            out = scope.child(f, name=name)(inp, **kwargs)
            outputs.append(out)
        return outputs
    return _parallel


import select
import socket

EOL1 = b'\n\n'
EOL2 = b'\r\n'
response = b'HTTP/1.0 200 OK\r\nDate: Mon, 1 Jan 1996 01:01:01 GMT\r\n'
response += b'Content-Type: text/plain\r\nContent-Length: 13\r\n\r\n'
response += b'Hello, world!'

serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
serversocket.bind(('0.0.0.0', 8080))
serversocket.listen(1)
serversocket.setblocking(0)

epoll = select.epoll()


import datetime
import logging

import matplotlib.pyplot as plt
import numpy as np
import numpy.typing as npt

from backend.data.measurements import MeasurementArray, Measurements

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

def xyz2blh(x, y, z):
    """_summary_
    Angle returned will be in radians
    """
    A = 6378137.0
    B = 6356752.314245
    e = np.sqrt(1 - (B ** 2) / (A ** 2))
    # calculate longitude, in radians
    longitude = np.arctan2(y, x)
    # calculate latitude, in radians
    xy_hypot = np.hypot(x, y)
    lat0 = np.zeros_like(x)
    latitude = np.arctan(z / xy_hypot)
    while np.any(np.abs(latitude - lat0) > 1e-9):
        lat0 = latitude
        N = A / np.sqrt(1 - e ** 2 * np.sin(lat0) ** 2)
        latitude = np.arctan((z + e ** 2 * N * np.sin(lat0)) / xy_hypot)
    # calculate height, in meters
    N = A / np.sqrt(1 - e ** 2 * np.sin(latitude) ** 2)
    small_angle_indices = np.abs(latitude) < np.pi / 4
    R, phi = np.hypot(xy_hypot[small_angle_indices], z[small_angle_indices]), np.arctan(z[small_angle_indices] / xy_hypot[small_angle_indices])
    height = np.zeros_like(x)
    height[small_angle_indices] = R * np.cos(phi) / np.cos(latitude[small_angle_indices]) - N[small_angle_indices]
    height[~small_angle_indices] = z[~small_angle_indices] / np.sin(latitude[~small_angle_indices]) - N[~small_angle_indices] * (1 - e ** 2)
    return latitude, longitude, height

class Position:
    """
    Position class to handle position analysis
    """

    def __init__(
        self,
        data: MeasurementArray = None,
        base: MeasurementArray = None,
        sitelist: list = None,
    ) -> None:
        self.data = data
        self.base = base
        self.sitelist = sitelist
        if self.base is not None:
            self.data = self.data - self.base

    def __iter__(self):
        return iter(self.data)

    def rotate_enu(self) -> None:
        """
        rotate Rotate the position to the ENU frame from the base
        """
        for data in self.data:
            for k in data.data:
                print(k)
            #locate the base with the same station id
            base = self.base.locate(site=data.id['site'])
            lat, lon, height = xyz2blh(base.data['x_0'], base.data['x_1'], base.data['x_2'])
            rot = np.zeros((3,3, len(lat)))
            rot[0,0] = -np.sin(lon)
            rot[0,1] = -np.sin(lat)*np.cos(lon)
            rot[0,2] = np.cos(lat)*np.cos(lon)
            rot[1,0] = np.cos(lon)
            rot[1,1] = -np.sin(lat)*np.sin(lon)
            rot[1,2] = np.cos(lat)*np.sin(lon)
            rot[2,0] = 0
            rot[2,1] = np.cos(lat)
            rot[2,2] = np.sin(lat)
            project = np.empty((len(data.data['x_0']),3))
            for i in range(3):
                project[:, i] = data.data[f'x_{i}']
            enu = np.matmul(rot.transpose(), project[:,:,np.newaxis])[:,:,0]
            for i in range(3):
                data.data[f'x_{i}'] = enu[:, i]


from .common import *  # NOQA
import pytest

project_detail = {"project": None, "namespace": None, "cluster": None,
                  "project2": None, "namespace2": None, "cluster2": None}
user_token = {"user_c1_p1_owner": {"user": None, "token": None},
              "user_c1_p1_member": {"user": None, "token": None},
              "user_c1_p2_owner": {"user": None, "token": None},
              "user_standard": {"user": None, "token": None}}

CATALOG_URL = "https://git.rancher.io/charts"
MYSQL_EXTERNALID_037 = "catalog://?catalog=library&template=mysql" \
                       "&version=0.3.7"
MYSQL_EXTERNALID_038 = "catalog://?catalog=library&template=mysql" \
                       "&version=0.3.8"
WORDPRESS_EXTID = "catalog://?catalog=library&template=wordpress" \
                  "&version=1.0.5"

def cluster_and_client(cluster_id, mgmt_client):
    cluster = mgmt_client.by_id_cluster(cluster_id)
    url = cluster.links.self + '/schemas'
    client = rancher.Client(url=url,
                            verify=False,
                            token=mgmt_client.token)
    return cluster, client

def wait_for_template_to_be_created(client, name, timeout=45):
    found = False
    start = time.time()
    interval = 0.5
    while not found:
        if time.time() - start > timeout:
            raise AssertionError(
                "Timed out waiting for templates")
        templates = client.list_template(catalogId=name)
        if len(templates) > 0:
            found = True
        time.sleep(interval)
        interval *= 2

def check_condition(condition_type, status):
    def _find_condition(resource):
        if not hasattr(resource, "conditions"):
            return False

        if resource.conditions is None:
            return False

        for condition in resource.conditions:
            if condition.type == condition_type and condition.status == status:
                return True
        return False

    return _find_condition

def test_tiller():
    name = random_test_name()
    admin_client = get_user_client()

    clusters = admin_client.list_cluster(name=CLUSTER_NAME).data
    assert len(clusters) > 0
    cluster_id = clusters[0].id

    p = admin_client. \
        create_project(name="test-" + random_str(),
                       clusterId=cluster_id,
                       resourceQuota={
                           "limit": {
                               "secrets": "1"}},
                       namespaceDefaultResourceQuota={
                           "limit": {
                               "secrets": "1"}}
                       )

    p = admin_client.reload(p)
    proj_client = rancher.Client(url=p.links.self +
                                 '/schemas', verify=False,
                                 token=USER_TOKEN)
    # need a cluster scoped client to create a namespace
    _cluster, cluster_client = cluster_and_client(cluster_id, admin_client)
    ns = cluster_client.create_namespace(name=random_str(),
                                         projectId=p.id,
                                         resourceQuota={
                                             "limit": {
                                                 "secrets": "1"
                                             }}
                                         )
    wait_for_template_to_be_created(admin_client, "library")
    app = proj_client.create_app(
        name=name,
        externalId=WORDPRESS_EXTID,
        targetNamespace=ns.name,
        projectId=p.id,
        answers=get_defaut_question_answers(admin_client, WORDPRESS_EXTID)
    )

    app = proj_client.reload(app)
    # test for tiller to be stuck on bad installs
    wait_for_condition(proj_client, app, check_condition('Installed', 'False'))
    # cleanup by deleting project
    admin_client.delete(p)

def test_app_deploy():
    admin_client = get_user_client()
    proj_client = get_project_client_for_token(
        project_detail["project"],
        USER_TOKEN)
    answer = get_defaut_question_answers(
        admin_client,
        MYSQL_EXTERNALID_037)
    wait_for_template_to_be_created(admin_client, "library")
    app = proj_client.create_app(
        name=random_test_name(),
        externalId=MYSQL_EXTERNALID_037,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=answer)
    print("App is active")
    validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037)
    proj_client.delete(app)

def test_app_delete():
    admin_client = get_user_client()
    proj_client = get_project_client_for_token(
        project_detail["project"],
        USER_TOKEN)
    wait_for_template_to_be_created(admin_client, "library")
    answer = get_defaut_question_answers(
        admin_client,
        MYSQL_EXTERNALID_037)
    app = proj_client.create_app(
        name=random_test_name(),
        externalId=MYSQL_EXTERNALID_037,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=answer)
    print("App is active")
    validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037)
    app = proj_client.delete(app)
    validate_app_deletion(proj_client, app.id)

def test_app_upgrade_version():
    admin_client = get_user_client()
    proj_client = get_project_client_for_token(
        project_detail["project"],
        USER_TOKEN)
    wait_for_template_to_be_created(admin_client, "library")
    answer = get_defaut_question_answers(
        admin_client,
        MYSQL_EXTERNALID_037)
    app = proj_client.create_app(
        name=random_test_name(),
        externalId=MYSQL_EXTERNALID_037,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=answer)
    print("App is active")
    validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037)
    new_answer = get_defaut_question_answers(
            admin_client,
            MYSQL_EXTERNALID_038)
    app = proj_client.update(
        obj=app,
        externalId=MYSQL_EXTERNALID_038,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=new_answer)
    app = proj_client.reload(app)
    validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_038)
    assert app.externalId == MYSQL_EXTERNALID_038, "incorrect template version"
    proj_client.delete(app)

def test_app_rollback():
    admin_client = get_user_client()
    proj_client = get_project_client_for_token(
        project_detail["project"],
        USER_TOKEN)
    wait_for_template_to_be_created(admin_client, "library")
    answer = get_defaut_question_answers(
        admin_client,
        MYSQL_EXTERNALID_037)
    app = proj_client.create_app(
        name=random_test_name(),
        externalId=MYSQL_EXTERNALID_037,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=answer)
    print("App is active")
    app = validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037)
    rev_id = app.appRevisionId
    new_answer = get_defaut_question_answers(
            admin_client,
            MYSQL_EXTERNALID_038)
    app = proj_client.update(
        obj=app,
        externalId=MYSQL_EXTERNALID_038,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=new_answer)
    app = proj_client.reload(app)
    app = validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_038)
    assert app.externalId == MYSQL_EXTERNALID_038, "incorrect template version"
    proj_client.action(obj=app,
                       action_name='rollback',
                       revisionId=rev_id)
    app = proj_client.reload(app)
    validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037)
    assert app.externalId == MYSQL_EXTERNALID_037, "incorrect template version"
    proj_client.delete(app)

def test_app_answer_override():
    admin_client = get_user_client()
    proj_client = get_project_client_for_token(
        project_detail["project"],
        USER_TOKEN)
    wait_for_template_to_be_created(admin_client, "library")
    answers = get_defaut_question_answers(
        admin_client,
        MYSQL_EXTERNALID_037)
    app = proj_client.create_app(
        name=random_test_name(),
        externalId=MYSQL_EXTERNALID_037,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=answers)
    print("App is active")
    app = validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037)
    answers["mysqlUser"] = "admin1234"
    app = proj_client.update(
        obj=app,
        externalId=MYSQL_EXTERNALID_037,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id,
        answers=answers)
    app = proj_client.reload(app)
    app = validate_catalog_app(proj_client, app, MYSQL_EXTERNALID_037, answers)
    assert app["answers"].mysqlUser == "admin1234", \
        "incorrect answer upgrade"
    proj_client.delete(app)

def test_rbac_app_project_scope_deploy():
    admin_client = get_user_client()
    proj_client = get_project_client_for_token(
        project_detail["project"],
        USER_TOKEN)
    catalog = admin_client.create_projectCatalog(
        name="projectcatalog",
        baseType="projectCatalog",
        branch="master",
        url=CATALOG_URL,
        projectId=project_detail["project"].id)
    time.sleep(5)
    pId = project_detail["project"].id.split(":")[1]
    catalog_proj_scoped_ext_id = "catalog://?catalog=" + pId + \
                                 "/projectcatalog&type=" \
                                 "projectCatalog&template=" \
                                 "mysql&version=0.3.8"
    answers = get_defaut_question_answers(
        admin_client,
        catalog_proj_scoped_ext_id)
    app = proj_client.create_app(
        name=random_test_name(),
        externalId=catalog_proj_scoped_ext_id,
        answers=answers,
        targetNamespace=project_detail["namespace"].name,
        projectId=project_detail["project"].id)
    validate_catalog_app(proj_client, app, catalog_proj_scoped_ext_id)
    p2, ns2 = create_project_and_ns(
        USER_TOKEN,
        project_detail["cluster"],
        random_test_name("testapp"))
    #Assign role
    assign_members_to_project(admin_client,
                              user_token["user_c1_p2_owner"]["user"],
                              p2,
                              "project-owner")
    #Verify "project-owner" of p1 can list the added catalog
    user1_client = get_client_for_token(
        user_token["user_c1_p1_owner"]["token"])
    catalogs_list = user1_client.list_projectCatalog()
    assert len(catalogs_list) == 1, \
        "Project catalog not found for the user"
    assert catalogs_list["data"][0]["name"] == \
           "projectcatalog", "Incorrect project catalog found"

    # Verify "project-member" of p1 can list the added catalog
    user2_client = get_client_for_token(
        user_token["user_c1_p1_member"]["token"])
    catalogs_list_2 = user2_client.list_projectCatalog()
    assert len(catalogs_list_2) == 1, \
        "Project catalog not found for the user"

    # Verify "project-owner" of p2 CANNOT list the added catalog
    user3_client = get_client_for_token(
        user_token["user_c1_p2_owner"]["token"])
    catalogs_list_3 = user3_client.list_projectCatalog()
    assert len(catalogs_list_3) == 0, \
        "Project catalog found for the user"

    # Verify A standard user CANNOT list the added catalog
    user4_client = get_client_for_token(
        user_token["user_standard"]["token"])
    catalogs_list_4 = user4_client.list_projectCatalog()
    assert len(catalogs_list_4) == 0, \
        "Project catalog found for the user"
    admin_client.delete(p2)

@pytest.fixture(scope='module', autouse="True")
def create_project_client(request):
    client = get_admin_client()
    clusters = client.list_cluster(name=CLUSTER_NAME).data
    assert len(clusters) > 0
    project_detail["project"], project_detail["namespace"] = \
        create_project_and_ns(USER_TOKEN, clusters[0],
                              random_test_name("testapp"))
    project_detail["cluster"] = clusters[0]
    #create users
    user_token["user_c1_p1_owner"]["user"], \
        user_token["user_c1_p1_owner"]["token"] = create_user(client)
    user_token["user_c1_p1_member"]["user"], \
        user_token["user_c1_p1_member"]["token"] = create_user(client)
    user_token["user_c1_p2_owner"]["user"], \
        user_token["user_c1_p2_owner"]["token"] = create_user(client)
    user_token["user_standard"]["user"], \
        user_token["user_standard"]["token"] = create_user(client)
    #Assign roles to the users
    assign_members_to_project(client,
                              user_token["user_c1_p1_owner"]["user"],
                              project_detail["project"],
                              "project-owner")
    assign_members_to_project(client,
                              user_token["user_c1_p1_member"]["user"],
                              project_detail["project"],
                              "project-member")

    def fin():
        client = get_user_client()
        client.delete(project_detail["project"])
    request.addfinalizer(fin)


def secondElem(a):
    return a[1]
def alphasort(a):
    a.sort(key=secondElem,reverse=True)
    for i in range(len(a)-1):
        if a[i][1] == a[i+1][1] and a[i][0] > a[i+1][0]:
            a[i],a[i+1] = a[i+1],a[i]
    return a
from collections import Counter
name = list(Counter(input()).items())
name = alphasort(name)
for i in range(3):
    print(name[i][0],name[i][1])


import netCDF4
import numpy as np
from matplotlib import pyplot as plt
from matplotlib.colors import Normalize
import cartopy.crs as ccrs
import matplotlib.colors as colors

# Land cover
lc = netCDF4.Dataset("../data/LandCover_half.nc")
lc.set_auto_mask(True)
lc.variables
land_cover = lc["Land Cover"][:]

# fix landcover
halfway = np.shape(land_cover)[1]//2
first_half = land_cover[:,:halfway]
second_half = land_cover[:,halfway:]
land_cover = np.concatenate((second_half,first_half), axis=1)
#land_cover[land_cover == 0] = np.nan
#land_cover[land_cover > 10] = np.nan

land_cover_int = land_cover.astype(int)  # Convert land_cover to int

unique_land_cover = np.unique(land_cover_int)

# plot
fig = plt.figure(figsize=(10, 6))
ax = plt.axes(projection=ccrs.PlateCarree())
image = ax.imshow(land_cover_int , origin="lower", extent=[-180, 180, -90, 90], transform=ccrs.PlateCarree())
ax.coastlines()
ax.set_global()
plt.show()

#ds = netCDF4.Dataset("../results/iso_result_mon_mean.nc")
ds = netCDF4.Dataset("../results/iso_result_OMI_filterd_mon_mean.nc")
ds.set_auto_mask(True)

dar = ds["J_times_IspS"][:]
dar[dar > 1000] = np.nan

dar_mean = np.nanmean(dar, axis=0)
dar_mean.shape

# Subset everything above 30 degrees North
def calculate_mean_subset(latitude_threshold,latitude, data):
    lat_indices = np.where(latitude > latitude_threshold)[0]
    data_subset = data[:, lat_indices, :]
    data_mean_subset = np.nanmean(data_subset, axis=0)
    anomaly_subset = np.subtract(data_subset, data_mean_subset)
    norm_subset = np.divide(anomaly_subset, data_mean_subset)
    mean_data_subset = np.column_stack((np.arange(12), [np.nanmean(norm_subset[i, :, :]) for i in range(12)]))
    return mean_data_subset

mean_iso_subset_30 = calculate_mean_subset(30, latitude_iso, dar)

print(mean_iso_subset_30)

calculate_mean_subset(30, latitude_iso, dar)

# Normalize the data between the maximum and 0
anomaly = np.subtract(dar, dar_mean)
norm = np.divide(anomaly, dar_mean)

np.nanmean(norm[5,:,:])

np.nanmin(norm[5,:,:])
np.nanmax(norm[5,:,:])

# Create a figure with multiple subplots
fig, axs = plt.subplots(3, 4, figsize=(16, 12), subplot_kw={'projection': ccrs.PlateCarree()})

# Iterate over the range of plots you want to display (0 to 11)
for i in range(12):
    # Calculate the subplot indices based on the iteration index
    row = i // 4  # Row index
    col = i % 4   # Column index

    # Plot the normalized data in the current subplot
    im = axs[row, col].imshow(norm[i, :, :], origin="lower", extent=[-180, 180, -90, 90], transform=ccrs.PlateCarree(), cmap='RdYlGn', vmin=-4, vmax=4, )

    axs[row, col].set_title("Iso_{:02d}".format(i))  # Format the title with leading zeros

    # Add coastlines
    axs[row, col].coastlines()

# Create a colorbar for the figure
fig.colorbar(im, ax=axs, fraction=0.022, pad=0.03, location='bottom')

# Adjust the spacing between subplots
plt.tight_layout()

# Move the colorbar below the subplots
fig.subplots_adjust(bottom=0.15)

# Display the plot
plt.show()
ds.close()

np.where(land_cover_int == 1)
norm[5, np.where(land_cover_int == 1)]
mean_norm_values = []
for lc in unique_land_cover:
    indices = np.where(land_cover_int == lc)  # Find indices of matching land_cover values
    mean_norm = np.mean(norm[indices])    # Calculate mean norm for the current land_cover value
    mean_norm_values.append(mean_norm)

print("Mean norm values for each land_cover group:")
for lc, mean_norm in zip(unique_land_cover, mean_norm_values):
    print(f"Land Cover {lc}: {mean_norm}")

#### Compare to the HCHO
HCHO = netCDF4.Dataset("../data/OMI_iso_estimate/mon_average_OMI.nc")
HCHO.set_auto_mask(False)

## Conveert isoprene values from the units kg/gridcell/month to kg/m2/month
def gridcell_to_m2(latitude):
    "Find the number of m2 in each grid cell in a 0.5 by 0.5 degree grid"
    half_degree_lat = 111111.1/2 # latitude lengths stay at about 111.1km per degree
    half_degree_lon = np.cos(np.deg2rad(latitude)) * (111111.1/2) # equation to get the length of each half degree of longitude (changes with latitude)
    meter_square_gridcell = half_degree_lat * half_degree_lon
    return meter_square_gridcell

latitudes = [float(lat) for lat in HCHO['lat'][:]] #get each latitude degree as a float
no_m2_in_grid = [gridcell_to_m2(lat) for lat in latitudes] #gets the number of m2 blocks in each grid cell
tiled = np.tile(no_m2_in_grid, (len(HCHO['lon'][:]), 1)).T #repeat across each longitude as distance remains the same across each latitude degree

# Get the isoprene emmited per m2
isoprene_per_m2 = (HCHO["EMworldC5H8"][:])/(tiled)

# Calculate the threshold value for the top 5%
threshold = np.percentile(isoprene_per_m2, 99.98)

isoprene_per_m2[isoprene_per_m2 == 0] = np.nan

# Set values above the threshold to NaN
isoprene_per_m2[isoprene_per_m2 > threshold] = np.nan

HCHO_mean = np.nanmean(isoprene_per_m2, axis=0)
HCHO_anomaly = np.subtract(isoprene_per_m2, HCHO_mean)
HCHO_norm = np.divide(HCHO_anomaly, HCHO_mean)

i = 6
np.nanmean(HCHO_norm[i,:,:])
np.nanmin(HCHO_norm[i,:,:])

# Subset everything above 30 degrees North
latitude_HCHO = HCHO.variables["lat"][:]

def HCHO_std_anomoly_subset(latitude_threshold, latitude, data):
    lat_indices = np.where(latitude > latitude_threshold)[0]
    data_subset = np.flipud(data)[:, lat_indices, :]
    data_mean_subset = np.nanmean(data_subset, axis=0)
    data_anomaly_subset = np.subtract(data_subset, data_mean_subset)
    data_norm_subset = np.divide(data_anomaly_subset, data_mean_subset)
    std_anomoly_subset = np.column_stack((np.arange(12), [np.nanmean(data_norm_subset[i, :, :]) for i in range(12)]))
    return std_anomoly_subset

mean_HCHO_subset_30 = HCHO_std_anomoly_subset(30, latitude_HCHO, isoprene_per_m2)

print(mean_HCHO_subset_30)
HCHO_std_anomoly_subset(0, latitude_HCHO, isoprene_per_m2)

# Create a figure and a single subplot
fig, ax = plt.subplots(figsize=(10, 4))

# Plot the first line with a red color
ax.plot(mean_HCHO_subset_30[:, 0], mean_HCHO_subset_30[:, 1], 'r-', label='HCHO')

# Plot the second line with a blue color
ax.plot(mean_iso_subset_30[:, 0], mean_iso_subset_30[:, 1], 'b-', label='Model')

# Set the title and legend
ax.set_title('Average anomaly in the Northern temperate zone (>30 degrees)')
ax.legend()

# Display the plot
plt.show()

np.nanmin(HCHO_norm)

# Create a figure with multiple subplots
fig, axs = plt.subplots(3, 4, figsize=(16, 12), subplot_kw={'projection': ccrs.PlateCarree()})

# Iterate over the range of plots you want to display (0 to 11)
for i in range(12):
    # Calculate the subplot indices based on the iteration index
    row = i // 4  # Row index
    col = i % 4   # Column index

    # Plot the normalized data in the current subplot
    im = axs[row, col].imshow(np.flipud(HCHO_norm[i, :, :]), origin="lower", extent=[-180, 180, -90, 90], transform=ccrs.PlateCarree(), cmap='RdYlGn')#, vmin=-4, vmax=4, )

    axs[row, col].set_title("OMI_{:02d}".format(i))  # Format the title with leading zeros

    # Add coastlines
    axs[row, col].coastlines()

# Create a colorbar for the figure
fig.colorbar(im, ax=axs, fraction=0.022, pad=0.03, location='bottom')

# Adjust the spacing between subplots
plt.tight_layout()

# Move the colorbar below the subplots
fig.subplots_adjust(bottom=0.15)

# Display the plot
plt.show()

HCHO.close()


import os
import shutil
import pickle
import glob
import cv2
import numpy as np
import matplotlib
import matplotlib.pyplot as plt

matplotlib.use('TkAgg')

from cam import create_dataset, Camera

def save_data(path, data):
    with open(path, 'wb') as handle:
        pickle.dump(data, handle)

    print("Saved")

def load_data(path):
    with open(path, 'rb') as handle:
        data = pickle.load(handle)

    return data

def camera_calibrate(images_folder='./img',
                     board_size=(6, 9),
                     world_scaling=1.,
                     debug=False):
    images_names = sorted(glob.glob(images_folder))
    images = []

    for imname in images_names:
        im = cv2.imread(imname, 1)
        images.append(im)

    # критерии, используемые детектором шахматной доски.
    # Измените это, если код не может найти шахматную доску
    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

    objp = np.zeros((1, board_size[0] * board_size[1], 3), np.float32)
    objp[0, :, :2] = np.mgrid[0:board_size[0], 0:board_size[1]].T.reshape(-1, 2)
    objp = world_scaling * objp

    width = images[0].shape[1]
    height = images[0].shape[0]

    imgpoints = []
    objpoints = []

    for frame in images:
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        ret, corners = cv2.findChessboardCorners(gray, (board_size[0], board_size[1]), None)

        if ret:

            corners = cv2.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)
            cv2.drawChessboardCorners(frame, (board_size[0], board_size[1]), corners, ret)

            if debug:
                cv2.imshow('img', frame)
                k = cv2.waitKey(500)

            objpoints.append(objp)
            imgpoints.append(corners)

    ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, (width, height), None, None)

    return mtx, dist

def stereo_camera_calibrate(images_folder1='./img',
                            images_folder2='./img',
                            board_size=(6, 9),
                            world_scaling=1.,
                            cameraMatrix1=None,
                            distCoeffs1=None,
                            cameraMatrix2=None,
                            distCoeffs2=None,
                            debug=False):
    cam1_path = sorted(glob.glob(images_folder1))
    cam2_path = sorted(glob.glob(images_folder2))

    c1_images = []
    c2_images = []

    for im1, im2 in zip(cam1_path, cam2_path):
        im = cv2.imread(im1, 1)
        c1_images.append(im)

        im = cv2.imread(im2, 1)
        c2_images.append(im)

    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.0001)

    objp = np.zeros((board_size[0] * board_size[1], 3), np.float32)
    objp[:, :2] = np.mgrid[0:board_size[0], 0:board_size[1]].T.reshape(-1, 2)
    objp = world_scaling * objp

    width = c1_images[0].shape[1]
    height = c1_images[0].shape[0]

    imgpoints_left = []
    imgpoints_right = []

    objpoints = []

    for frame1, frame2 in zip(c1_images, c2_images):
        gray1 = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
        gray2 = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
        c_ret1, corners1 = cv2.findChessboardCorners(gray1, board_size, None)
        c_ret2, corners2 = cv2.findChessboardCorners(gray2, board_size, None)

        if c_ret1 == True and c_ret2 == True:
            corners1 = cv2.cornerSubPix(gray1, corners1, (11, 11), (-1, -1), criteria)
            corners2 = cv2.cornerSubPix(gray2, corners2, (11, 11), (-1, -1), criteria)

            if debug:
                cv2.drawChessboardCorners(frame1, board_size, corners1, c_ret1)
                cv2.imshow('img', frame1)

                cv2.drawChessboardCorners(frame2, board_size, corners2, c_ret2)

                cv2.imshow('img2', frame2)
                cv2.waitKey(500)

            objpoints.append(objp)
            imgpoints_left.append(corners1)
            imgpoints_right.append(corners2)

    stereocalibration_flags = cv2.CALIB_FIX_INTRINSIC
    ret, CM1, dist1, CM2, dist2, R, T, E, F = cv2.stereoCalibrate(objectPoints=objpoints,
                                                                  imagePoints1=imgpoints_left,
                                                                  imagePoints2=imgpoints_right,
                                                                  cameraMatrix1=cameraMatrix1,
                                                                  distCoeffs1=distCoeffs1,
                                                                  cameraMatrix2=cameraMatrix2,
                                                                  distCoeffs2=distCoeffs2,
                                                                  imageSize=(width, height),
                                                                  criteria=criteria,
                                                                  flags=stereocalibration_flags)

    return R, T

def mousePoint(event, x, y, flag, params):
    if event == cv2.EVENT_LBUTTONDOWN:
        print(f"[{x}, {y}]")

def DLT(P1, P2, point1, point2):
    A = [point1[1] * P1[2, :] - P1[1, :],
         P1[0, :] - point1[0] * P1[2, :],
         point2[1] * P2[2, :] - P2[1, :],
         P2[0, :] - point2[0] * P2[2, :]
         ]
    A = np.array(A).reshape((4, 4))
    # print('A: ')
    # print(A)

    B = A.transpose() @ A
    # from scipy import linalg
    U, s, Vh = np.linalg.svd(B, full_matrices=False)

    # print('Triangulated point: ')
    # print(Vh[3, 0:3] / Vh[3, 3])

    return Vh[3, 0:3] / Vh[3, 3]

if __name__ == "__main__":
    camera1 = Camera(camera_id=0, show_frame=False, vertical_flip=True, save_video=False)
    camera2 = Camera(camera_id=1, show_frame=False, vertical_flip=True, save_video=False)
    #
    # create_screen(0)
    # create_screen(1)
    #
    # camera1.initialize()
    # camera2.initialize()

    # create_dataset([camera1, camera2], './img/split/')

    # ========================================== КАЛИБРОВКА КАМЕРЫ =====================================================
    # mtx1, dist1 = camera_calibrate('./img/split/camera 1/*.jpg', debug=False)
    # mtx2, dist2 = camera_calibrate('./img/split/camera 2/*.jpg', debug=False)
    #
    # R, T = stereo_camera_calibrate(images_folder1="./img/split/camera 1/*.jpg",
    #                                images_folder2="./img/split/camera 2/*.jpg",
    #                                cameraMatrix1=mtx1,
    #                                cameraMatrix2=mtx2,
    #                                distCoeffs1=dist1,
    #                                distCoeffs2=dist2,
    #                                debug=False)

    # ==================================== СОХРАНЕНИЕ ДАННЫХ ===========================================================
    # save_data('./data/matrix_camera_1080.pickle', mtx1)
    # save_data('./data/matrix_camera.pickle', mtx2)
    #
    # save_data('./data/dist_camera_1080.pickle', dist1)
    # save_data('./data/dist_camera.pickle', dist2)
    #
    # save_data('./data/stereo_R.pickle', R)
    # save_data('./data/stereo_T.pickle', T)

    # ============================================== ЗАГРУЗКА ДАННЫХ ===================================================
    mtx2 = load_data('./data/matrix_camera_1080.pickle')
    mtx1 = load_data('./data/matrix_camera.pickle')

    dist2 = load_data('./data/dist_camera_1080.pickle')
    dist1 = load_data('./data/dist_camera.pickle')

    R = load_data('./data/stereo_R.pickle')
    T = load_data('./data/stereo_T.pickle')

    print(f"Camera matrix 0:\n {mtx1}")
    print(f"Camera matrix 1:\n {mtx2}")

    print(f"Camera dist 0:\n {dist1}")
    print(f"Camera dist 1:\n {dist2}")

    print(f"R:\n {R}")
    print(f"T:\n {T}")

    # board_size = (6, 9)
    # world_scaling = 1.

    # =============================================== РУЧНАЯ РАЗМЕТКА ДАННЫХ ===========================================
    # count = 0
    # while True:
    #
    #     if not count:
    #         path = './img/1.jpg'
    #     else:
    #         path = './img/2.jpg'
    #
    #     img = cv2.imread(path, 1)
    #
    #     cv2.imshow("Img", img)
    #     cv2.setMouseCallback('Img', mousePoint)
    #
    #     if cv2.waitKey(0) & 0xFF == ord('q'):
    #         if not count:
    #             count += 1
    #             continue
    #
    #         cv2.destroyAllWindows()
    #         break
    #                   # Право     #Середина  # лево
    # uvs1 = np.array([[249, 175], [187, 177], [106, 166],
    #                  [67, 296], [163, 409], [257, 289],
    #                  [267, 408], [190, 405]])
    #
    # uvs2 = np.array([[506, 50], [408, 52], [321, 53],
    #                  [286, 196], [355, 320], [503, 189],
    #                  [494, 329], [398, 321]])

    # frame1 = cv2.imread('./img/1.jpg')
    # frame2 = cv2.imread('./img/2.jpg')
    #
    # plt.imshow(frame1[:, :, [2, 1, 0]])
    # plt.scatter(uvs1[:, 0], uvs1[:, 1])
    # plt.show()
    #
    # plt.imshow(frame2[:, :, [2, 1, 0]])
    # plt.scatter(uvs2[:, 0], uvs2[:, 1])
    # plt.show()
    # #
    # RT1 = np.concatenate([np.eye(3), [[0], [0], [0]]], axis=-1)
    # P1 = mtx1 @ RT1
    #
    # RT2 = np.concatenate([R, T], axis=-1)
    # P2 = mtx2 @ RT2
    #
    # from mpl_toolkits.mplot3d import Axes3D
    #
    # fig = plt.figure()
    # ax = fig.add_subplot(111, projection='3d')
    # ax.set_xlim3d(0, -40)
    # ax.set_ylim3d(-20, 20)
    # ax.set_zlim3d(50, 100)
    #
    # p3ds = []
    # for uv1, uv2 in zip(uvs1, uvs2):
    #     _p3d = DLT(P1, P2, uv1, uv2)
    #     p3ds.append(_p3d)
    # p3ds = np.array(p3ds)
    #
    # connections = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [1, 7]]
    # for _c in connections:
    #     # print(p3ds[_c[0]])
    #     # print(p3ds[_c[1]])
    #     ax.plot(xs=[p3ds[_c[0], 0], p3ds[_c[1], 0]], ys=[p3ds[_c[0], 1], p3ds[_c[1], 1]],
    #             zs=[p3ds[_c[0], 2], p3ds[_c[1], 2]], c='red')
    #
    # plt.show()

    # =============================================== НАХОЖДЕНИЕ АВТО =================================================

    import mediapipe as mp
    from mpl_toolkits.mplot3d import Axes3D

    def get_frame_keypoints(landmarks, frame):
        frame_keypoints = []
        print(landmarks)
        for face_landmarks in landmarks:
            for p in range(21):
                pxl_x = int(round(frame.shape[1] * face_landmarks.landmark[p].x))
                pxl_y = int(round(frame.shape[0] * face_landmarks.landmark[p].y))
                kpts = [pxl_x, pxl_y]
                frame_keypoints.append(kpts)

        return frame_keypoints

    mp_drawing = mp.solutions.drawing_utils
    # mp_face = mp.solutions.face_mesh
    mp_face = mp.solutions.hands

    # face1 = mp_face.FaceMesh(max_num_faces=1,
    #                          refine_landmarks=True,
    #                          min_detection_confidence=0.5,
    #                          min_tracking_confidence=0.5)
    # face2 = mp_face.FaceMesh(max_num_faces=1,
    #                          refine_landmarks=True,
    #                          min_detection_confidence=0.5,
    #                          min_tracking_confidence=0.5)

    face1 = mp_face.Hands(max_num_hands=1,
                          model_complexity=0,
                          min_detection_confidence=0.5,
                          min_tracking_confidence=0.5)
    face2 = mp_face.Hands(max_num_hands=1,
                          model_complexity=0,
                          min_detection_confidence=0.5,
                          min_tracking_confidence=0.5)

    camera1.initialize()
    camera2.initialize()

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    # connections = [[i, i+1] for i in range(467)]
    ax.view_init(-90, -90)

    mp_pose = mp.solutions.pose

    connections = mp_face.HAND_CONNECTIONS

    counter = 0
    global_kps1 = []
    global_kps2 = []

    while True:

        frame1 = camera1.read_frame()
        frame2 = camera2.read_frame()

        frame1_copy = cv2.cvtColor(frame1, cv2.COLOR_BGR2RGB)
        frame2_copy = cv2.cvtColor(frame2, cv2.COLOR_BGR2RGB)

        frame1_copy.flags.writeable = False
        frame2_copy.flags.writeable = False

        results1 = face1.process(frame1_copy)
        results2 = face2.process(frame2_copy)

        # if results1.multi_face_landmarks:
        if results1.multi_hand_landmarks:
            frame1_keypoints = get_frame_keypoints(results1.multi_hand_landmarks,
                                                   frame1)
        else:
            # frame1_keypoints = [[-1, -1]] * 468
            frame1_keypoints = [[-1, -1]] * 21

        if results2.multi_hand_landmarks:
            frame2_keypoints = get_frame_keypoints(results2.multi_hand_landmarks,
                                                   frame2)
        else:
            frame2_keypoints = [[-1, -1]] * 21

        global_kps1.append(frame1_keypoints)
        global_kps2.append(frame2_keypoints)

        # print("Frame kp 1:\n", frame1_keypoints)
        # print("Frame kp 2:\n", frame2_keypoints)

        for points1, points2 in zip(frame1_keypoints, frame2_keypoints):
            cv2.circle(frame1, points1, 1, (255, 0, 0), cv2.FILLED)
            cv2.circle(frame2, points2, 1, (255, 0, 0), cv2.FILLED)

        frames = Camera().stack_images(0.8, [[frame1, frame2]])

        cv2.imshow('Check', frames)

        if cv2.waitKey(1) & 0xFF == ord('q'):
            cv2.destroyAllWindows()
            camera1.release()
            camera2.release()
            save_data('data/glob1_kps.pickle', global_kps1)
            save_data('data/glob2_kps.pickle', global_kps2)
            break

        # @ - матричное умножение
        RT1 = np.concatenate([np.eye(3), [[0], [0], [0]]], axis=-1)
        P1 = mtx1 @ RT1

        RT2 = np.concatenate([R, T], axis=-1)
        P2 = mtx2 @ RT2

        p3ds = []
        for uv1, uv2 in zip(frame1_keypoints, frame2_keypoints):
            _p3d = DLT(P1, P2, uv1, uv2)
            p3ds.append(_p3d)
        p3ds = np.array(p3ds)

        for _c in connections:
            ax.plot(xs=[p3ds[_c[0], 0], p3ds[_c[1], 0]],
                    ys=[p3ds[_c[0], 1], p3ds[_c[1], 1]],
                    zs=[p3ds[_c[0], 2], p3ds[_c[1], 2]],
                    c='red')
            ax.scatter(xs=[p3ds[:, 0], p3ds[:, 0]],
                       ys=[p3ds[:, 1], p3ds[:, 1]],
                       zs=[p3ds[:, 2], p3ds[:, 2]],
                       c='green')

        # ax.set_axis_off()
        # ax.set_xticks([])
        # ax.set_yticks([])
        # ax.set_zticks([])

        plt.draw()
        plt.pause(.001)
        ax.clear()

    # save_data('./data/glob1_kps.pickle', global_kps1)
    # save_data('./data/glob2_kps.pickle', global_kps2)

    # # ax.set_xlim3d(-14, -24)
    # # ax.set_ylim3d(-5, 5)
    # # ax.set_zlim3d(-500, 500)
    #

    #
    # connections = [[0, 1], [1, 2], [2, 3], [3, 4],
    #                [0,5], [5,6], [6,7], [7,8],
    #                [5,9], [9,10], [10,11], [11,12],
    #                [9,13], [13,14], [14,15], [15,16],
    #                [13,17], [17,18], [18,19], [19,20], [17, 0]]
    #
    # for _c in connections:
    #     # print(p3ds[_c[0]])
    #     # print(p3ds[_c[1]])
    #     ax.plot(xs=[p3ds[_c[0], 0], p3ds[_c[1], 0]], ys=[p3ds[_c[0], 1], p3ds[_c[1], 1]],
    #             zs=[p3ds[_c[0], 2], p3ds[_c[1], 2]], c='red')
    #     ax.scatter(xs=[p3ds[_c[0], 0], p3ds[_c[1], 0]], ys=[p3ds[_c[0], 1], p3ds[_c[1], 1]],
    #             zs=[p3ds[_c[0], 2], p3ds[_c[1], 2]], c='green')
    #
    # def animate(i):
    #     print(i/360 * 100, "%")
    #     line = ax.view_init(210, i)
    #     return line
    #
    # import matplotlib.animation as animation
    #
    # #  Создаем объект анимации:
    # sin_animation = animation.FuncAnimation(fig,
    #                                         animate,
    #                                         frames=np.linspace(0, 360, 360),
    #                                         interval = 10,
    #                                         repeat = False)
    #
    # #  Сохраняем анимацию в виде gif файла:
    # sin_animation.save('моя анимация.gif',
    #                    writer='imagemagick',
    #                    fps=30)

    # for angle in range(0, 360):
    #     ax.view_init(210, angle)
    #     plt.draw()
    #     plt.pause(.001)

# %%


import torch
import torchvision
import random
import torch.nn as nn
import torch
from torch import tanh
import torch.nn.functional as F

# custom weights initialization
def weights_init_1st(m):
    classname = m.__class__.__name__
    if classname.find('Linear') != -1:
        m.weight.data.normal_(0.0, 0.15)
        #m.weight.data.uniform_(-0.15, 0.15)
        #m.weight.data.fill_(0.5)

def weights_init_2nd(m):
    classname = m.__class__.__name__
    if classname.find('Linear') != -1:
        m.weight.data.normal_(-0.3, 0.3)
        #m.weight.data.uniform_(0.01, 0.02)
        #m.weight.data.fill_(0.5)

def print_net(model):
    for name, param in model.named_parameters():
        if param.requires_grad:
            print(name, param.data.numpy())

def get_pi_net():
    net = pi_net()
    net.apply(weights_init_1st)
    return net

class pi_net(nn.Module):
    def __init__(self):
        super(pi_net, self).__init__()
        bias_on = True
        self.linear1 = nn.Linear(36, 64, bias=bias_on)
        self.linear2 = nn.Linear(64, 64, bias=bias_on)
        self.linear3 = nn.Linear(64, 36, bias=bias_on)
        #torch.nn.init.xavier_uniform_(self.linear1)
        #torch.nn.init.xavier_uniform_(self.linear2)

    def forward(self, x):

        # --- 0000 ---- 0000 >>>  z-score normalization
        x = self.linear1(x)
        # print("AFTER linear1 = = = = = = = = = =")
        # print(x)
        # print("AFTER linear1 = = = = = = = = = =")

        x_avg = torch.sum(x)  / 20
        # print("AVG " + str(x_avg) )
        # print("x - x_avg ~~~~~~~~~~~~~~")
        x_minus_x_avg = x - x_avg
        # print(x_minus_x_avg)
        # print("x - x_avg ~~~~~~~~~~~~~~")
        x_std = torch.sum(torch.pow(x_minus_x_avg, 2)) / 20
        # print("VAR " + str(x_std))
        epsilon = 0.0000001
        # print("STD " + str(torch.sqrt(x_std)))
        x_norm = (x_minus_x_avg) / (torch.sqrt(x_std) + epsilon)
        # print("BEFORE sigmoid = = = = = = = = = =")
        # print(x_norm)
        # print("BEFORE sigmoid = = = = = = = = = =")
        #x = F.sigmoid(x_norm)
        x = tanh(x_norm)

        x = self.linear2(x)

        x_avg = torch.sum(x) / 40
        x_minus_x_avg = x - x_avg
        x_std = torch.sum(torch.pow(x_minus_x_avg, 2)) / 40
        x_norm = (x_minus_x_avg) / (torch.sqrt(x_std) + epsilon)
        x = tanh(x_norm)

        # print("AFTER sigmoid = = = = = = = = = =")
        # print(x)
        # print("AFTER sigmoid = = = = = = = = = =")
        x = self.linear3(x)
        return x.view(-1, 36)

        # --- 0000 ---- 0000 >>>  feature scaling
        # x = self.linear1(x)
        # print("AFTER linear1 = = = = = = = = = =")
        # print(x)
        # print("AFTER linear1 = = = = = = = = = =")

        # x_max = torch.max(x)
        # x_min = torch.min(x)
        # epsilon = 0.00001
        # x_norm = ((x - x_min) / (x_max - x_min + epsilon))

        # print("BEFORE sigmoid = = = = = = = = = =")
        # print(x_norm)
        # print("BEFORE sigmoid = = = = = = = = = =")
        # x = F.sigmoid(x_norm)
        # print("AFTER sigmoid = = = = = = = = = =")
        # print(x)
        # print("AFTER sigmoid = = = = = = = = = =")
        # x = self.linear2(x)
        # return x.view(-1, 4)


from sys import exit

def get_hurt(message, severity):
    """
    A simple function that determines damage of mc based on severity level.
    Parameters:
    (String) message: message explaining how the mc got hurt.
    (int) severity: the severity level of how bad the damage inflicted is.
    """
    if severity == 1:
        print(message)
        print("You take 5 damage!")
        health -= int(5)
    elif severity == 2:
        print(message)
        print("You take 25 damage!")
        health -= int(25)
    else:
       print(message)
       print("You take 100 damage!")
       health -= int(100)

def dead(message):
    """
    Function that tells you the MC is dead.
    Parameters:
        (String) message: Message stating the character is dead.
    """
    print(message)
    exit()

def dungeon():
    print("It is dark there's no light")
    print("What do you do? ")
    print(" Go left or right?")
    print(" Or go straight?")
    mc_moved = False

    while True:
        choice = input("> ")
        if choice == "go left":
            get_hurt('Oscar was right if this message prints', 1)
            print('You slipped on some moldy surface while drake plays in' +
                 'the background. You get up with a bruised arm.')
            mc_moved = True
        elif choice =='go right':
            print("You found a small insufficient light source.")
            print("You can move forward.")
            mc_moved = True
        elif choice == "go straight":
            if not light:
                print("You found a police flashlight. Lights up the room decently.")
                #light_found()
            else:
                print("You stumble on some dark object. It falls through a " +
                "crevice in the floor.")

""" Main Program """

health = int(100)
light = False
health_kit = False
bandages = int(0)
name = "Sir Donkulus"
mc = ""
name_1 = mc
print(f"Hello my name is {name}.")
print(f"What is yours {mc} fellow adventurer?")
mc = input(name_1)
print(f"Well {mc} you have entered a dungeon")
dungeon() # Added by Oscar

def light_found():
      print("Where do I go next?")
      print("Do I enter the mysterious room?")
      print("Continue onward?")
      choice = input("> ")

      if choice == "Enter the mysterious room":
          dead("You encounter a dragon! It burned you're face off.")
      mc_moved = False
      if choice == "Continue onward":
          print("Freedom! You did it")
          mc_moved = True


import numpy as np
from astropy import table
from glob import glob
import pandas as pd
from scipy.stats import binned_statistic

def get_outlier_fraction(tbl, suffix='', bins=20):
    diff = np.array(np.abs(tbl['z_est'] - tbl['z']) > 0.15 * (1 + tbl['z']),
                    dtype=float)
    stat = binned_statistic(tbl['z%s' % suffix], diff, statistic='mean',
                            bins=bins)
    return stat.statistic

def get_diagnostics(z1, z2):
    diff = np.array(z1 - z2) / (1 + np.array((z1)))
    outlier_mask = np.abs(diff) < 0.15  # * (1 + z1)
    med = np.median(diff)
    mad = np.median(np.abs(diff - med))
    return 100*np.array((np.mean(diff[outlier_mask]),
                         np.std(diff[outlier_mask]),
                         med, mad, 1-outlier_mask.mean()))

def run_for_table_old(name, min=None):
    t = table.Table.from_pandas(pd.read_csv(name))
    tmax = t['mags'].max()
    t = t[t['z_est'] > 0]
    if min is None:
        max_mag = 2
        while max_mag <= max(max_mag, tmax):
            t_ = t[t['mags'] <= max_mag]
            if len(t_) > 0.9 * len(t):
                break
            max_mag += 1
        diag_old = get_diagnostics(t_['z'], t_['z_est'])
        max_outlier_rate = diag_old[-1]
        used_fraction = len(t_)*100 / len(t)
        i = 2
        for i in range(max_mag, tmax + 1):
            t_ = t[t['mags'] <= i]
            x = t_['z']
            y = t_['z_est']
            if len(t_) == 0:
                break
            diag = get_diagnostics(x, y)
            print(name, i, '%.3f' % diag[-1], len(t_), i,
                  '%.3f' % max_outlier_rate)
            if diag[-1] > max_outlier_rate:
                break
            diag_old = diag
            used_fraction = len(t_)*100 / len(t)
    else:
        i = min + 1
        t_ = t[t['mags'] <= int(min)]
        diag_old = get_diagnostics(t_['z'], t_['z_est'])
        used_fraction = len(t_)*100 / len(t)
    return len(t_['z']), diag_old, i - 1, used_fraction

def run_for_table(name):
    if name.endswith('csv'):
        df = pd.read_csv(name)
    elif name.endswith('parquet'):
        df = pd.read_parquet(name, columns=['mags', 'z', 'z_est'])
    else:
        return [0, [0]*5]
    x = df['z']
    y = df['z_est']
    diag = get_diagnostics(x, y)
    return len(df['z']), diag

def name_to_caption(name):
    output = name.split('/')[-1].replace('.csv', '').replace('.parquet', '')
    if '-' in output:
        output_parts = output.split('-')[1:-1]
        output = ' '.join([s.replace('_', ' ').replace('+', '')
                           for s in output_parts])
    output = output.replace('  ', ' ').replace(' ', ', ')
    return output

def get_stats_for_file(name, **kwargs):
    output = table.Table(names=['Name', 'Mean', 'Std', 'Median',
                                'MAD', 'Outliers',
                                'Count'],
                         dtype=[str, float, float, float, float, float,
                                int])
    row = run_for_table(name, **kwargs)
    output.add_row([name_to_caption(name), *row[1], row[0]])
    return output

def get_stats_for_folder(folder, **kwargs):
    output = table.Table(names=['Name', 'Mean', 'Std', 'Median',
                                'MAD', 'Outliers',
                                'Count'],
                         dtype=[str, float, float, float, float, float,
                                int])

    names = glob('%s/*.csv' % folder) + glob('%s/*.parquet' % folder)
    names.sort()
    for f in names:
        row = run_for_table(f, **kwargs)
        output.add_row([name_to_caption(f), *row[1], row[0]])
    return output


import datetime as dt

from rest_framework import status
from rest_framework.exceptions import NotAuthenticated, PermissionDenied
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.views import APIView

from bumblebee.core.exceptions import (
    MissingFieldsError,
    NoneExistenceError,
    UrlParameterError,
)
from bumblebee.core.helpers import create_400, create_500
from bumblebee.feeds.api.serializers.feed_serializers import (
    FeedBuzzSerializer,
    FeedRebuzzSerializer,
)
from bumblebee.feeds.api.serializers.user_serializers import FeedUserSerializer
from bumblebee.feeds.utils import (
    get_follow_suggestions_for_user,
    get_folowing_buzzes_for_user,
)
from bumblebee.users.utils import DbExistenceChecker

class FeedBuzzListView(APIView):
    """ """

    permission_classes = [IsAuthenticated]

    def get_posts(self, *args, **kwargs):
        """ """

        return get_folowing_buzzes_for_user(self.request.user)

    def get(self, request, *args, **kwargs):
        """ """

        try:

            post_instances = self.get_posts()
            user_serializer = FeedUserSerializer(self.request.user, many=False)
            buzz_serializer = FeedBuzzSerializer(
                post_instances.get("buzzes"), many=True
            )
            rebuzz_serializer = FeedRebuzzSerializer(
                post_instances.get("rebuzzes"), many=True
            )

            return Response(
                data=dict(
                    updated_time=dt.datetime.now(),
                    user=user_serializer.data,
                    post=buzz_serializer.data + rebuzz_serializer.data,
                ),
                status=status.HTTP_200_OK,
            )

        except (MissingFieldsError, UrlParameterError, NoneExistenceError) as error:
            return Response(error.message, status=error.message.get("status"))

        except (PermissionDenied, NotAuthenticated) as error:
            return Response(
                create_400(
                    error.status_code,
                    error.get_codes(),
                    error.get_full_details().get("message"),
                ),
                status=error.status_code,
            )

        except Exception as error:
            return Response(
                create_500(
                    cause=error.args[0] or None,
                    verbose=f"Could not get feed due to an unknown error",
                ),
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )

class FeedFollowSuggestionsListView(APIView):
    """ """

    permission_classes = [IsAuthenticated]

    def get_suggestions(self, *args, **kwargs):
        """ """

        return get_follow_suggestions_for_user(self.request.user)

    def get(self, request, *args, **kwargs):
        """ """

        try:
            suggestion_instances = self.get_suggestions()
            user_serializer = FeedUserSerializer(self.request.user, many=False)
            suggestion_serializer = FeedUserSerializer(suggestion_instances, many=True)

            return Response(
                data=dict(
                    updated_time=dt.datetime.now(),
                    user=user_serializer.data,
                    suggestions=suggestion_serializer.data,
                ),
                status=status.HTTP_200_OK,
            )

        except (MissingFieldsError, UrlParameterError, NoneExistenceError) as error:
            return Response(error.message, status=error.message.get("status"))

        except (PermissionDenied, NotAuthenticated) as error:
            return Response(
                create_400(
                    error.status_code,
                    error.get_codes(),
                    error.get_full_details().get("message"),
                ),
                status=error.status_code,
            )

        except Exception as error:
            return Response(
                create_500(
                    cause=error.args[0] or None,
                    verbose=f"Could not get suggestions due to an unknown error",
                ),
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )


# If this is the name of a readable file, the Python commands in that file are
# executed before the first prompt is displayed in interactive mode.
# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONSTARTUP
#
# Sample code which supports concurrent interactive sessions, by only
# appending the new history is taken from
# https://docs.python.org/3/library/readline.html?highlight=readline#example
#
# The goal is to store interactive Python shell history in
# $XDG_STATE_HOME/python/python_history instead of ~/.python_history.

import atexit
import os
import readline

histfile = os.path.join(
    os.getenv("XDG_STATE_HOME", "~/.local/state"), "python", "python_history"
)

if not os.path.exists(histfile):
    os.makedirs(os.path.dirname(histfile), exist_ok=True)

if not os.path.isfile(histfile):
    if os.path.exists(histfile):
        os.remove(histfile)
    # Create an empty file
    open(histfile, "a").close()

readline.read_history_file(histfile)
h_len = readline.get_current_history_length()

def save(prev_h_len, histfile):
    new_h_len = readline.get_current_history_length()
    # Make the history file much bigger for relative suggestions
    readline.set_history_length(int(os.getenv("HISTSIZE", 1000000)))
    readline.append_history_file(new_h_len - prev_h_len, histfile)

atexit.register(save, h_len, histfile)

# Map TAB to auto-completion instead of the TAB symbol
readline.parse_and_bind("tab: complete")


"""Contains the MetaCurriculum class."""

import os
from unitytrainers.curriculum import Curriculum
from unitytrainers.exception import MetaCurriculumError

import logging

logger = logging.getLogger('unitytrainers')

class MetaCurriculum(object):
    """A MetaCurriculum holds curriculums. Each curriculum is associated to a particular
    brain in the environment.
    """

    def __init__(self, curriculum_folder, default_reset_parameters):
        """Initializes a MetaCurriculum object.

        Args:
            curriculum_folder (str): The relative or absolute path of the
                folder which holds the curriculums for this environment.
                The folder should contain JSON files whose names are the
                brains that the curriculums belong to.
            default_reset_parameters (dict): The default reset parameters
                of the environment.
        """
        used_reset_parameters = set()
        self._brains_to_curriculums = {}

        try:
            for curriculum_filename in os.listdir(curriculum_folder):
                brain_name = curriculum_filename.split('.')[0]
                curriculum_filepath = \
                    os.path.join(curriculum_folder, curriculum_filename)
                curriculum = Curriculum(curriculum_filepath, default_reset_parameters)

                # Check if any two curriculums use the same reset params.
                if any([(parameter in curriculum.get_config().keys()) for parameter in used_reset_parameters]):
                    logger.warning('Two or more curriculums will '
                                'attempt to change the same reset '
                                'parameter. The result will be '
                                'non-deterministic.')

                used_reset_parameters.update(curriculum.get_config().keys())
                self._brains_to_curriculums[brain_name] = curriculum
        except NotADirectoryError:
            raise MetaCurriculumError(curriculum_folder + ' is not a '
                                      'directory. Refer to the ML-Agents '
                                      'curriculum learning docs.')

    @property
    def brains_to_curriculums(self):
        """A dict from brain_name to the brain's curriculum."""
        return self._brains_to_curriculums

    @property
    def lesson_nums(self):
        """A dict from brain name to the brain's curriculum's lesson number."""
        lesson_nums = {}
        for brain_name, curriculum in self.brains_to_curriculums.items():
            lesson_nums[brain_name] = curriculum.lesson_num

        return lesson_nums

    @lesson_nums.setter
    def lesson_nums(self, lesson_nums):
        for brain_name, lesson in lesson_nums.items():
            self.brains_to_curriculums[brain_name].lesson_num = lesson

    def increment_lessons(self, progresses):
        """Increments all the lessons of all the curriculums in this MetaCurriculum.

        Args:
            progresses (dict): A dict of brain name to progress.
        """
        for brain_name, progress in progresses.items():
            self.brains_to_curriculums[brain_name].increment_lesson(progress)

    def set_all_curriculums_to_lesson_num(self, lesson_num):
        """Sets all the curriculums in this meta curriculum to a specified lesson number.

        Args:
            lesson_num (int): The lesson number which all the curriculums will
                be set to.
        """
        for _, curriculum in self.brains_to_curriculums.items():
            curriculum.lesson_num = lesson_num

    def get_config(self):
        """Get the combined configuration of all curriculums in this MetaCurriculum.

        Returns:
            A dict from parameter to value.
        """
        config = {}

        for _, curriculum in self.brains_to_curriculums.items():
            curr_config = curriculum.get_config()
            config.update(curr_config)

        return config


from pulp_2to3_migration.app.plugin.api import (
    is_different_relative_url,
    Pulp2to3Importer,
    Pulp2to3Distributor,
)

from pulp_rpm.app.models import RpmRemote, RpmPublication, RpmDistribution
from pulp_rpm.app.tasks.publishing import publish

from urllib.parse import urlparse, urlunparse

class RpmImporter(Pulp2to3Importer):
    """
    Interface to migrate Pulp 2 RPM importer
    """

    pulp3_remote_models = [RpmRemote]

    @classmethod
    def migrate_to_pulp3(cls, pulp2importer):
        """
        Migrate importer to Pulp 3.

        Args:
            pulp2importer(Pulp2Importer): Pre-migrated pulp2 importer to migrate

        Return:
            remote(RpmRemote): RpmRemote in Pulp3
            created(bool): True if Remote has just been created; False if Remote is an existing one
        """
        pulp2_config = pulp2importer.pulp2_config
        base_config, name = cls.parse_base_config(pulp2importer, pulp2_config)
        sles_auth_token = pulp2_config.get("query_auth_token")
        if sles_auth_token:
            base_config["sles_auth_token"] = sles_auth_token
        else:
            url = urlparse(pulp2_config.get("feed", ""))
            if url.query and "=" not in url.query and "&" not in url.query:
                base_config["sles_auth_token"] = url.query
                base_config["url"] = urlunparse(url._replace(query=""))
        return RpmRemote.objects.update_or_create(name=name, defaults=base_config)

class RpmDistributor(Pulp2to3Distributor):
    """
    Interface to migrate Pulp 2 RPM distributor
    """

    pulp3_publication_models = [RpmPublication]
    pulp3_distribution_models = [RpmDistribution]

    @classmethod
    def migrate_to_pulp3(cls, pulp2distributor, repo_version, signing_service):
        """
        Migrate distributor to Pulp 3.

        Args:
            pulp2distributor(Pulp2distributor): Pre-migrated pulp2 distributor to migrate

        Return:
            publication(RpmPublication): publication in Pulp 3
            distribution(RpmDistribution): distribution in Pulp 3
            created(bool): True if a distribution has just been created; False if a distribution
                           is an existing one

        """
        pulp2_config = pulp2distributor.pulp2_config

        # this will go away with the simple-complex plan conversion work
        if not repo_version:
            repo = pulp2distributor.pulp2_repos.filter(not_in_plan=False, is_migrated=True)
            repo_version = repo[0].pulp3_repository_version
        publication = repo_version.publication_set.filter(complete=True).first()
        if not publication:
            pulp2_checksum_type = pulp2_config.get("checksum_type")
            checksum_types = None
            if pulp2_checksum_type:
                checksum_types = {
                    "metadata": pulp2_checksum_type,
                    "package": pulp2_checksum_type,
                }
            else:
                # Set the checksum type based on content in a repo, pulp 2 supports only one
                # checksum type for packages in a repo. It is important to set checksum type for
                # Pulp 3 to Pulp 2 sync use case.
                package_qs = repo_version.content.filter(pulp_type="rpm.package")
                if package_qs.count():
                    pkg_checksum_type = package_qs.first().cast().checksum_type
                    checksum_types = {
                        "metadata": pkg_checksum_type,
                        "package": pkg_checksum_type,
                    }
            sqlite = pulp2_config.get("generate_sqlite", False)
            try:
                publish(
                    repo_version.pk,
                    checksum_types=checksum_types,
                    sqlite_metadata=sqlite,
                )
            except TypeError:
                # hack, pulp_rpm <3.9 doesn't support sqlite_metadata kwarg
                publish(repo_version.pk, checksum_types=checksum_types)
            publication = repo_version.publication_set.filter(complete=True).first()

        # create distribution
        distribution_data = cls.parse_base_config(pulp2distributor, pulp2_config)

        # ensure that the base_path does not end with / in Pulp 3, it's often present in Pulp 2.
        base_path = pulp2_config.get("relative_url", pulp2distributor.pulp2_repo_id)
        distribution_data["base_path"] = base_path.rstrip("/")
        distribution_data["publication"] = publication
        distribution, created = RpmDistribution.objects.update_or_create(
            name=distribution_data["name"],
            base_path=distribution_data["base_path"],
            defaults=distribution_data,
        )
        return publication, distribution, created

    @classmethod
    def needs_new_publication(cls, pulp2distributor):
        """
        Check if a publication associated with the pre_migrated distributor needs to be recreated.

        Args:
            pulp2distributor(Pulp2Distributor): Pre-migrated pulp2 distributor to check

        Return:
            bool: True, if a publication needs to be recreated; False if no changes are needed

        """
        if not pulp2distributor.pulp3_publication:
            return True

        new_checksum_type = pulp2distributor.pulp2_config.get("checksum_type")
        current_checksum_type = pulp2distributor.pulp3_publication.cast().metadata_checksum_type

        is_default_checksum_type = new_checksum_type is None and current_checksum_type == "sha256"
        if new_checksum_type != current_checksum_type and not is_default_checksum_type:
            return True

        return False

    @classmethod
    def needs_new_distribution(cls, pulp2distributor):
        """
        Check if a distribution associated with the pre_migrated distributor needs to be recreated.

        Args:
            pulp2distributor(Pulp2Distributor): Pre-migrated pulp2 distributor to check

        Return:
            bool: True, if a distribution needs to be recreated; False if no changes are needed

        """
        return is_different_relative_url(pulp2distributor)


import csv

positives = 0
negatives = 0
i = 1
tn = 0
tp = 0
fn = 0
fp = 0
flag = 0
flag2 = 0
totalRawCount = 0
flag5 = 0
linelist = []

#################################################################################################################
#################################################################################################################
#################################################################################################################

folder_path = "C:\\Users\\Ayan Deep Hazra\\Desktop\\Repos\\Natural_Language_Processing_Research\\open this for text file " \
              "manipulation\\filtered_test_scibert.csv"
csvfile = open(folder_path, encoding="utf-8")
csvreader = csv.reader(csvfile)
rows = []
truelabel = []

for row in csvreader:
    rows.append(row[0])

for row in csvreader:
    truelabel.append(row[1])

#################################################################################################################
#################################################################################################################
#################################################################################################################

with open(folder_path, "r", encoding="utf-8") as filetemp:
    for line in filetemp:
        words = line.split()
        if len(words) > 0:
            linelist.append(line)

with open("file.txt", "r", encoding="utf-8") as file:
    for line in file:

        print(line)
        words = line.split()
        if len(words) > 0:
            # totalRawCount = totalRawCount + 1

            if words[0] == "input:":
                totalRawCount = totalRawCount + 1

                print(totalRawCount, words)
                if flag5 == 0:
                    a = totalRawCount
                flag5 = 1

                if float(words[len(words) - 1]) > 0.1:
                    positives = positives + 1
                    str = " ".join(words[1:len(words) - 2])

                    flag = 0
                    if str.find("bulk moduli") != -1 or str.find("bulk modulus") != -1 or str.find(
                            "K 0") != -1 or str.find("K0") != -1 or str.find("B=") != -1 or str.find("B =") != -1:
                        flag = 1
                        # tp = tp + 1

                    if flag == 1 and (
                            str.find("GPa") != -1 or str.find("gpa") != -1 or str.find("Gpa") != -1 or str.find("range of") != -1 or str.find(
                        "ranges of") != -1):
                        tp = tp + 1

                    # printing the positive sentences
                    # print(i, " ".join(words[1:len(words) - 3]))
                    i = i + 1

                else:
                    negatives = negatives + 1
                    str = " ".join(words[1:len(words) - 2])
                    flag2 = 0
                    if str.find("bulk moduli") != -1 or str.find("bulk modulus") != -1 or str.find("K 0") != -1 \
                            or str.find("K0") != -1 or str.find("B=") != -1 or str.find("B =") != -1:
                        flag2 = 1
                        # fn = fn + 1

                    if flag2 == 1 and (
                            str.find("GPa") != -1 or str.find("gpa") != -1 or str.find("Gpa") != -1 or str.find("range of") != -1 or str.find(
                            "ranges of") != -1):
                        fn = fn + 1

                string = " ".join(words[1:len(words) - 3])

print(positives)
print(negatives)
fp = positives - tp
tn = negatives - fn
print("tn:", tn)
print("tp:", tp)
print("fp:", fp)
print("fn:", fn)

recall = tp / (tp + fn)
precision = tp / (tp + fp)

print("recall: ", recall)
print("precision: ", precision)
print("F score:", 2 * precision * recall / (precision + recall))
print(totalRawCount)

open this for text file manipulation/manualClassifier.py


# -*- coding: utf-8 -*-
"""
Sihoo Celery Worker 模块

@author: AZLisme
@email:  helloazl@icloud.com
"""

from celery import Celery

celery_app = Celery('SihooWorker')

def configure(app):
    celery_app.config_from_object('sihoo.settings.celery-setting')
    celery_app.config_from_envvar('SIHOO_CELERY_SETTINGS', silent=True)
    app.config['CELERY'] = celery_app


# (C) StackState 2020
# All rights reserved
# Licensed under a 3-clause BSD style license (see LICENSE)
import pytest

@pytest.fixture(scope='session')
def sts_environment():
    return {
        'type': 'csv',
        'health_file': '/home/static_health/health.csv',
        'delimiter': ',',
        'collection_interval': 15
    }

@pytest.fixture(scope="class")
def instance(request):
    cfg = {
        'type': 'csv',
        'health_file': 'health.csv',
        'delimiter': ',',
        'collection_interval': 15
    }
    request.cls.instance = cfg


import numpy as np
import tensorflow as tf

from eventgen import CEvent
from nets.net import Net
from nets.utils import get_trainable_vars, prep_data_cells

class PPOSinghNet(Net):
    def __init__(self, pre_conv=False, double_net=False, *args, **kwargs):
        """
        Afterstate value net
        """
        self.name = "SinghNet"
        self.pre_conv = pre_conv
        self.double_net = double_net
        super().__init__(name=self.name, *args, **kwargs)
        self.weight_beta = self.pp['weight_beta']
        self.weight_beta_decay = self.pp['weight_beta_decay']
        self.avg_reward = 0

    def _build_vnet(self, freps, name):
        with tf.variable_scope('model/' + name) as scope:
            value_layer = tf.layers.Dense(
                units=1,
                kernel_initializer=tf.zeros_initializer(),
                kernel_regularizer=self.dense_regularizer,
                use_bias=False,
                activation=None)
            value = value_layer.apply(tf.layers.flatten(freps))
            self.weight_vars.append(value_layer.kernel)
            self.weight_names.append(value_layer.name)
            # NOTE TODO either gotta have 7x7x70 outputs, or input cell
            # also gotta think about a hidden layer before value/policy
            trainable_vars = get_trainable_vars(scope)
        return value, trainable_vars

    def _build_pnet(self, freps, name):
        with tf.variable_scope('model/' + name) as scope:
            # policy = tf.keras.layers.LocallyConnected2D(
            #     filters=70,
            #     kernel_size=1,
            #     padding="valid",
            #     kernel_initializer=tf.zeros_initializer(),
            #     use_bias=self.pp['conv_bias'],
            #     activation=None)(freps)
            # print(policy.shape)
            policy_layer = tf.layers.Dense(
                units=70,
                kernel_initializer=tf.zeros_initializer(),
                kernel_regularizer=self.dense_regularizer,
                use_bias=False,
                activation=None)
            policy = policy_layer.apply(tf.layers.flatten(freps))
            # self.weight_vars.append(policy_layer.kernel)
            # self.weight_names.append(policy_layer.name)
            trainable_vars = get_trainable_vars(scope)
        return policy, trainable_vars

    def build(self):
        # frepshape = [None, self.rows, self.cols, self.n_channels * 3 + 1]
        frepshape = [None, self.rows, self.cols, self.n_channels + 1]
        self.freps = tf.placeholder(tf.float32, frepshape, "feature_reps")
        self.value_target = tf.placeholder(tf.float32, [None], "value_target")
        oh_cellshape = [None, self.rows, self.cols, 1]
        self.cells = tf.placeholder(tf.bool, oh_cellshape, "oh_cell")
        cells = tf.cast(self.cells, tf.float32)
        # self.cells = tf.placeholder(tf.int32, [None, 2], "cell")
        self.action = tf.placeholder(tf.int32, [None], "action")
        self.policy_in = tf.placeholder(tf.float32, [70], "pol_in")
        self.old_neglogpac = tf.placeholder(tf.float32, [None], "old_neglogpac")
        inp = tf.concat([self.freps, cells], axis=3)
        self.value, online_vf_vars = self._build_vnet(self.freps, "online-vf")
        self.policy, online_pg_vars = self._build_pnet(inp, "online-pg")
        # nrange = tf.range(tf.shape(self.freps)[0], name="cellrange")
        # ncells = tf.concat([tf.expand_dims(nrange, axis=1), self.cells], axis=1)
        # self.policy = tf.gather_nd(self.conv_policy, ncells)

        self.err = self.value_target - self.value
        self.vf_loss = tf.losses.mean_squared_error(
            labels=tf.expand_dims(self.value_target, axis=1), predictions=self.value)

        CLIPRANGE = 0.2
        self.neglogpac_out = self.neglogp(self.policy_in, self.action)
        self.neglogpac = self.neglogp(self.policy, self.action)
        ratio = tf.exp(self.old_neglogpac - self.neglogpac)
        pg_losses = -self.value_target * ratio
        pg_losses2 = -self.value_target * tf.clip_by_value(ratio, 1.0 - CLIPRANGE,
                                                           1.0 + CLIPRANGE)
        pg_loss = tf.reduce_mean(tf.maximum(pg_losses, pg_losses2))
        # entropy = self.entropy(self.policy)
        # pgnet_loss = pg_loss + 0.01 * entropy
        # trainer = tf.train.AdamOptimizer(learning_rate=self.pp, epsilon=1e-5)
        trainer = tf.train.GradientDescentOptimizer(
            learning_rate=1e-6)  #self.pp['net_lr'])
        grads = trainer.compute_gradients(pg_loss, online_pg_vars)
        self.do_train_pg = trainer.apply_gradients(grads)
        return self.vf_loss, online_vf_vars

    @staticmethod
    def entropy(logits):
        a0 = logits - tf.reduce_max(logits, axis=-1, keep_dims=True)
        ea0 = tf.exp(a0)
        z0 = tf.reduce_sum(ea0, axis=-1, keep_dims=True)
        p0 = ea0 / z0
        return tf.reduce_sum(p0 * (tf.log(z0) - a0), axis=-1)

    @staticmethod
    def neglogp(logits, x):
        one_hot_actions = tf.one_hot(x, logits.get_shape().as_list()[-1])
        return tf.nn.softmax_cross_entropy_with_logits(
            logits=logits, labels=one_hot_actions)

    def forward_action(self, frep, cell, ce_type, chs):
        # u = tf.random_uniform(tf.shape(self.policy))
        # self.sample_action = tf.argmax(elig_policy - tf.log(-tf.log(u)), axis=-1)
        policy = self.sess.run(self.policy, {
            self.freps: [frep],
            self.cells: prep_data_cells(cell),
        })[0]

        # u = np.random.uniform(policy.shape)
        # policy_ent = policy - np.log(-np.log(u))
        # NOTE TODO should this be argmin for END?
        if ce_type == CEvent.END:
            idx = np.argmin(policy[chs])
        else:
            idx = np.argmax(policy[chs])
        ch = chs[idx]
        neglogpac = self.sess.run(self.neglogpac_out, {
            self.policy_in: policy,
            self.action: [ch]
        })
        return ch, neglogpac

    def get_neglogpac(self, frep, cell, ch):
        policy = self.sess.run(self.policy, {
            self.freps: [frep],
            self.cells: prep_data_cells(cell),
        })[0]
        neglogpac = self.sess.run(self.neglogpac_out, {
            self.policy_in: policy,
            self.action: [ch]
        })
        return neglogpac

    def forward_value(self, freps):
        value = self.sess.run(
            self.value,
            feed_dict={
                self.freps: freps
            },
        ).squeeze()
        return value

    def backward(self, step, buf, n_step):
        # TODO:
        # - collect nsteps of data. 16-128
        # - train noptepochs consecutive times on pg net. 4
        # next_values = self.sess.run(
        #     self.value, feed_dict={
        #         self.freps: [e.next_frep for e in buf]
        #     }).squeeze()
        value_target = step.reward - self.avg_reward + step.next_val
        loss, lr, err = self.backward_vf([step.frep], [value_target])
        if len(buf) != n_step:
            return loss, lr, err
        # np.random.shuffle(buf)
        next_values = np.array([e.next_val for e in buf])
        rewards = [e.reward for e in buf]
        value_targets = rewards + next_values - self.avg_reward
        freps = [e.frep for e in buf]
        cells = [e.cell for e in buf]
        neglogpacs = [e.neglogpac for e in buf]
        chs = [e.ch for e in buf]
        for _ in range(4):
            self.sess.run(
                [self.do_train_pg], {
                    self.freps: freps,
                    self.cells: prep_data_cells(cells),
                    self.value_target: value_targets,
                    self.old_neglogpac: neglogpacs,
                    self.action: chs
                })
        return loss, lr, err

    def backward_vf(self, freps, value_target):
        data = {self.freps: freps, self.value_target: value_target}
        _, loss, lr, err = self.sess.run([self.do_train, self.vf_loss, self.lr, self.err],
                                         data)
        self.avg_reward += self.weight_beta * np.mean(err)
        return loss, lr, err


import newspaper

# Declare the url
url = "https://ktechhub.com/tutorials/completely-deploy-your-laravel-application-on-ubuntu-linux-server-60a51098a8bf2"

#Extract web content
url = newspaper.Article(url="%s" % (url), language='en')

url.download()
url.parse()

# Display scraped data
print(url.text)


"""Define custom dataset class extending the Pytorch Dataset class"""

import os
from typing import Dict, List, Tuple

import numpy as np
import pandas as pd
from PIL import Image
import torch
from torch.utils.data import DataLoader, Dataset
import torchvision.transforms as tvt

from utils.utils import Params

class SketchesDataset(Dataset):
    """Custom class for Sketches dataset"""

    def __init__(self, root: str, csv_file: str, transform: tvt = None) -> None:
        """Get the filenames and labels of images from a csv file.
        Args:
            root: Directory containing the data
            csv_file: file containing the data
            transform: Transformation to apply on images
        """
        self.root = root
        self.data = pd.read_csv(os.path.join(root, csv_file))
        self.transform = transform

    def __len__(self) -> int:
        """Return the size of the dataset.
        """
        return len(self.data)

    def __getitem__(self, idx: int) -> Tuple[Image.Image, np.ndarray]:
        """Get an item from the dataset given the index idx"""
        row = self.data.iloc[idx]

        im_name = row["Image Id"] + ".png"
        im_path = os.path.join(self.root, "images", im_name)
        img = Image.open(im_path).convert("RGB")

        labels = torch.tensor(row[1:], dtype=torch.float32)

        if self.transform is not None:
            img = self.transform(img)

        return img, labels

def get_transform(mode: str, params: Params) -> tvt.Compose:
    """Data augmentation
    Args:
        is_train: If the dataset is training
    Returns:
        Composition of all the data transforms
    """
    trans = [
        tvt.Resize((params.height, params.width)),
        tvt.ToTensor(),
        tvt.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
    ]
    if mode == "train":
        trans += [
            tvt.RandomHorizontalFlip(params.flip),
            tvt.ColorJitter(
                brightness=params.brightness,
                contrast=params.contrast,
                saturation=params.saturation,
                hue=params.hue
            ),
            tvt.RandomRotation(params.degree)
        ]
    return tvt.Compose(trans)

def collate_fn(batch: List[Tuple[torch.tensor, torch.tensor]]) -> Tuple[torch.tensor, torch.tensor]:
    """Collate function to create a batch of data
    Args:
        batch: List of data generated by dataset
    Returns:
        Batch of images and labels
    """
    data = list(zip(*batch))
    imgs = torch.stack(data[0], 0)
    labels = torch.stack(data[1], 0)

    return imgs, labels

def get_dataloader(
    modes: List[str],
    params: Params,
) -> Dict[str, DataLoader]:
    """Get DataLoader objects.
    Args:
        modes: Mode of operation i.e. 'train', 'val', 'test'
        params: Hyperparameters
    Returns:
        DataLoader object for each mode
    """
    dataloaders = {}

    for mode in modes:
        if mode == "train":
            trans = get_transform(mode, params)
            shuf = True
        else:
            trans = get_transform(mode, params)
            shuf = False

        dataset = SketchesDataset(
            root=params.data_dir,
            csv_file=mode + "_sketches_" + params.type + ".csv",
            transform=trans
        )
        dataloaders[mode] = DataLoader(
            dataset,
            batch_size=params.batch_size,
            num_workers=params.num_workers,
            pin_memory=params.pin_memory,
            collate_fn=collate_fn,
            shuffle=shuf
        )

    return dataloaders


#!/usr/bin/env python3

""" basic Flask app """

from flask import Flask, render_template, request, g
from flask_babel import Babel
import pytz

app = Flask(__name__)
babel = Babel(app)
users = {
    1: {"name": "Balou", "locale": "fr", "timezone": "Europe/Paris"},
    2: {"name": "Beyonce", "locale": "en", "timezone": "US/Central"},
    3: {"name": "Spock", "locale": "kg", "timezone": "Vulcan"},
    4: {"name": "Teletubby", "locale": None, "timezone": "Europe/London"},
}

class Config(object):
    """ Babel configuration class """
    LANGUAGES = ["en", "fr"]
    BABEL_DEFAULT_LOCALE = 'en'
    BABEL_DEFAULT_TIMEZONE = 'UTC'

def get_user() -> dict:
    """ returns a user dictionary or None """
    user_logged = request.args.get('login_as')
    if user_logged and int(user_logged) in users:
        return users[int(user_logged)]
    return None

@app.before_request
def before_request():
    """ find a user if any """
    user = get_user()
    g.user = user

@babel.localeselector
def get_locale():
    """ determine the best match """
    locale = request.args.get('locale')
    if locale and locale in Config.LANGUAGES:
        return locale
    if g.user:
        locale = g.user.get('locale')
        if locale and locale in Config.LANGUAGES:
            return locale
    locale = request.headers.get('locale')
    if locale and locale in Config.LANGUAGES:
        return locale
    return request.accept_languages.best_match(Config.LANGUAGES)

@babel.timezoneselector
def get_timezone():
	""" get timezone selector """
	try:
		if request.args.get('timezone'):
			time_zone = request.args.get('timezone')
			pytz.timezone(time_zone)
		elif g.user and g.user.get('timezone'):
			time_zone = g.user.get('timezone')
			pytz.timezone(time_zone)
		else:
			time_zone = app.config["BABEL_DEFAULT_TIMEZONE"]
			pytz.timezone(time_zone)
	except pytz.exceptions.UnknownTimeZoneError:
		time_zone = "UTC"
	return time_zone

app.config.from_object(Config)

@app.route("/", methods=["GET"])
def index():
    """ returns the index """
    return render_template('7-index.html')

if __name__ == "__main__":
    app.run(host="0.0.0.0", port="5000")


from typing import List
import copy

class Solution:
    def calcEquation(self, equations: List[List[str]], values: List[float], queries: List[List[str]]) -> List[float]:
        graph = {}
        s = set()
        for i, each in enumerate(equations):
            nominator = each[0]
            denominator = each[1]
            s.add(nominator)
            s.add(denominator)
            if nominator not in graph:
                graph[nominator] = {denominator: values[i]}
            else:
                graph[nominator][denominator] = values[i]

            if denominator not in graph:
                graph[denominator] = {nominator: 1.0 / values[i]}
            else:
                graph[denominator][nominator] = 1.0 / values[i]
        print(graph)
        ans = []
        for query in queries:
            nominator = query[0]
            denominator = query[1]

            if nominator not in s or denominator not in s:
                ans.append(-1)
            elif nominator == denominator:
                ans.append(1.0)
            else:
                r = self.solve(graph, nominator,denominator, [], [])
                ans.append(r)
        return ans

    def solve(self, graph, nominator, denominator, path, ratio):
        # print(path)
        if nominator == denominator:
            res = 1
            for each in ratio:
                res *= each
            return res

        if len(path) == 0:
            path.append(nominator)

        for each in graph[nominator]:
            if each not in path:
                p = copy.deepcopy(path)
                r = copy.deepcopy(ratio)
                p.append(each)
                r.append(graph[nominator][each])
                ans = self.solve(graph, each, denominator, p, r)
                if ans != -1:
                    return ans
                # path.pop()
                # ratio.pop()
        return -1

equations = [["x1","x2"],["x2","x3"],["x3","x4"],["x4","x5"]]
values =  [3.0,4.0,5.0,6.0]
queries =[["x1","x5"],["x5","x2"],["x2","x4"],["x2","x2"],["x2","x9"],["x9","x9"]]

s = Solution()
r = s.calcEquation(equations, values, queries)
print(r)