# coding: utf-8

# In[1]:

import numpy as np
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Input,Dense,Dropout,Conv2D,MaxPool2D,Flatten,GlobalAvgPool2D,concatenate,BatchNormalization,Activation,Add,ZeroPadding2D,Lambda
from tensorflow.keras.optimizers import Adam
import matplotlib.pyplot as plt
from tensorflow.keras.callbacks import LearningRateScheduler
from tensorflow.keras.models import Model

# In[2]:

# 类别数
num_classes = 17
# 批次大小
batch_size = 32
# 周期数
epochs = 100
# 图片大小
image_size = 224

# In[3]:

# 训练集数据进行数据增强
train_datagen = ImageDataGenerator(
    rotation_range = 20,     # 随机旋转度数
    width_shift_range = 0.1, # 随机水平平移
    height_shift_range = 0.1,# 随机竖直平移
    rescale = 1/255,         # 数据归一化
    shear_range = 10,       # 随机错切变换
    zoom_range = 0.1,        # 随机放大
    horizontal_flip = True,  # 水平翻转
    brightness_range=(0.7, 1.3), # 亮度变化
    fill_mode = 'nearest',   # 填充方式
)
# 测试集数据只需要归一化就可以
test_datagen = ImageDataGenerator(
    rescale = 1/255,         # 数据归一化
)

# In[4]:

# 训练集数据生成器，可以在训练时自动产生数据进行训练
# 从'data/train'获得训练集数据
# 获得数据后会把图片resize为image_size×image_size的大小
# generator每次会产生batch_size个数据
train_generator = train_datagen.flow_from_directory(
    'data/train',
    target_size=(image_size,image_size),
    batch_size=batch_size,
    )

# 测试集数据生成器
test_generator = test_datagen.flow_from_directory(
    'data/test',
    target_size=(image_size,image_size),
    batch_size=batch_size,
    )

# In[5]:

# 字典的键为17个文件夹的名字，值为对应的分类编号
train_generator.class_indices

# In[6]:

# 定义分组卷积
# g_channels 每组的通道数
# groups 多少组
def grouped_convolution_block(init_x, strides, groups, g_channels):
    group_list = []
    # 分组进行卷积
    for c in range(groups):
        # 分组取出数据
        x = Lambda(lambda x: x[:, :, :, c*g_channels:(c+1)*g_channels])(init_x)
        # 分组进行卷积
        x = Conv2D(filters=g_channels,kernel_size=3,strides=strides,padding='same',use_bias=False)(x)
        # 存入list
        group_list.append(x)
    # 合并list中的数据
    group_merge = concatenate(group_list, axis=3)
    x = BatchNormalization(epsilon=1.001e-5)(group_merge)
    x = Activation('relu')(x)
    return x

# 定义残差单元
def block(x, filters, strides=1, groups=32, conv_shortcut=True):
    # projection shortcut
    if conv_shortcut == True:
        shortcut = Conv2D(filters*2,kernel_size=1,strides=strides,padding='same')(x)
        # epsilon为BN公式中防止分母为零的值
        shortcut = BatchNormalization(epsilon=1.001e-5)(shortcut)
    else:
        # identity_shortcut
        shortcut = x
    # 3个卷积层
    x = Conv2D(filters=filters,kernel_size=1,strides=1,padding='same')(x)
    x = BatchNormalization(epsilon=1.001e-5)(x)
    x = Activation('relu')(x)
    # 计算每组的通道数
    g_channels = int(filters / groups)
    # 进行分组卷积
    x = grouped_convolution_block(x, strides, groups, g_channels)

    x = Conv2D(filters=filters*2,kernel_size=1,strides=1,padding='same')(x)
    x = BatchNormalization(epsilon=1.001e-5)(x)
    x = Add()([x, shortcut])
    x = Activation('relu')(x)
    return x

# 堆叠残差单元
def stack(x, filters, blocks, strides, groups=32):
    x = block(x, filters, strides=strides, groups=groups)
    for i in range(blocks):
        x = block(x, filters, groups=groups, conv_shortcut=False)
    return x

# 定义ResNeXt50
inputs = Input(shape=(image_size,image_size,3))
# 填充3圈0，填充后图像从224×224变成230×230
x = ZeroPadding2D((3, 3))(inputs)
x= Conv2D(filters=64,kernel_size=7,strides=2,padding='valid')(x)
x = BatchNormalization(epsilon=1.001e-5)(x)
x = Activation('relu')(x)
# 填充1圈0
x = ZeroPadding2D((1, 1))(x)
x = MaxPool2D(pool_size=3,strides=2,padding='valid')(x)
# 堆叠残差结构
# blocks表示堆叠数量
x = stack(x, filters=128, blocks=2, strides=1)
x = stack(x, filters=256, blocks=3, strides=2)
x = stack(x, filters=512, blocks=5, strides=2)
x = stack(x, filters=1024, blocks=2, strides=2)
# 根据特征图大小进行平均池化，池化后得到2维数据
x = GlobalAvgPool2D()(x)
x = Dense(num_classes, activation='softmax')(x)
# 定义模型
model = Model(inputs=inputs,outputs=x)

# 电脑配置不好的话不要运行summary或者plot_model
# model.summary()

# In[7]:

# 学习率调节函数，逐渐减小学习率
def adjust_learning_rate(epoch):
    # 前40周期
    if epoch<=40:
        lr = 1e-4
    # 前40到80周期
    elif epoch>40 and epoch<=80:
        lr = 1e-5
    # 80到100周期
    else:
        lr = 1e-6
    return lr

# In[8]:

# 定义优化器
adam = Adam(lr=1e-4)

# 定义学习率衰减策略
callbacks = []
callbacks.append(LearningRateScheduler(adjust_learning_rate))

# 定义优化器，loss function，训练过程中计算准确率
model.compile(optimizer=adam,loss='categorical_crossentropy',metrics=['accuracy'])

# Tensorflow2.1版本之前可以使用fit_generator训练模型
# history = model.fit_generator(train_generator,steps_per_epoch=len(train_generator),epochs=epochs,validation_data=test_generator,validation_steps=len(test_generator))

# Tensorflow2.1版本(包括2.1)之后可以直接使用fit训练模型
history = model.fit(x=train_generator,epochs=epochs,validation_data=test_generator,callbacks=callbacks)

# In[9]:

# 画出训练集准确率曲线图
plt.plot(np.arange(epochs),history.history['accuracy'],c='b',label='train_accuracy')
# 画出验证集准确率曲线图
plt.plot(np.arange(epochs),history.history['val_accuracy'],c='y',label='val_accuracy')
# 图例
plt.legend()
# x坐标描述
plt.xlabel('epochs')
# y坐标描述
plt.ylabel('accuracy')
# 显示图像
plt.show()
# 模型保存
model.save('ResNeXt50.h5')

程序数据整理/第12章-图像识别项目实战/12-11 ResNeXt50(32×4d)图像识别.py


import re

from turbogears.validators import Invalid, FancyValidator, Set, Regex, \
        UnicodeString
from sqlalchemy.exceptions import InvalidRequestError

try:
    from fedora.textutils import to_unicode
except ImportError:
    from pkgdb.lib.utils import to_unicode

from pkgdb.model import Collection
from pkgdb.lib.utils import STATUS

#pylint:disable-msg=W0622
def _(string):
    ''' *HACK*:  TurboGears/FormEncode requires that we use a dummy _ function.

    Internationalizing error messages won't work otherwise.
    http://docs.turbogears.org/1.0/Internationalization#id13
    '''
    return string
#pylint:enable-msg=W0622

#
# SetOf validator can validate its elements
#
class SetOf(Set):
    '''formencode Set() validator with the ability to validate its elements.

    :kwarg element_validator: Validator to run on each of the elements of the set.
    '''
    element_validator = None
    messages = {'incorrect_value': 'list values did not satisfy the element_validator'}

    def validate_python(self, value, state):
        if self.element_validator:
            try:
                value = map(self.element_validator.to_python, value)
            except Invalid:
                raise
            except:
                # Just in case the element validator doesn't throw an Invalid
                # exception
                raise Invalid(self.message('incorrect_value', state),
                        value, state)

#
# Three sorts of validators:
#
# 1) does minimal checking that a string looks sort of right
#    - For these we'll mostly just use the standard tg and formencode
#      validators.
# 2) Hits the db to verify that the string exists in the proper field
#    - These are appropriate where we're going to use the string anyways.  For
#      instance, in a select statement.
#    - These should be checked by making calls against something that's easily
#      sent to a memcached or redis server.
# 3) Looks in the db and transforms the string into the type of thing that it
#    is a key for
#    - This will do an actual call into the database and load an ORM mapped
#      object.
#

class IsCollectionSimpleNameRegex(Regex):
    '''Test the collection simple name against a simple heuristic

    :kwarg strip: If True, strips whitespace from the beginnng and end of the
        value.  (default True)
    :kwarg regex: regular expression object or string to be compiled to match
        the simple name against. Default: r'^[A-Z]+-([0-9]+|devel)$'
    '''
    strip = True
    regex = re.compile(r'^((FC|fc|f|F|EL|el|)-?[0-9]+|devel)$')

    messages = {'no_collection': _('%(collection)s does not match the pattern'
        ' for collection names')}

    def _to_python(self, value, state):
        value = Regex._to_python(self, value, state)
        return to_unicode(value)

    def validate_python(self, value, state):
        if not self.regex.match(value):
            raise Invalid(self.message('no_collection', state,
                collection=value), value, state)

class IsCollectionSimpleName(UnicodeString):
    '''Test that the value is a recognized collection short name.

    :kwarg eol: If True, include eol releases. (default False)
    :kwarg strip: If True, strips whitespace from the beginnng and end of the
        value.  (default True)
    '''
    strip = True
    eol = False

    messages = {'no_collection': _('A collection named %(collection)s does'
                    ' not exist'),
                'eol_collection': _('Collection named %(collection)s is eol')
                }

    def validate_python(self, value, state):
        try:
            collection = Collection.by_simple_name(value)
        except InvalidRequestError:
            raise Invalid(self.message('no_collection', state,
                collection=value), value, state)
        if not self.eol and (collection.statuscode ==
                STATUS['EOL']):
            raise Invalid(self.message('eol_collection', state,
                collection=value), value, state)
        return value

class IsCollection(IsCollectionSimpleName):
    '''Transforms a Collection simplename into a Collection.

    :kwarg eol: If True, include eol releases. (default False)
    :kwarg strip: If True, strips whitespace from the beginnng and end of the
        value.  (default True)
    :rtype: Collection
    :returns: Collection that the simplename we were given references.
    '''
    messages = {'no_collection': _('A collection named %(collection)s does'
                    ' not exist'),
                'eol_collection': _('Collection named %(collection)s is eol')
                }

    def validate_python(self, value, state):
        try:
            collection = Collection.by_simple_name(value)
        except InvalidRequestError:
            raise Invalid(self.message('no_collection', state,
                collection=value), value, state)
        if not self.eol and (collection.statuscode ==
                STATUS['EOL']):
            raise Invalid(self.message('eol_collection', state,
                collection=value), value, state)
        return collection

#
# Legacy -- Remove when we update the API
#

class CollectionName(FancyValidator):
    '''Test that the value is a recognized collection name.'''
    messages = {'no_collection': _('A collection named %(collection)s does'
                    ' not exist.')}

    def _to_python(self, value, state):
        '''Just remove leading and trailing whitespace.'''
        return value.strip()

    def validate_python(self, value, state):
        '''Make sure the collection is in the database.'''
        #pylint:disable-msg=E1101
        try:
            Collection.query.filter_by(name=value).first()
        except InvalidRequestError:
            raise Invalid(self.message('no_collection', state,
                collection=value), value, state)
        #pylint:enable-msg=E1101

#
# Chained Validators
#

# Note: Chained validators take different params so they are not interchangable
# with normal validators:
# validate_python: field_dict instead of value.  This is a dictionary of the
# fields passed into the schema.
#
# raising Invalid: error_dict.  In addition to the other values to Invalid()
# we send an error_dict that maps the field to display an error with to the
# message.

class CollectionNameVersion(FancyValidator):
    '''Test the combination of a Collection and Version for validity.'''
    messages = {'nameless_version': _('Version specified without a collection'
                    ' name'),
            'no_version': _('There is no collection for %(name)s-%(version)s'),
            'no_collection': _('Collection named %(name)s does not exist')}

    def validate_python(self, field_dict, state):
        '''Make sure the Collection with the given `name` and `version` exists.

        We want to allow for:
          1) Neither to be set
          2) Name to exist in the db and version unset
          3) Name and version to exist in the db
        '''
        if not field_dict:
            # It's okay for both to be none
            return

        errors = {}
        name = field_dict.get('name')
        version = field_dict.get('version')
        if (not name) and version:
            #pylint:disable-msg=E1101
            errors['version'] = self.message('nameless_version', state)
        elif name and version:
            #pylint:disable-msg=E1101
            try:
                Collection.query.filter_by(name=name, version=version).one()
            except InvalidRequestError:
                errors['version'] = self.message('no_version', state,
                        name=name, version=version)
        elif name and not version:
            #pylint:disable-msg=E1101
            try:
                Collection.query.filter_by(name=name).first()
            except InvalidRequestError:
                errors['name'] = self.message('no_collection', state, name=name)

        if errors:
            error_list = sorted(errors.iteritems())
            error_message = '\n'.join([u'%s: %s' % (error, msg)
                    for error, msg in error_list])
            raise Invalid(error_message, field_dict, state,
                    error_dict=errors)


import math
import os
import itertools
import numpy as np

import dwavebinarycsp
import dwave.inspector
from dwave.system import EmbeddingComposite, DWaveSampler

from collections import defaultdict
import matplotlib
matplotlib.use("agg")
import matplotlib.pyplot as plt

class Coordinate:
    def __init__(self, x, y):
        self.x = x
        self.y = y

        # coordinate labels for groups red, green, and blue
        label = "{0},{1}_".format(x, y)
        self.r = label + "r"
        self.g = label + "g"
        self.b = label + "b"

def get_distance(coordinate_0, coordinate_1):
    diff_x = coordinate_0.x - coordinate_1.x
    diff_y = coordinate_0.y - coordinate_1.y

    return math.sqrt(diff_x**2 + diff_y**2)

def get_max_distance(coordinates):
    max_distance = 0
    for i, coord0 in enumerate(coordinates[:-1]):
        for coord1 in coordinates[i+1:]:
            distance = get_distance(coord0, coord1)
            max_distance = max(max_distance, distance)

    return max_distance

def get_groupings(sample, nodes):
    """Grab selected items and group them by color"""
    colored_points = defaultdict(list)

    for label, bool_val in sample.items():
        # Skip over items that were not selected
        if not bool_val:
            continue

        # Parse selected items
        # Note: label look like "<x_coord>,<y_coord>_<color>"
        coord, color = label.split("_")
        coord_tuple = tuple(map(float, coord.split(",")))
        colored_points[color].append(coord_tuple)
    all_groups = dict(colored_points)
    final_groups = []
    for color in all_groups:
        group = all_groups[color]
        final_group = []
        for point in group:
            final_group.append(nodes.tolist().index(list(point)))
        final_groups.append(final_group)

    return all_groups, final_groups

def visualize_groupings(groupings_dict, filename):
    """
    Args:
        groupings_dict: key is a color, value is a list of x-y coordinate tuples.
          For example, {'r': [(0,1), (2,3)], 'b': [(8,3)]}
        filename: name of the file to save plot in
    """
    for color, points in groupings_dict.items():
        # Ignore items that do not contain any coordinates
        if not points:
            continue

        # Populate plot
        point_style = color + "o"
        plt.plot(*zip(*points), point_style)

    plt.savefig(filename)

def visualize_scatterplot(x_y_tuples_list, filename):
    """Plotting out a list of x-y tuples

    Args:
        x_y_tuples_list: A list of x-y coordinate values. e.g. [(1,4), (3, 2)]
    """
    plt.plot(*zip(*x_y_tuples_list), "o")
    plt.savefig(filename)

def cluster_points(scattered_points, chainstrength=4, numruns=1000, problem_inspector=False):
    """Perform clustering analysis on given points

    Args:
        scattered_points (list of tuples):
            Points to be clustered
        filename (str):
            Output file for graphic
        problem_inspector (bool):
            Whether to show problem inspector
    """
    # Set up problem
    # Note: max_distance gets used in division later on. Hence, the max(.., 1)
    #   is used to prevent a division by zero
    coordinates = [Coordinate(x, y) for x, y in scattered_points]
    max_distance = max(get_max_distance(coordinates), 1)

    # Build constraints
    csp = dwavebinarycsp.ConstraintSatisfactionProblem(dwavebinarycsp.BINARY)

    # Apply constraint: coordinate can only be in one colour group
    choose_one_group = {(0, 0, 1), (0, 1, 0), (1, 0, 0)}
    for coord in coordinates:
        csp.add_constraint(choose_one_group, (coord.r, coord.g, coord.b))

    # Build initial BQM
    bqm = dwavebinarycsp.stitch(csp)

    # Edit BQM to bias for close together points to share the same color
    for i, coord0 in enumerate(coordinates[:-1]):
        for coord1 in coordinates[i+1:]:
            # Set up weight
            d = get_distance(coord0, coord1) / max_distance  # rescale distance
            weight = -math.cos(d*math.pi)

            # Apply weights to BQM
            bqm.add_interaction(coord0.r, coord1.r, weight)
            bqm.add_interaction(coord0.g, coord1.g, weight)
            bqm.add_interaction(coord0.b, coord1.b, weight)

    # Edit BQM to bias for far away points to have different colors
    for i, coord0 in enumerate(coordinates[:-1]):
        for coord1 in coordinates[i+1:]:
            # Set up weight
            # Note: rescaled and applied square root so that far off distances
            #   are all weighted approximately the same
            d = math.sqrt(get_distance(coord0, coord1) / max_distance)
            weight = -math.tanh(d) * 0.1

            # Apply weights to BQM
            bqm.add_interaction(coord0.r, coord1.b, weight)
            bqm.add_interaction(coord0.r, coord1.g, weight)
            bqm.add_interaction(coord0.b, coord1.r, weight)
            bqm.add_interaction(coord0.b, coord1.g, weight)
            bqm.add_interaction(coord0.g, coord1.r, weight)
            bqm.add_interaction(coord0.g, coord1.b, weight)

    # Submit problem to D-Wave sampler
    sampler = EmbeddingComposite(DWaveSampler())
    sampleset = sampler.sample(bqm,
                               chain_strength=chainstrength,
                               num_reads=numruns,
                               label='Example - Clustering')
    best_sample = sampleset.first.sample

    # Visualize graph problem
    if problem_inspector:
        dwave.inspector.show(bqm, sampleset)

    # Visualize solution
    return get_groupings(best_sample, scattered_points)

def solver(graph, output_dir, save_fig=False,
           print_result=False, config=None, return_edge=False):

    scattered_points = graph["nodes"]
    count = 0
    while True:
        filename = f"Input Graph {count}.png"
        orig_filename = os.path.join(output_dir, filename)
        if os.path.exists(orig_filename):
            count += 1
        else:
            break
    if save_fig:
        plt.figure()
        visualize_scatterplot(scattered_points, orig_filename)

    filename = f"Output Graph {count}.png"
    clustered_filename = os.path.join(output_dir, filename)

    try:
        chainstrength = config.chain_strength
    except:
        chainstrength = 4
    try:
        numruns = config.num_runs
    except:
        numruns = 1000

    color_groups, groups = cluster_points(scattered_points,
                                          chainstrength=chainstrength,
                                          numruns=numruns)
    if save_fig:
        visualize_groupings(color_groups, clustered_filename)
    if return_edge:
        all_edges = []
        for group in groups:
            all_edges.extend(itertools.combinations(group, 2))
        return all_edges
    return groups


"""Admin routes for presentations."""

import json

from uuid import uuid4

from aiohttp import web
from aiohttp_apispec import docs, request_schema, response_schema
from marshmallow import fields, Schema

from ....holder.base import BaseHolder
from ....storage.error import StorageNotFoundError

from ...connections.models.connection_record import ConnectionRecord
from ...decorators.attach_decorator import AttachDecorator
from ...valid import (
    INDY_CRED_DEF_ID,
    INDY_DID,
    INDY_PREDICATE,
    INDY_SCHEMA_ID,
    INDY_VERSION,
    INT_EPOCH,
    UUIDFour,
)

from .manager import PresentationManager
from .messages.inner.presentation_preview import (
    PresentationPreview,
    PresentationPreviewSchema
)
from .messages.presentation_proposal import PresentationProposal
from .messages.presentation_request import PresentationRequest
from .models.presentation_exchange import (
    V10PresentationExchange,
    V10PresentationExchangeSchema,
)

class V10PresentationExchangeListSchema(Schema):
    """Result schema for an Aries#0037 v1.0 presentation exchange query."""

    results = fields.List(
        fields.Nested(V10PresentationExchangeSchema()),
        description="Aries#0037 v1.0 presentation exchange records"
    )

class V10PresentationProposalRequestSchema(Schema):
    """Request schema for sending a presentation proposal admin message."""

    connection_id = fields.UUID(
        description="Connection identifier",
        required=True,
        example=UUIDFour.EXAMPLE,
    )
    comment = fields.Str(
        description="Human-readable comment",
        required=False,
        default="")
    presentation_proposal = fields.Nested(PresentationPreviewSchema(), required=True)
    auto_present = fields.Boolean(
        description=(
            "Whether to respond automatically to presentation requests, building "
            "and presenting requested proof"
        ),
        required=False,
        default=False
    )

class IndyProofReqSpecRestrictionsSchema(Schema):
    """Schema for restrictions in attr or pred specifier indy proof request."""

    credential_definition_id = fields.Str(
        description="Credential definition identifier",
        required=True,
        **INDY_CRED_DEF_ID
    )
    schema_id = fields.String(
        description="Schema identifier",
        required=False,
        **INDY_SCHEMA_ID
    )
    schema_issuer_did = fields.String(
        description="Schema issuer (origin) DID",
        required=False,
        **INDY_DID
    )
    schema_name = fields.String(
        example="transcript",
        description="Schema name",
        required=False
    )
    schema_version = fields.String(
        description="Schema version",
        required=False,
        **INDY_VERSION
    )
    issuer_did = fields.String(
        description="Credential issuer DID",
        required=False,
        **INDY_DID
    )
    cred_def_id = fields.String(
        description="Credential definition identifier",
        required=False,
        **INDY_CRED_DEF_ID
    )

class IndyProofReqNonRevoked(Schema):
    """Non-revocation times specification in indy proof request."""

    from_epoch = fields.Int(
        description="Earliest epoch of interest for non-revocation proof",
        required=True,
        **INT_EPOCH
    )
    to_epoch = fields.Int(
        description="Latest epoch of interest for non-revocation proof",
        required=True,
        **INT_EPOCH
    )

class IndyProofReqAttrSpecSchema(Schema):
    """Schema for attribute specification in indy proof request."""

    name = fields.String(
        example="favouriteDrink",
        description="Attribute name",
        required=True)
    restrictions = fields.List(
        fields.Nested(IndyProofReqSpecRestrictionsSchema()),
        description="If present, credential must satisfy one of given restrictions",
        required=False
    )
    non_revoked = fields.Nested(
        IndyProofReqNonRevoked(),
        required=False
    )

class IndyProofReqPredSpecSchema(Schema):
    """Schema for predicate specification in indy proof request."""

    name = fields.String(
        example="index",
        description="Attribute name",
        required=True)
    p_type: fields.String(
        description="Predicate type (indy currently supports only '>=')",
        required=True,
        **INDY_PREDICATE
    )
    p_value: fields.Integer(
        description="Threshold value",
        required=True,
    )
    restrictions = fields.List(
        fields.Nested(IndyProofReqSpecRestrictionsSchema()),
        description="If present, credential must satisfy one of given restrictions",
        required=False
    )
    non_revoked = fields.Nested(
        IndyProofReqNonRevoked(),
        required=False
    )

class IndyProofRequestSchema(Schema):
    """Schema for indy proof request."""

    nonce = fields.String(
        description="Nonce",
        required=False,
        example="1234567890"
    )
    name = fields.String(
        description="Proof request name",
        required=False,
        example="Proof request",
        default="Proof request"
    )
    version = fields.String(
        description="Proof request version",
        required=False,
        default="1.0",
        **INDY_VERSION
    )
    requested_attributes = fields.Dict(
        description=("Requested attribute specifications of proof request"),
        required=True,
        keys=fields.Str(example="0_attr_uuid"),  # marshmallow/apispec v3.0 ignores
        values=fields.Nested(IndyProofReqAttrSpecSchema())
    )
    requested_predicates = fields.Dict(
        description=("Requested predicate specifications of proof request"),
        required=True,
        keys=fields.Str(example="0_age_GE_uuid"),  # marshmallow/apispec v3.0 ignores
        values=fields.Nested(IndyProofReqPredSpecSchema())
    )

class V10PresentationRequestRequestSchema(Schema):
    """Request schema for sending a proof request."""

    connection_id = fields.UUID(
        description="Connection identifier",
        required=True,
        example=UUIDFour.EXAMPLE,
    )
    proof_request = fields.Nested(IndyProofRequestSchema(), required=True)

class IndyRequestedCredsRequestedAttrSchema(Schema):
    """Schema for requested attributes within indy requested credentials structure."""

    cred_id = fields.Str(
        example="3fa85f64-5717-4562-b3fc-2c963f66afa6",
        description=(
            "Wallet credential identifier (typically but not necessarily a UUID)"
        )
    )
    revealed = fields.Bool(
        description="Whether to reveal attribute in proof",
        default=True
    )

class IndyRequestedCredsRequestedPredSchema(Schema):
    """Schema for requested predicates within indy requested credentials structure."""

    cred_id = fields.Str(
        example="3fa85f64-5717-4562-b3fc-2c963f66afa6",
        description=(
            "Wallet credential identifier (typically but not necessarily a UUID)"
        )
    )

class V10PresentationRequestSchema(Schema):
    """Request schema for sending a presentation."""

    self_attested_attributes = fields.Dict(
        description=("Self-attested attributes to build into proof"),
        required=True,
        keys=fields.Str(example="attr_name"),  # marshmallow/apispec v3.0 ignores
        values=fields.Str(
            example="self_attested_value",
            description=(
                "Self-attested attribute values to use in requested-credentials "
                "structure for proof construction"
            )
        )
    )
    requested_attributes = fields.Dict(
        description=(
            "Nested object mapping proof request attribute referents to "
            "requested-attribute specifiers"
        ),
        required=True,
        keys=fields.Str(example="attr_referent"),  # marshmallow/apispec v3.0 ignores
        values=fields.Nested(IndyRequestedCredsRequestedAttrSchema())
    )
    requested_predicates = fields.Dict(
        description=(
            "Nested object mapping proof request predicate referents to "
            "requested-predicate specifiers"
        ),
        required=True,
        keys=fields.Str(example="pred_referent"),  # marshmallow/apispec v3.0 ignores
        values=fields.Nested(IndyRequestedCredsRequestedPredSchema())
    )

@docs(
    tags=["present-proof"],
    summary="Fetch all present-proof exchange records"
)
@response_schema(V10PresentationExchangeListSchema(), 200)
async def presentation_exchange_list(request: web.BaseRequest):
    """
    Request handler for searching presentation exchange records.

    Args:
        request: aiohttp request object

    Returns:
        The presentation exchange list response

    """
    context = request.app["request_context"]
    tag_filter = {}
    for param_name in (
        "connection_id",
        "thread_id",
        "initiator",
        "state",
        "verified"
    ):
        if param_name in request.query and request.query[param_name] != "":
            tag_filter[param_name] = request.query[param_name]
    records = await V10PresentationExchange.query(context, tag_filter)
    return web.json_response({"results": [record.serialize() for record in records]})

@docs(
    tags=["present-proof"],
    summary="Fetch a single presentation exchange record"
)
@response_schema(V10PresentationExchangeSchema(), 200)
async def presentation_exchange_retrieve(request: web.BaseRequest):
    """
    Request handler for fetching a single presentation exchange record.

    Args:
        request: aiohttp request object

    Returns:
        The presentation exchange record response

    """
    context = request.app["request_context"]
    presentation_exchange_id = request.match_info["pres_ex_id"]
    try:
        record = await V10PresentationExchange.retrieve_by_id(
            context,
            presentation_exchange_id
        )
    except StorageNotFoundError:
        raise web.HTTPNotFound()
    return web.json_response(record.serialize())

@docs(
    tags=["present-proof"],
    summary="Fetch credentials for a presentation request from wallet",
    parameters=[
        {
            "name": "start",
            "in": "query",
            "schema": {"type": "string"},
            "required": False,
        },
        {
            "name": "count",
            "in": "query",
            "schema": {"type": "string"},
            "required": False,
        },
        {
            "name": "extra_query",
            "in": "query",
            "schema": {"type": "string"},
            "required": False,
        },
    ]
)
async def presentation_exchange_credentials_list(request: web.BaseRequest):
    """
    Request handler for searching applicable credential records.

    Args:
        request: aiohttp request object

    Returns:
        The credential list response

    """
    context = request.app["request_context"]

    presentation_exchange_id = request.match_info["pres_ex_id"]
    presentation_referent = request.match_info.get("referent")

    try:
        presentation_exchange_record = await V10PresentationExchange.retrieve_by_id(
            context,
            presentation_exchange_id
        )
    except StorageNotFoundError:
        raise web.HTTPNotFound()

    start = request.query.get("start")
    count = request.query.get("count")

    # url encoded json extra_query
    encoded_extra_query = request.query.get("extra_query") or "{}"
    extra_query = json.loads(encoded_extra_query)

    # defaults
    start = int(start) if isinstance(start, str) else 0
    count = int(count) if isinstance(count, str) else 10

    holder: BaseHolder = await context.inject(BaseHolder)
    credentials = await holder.get_credentials_for_presentation_request_by_referent(
        presentation_exchange_record.presentation_request,
        (presentation_referent,) if presentation_referent else (),
        start,
        count,
        extra_query
    )

    presentation_exchange_record.log_state(
        context,
        "Retrieved presentation credentials",
        {
            "presentation_exchange_id": presentation_exchange_id,
            "referent": presentation_referent,
            "extra_query": extra_query,
            "credentials": credentials
        }
    )
    return web.json_response(credentials)

@docs(
    tags=["present-proof"],
    summary="Sends a presentation proposal"
)
@request_schema(V10PresentationProposalRequestSchema())
@response_schema(V10PresentationExchangeSchema(), 200)
async def presentation_exchange_send_proposal(request: web.BaseRequest):
    """
    Request handler for sending a presentation proposal.

    Args:
        request: aiohttp request object

    Returns:
        The presentation exchange details

    """
    context = request.app["request_context"]
    outbound_handler = request.app["outbound_message_router"]

    body = await request.json()

    connection_id = body.get("connection_id")
    try:
        connection_record = await ConnectionRecord.retrieve_by_id(
            context,
            connection_id
        )
    except StorageNotFoundError:
        raise web.HTTPBadRequest()

    if not connection_record.is_ready:
        raise web.HTTPForbidden()

    comment = body.get("comment")
    # Aries#0037 calls it a proposal in the proposal struct but it's of type preview
    presentation_preview = body.get("presentation_proposal")
    presentation_proposal_message = PresentationProposal(
        comment=comment,
        presentation_proposal=PresentationPreview.deserialize(presentation_preview)
    )
    auto_present = body.get(
        "auto_present",
        context.settings.get("debug.auto_respond_presentation_request")
    )

    presentation_manager = PresentationManager(context)

    presentation_exchange_record = (
        await presentation_manager.create_exchange_for_proposal(
            connection_id=connection_id,
            presentation_proposal_message=presentation_proposal_message,
            auto_present=auto_present
        )
    )
    await outbound_handler(presentation_proposal_message, connection_id=connection_id)

    return web.json_response(presentation_exchange_record.serialize())

@docs(
    tags=["present-proof"],
    summary="Sends a free presentation request not bound to any proposal"
)
@request_schema(V10PresentationRequestRequestSchema())
@response_schema(V10PresentationExchangeSchema(), 200)
async def presentation_exchange_send_free_request(request: web.BaseRequest):
    """
    Request handler for sending a presentation request free from any proposal.

    Args:
        request: aiohttp request object

    Returns:
        The presentation exchange details

    """
    context = request.app["request_context"]
    outbound_handler = request.app["outbound_message_router"]

    body = await request.json()

    connection_id = body.get("connection_id")
    try:
        connection_record = await ConnectionRecord.retrieve_by_id(
            context,
            connection_id
        )
    except StorageNotFoundError:
        raise web.HTTPBadRequest()

    if not connection_record.is_ready:
        raise web.HTTPForbidden()

    comment = body.get("comment")
    indy_proof_request = body.get("proof_request")
    if not indy_proof_request.get("nonce"):
        indy_proof_request["nonce"] = str(uuid4().int)

    presentation_request_message = PresentationRequest(
        comment=comment,
        request_presentations_attach=[
            AttachDecorator.from_indy_dict(indy_proof_request)
        ]
    )

    presentation_manager = PresentationManager(context)

    presentation_exchange_record = (
        await presentation_manager.create_exchange_for_request(
            connection_id=connection_id,
            presentation_request_message=presentation_request_message
        )
    )

    await outbound_handler(presentation_request_message, connection_id=connection_id)

    return web.json_response(presentation_exchange_record.serialize())

@docs(
    tags=["present-proof"],
    summary="Sends a presentation request in reference to a proposal"
)
@request_schema(V10PresentationRequestRequestSchema())
@response_schema(V10PresentationExchangeSchema(), 200)
async def presentation_exchange_send_bound_request(request: web.BaseRequest):
    """
    Request handler for sending a presentation request free from any proposal.

    Args:
        request: aiohttp request object

    Returns:
        The presentation exchange details

    """
    context = request.app["request_context"]
    outbound_handler = request.app["outbound_message_router"]

    presentation_exchange_id = request.match_info["pres_ex_id"]
    presentation_exchange_record = await V10PresentationExchange.retrieve_by_id(
        context,
        presentation_exchange_id
    )
    assert presentation_exchange_record.state == (
        V10PresentationExchange.STATE_PROPOSAL_RECEIVED
    )
    body = await request.json()

    connection_id = body.get("connection_id")
    try:
        connection_record = await ConnectionRecord.retrieve_by_id(
            context,
            connection_id
        )
    except StorageNotFoundError:
        raise web.HTTPBadRequest()

    if not connection_record.is_ready:
        raise web.HTTPForbidden()

    presentation_manager = PresentationManager(context)

    (
        presentation_exchange_record,
        presentation_request_message
    ) = await presentation_manager.create_bound_request(
        presentation_exchange_record
    )

    await outbound_handler(presentation_request_message, connection_id=connection_id)

    return web.json_response(presentation_exchange_record.serialize())

@docs(
    tags=["present-proof"],
    summary="Sends a proof presentation"
)
@request_schema(V10PresentationRequestSchema())
@response_schema(V10PresentationExchangeSchema())
async def presentation_exchange_send_presentation(request: web.BaseRequest):
    """
    Request handler for sending a presentation.

    Args:
        request: aiohttp request object

    Returns:
        The presentation exchange details

    """

    context = request.app["request_context"]
    outbound_handler = request.app["outbound_message_router"]
    presentation_exchange_id = request.match_info["pres_ex_id"]
    presentation_exchange_record = await V10PresentationExchange.retrieve_by_id(
        context, presentation_exchange_id
    )

    body = await request.json()

    connection_id = presentation_exchange_record.connection_id
    try:
        connection_record = await ConnectionRecord.retrieve_by_id(
            context,
            connection_id
        )
    except StorageNotFoundError:
        raise web.HTTPBadRequest()

    if not connection_record.is_ready:
        raise web.HTTPForbidden()

    assert (
        presentation_exchange_record.state
    ) == V10PresentationExchange.STATE_REQUEST_RECEIVED

    presentation_manager = PresentationManager(context)

    (
        presentation_exchange_record,
        presentation_message,
    ) = await presentation_manager.create_presentation(
        presentation_exchange_record,
        {
            "self_attested_attributes": body.get("self_attested_attributes"),
            "requested_attributes": body.get("requested_attributes"),
            "requested_predicates": body.get("requested_predicates")
        },
        comment=body.get("comment")
    )

    await outbound_handler(presentation_message, connection_id=connection_id)
    return web.json_response(presentation_exchange_record.serialize())

@docs(
    tags=["present-proof"],
    summary="Verify a received presentation"
)
@response_schema(V10PresentationExchangeSchema())
async def presentation_exchange_verify_presentation(
    request: web.BaseRequest
):
    """
    Request handler for verifying a presentation request.

    Args:
        request: aiohttp request object

    Returns:
        The presentation exchange details

    """
    context = request.app["request_context"]
    presentation_exchange_id = request.match_info["pres_ex_id"]

    presentation_exchange_record = await V10PresentationExchange.retrieve_by_id(
        context,
        presentation_exchange_id
    )
    connection_id = presentation_exchange_record.connection_id

    try:
        connection_record = await ConnectionRecord.retrieve_by_id(
            context,
            connection_id
        )
    except StorageNotFoundError:
        raise web.HTTPBadRequest()

    if not connection_record.is_ready:
        raise web.HTTPForbidden()

    assert (
        presentation_exchange_record.state
    ) == V10PresentationExchange.STATE_PRESENTATION_RECEIVED

    presentation_manager = PresentationManager(context)

    presentation_exchange_record = await presentation_manager.verify_presentation(
        presentation_exchange_record
    )

    return web.json_response(presentation_exchange_record.serialize())

@docs(
    tags=["present-proof"],
    summary="Remove an existing presentation exchange record",
)
async def presentation_exchange_remove(request: web.BaseRequest):
    """
    Request handler for removing a presentation exchange record.

    Args:
        request: aiohttp request object

    """
    context = request.app["request_context"]
    presentation_exchange_id = request.match_info["pres_ex_id"]
    try:
        presentation_exchange_record = await V10PresentationExchange.retrieve_by_id(
            context,
            presentation_exchange_id
        )
    except StorageNotFoundError:
        raise web.HTTPNotFound()

    await presentation_exchange_record.delete_record(context)
    return web.json_response({})

async def register(app: web.Application):
    """Register routes."""

    app.add_routes(
        [
            web.get(
                "/present-proof/records",
                presentation_exchange_list
            ),
            web.get(
                "/present-proof/records/{pres_ex_id}",
                presentation_exchange_retrieve
            ),
            web.get(
                "/present-proof/records/{pres_ex_id}/credentials",
                presentation_exchange_credentials_list,
            ),
            web.get(
                "/present-proof/records/{pres_ex_id}/credentials/{referent}",
                presentation_exchange_credentials_list,
            ),
            web.post(
                "/present-proof/send-proposal",
                presentation_exchange_send_proposal,
            ),
            web.post(
                "/present-proof/send-request",
                presentation_exchange_send_free_request,
            ),
            web.post(
                "/present-proof/records/{pres_ex_id}/send-request",
                presentation_exchange_send_bound_request,
            ),
            web.post(
                "/present-proof/records/{pres_ex_id}/send-presentation",
                presentation_exchange_send_presentation,
            ),
            web.post(
                "/present-proof/records/{pres_ex_id}/verify-presentation",
                presentation_exchange_verify_presentation,
            ),
            web.post(
                "/present-proof/records/{pres_ex_id}/remove",
                presentation_exchange_remove
            ),
        ]
    )


# Get top ten words from JSON file
import json

def get_news():
    with open('newsafr.json', encoding='utf-8') as news_obj:
        news = json.load(news_obj)
    return news

def get_items_list(news):
    return news['rss']['channel']['items']

def get_words_dict(items):
    words_dict = {}
    for item in items:
        splited_words = item['description'].split()
        for word in splited_words:
            word = word.lower()
            if word.isdigit() or len(word) <= 6:
                continue
            if words_dict.get(word):
                words_dict[word] += 1
            else:
                words_dict[word] = 1
    return words_dict

def get_top_ten_words(dictionary):
    return sorted(dictionary.items(), key=lambda x: x[1], reverse=True)[:10]

def print_top_ten_words(words):
    for word in words:
        print(f'Word: {word[0]} - meets {word[1]} times')

all_news = get_news()
items_list = get_items_list(all_news)
all_words_dict = get_words_dict(items_list)
top_ten_words = get_top_ten_words(all_words_dict)
print_top_ten_words(top_ten_words)


from django.urls import path
from .views import (
    ProveedorListView,
    ProveedorDetailView,
    ProveedorCreateView,
    ProveedorUpdateView,
    ProveedorDeleteView,
)

from .views_autocomplete import ProveedorAutocomplete

app_name = "proveedores"
urlpatterns = [
    path("", ProveedorListView.as_view(), name="list"),
    path("new/", ProveedorCreateView.as_view(), name="create"),
    path("<int:pk>/", ProveedorDetailView.as_view(), name="detail"),
    path("<int:pk>/delete/", ProveedorDeleteView.as_view(), name="delete"),
    path("<int:pk>/update/", ProveedorUpdateView.as_view(), name="update"),
    path("proveedor-autocomplete/", ProveedorAutocomplete.as_view(), name="autocomplete"),
]


import numpy as np
import networkx as nx
import matplotlib.pyplot as plt

def maximum_entropy_normalization(y):
    #MaximumEntropyNormalization   Normalize Data using principle of maximum entropy.
    # n=1/(1+exp(-(y-mean(y))/std(y)));
    m=y.mean()
    s=y.std()
    n=1/(1+np.exp(-1*(y-m)/s))
    return n

def get_topk_blink_matrix(D,k=5,score_col=4,query_col=1):
    """
    sort by score_col and query_col
    get top k for each query_col entry

    returns filtered blink matrix
    """

    # Do a quick hand calc to make sure this is working:
    # https://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.ndarray.sort.html
    D = D[np.lexsort((D[:, query_col], -D[:, score_col])),:]
    # idx = np.argsort(D[:,score_col])[::-1] #sort in descending order by blink score
    # D = D[idx,:]
    # idx = np.argsort(D[:,query_col]) # sort in ascending order by query number
    # D = D[idx,:]

    #return indices of first instance of each query number
    u,u_idx = np.unique(D[:,query_col],return_index=True)

    #for each query number get k next best hits
    hits = []
    for i,idx in enumerate(u_idx[:-1]):
        if (u_idx[i+1]-idx)>=k: #check the >=
            hits.append(np.arange(k)+idx)
        else:
            hits.append(np.arange(u_idx[i+1]-idx)+idx)
    hits = np.concatenate(hits)
    D = D[hits,:]
    return D

def filter_component_additive(G, max_component_size,edge_score='score'):
    if max_component_size == 0:
        return

    all_edges = list(G.edges(data=True))
    G.remove_edges_from(list(G.edges))

    all_edges = sorted(all_edges, key=lambda x: x[2][edge_score], reverse=True)
    counter = 0
    for i,edge in enumerate(all_edges):
        G.add_edges_from([edge])
        largest_cc = max(nx.connected_components(G), key=len)
        if len(largest_cc) > max_component_size:
            G.remove_edge(edge[0], edge[1])
        counter +=1
        if counter==1000:
            print(i,len(list(G.edges(data=True))))
            counter=0

def filter_top_k(G, top_k,edge_score='score'):
    print("Filter Top_K", top_k)
    #Keeping only the top K scoring edges per node
    print("Starting Numer of Edges", len(G.edges()))

    node_cutoff_score = {}
    for node in G.nodes():
        node_edges = G.edges((node), data=True)
        node_edges = sorted(node_edges, key=lambda edge: edge[2][edge_score], reverse=True)

        edges_to_delete = node_edges[top_k:]
        edges_to_keep = node_edges[:top_k]

        if len(edges_to_keep) == 0:
            continue

        node_cutoff_score[node] = edges_to_keep[-1][2][edge_score]

        #print("DELETE", edges_to_delete)

        #for edge_to_remove in edges_to_delete:
        #    G.remove_edge(edge_to_remove[0], edge_to_remove[1])

    #print("After Top K", len(G.edges()))
    #Doing this for each pair, makes sure they are in each other's top K
    edge_to_remove = []
    for edge in G.edges(data=True):
        cosine_score = edge[2][edge_score]
        threshold1 = node_cutoff_score[edge[0]]
        threshold2 = node_cutoff_score[edge[1]]

        if cosine_score < threshold1 or cosine_score < threshold2:
            edge_to_remove.append(edge)

    for edge in edge_to_remove:
        G.remove_edge(edge[0], edge[1])

    print("After Top K Mutual", len(G.edges()))

def make_mirror_plot(r,q,figsize=(12,6),mz_tol=0.01,color='black',match_color='red',fontsize=20,grid=True,ax=None,fig=None,vshade=None):
    """
    Make a mirror plot
    You should normalize the spectra before passing to this.
    q = query spectrum np.array([mzs,intensities])
    r = reference spectrum np.array([mzs,intensities])
    returns figure and axes handles
    """
    # return_fig = False
    if ax is None:
        fig,ax = plt.subplots(figsize=figsize)
        # return_fig = True

    dd = abs(np.subtract(r[0][None,:], q[0][:, None],))<mz_tol

    ax.vlines(q[0],q[1]*0,q[1],color,alpha=0.6,linewidth=2)
    idx = dd.any(axis=1)
    ax.vlines(q[0][idx],q[1][idx]*0,q[1][idx],match_color,linewidth=2)

    ax.vlines(r[0],r[1]*0,-1*r[1],color,alpha=0.6,linewidth=2)
    idx = dd.any(axis=0)
    ax.vlines(r[0][idx],r[1][idx]*0,-1*r[1][idx],match_color,linewidth=2)
    ax.set_ylabel('Normalized intensity (au)',fontsize=fontsize)
    ax.set_xlabel('m/z',fontsize=fontsize)
    ax.tick_params(axis='both', which='major', labelsize=(fontsize-2))
    if vshade is not None:
        ax.axvline(vshade[0],linewidth=vshade[1],alpha=vshade[2],color=vshade[3])
    if grid==True:
        ax.grid()
    # if return_fig==True
    # return fig,ax


from pyramid.config import Configurator
import os

def main(global_config, **settings):
    """ This function returns a Pyramid WSGI application.
    """
    config = Configurator(settings=settings)
    config.include('pyramid_jinja2')
    config.include('pyramid_fanstatic')
    config.add_jinja2_renderer('.html')
    config.add_static_view('static', 'static', cache_max_age=3600)
    config.add_route('home', '/')
    config.add_route('rdm', '/rdm')
    config.add_route('rdmdocs', '/rdm/docs')
    config.add_route('meta', '/rdm/meta')
    config.add_route('odktools', '/rdm/odktools')

    templatesPath = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'templates')
    config.add_jinja2_search_path(templatesPath)

    config.scan()
    return config.make_wsgi_app()


import pygame

from src.game.GameController import GameController
from src.gui.CommonSurface import CommonSurface
from src.players.opponents.Human import Human

class TrainBoard(CommonSurface):

    def __init__(self, main_surface, board_shape):
        super().__init__(main_surface)
        self.board_shape = board_shape
        self.game_controller = None

    def generate_surface(self, game_controller=None) -> str:
        self.clear_surface()

        self.current_screen = "TRAIN_BOARD"
        self.game_controller = game_controller

        current_episode_text = self.font_big.render("EPISODE : ", False, CommonSurface.BLUE)
        self.main_surface.blit(current_episode_text, (10, 10))
        current_episode_value = self.font_big.render(f"{self.game_controller.episode}", False, CommonSurface.WHITE)
        self.main_surface.blit(current_episode_value, (240, 10))

        stats_text = self.font_big.render("STATS", False, CommonSurface.BLUE)
        self.main_surface.blit(stats_text, (10, 120))

        win_text = self.font_big.render("WIN", False, CommonSurface.WHITE)
        self.main_surface.blit(win_text, (190, 120))

        win_learner_value = self.font_medium.render(
            f"{self.game_controller.stats[self.game_controller.players[0]]['win']}",
            False, CommonSurface.BLUE)
        self.main_surface.blit(win_learner_value, (190, 200))

        win_trainer_value = self.font_medium.render(
            f"{self.game_controller.stats[self.game_controller.players[1]]['win']}", False, CommonSurface.WHITE)
        self.main_surface.blit(win_trainer_value, (190, 260))

        draw_text = self.font_big.render("DRAW", False, CommonSurface.WHITE)
        self.main_surface.blit(draw_text, (310, 120))

        draw_learner_value = self.font_medium.render(
            f"{self.game_controller.stats[self.game_controller.players[0]]['draw']}",
            False, CommonSurface.BLUE)
        self.main_surface.blit(draw_learner_value, (310, 200))

        draw_trainer_value = self.font_medium.render(
            f"{self.game_controller.stats[self.game_controller.players[1]]['draw']}",
            False, CommonSurface.WHITE)
        self.main_surface.blit(draw_trainer_value, (310, 260))

        loss_text = self.font_big.render("LOSS", False, CommonSurface.WHITE)
        self.main_surface.blit(loss_text, (460, 120))

        loss_learner_value = self.font_medium.render(
            f"{self.game_controller.stats[self.game_controller.players[0]]['loss']}", False, CommonSurface.BLUE)
        self.main_surface.blit(loss_learner_value, (460, 200))

        loss_trainer_value = self.font_medium.render(
            f"{self.game_controller.stats[self.game_controller.players[1]]['loss']}", False, CommonSurface.WHITE)
        self.main_surface.blit(loss_trainer_value, (460, 260))

        algorithm_value = self.font_medium.render(f"{self.game_controller.players[0].name}", False,
                                                  CommonSurface.WHITE)
        self.main_surface.blit(algorithm_value, (10, 200))

        algorithm_value = self.font_medium.render(f"{self.game_controller.players[1].name}", False,
                                                  CommonSurface.WHITE)
        self.main_surface.blit(algorithm_value, (10, 260))

        pygame.display.flip()

        # running a training
        self.game_controller.game_loop(update_stats=self.update_stats)

        # generating plots
        game_controller.generate_plot()

        # displaying buttons after training was completed

        test_button_text = self.font_small.render("Test performance", False, CommonSurface.WHITE)
        pygame.draw.rect(self.main_surface, CommonSurface.BLUE, (440, 350, 150, 40))
        self.main_surface.blit(test_button_text, (450, 360))

        menu_button_text = self.font_small.render("Main Menu", False, CommonSurface.BLUE)
        pygame.draw.rect(self.main_surface, CommonSurface.WHITE, (10, 350, 150, 40))
        self.main_surface.blit(menu_button_text, (45, 360))

        return self.current_screen

    def handle_clicks(self, mouse_position, class_to_switch) -> str:

        # test performance button
        if 590 > mouse_position[0] > 440 and 390 > mouse_position[1] > 350:

            learner = self.game_controller.players[0]
            learner.epsilon = 0

            self.game_controller = GameController(players=[learner, Human("Human")], shape=self.game_controller.shape,
                                                  number_of_games=1)
            self.current_screen = class_to_switch[1].generate_surface(self.game_controller)

        # main menu button
        elif 160 > mouse_position[0] > 10 and 390 > mouse_position[1] > 40:
            self.current_screen = class_to_switch[0].generate_surface()

        return self.current_screen

    def update_stats(self):
        self.main_surface.fill(CommonSurface.GREY, (240, 10, 300, 50))
        current_episode_value = self.font_big.render(f"{self.game_controller.episode}", False, CommonSurface.WHITE)
        self.main_surface.blit(current_episode_value, (240, 10))

        self.main_surface.fill(CommonSurface.GREY, (310, 200, 100, 50))
        draw_learner_value = self.font_medium.render(
            f"{self.game_controller.stats[self.game_controller.players[0]]['draw']}",
            False, CommonSurface.BLUE)
        self.main_surface.blit(draw_learner_value, (310, 200))

        self.main_surface.fill(CommonSurface.GREY, (310, 260, 100, 50))
        draw_trainer_value = self.font_medium.render(
            f"{self.game_controller.stats[self.game_controller.players[1]]['draw']}",
            False, CommonSurface.WHITE)
        self.main_surface.blit(draw_trainer_value, (310, 260))

        self.main_surface.fill(CommonSurface.GREY, (190, 200, 100, 50))
        win_learner_value = self.font_medium.render(
            f"{self.game_controller.stats[self.game_controller.players[0]]['win']}",
            False, CommonSurface.BLUE)
        self.main_surface.blit(win_learner_value, (190, 200))

        self.main_surface.fill(CommonSurface.GREY, (190, 260, 100, 50))
        win_trainer_value = self.font_medium.render(
            f"{self.game_controller.stats[self.game_controller.players[1]]['win']}", False, CommonSurface.WHITE)
        self.main_surface.blit(win_trainer_value, (190, 260))

        self.main_surface.fill(CommonSurface.GREY, (460, 200, 100, 50))
        loss_learner_value = self.font_medium.render(
            f"{self.game_controller.stats[self.game_controller.players[0]]['loss']}", False, CommonSurface.BLUE)
        self.main_surface.blit(loss_learner_value, (460, 200))

        self.main_surface.fill(CommonSurface.GREY, (460, 260, 100, 50))
        loss_trainer_value = self.font_medium.render(
            f"{self.game_controller.stats[self.game_controller.players[1]]['loss']}", False, CommonSurface.WHITE)
        self.main_surface.blit(loss_trainer_value, (460, 260))


import argparse
import sys

import gym
from gym import wrappers, logger

class RandomAgent(object):
    """The world's simplest agent!"""
    def __init__(self, action_space):
        self.action_space = action_space

    def act(self, observation, reward, done):
        return self.action_space.sample()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=None)
    parser.add_argument('env_id', nargs='?', default='CartPole-v0', help='Select the environment to run')
    args = parser.parse_args()

    # You can set the level to logger.DEBUG or logger.WARN if you
    # want to change the amount of output.
    logger.set_level(logger.INFO)

    env = gym.make(args.env_id)

    # You provide the directory to write to (can be an existing
    # directory, including one with existing data -- all monitor files
    # will be namespaced). You can also dump to a tempdir if you'd
    # like: tempfile.mkdtemp().
    outdir = '/tmp/random-agent-results'
    env = wrappers.Monitor(env, directory=outdir, force=True)
    env.seed(0)
    agent = RandomAgent(env.action_space)

    episode_count = 100
    reward = 0
    done = False

    for i in range(episode_count):
        ob = env.reset()
        while True:
            action = agent.act(ob, reward, done)
            ob, reward, done, _ = env.step(action)
            if done:
                break
            # Note there's no env.render() here. But the environment still can open window and
            # render if asked by env.monitor: it calls env.render('rgb_array') to record video.
            # Video is not recorded every episode, see capped_cubic_video_schedule for details.

    # Close the env and write monitor result info to disk
    env.close()


from pathlib import Path
import pandas as pd
import numpy as np
import os
import shutil

## THESE FUNCTIONS ARE NOT CURRENTLY USED IN THE BACKGROUND TOOL - THEY WERE CREATED FOR A NOW UNUSED GRAL WIND ROSE PULLING PROGRAM

def check_file_exists(file: str) -> bool:
    """
    CHecks if file existis
    :param file: path to check
    :return: True if exists
    """
    if os.path.isfile(file):
        return True
    else:
        return False

def copy_files_as_zip(dir: str, filetype: str):
    """

    :param dir:
    :param filetype:
    :return:
    """
    files = get_files(dir, filetype)
    new_folder = dir + "\\unzipped_gff_files\\"
    if os.path.isdir(new_folder):
        pass
    else:
        os.mkdir(new_folder)
    count = 1
    for file in files:
        file_path = Path(file)
        file_name = file_path.name
        shutil.copy(dir + "\\" + file_name, new_folder + file_path.stem + ".zip")
        print(f"Copied file # {count} as zip file")
        count += 1

def unzip_files(dir: str, type: str):
    files = get_files(dir, type)
    import zipfile
    count = 1
    for file in files:
        print(f"Unzipping file # {count}")
        count += 1
        with zipfile.ZipFile(file, 'r') as zip_ref:
            zip_ref.extractall(dir)

def get_files(dir: str, extension: str) -> list:
    file_list = os.listdir(dir)
    return [dir + "\\" + filename for filename in file_list if filename.endswith(extension)]

def convert_con_file(file: str) -> np.array:
    # structure of con file datatypes according to GRAL
    dtp = np.dtype([('x', np.dtype('i4')), ('y', np.dtype('i4')), ('con', np.dtype('f4'))])
    # open confile, skip 4 byte header and convert it to numpy array
    with open(file, "rb") as f:
        f.seek(4)
        arr = np.fromfile(f, dtype=dtp)
    return arr

def convert_gff_file(file: str) -> (np.array, np.array):
    # gff file datatypes for header and data
    hdr = np.dtype([('nk', np.dtype('i4')), ('nj', np.dtype('i4')), ('ni', np.dtype('i4')), ('dir', np.dtype('f4')),
                    ('spd', np.dtype('f4')), ('ak', np.dtype('i4')), ('cs', np.dtype('f4')),
                    ('hd', np.dtype('i4'))])
    # data type is integer for u,v,w
    dtw = np.dtype([('u', np.dtype('i2')), ('v', np.dtype('i2')), ('w', np.dtype('i2'))])
    # convert binary gff to numpy array
    with open(file, "rb") as f:
        # header
        h = np.fromfile(f, dtype=hdr, count=1)

        # uvw values are stored as integers, convert to float array
        c = np.fromfile(f, dtype=dtw).astype(np.dtype([('u', np.dtype('<f2')), ('v', np.dtype('<f2')), ('w', np.dtype('<f2'))]))
    return h, c

def print_grid_params(file: str) -> (str, list):
    geb_df = pd.read_fwf(file)
    geb_df = geb_df.iloc[:,0]
    cell_size = geb_df.iloc[0]
    numx = geb_df.iloc[2]
    numy = geb_df.iloc[3]
    sw_corner_x = geb_df.iloc[6]
    sw_corner_y = geb_df.iloc[8]
    print_text = "\nGrid Parameters for the selected GRAL model are:\n--- Grid spacing = " + str(cell_size) + "\n" \
              "--- Number of grid cells = " + str(numx) + "(X) and " + str(numy) + "(Y)\n" \
                "--- SW Corner coordinates = " + str(sw_corner_x) + " " + str(sw_corner_y) + "\n"
    return print_text, [float(cell_size), int(numx), int(numy), int(sw_corner_x), int(sw_corner_y)]


import pandas as pd
import pickle
from tools.get_prediction_data import get_bid_results

import torch
from tools.DLinear import Model as Dlinear
device = torch.device('cuda')

# data preparation
today = pd.Timestamp.now().strftime('%Y-%m-%d')
errors_x = get_bid_results().iloc[:, 1:].values
errors_x = torch.FloatTensor(errors_x).reshape(1, -1, 5).to(device)

# errors prediction
class Config:
    def __init__(self, seq_len=10, pred_len=2, individual=True, enc_in=10):
        self.seq_len = seq_len
        self.pred_len = pred_len
        self.individual = individual
        self.enc_in = enc_in  # channels
        return

configs = Config(seq_len=72, pred_len=48, individual=True, enc_in=5)
dlinear = Dlinear(configs).to(device)
dlinear.load_state_dict(torch.load('checkpoints/errors_to_errors_1.pt'))
print('dlinear model load complete.')

with torch.no_grad():
    errors_prediction = dlinear(errors_x).squeeze(dim=0)
    errors_prediction = errors_prediction.cpu().numpy()

# saving
with open(f'data_prediction/{today}-errors_prediction.pickle', 'wb') as f:
    pickle.dump(errors_prediction, f)
print('errors prediction and saving process complete.')


from django.conf.urls import url
from django.contrib.auth import views as auth_views
from . import views
app_name='pachangapp'
urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^stats/(?P<usernameID>[\w.@+-]+)/$', views.stats, name='stats'),
    url(r'^stats/$', views.stats, name='stats'),
    url(r'^profile/$', views.profile, name='profile'),
    url(r'^login/$', views.login, name='login'),
    url(r'^logout/$', auth_views.logout, {'next_page': '/pachangapp/'}, name='logout')
]


from flask import Flask, request, jsonify
import scanner
from flask_cors import CORS

app = Flask(__name__)
CORS(app)

@app.route('/', methods=['POST'])
def analysis():
    try:
        data = request.json
        code = data.get('code')
        response = jsonify(scanner.analyze(code))
    except Exception as e:
        response = jsonify({'error': 'Could not analyze the code: ' + str(e)})
    return response

if __name__ == '__main__':
    app.run()


#!/usr/bin/env python
import pika
import sys
import json

timeout=30

def callbackresponse(ch, method, properties, body):
	body = body.decode('utf-8')
	print(json.dumps(json.loads(body),sort_keys=True,indent=4, separators=(',', ': ')))
	connection.close()
	exit()

def callbacktimeout():
	print("Timeout")
	connection.close()
	exit()

connection = pika.BlockingConnection(pika.ConnectionParameters(
        host='10.0.0.103'))
connection.add_timeout(timeout,callbacktimeout)
channel = connection.channel()

# create the topic on the rabbitmq server, if not already done
channel.exchange_declare(exchange='lxc',type='topic',durable=True,auto_delete=True)

#declare the queue on the rabbitmq server, if not already done
result = channel.queue_declare(durable=False, exclusive=False, auto_delete=True)
queue_name = result.method.queue

routing_key = sys.argv[1] if len(sys.argv) > 1 else 'anonymous.info'
message = sys.argv[2]

# send to the routing key from the commandline the message from the commandline
channel.basic_publish(exchange='lxc',
                      routing_key=routing_key,
                      body=message)
print ("[x] Sent %r:%r  -- waiting for reply" % (routing_key, message))

#wait the reply for [timeout] seconds
channel.queue_bind(exchange='lxc',
                       queue=queue_name,
                       routing_key="lxc.master")

channel.basic_consume(callbackresponse,
                      queue=queue_name,
                      no_ack=True)

channel.start_consuming()
connection.close()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Date: '2018/09/12 9:34'

from django.conf.urls import url

from .views import UserInfoView, UploadImageView, UpdatePwdView, SendEmailCodeView, UpdateEmailView, MycourseView
from .views import MyFavCourseView, MyFavOrgView, MyFavTeacherView, MyMessageView

urlpatterns = [
    url(r'^info/$', UserInfoView.as_view(), name='user_info'),
    url(r'^upload/image/$', UploadImageView.as_view(), name='upload_image'),
    url(r'^update/pwd/$', UpdatePwdView.as_view(), name='update_pwd'),
    url(r'^sendemail_code/$', SendEmailCodeView.as_view(), name='sendemail_code'),
    url(r'^update_email/$', UpdateEmailView.as_view(), name='update_email'),
    url(r'^courses/$', MycourseView.as_view(), name='my_courses'),
    url(r'^fav_courses/$', MyFavCourseView.as_view(), name='my_fav_courses'),
    url(r'^fav_orgs/$', MyFavOrgView.as_view(), name='my_fav_orgs'),
    url(r'^fav_teachers/$', MyFavTeacherView.as_view(), name='my_fav_teachers'),
    url(r'^my_messages/$', MyMessageView.as_view(), name='my_messages'),
]


# -*- coding: utf-8 -*-
#
# Release Notes build configuration file, created by
# sphinx-quickstart on Thu Feb 12 02:10:34 2009.
#
# This file is execfile()d with the current directory set to its containing
# dir.
#
# The contents of this file are pickled, so don't put values in the namespace
# that aren't pickleable (module imports are okay, they're removed
# automatically).
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If your extensions are in another directory, add it here. If the directory
# is relative to the documentation root, use os.path.abspath to make it
# absolute, like shown here.
import os
import sys
from datetime import datetime

cur_dir = os.path.abspath(os.path.join(__file__, '..'))

sys.path.append(os.path.join(cur_dir, '_ext'))
sys.path.append(os.path.abspath(os.path.join(cur_dir, '..', 'manual', '_ext')))

import reviewboard
from reviewboard.dependencies import django_doc_major_version

# General configuration
# ---------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
    'sphinx.ext.intersphinx',
    'beanbag_docutils.sphinx.ext.django_utils',
    'beanbag_docutils.sphinx.ext.extlinks',
    'beanbag_docutils.sphinx.ext.http_role',
    'beanbag_docutils.sphinx.ext.image_srcsets',
    'beanbag_docutils.sphinx.ext.intersphinx_utils',
    'beanbag_docutils.sphinx.ext.json_writer',
    'beanbag_docutils.sphinx.ext.metadata',
    'beanbag_docutils.sphinx.ext.retina_images',
    'extralinks',
    'rbwebsite_refs',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'Release Notes'
copyright = '2009-%s, Beanbag, Inc.' % datetime.now().year
bugtracker_url = 'https://www.reviewboard.org/bugs/%s'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '.'.join([str(i) for i in reviewboard.__version_info__[:2]])
# The full version, including alpha/beta/rc tags.
release = reviewboard.get_version_string()

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

suppress_warnings = ['ref.option']

# Options for HTML output
# -----------------------

html_theme = 'classic'

# The style sheet to use for HTML and HTML Help pages. A file of that name
# must exist either in Sphinx' static/ path, or in one of the custom paths
# given in html_static_path.
html_style = 'classic.css'

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
html_title = "Release Notes"

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If false, no index is generated.
html_use_index = False

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, the reST sources are included in the HTML build as _sources/<name>.
html_copy_source = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'ReleaseNotes'

# Options for LaTeX output
# ------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class
# [howto/manual]).
latex_documents = [
  ('index', 'ReleaseNotes.tex', 'Release Notes', 'Beanbag, Inc.', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_use_modindex = True

# Check whether reviewboard.org intersphinx lookups should use the local
# server.
if os.getenv('DOCS_USE_LOCAL_RBWEBSITE') == '1':
    rbwebsite_url = 'http://localhost:8081'
else:
    rbwebsite_url = 'https://www.reviewboard.org'

# Add references for intersphinx and custom roles.
django_doc_base_url_fmt = 'http://django.readthedocs.io/en/%s.x/'
django_doc_base_url = django_doc_base_url_fmt % django_doc_major_version

github_djblets_src_base_url = 'https://github.com/djblets/djblets/blob/'
github_rb_src_base_url = 'https://github.com/reviewboard/reviewboard/blob/'

intersphinx_mapping = {
    'django': (django_doc_base_url, None),
    'django1.6': (django_doc_base_url_fmt % '1.6', None),
    'django1.11': (django_doc_base_url_fmt % '1.11', None),
    'djblets0.9': ('%s/docs/djblets/0.9/' % rbwebsite_url, None),
    'djblets0.10': ('%s/docs/djblets/1.0/' % rbwebsite_url, None),
    'djblets1.0': ('%s/docs/djblets/1.0/' % rbwebsite_url, None),
    'djblets2.0': ('%s/docs/djblets/2.0/' % rbwebsite_url, None),
    'djblets2.x': ('%s/docs/djblets/2.x/' % rbwebsite_url, None),
    'djblets3.x': ('%s/docs/djblets/3.x/' % rbwebsite_url, None),
    'djblets4.x': ('%s/docs/djblets/4.x/' % rbwebsite_url, None),
    'python': ('https://docs.python.org/2.7', None),
    'python2': ('https://docs.python.org/2.7', None),
    'python3': ('https://docs.python.org/3', None),
    'rbt0.6': ('%s/docs/rbtools/0.6/' % rbwebsite_url, None),
    'rbt0.7': ('%s/docs/rbtools/0.7/' % rbwebsite_url, None),
    'rbt-latest': ('%s/docs/rbtools/latest/' % rbwebsite_url, None),
    'rb1.7': ('%s/docs/manual/1.7/' % rbwebsite_url, None),
    'rb2.0': ('%s/docs/manual/2.0/' % rbwebsite_url, None),
    'rb2.5': ('%s/docs/manual/2.5/' % rbwebsite_url, None),
    'rb3.0': ('%s/docs/manual/3.0/' % rbwebsite_url, None),
    'rb4.0': ('%s/docs/manual/4.0/' % rbwebsite_url, None),
    'rb5.x': ('%s/docs/manual/5.0/' % rbwebsite_url, None),
    'rb6.x': ('%s/docs/manual/6.x/' % rbwebsite_url, None),
}

extlinks = {
    'djangodoc': ('%s%%s.html' % django_doc_base_url, '%s'),
    'backbonejs': ('http://backbonejs.org/#%s', 'Backbone.%s'),
    'pypi': ('https://pypi.org/project/%s/', '%s'),
    'rbintegration': ('%s/integrations/%%s' % rbwebsite_url, '%s'),
    'rbintegrations-relnotes': (
        '%s/docs/releasenotes/rbintegrations/%%s/' % rbwebsite_url,
        'rbintegrations %s'),
    'rbsrc-4.0.x': ('%srelease-4.0.x/%%s' % github_rb_src_base_url, '%s'),
    'djbletssrc-2.0.x': ('%srelease-2.0.x/%%s' % github_djblets_src_base_url,
                         '%s'),
}


from dataclasses import dataclass
from pathlib import Path
from typing import Callable, cast
from uuid import UUID, uuid4

import pytest

# use async-asgi-testclient instead of the starlette one because of
# https://github.com/encode/starlette/issues/472
from async_asgi_testclient import TestClient
from fastapi import Depends, Header
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
from sqlalchemy.orm import sessionmaker

from ..app import app
from ..auth import current_user
from ..config import Env, settings
from ..crud import CRUD
from ..database import Base
from ..db import get_db

assets = Path(__file__).parent / "assets"

settings.env = Env.TESTING

SQLALCHEMY_DATABASE_URL = "sqlite+aiosqlite:///:memory:"

engine = create_async_engine(
    SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}
)
TestingSessionLocal = cast(
    Callable[[], AsyncSession],
    sessionmaker(
        autocommit=False,
        autoflush=False,
        bind=engine,
        class_=AsyncSession,  # type: ignore
    ),
)

async def override_get_db():
    session: AsyncSession
    async with engine.begin() as session:  # type: ignore
        await session.run_sync(Base.metadata.create_all)
    async with TestingSessionLocal() as session:
        yield session

app.dependency_overrides[get_db] = override_get_db

async def override_current_user(
    authorization: str | None = Header(None),
    crud: CRUD = Depends(),
):
    if authorization:
        uname, userid = authorization.split(":")
        uid = UUID(userid)
        return await crud.get_or_create_user(uid, uname)
    return None

app.dependency_overrides[current_user] = override_current_user

@pytest.fixture
def client(tmpdir: Path):
    settings.textures_fs = f"file://{tmpdir}"
    return TestClient(app)

@pytest.fixture(autouse=True)
def anyio_backend():
    return "asyncio"

@dataclass
class TestUser:
    uuid: UUID
    name: str

    __test__ = False

    @property
    def auth_header(self):
        return {"authorization": f"{self.name}:{self.uuid}"}

@pytest.fixture
def user():
    return TestUser(uuid4(), "TestUser")

@pytest.fixture
def users():
    """Fixture to get a list of random users"""
    return [TestUser(uuid4(), f"TestUser{n}") for n in range(1, 11)]


#!/usr/bin/env python2.7
from __future__ import unicode_literals

import simplejson as json
import sys, os
from time import sleep, time
import subprocess

txt_launch_error = '''No path given.

Example:
    %s /tmp/config.json
    \n''' % sys.argv[0]

def main():
    ## defaults:
    if 'jingle_period' in config and type(config['jingle_period']) == int and config['jingle_period'] > 0:
        jingle_period = config['jingle_period']
    else:
        jingle_period = 30000000

    ## Prepare jingles list ##
    if 'jingles' in config:
        if type(config['jingles']) == list and len(config['jingles']) > 0:
            jingles = config['jingles']
        elif type(config['jingles']) == str and os.path.isfile(config['jingles']):
            # Its a path to a json file, load it
            with open(config['jingles'], 'r') as f:
                jingles = json.load(f)
        else:
            jingles = []
    else:
        jingles = []

    if len(jingles) > 0:
        add_j = True
        if len(jingles) > 1:
            jn = 1
        else:
            jn = 0
    else:
        add_j = False
    ## end of prepare jingles ##

    dirname = os.path.dirname(config_file)
    cachedir = '%s/cache' % dirname
    cachelist_file = '%s/%s.playlist.json' % (cachedir, config['name'])
    with open(cachelist_file, 'r') as f:
        cachelist = json.load(f)

    #files = []
    #for item in cachelist

    ## Generate liquidsoap playlist
    lq_playlist_file = '%s/%s.playlist' % (dirname, config['name'])
    with open(lq_playlist_file, 'w') as f:
        if add_j:
            # Add a jingle in the beginning
            jingle = u'request.push ' + jingles[0] + '\n\n'
            f.write(jingle.encode('utf-8'))
        size_count = 0
        for item in cachelist:
            size_count += item[2]
            filename = '%s/%s' % (cachedir, os.path.basename(item[0]))
            playlist_item = u'request.push annotate:artist="%s",title="%s (by %s)":%s\n\n' % (config['title'], item[1], item[3], filename)
            f.write(playlist_item.encode('utf-8'))
            if add_j and size_count > jingle_period:
                jingle = u'request.push ' + jingles[jn] + '\n\n'
                f.write(jingle.encode('utf-8'))
                size_count = 0
                jn += 1
                if jn >= len(jingles):
                    jn = 0
        if add_j:
            # Add last jingle to the end of playlist
            jingle = u'request.push ' + jingles[len(jingles)-1] + '\n\n'
            f.write(jingle.encode('utf-8'))

    ## add playlist to liquidsoap queue
    subprocess.call(['/usr/local/bin/autodj-play', 'list', lq_playlist_file])

if __name__ == '__main__':
    if len(sys.argv) < 2:
        sys.stderr.write(txt_launch_error)
        sys.exit(0)

    config_file = os.path.abspath(sys.argv[1])
    with open(config_file, 'r') as f:
        config = json.load(f)
    try:
        main()
    except KeyboardInterrupt:
        sys.exit(0)


# @Author:Liu Yue (v3u.cn)
# @Software:Vscode
# @Time:2022/12/30

import base64
import hmac
import time
from hashlib import sha1
import json
import httpx
import aiofiles

class Qiniu:

    def __init__(self, access_key, secret_key):
        """初始化"""
        self.__checkKey(access_key, secret_key)
        self.__access_key = access_key
        self.__secret_key = secret_key.encode('utf-8')

    def get_access_key(self):
        return self.__access_key

    def get_secret_key(self):
        return self.__secret_key

    def __token(self, data):
        hashed = hmac.new(self.__secret_key,data.encode('utf-8'), sha1)
        return self.urlsafe_base64_encode(hashed.digest())

    def token(self, data):
        return '{0}:{1}'.format(self.__access_key, self.__token(data))

    def token_with_data(self, data):
        data = self.urlsafe_base64_encode(data)
        return '{0}:{1}:{2}'.format(
            self.__access_key, self.__token(data), data)

    def urlsafe_base64_encode(self,data):

        if isinstance(data, str):
            data = data.encode('utf-8')

        ret = base64.urlsafe_b64encode(data)

        data = ret.decode('utf-8')

        return data

    @staticmethod
    def __checkKey(access_key, secret_key):
        if not (access_key and secret_key):
            raise ValueError('invalid key')

    def upload_token(
            self,
            bucket,
            key=None,
            expires=3600,
            policy=None,
            strict_policy=True):
        """生成上传凭证

        Args:
            bucket:  上传的空间名
            key:     上传的文件名，默认为空
            expires: 上传凭证的过期时间，默认为3600s
            policy:  上传策略，默认为空

        Returns:
            上传凭证
        """
        if bucket is None or bucket == '':
            raise ValueError('invalid bucket name')

        scope = bucket
        if key is not None:
            scope = '{0}:{1}'.format(bucket, key)

        args = dict(
            scope=scope,
            deadline=int(time.time()) + expires,
        )

        return self.__upload_token(args)

    @staticmethod
    def up_token_decode(up_token):
        up_token_list = up_token.split(':')
        ak = up_token_list[0]
        sign = base64.urlsafe_b64decode(up_token_list[1])
        decode_policy = base64.urlsafe_b64decode(up_token_list[2])
        decode_policy = decode_policy.decode('utf-8')
        dict_policy = json.loads(decode_policy)
        return ak, sign, dict_policy

    def __upload_token(self, policy):
        data = json.dumps(policy, separators=(',', ':'))
        return self.token_with_data(data)

    @staticmethod
    def __copy_policy(policy, to, strict_policy):
        for k, v in policy.items():
            if (not strict_policy) or k in _policy_fields:
                to[k] = v

    # 上传文件流
    async def upload_data(self,up_token, key,data,url="http://up-z1.qiniup.com",params=None,mime_type='application/octet-stream',file_name=None,metadata=None):

        data.encode('utf-8')

        fields = {}
        if params:
            for k, v in params.items():
                fields[k] = str(v)

        if key is not None:
            fields['key'] = key
        fields['token'] = up_token

        fname = file_name
        if not fname or not fname.strip():
            fname = 'file_name'

        async with httpx.AsyncClient() as client:

            # 调用异步使用await关键字
            res = await client.post(url,data=fields,files={'file': (fname,data,mime_type)})

            print(res.text)

    # 上传文件实体
    async def upload_file(self,up_token,key,path,url="http://up-z1.qiniup.com",params=None,mime_type='application/octet-stream',file_name=None,metadata=None):

        async with aiofiles.open(path, mode='rb') as f:
            contents = await f.read()

        fields = {}
        if params:
            for k, v in params.items():
                fields[k] = str(v)

        if key is not None:
            fields['key'] = key
        fields['token'] = up_token

        fname = file_name
        if not fname or not fname.strip():
            fname = 'file_name'

        async with httpx.AsyncClient() as client:

            # 调用异步使用await关键字
            res = await client.post(url,data=fields,files={'file': (fname,contents,mime_type)})

            print(res.text)


import numpy as np
import pandas as pd
from scipy.sparse import csr_matrix
from sklearn.neighbors import NearestNeighbors
import pickle
import json

data = pd.read_csv('C:/Users/hp/Documents/need-for-code-7/ml/data/kids.csv')

kidsData = pd.read_csv('C:/Users/hp/Documents/need-for-code-7/ml/data/kids.csv')
menData = pd.read_csv('C:/Users/hp/Documents/need-for-code-7/ml/data/men.csv')
womenData = pd.read_csv('C:/Users/hp/Documents/need-for-code-7/ml/data/women.csv')
shoesData = pd.read_csv('C:/Users/hp/Documents/need-for-code-7/ml/data/shoes.csv')

row_subcategory_drop = ['Brassières de grossesse',
       'Costumes pour bébé', 'Salopettes & Combinaisons',
       'Maillots de bain', 'Coutumes & Co-ords',
       'Costume & Jupe-culotte',
       'Blousons & Vestes', 'Tops', 'Pantalons', 'Chaussures pour fille',
       'Vêtements de nuit',
       'Sous-vêtements de maintien de grossesse', 'Trainers',
       'Bas', 'Chaussures pour bébé',
       'Chapeaux',
       'Trousses', "Vêtements d'extérieur & Manteaux",
       'Robes & Sacs de couchage', 'Chaussures pour enfant',
       'Lingerie de grossesse', 'Accessoires pour cheveux', 'Sacs à dos',
       'Accesssoires des vêtements', 'Écharpes', 'Chaussettes',
       'Sets de coutumes', 'Lunettes de soleil', 'Sacs des couches',
       'Sacs à main', 'Swimwear', 'Vêtements pour fille',
       'Vêtements de nuit & Robes',
       'Colliers & Pendentifs', 'Imperméables', 'Accessoires pour enfant',
       'Accessoires']

drop_cloumns = ['discount', 'currency', 'brand', 'brand_url', 'codCountry', 'variation_0_color', 'variation_1_color', 'variation_0_thumbnail', 'variation_0_image', 'variation_1_thumbnail', 'variation_1_image', 'image_url', 'model', 'url']

df = data.drop(drop_cloumns, axis=1)

for i in range(len(row_subcategory_drop)):
    index_names = df[ df['subcategory'] == row_subcategory_drop[i] ].index
    df.drop(index_names, inplace = True)

likes_features_df = df.pivot_table(index='name', columns='subcategory', values=['likes_count','current_price', 'is_new', 'raw_price']).fillna(0)
likes_features_df_matrix = csr_matrix(likes_features_df.values)

# model_kids_knn = NearestNeighbors(metric='cosine', algorithm='brute')
# model_kids_knn.fit(likes_features_df_matrix)
model_kids_knn = pickle.load(open('C:/Users/hp/Documents/need-for-code-7/ml/models/kids_model.pkl', 'rb'))

# query_index = np.random.choice(likes_features_df.shape[0])
# print(query_index)

def getSimilarKidsProducts(productName):
    query_index = 0
    for i in range(likes_features_df.index.shape[0]):
        if likes_features_df.index[i] == productName:
            query_index = i

    distances, indices = model_kids_knn.kneighbors(likes_features_df.iloc[query_index, :].values.reshape(1,-1), n_neighbors=11)

    ls = []
    for i in range(0, len(distances.flatten())):
        if i == 0:
            print("Recommendations for {0}".format(likes_features_df.index[query_index]))
        else:
            print("{0}: {1}, with distance of {2}".format(i, likes_features_df.index[indices.flatten()[i]], distances.flatten()[i]))
            ls.append(likes_features_df.index[indices.flatten()[i]])

    new_ls = data['name'].isin(ls)
    similarProducts = data[new_ls]

    return json.dumps(ls)

# def getProducts():
#     productsData = "Data Not Available"
#     print("Request Recieved!")
#     # recievedData = request.json
#     categoryRequested = "shoes"

#     if categoryRequested == "kids":
#         # kidsData = kidsData.sort_values(by=['likes_count'], ascending=False)
#         print(kidsData.sort_values(by=['likes_count'], ascending=False)['likes_count'])
#     elif categoryRequested == "men":
#         # menData = menData.sort_values(by=['likes_count'], ascending=False)
#         print(menData.sort_values(by=['likes_count'], ascending=False)['likes_count'])
#     elif categoryRequested == "women":
#         # womenData = womenData.sort_values(by=['likes_count'], ascending=False)
#         print(womenData.sort_values(by=['likes_count'], ascending=False)['likes_count'])
#     elif categoryRequested == "shoes":
#         # shoesData = shoesData.sort_values(by=['likes_count'], ascending=False)
#         print(shoesData.sort_values(by=['likes_count'], ascending=False)['likes_count'])

#     print(productsData)

# getProducts()


import os
import json
from glob import glob
from omegaconf import ListConfig
from hydra.utils import to_absolute_path

import numpy as np
import pandas as pd
import uproot

def tau_vs_other(prob_tau, prob_other):
    return np.where(prob_tau > 0, prob_tau / (prob_tau + prob_other), np.zeros(prob_tau.shape))

def create_df(path_to_preds, pred_samples, tau_type_to_select, selection,
              pred_group_name, pred_column_prefix, target_group_name, target_column_prefix,
              add_columns_from, add_columns, group_or_tree_name,
              **kwargs):
    df = []
    path_to_preds = os.path.abspath(to_absolute_path(path_to_preds))

    # loop over input samples
    for sample_name, sample_cfg in pred_samples.items():

        # make list of files with predictions depending on the specified format
        filename_pattern = sample_cfg['filename_pattern']
        if isinstance(filename_pattern, ListConfig):
            pred_files = [f'{path_to_preds}/{sample_name}/{filename}' for filename in filename_pattern]
        elif isinstance(filename_pattern, str):
            pred_files = glob(f'{path_to_preds}/{sample_name}/{filename_pattern}')
        else:
            raise Exception(f"unknown type of filename_pattern: {type(filename_pattern)}")

        df_sample = []
        for pred_file in pred_files:
            # read predictions and labels
            l_ = []
            for group in [pred_group_name, target_group_name]:
                df_ = pd.read_hdf(pred_file, group)
                df_ = df_.rename(columns={column: f'{group}_{column}' for column in df_.columns})
                l_.append(df_)
            assert l_[0].shape[0] == l_[1].shape[0], "Sizes of prediction and target dataframes don't match."
            df_pred = pd.concat(l_, axis=1)

            if add_columns_from == 'inputs':# read add_columns from the corresponding input file
                json_filemap_name = f'{path_to_preds}/{sample_name}/pred_input_filemap.json'
                with open(json_filemap_name, 'r') as json_file:
                    target_input_map = json.load(json_file)

                with uproot.open(target_input_map[pred_file]) as f:
                    df_add = f[group_or_tree_name].arrays(add_columns, library='pd')

                # concatenate input branches and predictions/labels
                assert df_pred.shape[0] == df_add.shape[0], "Sizes of prediction and input dataframes don't match."
            elif add_columns_from == 'predictions':
                df_add = pd.read_hdf(pred_file, group_or_tree_name)
                df_add = df_add[add_columns]
            else:
                raise RuntimeError(f'add_columns_from should be either predictions or inputs, got {add_columns_from}')

            df_sample_ = pd.concat([df_pred, df_add], axis=1)
            assert not any(df_sample_.isna().any(axis=0)), 'found NaNs!'
            df_sample.append(df_sample_)

        # concat together files for a given sample and apply tau_type matching & selection
        df_sample = pd.concat(df_sample, axis=0)
        df_sample = df_sample.query(f'{target_group_name}_{target_column_prefix}{tau_type_to_select}==1')
        if selection is not None:
            df_sample = df_sample.query(selection)

        # compute weight variable
        df_sample['weight'] = 1.
        if sample_cfg['reweight_to_lumi'] is not None:
            df_sample['weight'] *= sample_cfg['sample_lumi'] / (sample_cfg['reweight_to_lumi'] * df_sample.shape[0])

        df.append(df_sample)

    # concat across samples
    df = pd.concat(df, axis=0)

    # compute vs_type discriminator scores
    vs_types = ['e', 'mu', 'jet']
    for vs_type in vs_types:
        df['score_vs_' + vs_type] = tau_vs_other(df[f'{pred_group_name}_{pred_column_prefix}tau'].values, df[f'{pred_group_name}_{pred_column_prefix}' + vs_type].values)

    print(f'\n-> Selected {df.shape[0]} {tau_type_to_select}s')
    return df