import os
import torch

import numpy as np
import pandas as pd
import torch.nn as nn
from torch.distributions.normal import Normal

from tensorboardX import SummaryWriter
from torch.utils.data.dataloader import DataLoader

import sys
sys.path.append("..")
from models.model import Model
from runners.base_run import Runner
from tools.utils import import_class
from models.classifier import Classifier
from data.dataset import ShapeNetPart

class SegmentationRunner(Runner):
    def __init__(self,args):
        super(SegmentationRunner, self).__init__(args)
        # loss
        self.loss = nn.NLLLoss().to(self.output_dev)

    def load_dataset(self,args):
        feeder = ShapeNetPart(args, phase='train')
        self.print_log("Train data is loading:...")

        train_loader = DataLoader(
                dataset=feeder,
                batch_size=args.train_batch_size*len(args.device),
                shuffle=True,
                num_workers=8
            )

        testfd = ShapeNetPart(args, phase='test')
        self.print_log("Test data is loading:...")
        test_loader = DataLoader(
                dataset=testfd,
                batch_size=args.test_batch_size*len(args.device),
                shuffle=False,
                num_workers=8
            )

        self.dataset['train'] = train_loader

        self.shape_names = feeder.shape_names
        self.num_classes = feeder.num_classes
        self.num_parts = feeder.num_parts
        self.num_points = feeder.num_points
        self.print_log(f'Train data loaded: {len(feeder)} samples.')

        self.dataset['test'] = test_loader
        self.print_log(f'Test data loaded: {len(feeder)} samples.')

    def load_model(self, args):
        self.output_dev = self.output_dev
        classifier = Classifier(
            args.detail, num_points=self.num_points, num_classes=self.num_classes,
            num_parts=self.num_parts
        )
        classifier = classifier.to(self.output_dev)
        self.model['train'] = classifier
        backbone = Model(
            k=20, out_features=3
        )
        backbone = backbone.to(self.output_dev)
        self.model['test'] = backbone
        if (args.detail=='4fc-seg' or args.detail=='semi-ft') and self.eval_models==True:
            self.model['test'] = self.model['test'].encoder

    def initialize_model(self, args):
        if args.detail=='4fc-seg':
            if self.eval_models==False:
                self.load_model_weights(self.model['test'],args.backbone_path)
            else:
                self.load_whole_model(self.model['train'], self.model['test'], './log/pretrained_models/4fc_seg.pt')
        elif args.detail=='1fc-seg':
            self.load_model_weights(
                self.model['test'],
                './log/pretrained_models/1fc_seg_b.pt'
            )
            if self.eval_models == True:
                self.load_model_weights(
                    self.model['train'],
                    './log/pretrained_models/1fc_seg_c.pt'
                )
        elif args.detail=='semi-seg':
            self.load_model_weights(
                self.model['test'],
                './log/pretrained_models/semi_seg_b.pt'
            )
            if self.eval_models == True:
                self.load_model_weights(
                    self.model['train'],
                    './log/pretrained_models/semi_seg_c.pt'
                )
        elif args.detail=='semi-ft':
            if self.eval_models==False:
                self.load_model_weights(self.model['test'], args.backbone_path)
            else:
                self.load_whole_model(self.model['train'], self.model['test'], './log/pretrained_models/semi_ft.pt')

    def run(self, args):
        if self.eval_models==False:
            self.model['test'] = self.model['test'].encoder
        best_epoch = -1
        best_acc = 0.0
        self.epoch = 0
        if self.eval_models == False:
            self.end = 100
        else:
            self.end = 1
        for epoch in range(self.epoch, self.end):
            if self.eval_models == False:
                if self.epoch % 15 == 0 and self.epoch > 0:
                    self.print_log("lr decay...")
                    self.optimizer.param_groups[0]['lr'] *= 0.5
                else:
                    self.print_log("end...")
                self._train_classifier(epoch)
            if self.eval_models == True:
                if args.detail!='4fc-seg' and args.detail!='semi-ft':
                    self.model['test'] = self.model['test'].encoder
                acc = self._eval_classifier(epoch)
                if acc > best_acc:
                    best_acc = acc
                    best_epoch = epoch
                self.print_log(
                    'Best IoU: {:.2f}%, best model: model{}.pt'.format(
                        best_acc * 100.0, best_epoch + 1
                    ))
            self.epoch += 1

    def _train_classifier(self, epoch):
        self.print_log(f'Train Classifier Epoch: {epoch + 1}')
        tlr = self.optimizer.param_groups[0]['lr']
        self.print_log(f'lr: {tlr}')
        self.model['test'].train()
        self.model['train'].train()

        loader = self.dataset['train']
        loss_values = []

        self.record_time()
        timer = dict(data=0.0, model=0.0, statistic=0.0)
        for batch_id, (x, labels, target) in enumerate(loader):
            # get data
            x = x.float().to(self.output_dev)
            labels = labels.float().to(self.output_dev)
            target = target.long().to(self.output_dev)
            target = target.view(-1, 1)[:, 0]
            timer['data'] += self.tick()

            # forward
            features = self.model['test'](x)
            pred = self.model['train'](features, labels)
            pred = pred.contiguous().view(-1, self.num_parts)
            loss = self.loss(pred, target)

            # backward
            self.optimizer.zero_grad()
            loss.backward()
            self.optimizer.step()
            timer['model'] += self.tick()

            # statistic
            loss_values.append(loss.item())
            if (batch_id + 1) % 50 == 0:
                self.print_log(
                    'Batch({}/{}) done. Loss: {:.4f}, lr: {:.5f}'.format(
                        batch_id + 1, len(loader), loss.item(),
                        self.optimizer.param_groups[0]['lr']
                    ))
            timer['statistic'] += self.tick()

            if batch_id % 25 == 0:
                self.print_log("batchid {}".format(batch_id))

        mean_loss = np.mean(loss_values)
        self.print_log("The epoch {}".format(epoch)+" loss is {}".format(mean_loss))

        self.print_log('Mean training loss: {:.4f}.'.format(mean_loss))
        self.print_log(
            'Time consumption: [Data] {:.1f} min, [Model] {:.1f} min'.format(
                timer['data'] / 60.0, timer['model'] / 60.0
            ))
        if self.eval_models == False:
            model2_path = os.path.join(
                self.classifier_path, f'model{epoch}.pt'
            )

            self.save_whole_weights(
                epoch, self.model['train'], self.model['test'], self.optimizer,
                model2_path
            )

    def _eval_classifier(self, epoch):
        noise_level = 0  # modify the level
        m = Normal(loc=0, scale=noise_level)
        self.print_log(f'Eval Classifier Epoch: {epoch + 1}')
        self.model['train'].eval()
        self.model['test'].eval()
        self.logs = torch.zeros([16])

        loader = self.dataset['test']
        loss_values = []
        accuracy_values = []
        iou_values = []
        iou_table = np.zeros(shape=(len(self.shape_names), 3))
        with torch.no_grad():
            for batch_id, (x, norm, labels, target) in enumerate(loader):
                x = x.float().to(self.output_dev)
                labels = labels.float().to(self.output_dev)
                target = target.long().to(self.output_dev)

                # forward
                features = self.model['test'](x)
                pred = self.model['train'](features, labels)

                # statistic
                iou_table, iou = self._compute_cat_iou(pred, target, iou_table)
                iou_values += iou
                pred = pred.contiguous().view(-1, self.num_parts)
                target = target.view(-1, 1)[:, 0]
                loss = self.loss(pred, target)
                loss_values.append(loss.item())
                pred_indices = pred.data.max(1)[1]
                corrected = pred_indices.eq(target.data).cpu().sum()
                batch_size, _, num_points = x.size()
                accuracy_values.append(
                    corrected.item() / (batch_size * num_points))
                if (batch_id + 1) % 50 == 0:
                    self.print_log(
                        'Batch({}/{}) done. Loss: {:.4f}'.format(
                            batch_id + 1, len(loader), loss.item()
                        ))

        print(loss.values)
        mean_loss = np.mean(loss_values)
        mean_accuracy = np.mean(accuracy_values)
        mean_iou = np.mean(iou_values)
        self.print_log('Mean testing loss: {:.4f}.'.format(mean_loss))
        self.print_log('Mean accuracy: {:.2f}%.'.format(mean_accuracy * 100.0))
        self.print_log('Mean IoU: {:.2f}%.'.format(mean_iou * 100.0))
        return mean_iou

    @staticmethod
    def _compute_cat_iou(pred, target, iou_table):
        iou_list = []
        target = target.cpu().data.numpy()
        for j in range(pred.size(0)):
            batch_pred = pred[j]
            batch_target = target[j]
            batch_choice = batch_pred.data.max(1)[1].cpu().data.numpy()
            for cat in np.unique(batch_target):
                intersection = np.sum(
                    np.logical_and(batch_choice == cat, batch_target == cat)
                )
                union = np.sum(
                    np.logical_or(batch_choice == cat, batch_target == cat)
                )
                iou = intersection / float(union) if union != 0 else 1.0
                iou_table[cat, 0] += iou
                iou_table[cat, 1] += 1
                iou_list.append(iou)
        return iou_table, iou_list

def main():
    runner = SegmentationRunner()
    runner.run()

if __name__ == '__main__':
    main()


"""create user table

Revision ID: 726f693370c6
Revises:
Create Date: 2021-01-14 19:13:38.354021

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '726f693370c6'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    op.create_table(
        'users',
        sa.Column('id', sa.Integer, primary_key=True, index=True),
        sa.Column('spotify_id', sa.String, unique=True, index=True),
        sa.Column('is_active', sa.Boolean, default=True),
        sa.Column('name', sa.String(255)),
        sa.Column('image_url', sa.String),
        sa.Column('refresh_token', sa.String),
        sa.Column('access_token', sa.String),
        sa.Column('token_expires', sa.Integer)
    )

def downgrade():
    op.drop_table('users')


from django.shortcuts import render_to_response
from django.core.context_processors import csrf
from django.template import RequestContext # For CSRF
from django.forms.formsets import formset_factory, BaseFormSet
from django.http import HttpResponse, HttpResponseRedirect

from dynamicform.todo.forms import *

def index(request):
    # This class is used to make empty formset forms required
    # See http://stackoverflow.com/questions/2406537/django-formsets-make-first-required/4951032#4951032
    class RequiredFormSet(BaseFormSet):
        def __init__(self, *args, **kwargs):
            super(RequiredFormSet, self).__init__(*args, **kwargs)
            for form in self.forms:
                form.empty_permitted = False

    TodoItemFormSet = formset_factory(TodoItemForm, max_num=10, formset=RequiredFormSet)

    if request.method == 'POST': # If the form has been submitted...
        todo_list_form = TodoListForm(request.POST) # A form bound to the POST data
        # Create a formset from the submitted data
        todo_item_formset = TodoItemFormSet(request.POST, request.FILES)

        if todo_list_form.is_valid() and todo_item_formset.is_valid():
            todo_list = todo_list_form.save()
            for form in todo_item_formset.forms:
                todo_item = form.save(commit=False)
                todo_item.list = todo_list
                todo_item.save()

            return HttpResponseRedirect('thanks') # Redirect to a 'success' page
    else:
        todo_list_form = TodoListForm()
        todo_item_formset = TodoItemFormSet()

    # For CSRF protection
    # See http://docs.djangoproject.com/en/dev/ref/contrib/csrf/
    c = {'todo_list_form': todo_list_form,
         'todo_item_formset': todo_item_formset,
        }
    c.update(csrf(request))

    return render_to_response('todo/index.html', c)


import socket, pickle, argparse
import asyncio

async def client(args):
    host = args.host
    port = int(args.port)
    reader, writer = await asyncio.open_connection(
        host, port)

    while True:
        msg1 = input("> ")
        msg1 = pickle.dumps(msg1)
        writer.write(msg1)
        await writer.drain()

        msg2 = await reader.read(1024)
        msg2 = pickle.loads(msg2)

        if pickle.loads(msg1).lower() == "exit":
            print(f"Server: {msg2}")

            writer.close()
            await writer.wait_closed()
            exit()

        if msg2["error"] == "":
            print("OK\n", msg2["output"])
        else:
            print("ERROR\n", msg2["error"])

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-hs", '--host', required=True, help='host del sever')
    parser.add_argument("-p", '--port', required=True, help='Puerto de conexion')
    args = parser.parse_args()
    asyncio.run(client(args))


from collections import Counter

def solution(s):
    answer = []
    word = s.replace("{","").replace("}", "").split(",")
    count_word = Counter(word).most_common()
    for i in count_word:
        answer.append(int(i[0]))
    return answer

def solution2(s:str):
    word_list = list(map(int, s.replace("}", "").replace("{", "").split(",")))
    count_dict = Counter(word_list).most_common()
    answer = [item[0] for item in count_dict]
    return answer

if __name__ == '__main__':

    print(solution2("{{2},{2,1},{2,1,3},{2,1,3,4}}"))
    print(solution2("{{1,2,3},{2,1},{1,2,4,3},{2}}"))
    print(solution2("{{20,111},{111}}"))
    print(solution2("{{123}}"))
    print(solution2("{{4,2,3},{3},{2,3,4,1},{2,3}}"))

# print(solution(s))


from typing import *
from dataclasses import dataclass

from verification.src.controller import *
from verification.settings.content import *
from verification.settings.panels_inline import *
from verification.src.sender import *
from verification.src.error_handlers import *
from telebot import types

@dataclass
class Position:
    current: Optional[Status] = None
    panel: Optional[Callable] = None
    transitions: Optional[Dict[str, Status]] = None
    generate_text: Optional[Callable] = None
    handle_error: Optional[Callable] = None

POSITIONS = {
    Status.READY: Position(
        current=Status.READY,
        panel=lambda sample: info_panel,
        transitions={
            ESTIMATE: Status.IN_PROGRESS_FLUENCY_SOURCE,
        },
        generate_text=lambda controller, user: INSTRUCTIONS,
        handle_error=lambda user, sample, correction: save_sql_correct(user, sample, correction)
    ),
    Status.IN_PROGRESS_FLUENCY_SOURCE: Position(
        current=Status.IN_PROGRESS_FLUENCY_SOURCE,
        panel=lambda sample: fluency_source_panel,
        transitions={
            CALL_OK: Status.IN_PROGRESS_FLUENCY_SUBSTITUTION,
            CALL_WRONG: Status.ERROR_DESCRIBING_FLUENCY_SOURCE,
            CALL_SKIP: Status.IN_PROGRESS_FLUENCY_SOURCE,
            CALL_DB: Status.CHOOSING_TABLE,
            CALL_INFO: Status.INFO_READING
        },
        generate_text=lambda controller, user: generate_fluency_source_msg(controller, user),
        handle_error=lambda user, sample, correction: sample
    ),
    Status.IN_PROGRESS_FLUENCY_SUBSTITUTION: Position(
        current=Status.IN_PROGRESS_FLUENCY_SUBSTITUTION,
        panel=lambda sample: fluency_substitution_panel,
        transitions={
            CALL_OK: Status.IN_PROGRESS_EQUIVALENT,
            CALL_WRONG: Status.ERROR_DESCRIBING_FLUENCY_SUBSTITUTION,
            CALL_SKIP: Status.IN_PROGRESS_FLUENCY_SOURCE,
            CALL_DB: Status.CHOOSING_TABLE,
            CALL_INFO: Status.INFO_READING
        },
        generate_text=lambda controller, user: generate_fluency_substitution_msg(controller, user),
        handle_error=lambda user, sample, correction: save_fluency_source_correct(user, sample, correction)
    ),
    Status.IN_PROGRESS_EQUIVALENT: Position(
        current=Status.IN_PROGRESS_EQUIVALENT,
        panel=lambda sample: equivalent_panel,
        transitions={
            CALL_OK: Status.IN_PROGRESS_SQL,
            CALL_WRONG: Status.ERROR_DESCRIBING_EQUIVALENT,
            CALL_SKIP: Status.IN_PROGRESS_FLUENCY_SOURCE,
            CALL_DB: Status.CHOOSING_TABLE,
            CALL_INFO: Status.INFO_READING
        },
        generate_text=lambda controller, user: generate_equivalent_msg(controller, user),
        handle_error=lambda user, sample, correction: save_fluency_substitution_correct(user, sample, correction)
    ),
    Status.IN_PROGRESS_SQL: Position(
        current=Status.IN_PROGRESS_SQL,
        panel=lambda sample: sql_panel,
        transitions={
            CALL_OK: Status.IN_PROGRESS_FLUENCY_SOURCE,
            CALL_WRONG: Status.ERROR_DESCRIBING_SQL,
            CALL_SKIP: Status.IN_PROGRESS_FLUENCY_SOURCE,
            CALL_DB: Status.CHOOSING_TABLE,
            CALL_INFO: Status.INFO_READING
        },
        generate_text=lambda controller, user: generate_sql_msg(controller, user),
        handle_error=lambda user, sample, correction: save_equivalent_correct(user, sample, correction)
    ),
    Status.ERROR_DESCRIBING_FLUENCY_SOURCE: Position(
        current=Status.ERROR_DESCRIBING_FLUENCY_SOURCE,
        panel=lambda sample: error_fluency_source_panel,
        transitions={
            CALL_SKIP: Status.IN_PROGRESS_FLUENCY_SOURCE,
            CALL_INFO: Status.INFO_READING,
            TEXT_TYPED: Status.IN_PROGRESS_FLUENCY_SUBSTITUTION
        },
        generate_text=lambda controller, user: generate_error_fluency_source_msg(controller, user),
        handle_error=lambda user, sample, correction: save_fluency_source_error(user, sample, correction)
    ),
    Status.ERROR_DESCRIBING_FLUENCY_SUBSTITUTION: Position(
        current=Status.ERROR_DESCRIBING_FLUENCY_SUBSTITUTION,
        panel=lambda sample: error_fluency_substitution_panel,
        transitions={
            CALL_SKIP: Status.IN_PROGRESS_FLUENCY_SOURCE,
            CALL_INFO: Status.INFO_READING,
            TEXT_TYPED: Status.IN_PROGRESS_EQUIVALENT
        },
        generate_text=lambda controller, user: generate_error_fluency_substitution_msg(controller, user),
        handle_error=lambda user, sample, correction: save_fluency_substitution_error(user, sample, correction)
    ),
    Status.ERROR_DESCRIBING_EQUIVALENT: Position(
        current=Status.ERROR_DESCRIBING_EQUIVALENT,
        panel=lambda sample: error_equivalent_panel,
        transitions={
            CALL_SKIP: Status.IN_PROGRESS_FLUENCY_SOURCE,
            CALL_INFO: Status.INFO_READING,
            TEXT_TYPED: Status.IN_PROGRESS_SQL
        },
        generate_text=lambda controller, user: generate_error_equivalent_msg(controller, user),
        handle_error=lambda user, sample, correction: save_equivalent_error(user, sample, correction)
    ),
    Status.ERROR_DESCRIBING_SQL: Position(
        current=Status.ERROR_DESCRIBING_SQL,
        panel=lambda sample: error_sql_panel,
        transitions={
            CALL_SKIP: Status.IN_PROGRESS_FLUENCY_SOURCE,
            CALL_INFO: Status.INFO_READING,
            TEXT_TYPED: Status.IN_PROGRESS_FLUENCY_SOURCE
        },
        generate_text=lambda controller, user: generate_error_sql_msg(controller, user),
        handle_error=lambda user, sample, correction: save_sql_error(user, sample, correction)
    ),
    Status.INFO_READING: Position(
        current=Status.INFO_READING,
        panel=lambda sample: in_progress_info_panel,
        transitions={
            RETURN: Status.LAST,
        },
        generate_text=lambda controller, user: (INSTRUCTIONS, None),
        handle_error=lambda user, sample, correction: sample
    ),
    Status.CHOOSING_TABLE: Position(
        current=Status.CHOOSING_TABLE,
        panel=lambda sample: generate_tables_panel(sample),
        transitions={
            RETURN: Status.LAST,
            TABLE: Status.VIEW_TABLE
        },
        generate_text=lambda controller, user: generate_choosing_table(controller, user),
        handle_error=lambda user, sample, correction: sample
    ),
    Status.VIEW_TABLE: Position(
        current=Status.VIEW_TABLE,
        panel=lambda sample: view_panel,
        transitions={
            RETURN_TO_TABLES: Status.CHOOSING_TABLE,
            RETURN: Status.LAST
        },
        generate_text=lambda controller, user: generate_view_table_msg(controller, user),
        handle_error=lambda user, sample, correction: sample
    )
}

TEXT_STATE_MAP = {
    # Status.ERROR_DESCRIBING_FLUENCY: Status.IN_PROGRESS_EQUIVALENT,
    # Status.ERROR_DESCRIBING_EQUIVALENT: Status.IN_PROGRESS_SQL,
    # Status.ERROR_DESCRIBING_SQL: Status.ERROR_DESCRIBING_EQUIVALENT
}


import sys
assert sys.version_info >= (3, 5) # make sure we have Python 3
import datetime
from pyspark.sql import SparkSession, functions, types
spark = SparkSession.builder.appName('tmax model tester').getOrCreate()
assert spark.version >= '2.3' # make sure we have Spark 2.3+
spark.sparkContext.setLogLevel('WARN')
from pyspark.ml import Pipeline
from pyspark.ml import PipelineModel
from pyspark.ml.evaluation import RegressionEvaluator
from pyspark.ml.feature import SQLTransformer, VectorAssembler
from pyspark.ml.evaluation import RegressionEvaluator
from pyspark.ml.tuning import TrainValidationSplit, ParamGridBuilder
from pyspark.ml.regression import GBTRegressor
from pyspark.sql.types import DateType as dt
tmax_schema = types.StructType([
    types.StructField('station', types.StringType()),
    types.StructField('date', types.DateType()),
    types.StructField('latitude', types.FloatType()),
    types.StructField('longitude', types.FloatType()),
    types.StructField('elevation', types.FloatType()),
    types.StructField('tmax', types.FloatType()),
])

def main(out_model):
    #new_d=datetime.date(2018-11-12)
    new_d=datetime.date(2018, 11, 12)
    new_d2=datetime.date(2018, 11, 13)
    tom_df=spark.createDataFrame([('RSM00031235', new_d,49.2771,-122.9146,330.0,12.0),\
        ('RSM00031235', new_d2,49.2771,-122.9146,330.0,13.0)],\
        tmax_schema)

    tom_df.show()
    model = PipelineModel.load(out_model)

    predictions = model.transform(tom_df)
    predictions.show()

    result=predictions.select("prediction").collect()
    prediction=result[0]['prediction']
    print('Predicted tmax tomorrow:',prediction)

if __name__ == '__main__':
    out_model=sys.argv[1]
    main(out_model)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 20 18:22:58 2021

@author: sguan
"""
import os
import numpy as np
import pandas as pd
import skimage
import skimage.io
from skimage.measure import label, regionprops
import large_image
import openslide
import matplotlib.pyplot as plt
from sys import argv
import json
import scipy.sparse
import cv2

def process_svs(file, fpath, annotation_path, output_path, over_lap=1):
    main_tag = file.replace('.svs', '')
    main_tag = main_tag.replace(' ', '_')
    ts = large_image.getTileSource(fpath)

    # Create tile folder
    tile_folder = output_path + main_tag
    if not os.path.exists(tile_folder):
        os.makedirs(tile_folder)

    # Load annotation
    anno_fpath = annotation_path + file.replace('.svs', '.json')
    mask_fpath = annotation_path + file.replace('.svs', '.npz')
    f = open(anno_fpath)
    anno = json.loads(f.read())
    mask = scipy.sparse.load_npz(mask_fpath)
    mask = mask.toarray()

    # Remove tissue components
    for ii in anno['not_synovium']:
        mask[mask==ii] = 0

    # Calculate allowable bounding box overlap
    ol = np.floor(over_lap * 1024)

    # Calculate region properties
    props = regionprops(mask)

    bbox = []
    magnification = 20
    for pp in props:
        bb = np.asarray(pp.bbox) * magnification/anno['magnification']
        bb = bb - np.asarray([ol, ol, -ol, -ol])
        bbox.append(bb)

    tile_iterator = ts.tileIterator(
        scale=dict(magnification=magnification),
        tile_size=dict(width=1024, height=1024),
        tile_overlap=dict(x=0, y=0),
        format=large_image.tilesource.TILE_FORMAT_NUMPY)

    for tile_info in tile_iterator:

        col = tile_info['level_x']
        row = tile_info['level_y']

        col_wsi_start = col * tile_info['width']
        row_wsi_start = row * tile_info['height']
        col_wsi_end = (col+1) * tile_info['width']
        row_wsi_end = (row+1) * tile_info['height']

        keep = False

        for bb in bbox:
            if (row_wsi_start > bb[0] and
                row_wsi_end < bb[2] and
                col_wsi_start > bb[1] and
                col_wsi_end < bb[3]):
                keep = True

        if (keep):
            tile = np.array(tile_info['tile'])[:,:,:3]
            plt.imshow(tile)

            save_name = main_tag + '__R' + str(row) + '_C'+ str(col) + '_TS' + str(tile_size) + '_OL' + str(overlap)+'.jpg'
            output_path = tile_folder + '/' + save_name
            skimage.io.imsave(output_path, tile, quality=100)

def stringFilter(list1, list2):
    return [n for n in list1 if
             any(m in n for m in list2)]

## Define slide location and tile parameters
#file = argv[1]
#output_path = './tiles/'
#file_path = '../OA_slides/'
file = '004_SYN_207643.svs'
output_path = '../../data/processed/OA_tiles/'
file_path = '../../data/raw/OA/'
annotation_path = '../../data/processed/OA_annotations/'
lowres_path = '../../data/processed/OA_low_res/'

tile_size=1024
overlap=0
tissue_thresh=0.01

## Perform tiling
fpath = file_path + file
file_type = file[-3:]

try:
    process_svs(file, fpath, annotation_path, output_path)
except Exception as e:
    print('error: ' + file)
    print(e)

## Create image thumbnail
main_tag = file.replace('.svs', '')
main_tag = main_tag.replace(' ', '_')
tile_folder = output_path + main_tag

images = os.listdir(tile_folder)
images = stringFilter(images, ['.jpg'])

for ii, fname in enumerate(images):
    params = fname.split('__')[1].split('_')

    row = int(params[0][1:])
    col = int(params[1][1:])

    if ii == 0:
        min_row = row
        max_row = row
        min_col = col
        max_col = col

    if (row < min_row):
        min_row = row

    if (row > max_row):
        max_row = row

    if (col < min_col):
        min_col = col

    if (col > max_col):
        max_col = col

for row in range(min_row, max_row):
    for col in range(min_col, max_col):
        fname = main_tag + '__R' + str(row) + '_C'+ str(col) + '_TS' + str(tile_size) + '_OL' + str(overlap)+'.jpg'
        fpath = tile_folder + '/' + fname
        try:
            im_orig = skimage.io.imread(fpath)
            im = cv2.resize(im_orig, (256,256))
        except:
            im = np.zeros([256,256,3])
        if col==min_col:
            image_col = im
        else:
            image_col = np.concatenate([image_col, im], axis=1)

    if row==min_row:
        image = image_col
    else:
        image = np.concatenate([image, image_col], axis=0)

# Create thumbnail folder
if not os.path.exists(lowres_path):
    os.makedirs(lowres_path)

output_path = lowres_path + main_tag + '.jpg'

skimage.io.imsave(output_path, image, quality=100)


import datetime
import json
import logging
import os.path
import sys
import time
import uuid
import vesna.alh
import vesna.alh.spectrumsensor
import vesna.alh.signalgenerator
import vesna.alh.common

log = logging.getLogger(__name__)

def isstring(v):
	if sys.version_info[0] >= 3:
		return isinstance(v, str)
	else:
		return isinstance(v, str) or isinstance(v, unicode)

def force_list(v):
	if v is None:
		return []
	elif isstring(v):
		return [ v ]
	else:
		return v

class CDFError(Exception): pass

class CDFInterfererProgram:
	def __init__(self, center_hz, power_dbm, start_time, end_time, device_id=None, config_id=None):
		self.center_hz = center_hz
		self.power_dbm = power_dbm
		self.device_id = device_id
		self.config_id = config_id

		assert end_time > start_time

		self.start_time = start_time
		self.end_time = end_time

class CDFInterferer:
	def __init__(self, device):
		self.device = device
		self.programs = []

	def add_program(self, program):
		self.programs.append(program)

class CDFDevice:
	def __init__(self, base_url, cluster_id, addr):
		self.base_url = base_url
		self.cluster_id = cluster_id
		self.addr = addr

	def key(self):
		return (self.base_url, self.cluster_id, self.addr)

	def __str__(self):
		return "<CDFDevice base_url=%s cluster_id=%d addr=%d>" % (
				self.base_url, self.cluster_id, self.addr)

class CDFAuthor:
	def __init__(self, name, email, address=None, phone=None, institution=None):
		self.name = name
		self.email = email
		self.address = force_list(address)
		self.phone = force_list(phone)
		self.institution = force_list(institution)

class CDFDocument:
	def __init__(self, description=None, bibtex=None):
		self.description = force_list(description)
		self.bibtex = force_list(bibtex)

class CDFExperimentIteration:
	def __init__(self, slot_id=10):
		self.slot_id = slot_id

		self.sensors = []
		self.interferers = []

		self.start_time = None
		self.end_time = None

		self.tracefiles = []

class CDFExperimentSensor:
	def __init__(self, sensor):
		self.sensor = sensor

class CDFExperimentInterferer:
	def __init__(self, generator):
		self.generator = generator

		self.program_list = []

class CDFExperiment:
	def __init__(self, title, summary, related_experiments, notes, methodology=None,
			tag=None, release_date=None, authors=None, documentation=None, devices=None,
			interferers=None):

		self.title = title

		if tag is not None:
			self.tag = tag
		else:
			self.tag = "vesna-alh-tools-" + str(uuid.uuid4())

		self.authors = force_list(authors)

		self.release_date = release_date

		self.summary = summary

		self.methodology = force_list(methodology)

		self.documentation = force_list(documentation)

		self.related_experiments = related_experiments

		self.notes = force_list(notes)

		self.start_hz = None
		self.stop_hz = None
		self.step_hz = None

		self.duration = None

		self.devices = force_list(devices)
		self.interferers = force_list(interferers)

		self.iterations = []

	def set_frequency_range(self, start_hz, stop_hz, step_hz):
		self.start_hz = start_hz
		self.stop_hz = stop_hz
		self.step_hz = step_hz

	def set_duration(self, duration):
		self.duration = duration

	def add_author(self, author):
		self.authors.append(author)

	def add_document(self, document):
		self.documentation.append(document)

	def add_device(self, device):
		self.devices.append(device)

	def add_interferer(self, interferer):
		self.interferers.append(interferer)

	def iter_all_devices(self):
		for device in self.devices:
			yield device

		for interferer in self.interferers:
			yield interferer.device

	def _get_coordinators(self):
		coordinators = {}

		for device in self.iter_all_devices():
			args = (device.base_url, device.cluster_id)
			if args not in coordinators:
				coordinator = vesna.alh.ALHWeb(*args)

				coordinator.post("prog/firstCall", "1")

				coordinators[args] = coordinator

		return coordinators

	def _get_nodes(self):
		coordinators = self._get_coordinators()

		nodes = {}

		for device in self.iter_all_devices():
			if device.key() not in nodes:
				coordinator = coordinators[device.base_url, device.cluster_id]

				node = vesna.alh.ALHProxy(coordinator, device.addr)

				node.post("prog/firstCall", "1")

				nodes[device.key()] = node
			else:
				raise CDFError("Device %s used more than once" % device)

		return nodes

	def run(self, iteration):
		sensors = iteration.sensors
		interferers = iteration.interferers

		nodes = self._get_nodes()

		start_time = time.time() + 5.0 * len(nodes)
		end_time = start_time + self.duration

		iteration.start_time = datetime.datetime.fromtimestamp(start_time)
		iteration.end_time = datetime.datetime.fromtimestamp(end_time)

		for device in self.devices:
			node = nodes[device.key()]

			sensor = CDFExperimentSensor(vesna.alh.spectrumsensor.SpectrumSensor(node))

			config_list = sensor.sensor.get_config_list()

			sweep_config = config_list.get_sweep_config(
					start_hz=self.start_hz,
					stop_hz=self.stop_hz,
					step_hz=self.step_hz)

			if sweep_config is None:
				raise CDFError("Device %s cannot scan desired frequency range" % device)

			sensor.program = vesna.alh.spectrumsensor.SpectrumSensorProgram(
					sweep_config,
					start_time,
					end_time - start_time,
					slot_id=iteration.slot_id)

			sensors.append(sensor)

		for interferer in self.interferers:
			node = nodes[interferer.device.key()]

			einterferer = CDFExperimentInterferer(
					vesna.alh.signalgenerator.SignalGenerator(node))

			config_list = einterferer.generator.get_config_list()

			for program in interferer.programs:
				tx_config = config_list.get_tx_config(
						f_hz=program.center_hz,
						power_dbm=program.power_dbm)

				if tx_config is None:
					raise CDFError("Device %s cannot transmit at desired "
							"frequency range" % interferer.device)

				einterferer.program_list.append(
						vesna.alh.signalgenerator.SignalGeneratorProgram(
								tx_config,
								start_time + program.start_time,
								program.end_time - program.start_time))

			interferers.append(einterferer)

		for sensor in sensors:
			sensor.sensor.program(sensor.program)
		for interferer in interferers:
			interferer.generator.program_list(interferer.program_list)

		for sensor in sensors:
			while not sensor.sensor.is_complete(sensor.program):
				log.info("waiting")
				time.sleep(2)

				if time.time() > (end_time + 30):
					raise Exception("Something went wrong")

			log.info("experiment is finished. retrieving data.")

			sensor.result = sensor.sensor.retrieve(sensor.program)

		self.iterations.append(iteration)


#!/usr/bin/python3

import time
import queue
import pygame
import socket
import random
import threading
from pygame.locals import *
from random import randrange

host = socket.gethostname()
myip = socket.gethostbyname(host)
port = 16161          # porta para estabelecer conexão servidor/ seu valor muda durante execução (servidor manda porta as quais mensagens referentes ao jogo devem ser madadas)
port_to_play = 16661  # porta que recebe mensagens do servidor
s_ip = '192.168.0.12' # ip do servidor

### essa parte nao recebe nem envia mensagens
synchronized_queue    = queue.Queue(100) # fila bloqueante
vel_dificul           = 1
explodir_nave	      = False
collided	      = False # responsavel por parar o jogo
collided_2            = False # responsavel por parar o jogo
pontos		      = 0 #VARIAVEL PONTUACAO
pontuacaototal	      = 0 #VARIAVEL DE CONTROLE DE NIVEIS
tick_musica	      = 0 #VARIAVEL CONTROLADORA DE MUSICA
spawn_de_asteroides   = 5 #SPAWN DE ASTEROIDES DE ACORDO COM PONTUAÇÃO TOTAL
asteroides	      = [] #LISTA DE ASTEROIDES
background_filename   = './Assets/galaxy2.png'

# SD --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

# Conecta ao servidor
#	cliente envia o seu ip (provavelment não prescisa)
#	cliente recebe a porta que ele deve enviar as mensagens para o servidor (posição da nave)
def connect_to_server():
	global port

	s = socket.socket() # Socket para estabelecer conexão com servidor

	s.connect((s_ip, port))
	s.send(myip.encode())
	data = s.recv(1024)
	port = int(data.decode())
	s.close()

# Thread que recebe mensagens do servidor
#	coloca as mensagens em uma fila bloqueantes
# 	Uso de protocolo UDP
#	para quando receber do servidor mensagem que alguem colidiu
def receive_messages():
	global host
	global g_list
	global collided
	global collided_2
	global port_to_play

	sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) # Socket para receber dados do jogo
	print("Bind in ",(host,port_to_play))
	sock.bind((host,port_to_play))

	while True:
		data, addr = sock.recvfrom(4096)
		d_list = data.decode().split(";")

		synchronized_queue.put({
			'asteroide' : int(d_list[0]),
			'nave1'     : float(d_list[1]),
			'nave2'     : float(d_list[2]),
			'colidiu_p1': d_list[3],
			'colidiu_p2': d_list[4],
		})

		print("I received : ",data.decode())
		if d_list[3] == 'T' or d_list[4] == 'T':
			break

	print("1 - Im dead")
	sock.close()

# Thread de envio de mensagens
#	Função deve ser chamada a cada milesimo de segundo
#	Pega a posicao da nave e a envia em conjunto com collided (que indica se a nave colidiu ou não)
#	Envia estado periodicamente
#	para quando alguma das nave colide
def send_message():
	global collided
	global collided_2

	sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)

	while True:
		data = str(nave['posicao'][0])

		if collided :
			print("True - T")
			data = data + ";T"
		else :
			print("False - F")
			data = data + ";F"

		print("I sent : ",data)
		sent = sock.sendto(data.encode(),(s_ip,port))

		if collided or collided_2:
			print("2 - I'm dead")
			sent = sock.sendto(data.encode(),(host,port))
			print("I sent : ",data)
			sent = sock.sendto(data.encode(),(host,port))
			print("I sent : ",data)
			sent = sock.sendto(data.encode(),(host,port))
			print("I sent : ",data)
			sock.close()
			break

		time.sleep(.1)

# Thread responsavel por atualizar variaveis
#	Le fila bloqueante
#	Cria meteoro se necessário
#	Atualiza posição da nave e variavel collided_2
def atualiza_variaveis():
	global nave2
	global asteroides
	global collided
	global collided_2

	print("Time to att")
	while True :
		try:
			newstate = synchronized_queue.get(timeout=.5)  ## retira  a mensagem da fila

			# Cria asteroide
			if newstate['asteroide'] > 0:
				asteroidesNewPosition = newstate['asteroide']
				asteroides.append(create_asteroide(vel_dificul,asteroidesNewPosition))

			# 'Move' nave 2
			nave2['posicao'][0] = newstate['nave2']
			if newstate['colidiu_p2'] == 'T': ## Servidor vai mandar mensagem para os clientes de modo que o cliente sempre sera a nave1
				collided_2 = True
			print("Nave 2 : ", nave2['posicao'][0]," [",collided_2,"]")

			synchronized_queue.task_done() # sinaliza que a mensagem retirada foi tratada
		except Exception as e:
			print("Exception : ",e)
			pass

		if collided or collided_2:
			break

	print("3 - I'm dead")

# Game ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

def render_scene():
	global screen
	global score_font
	global asteroides
	global pontuacaototal
	global dificuldade_font

	screen.blit(background, (0, 0))
	textopontos = score_font.render('PONTUAÇÃO:'+str(pontos)+' ',1 ,(250,250,250))

	for asteroide in asteroides: #BLIT DOS ASTEROIDES NA TELA
		screen.blit(asteroide['tela'], asteroide['posicao'])

	if collided == False and collided_2 == False :
		screen.blit(textopontos, (0,0)) #TXT DO SCORE
	else:
		screen.blit(textopontos, (450, 300)) #TXT DO SCORE QUANDO ACABA O JOGO

	if (pontuacaototal<=9999): #CORES DOS LETREIROS DE DIFICULDADE
		dif1 = dificuldade_font.render('Dificuldade: Iniciante', 1, (255, 255, 255))
		screen.blit(dif1, (0, 22))
	elif (pontuacaototal<=20000):
		dif1 = dificuldade_font.render('Dificuldade: Amador', 1, (30,144,255))
		screen.blit(dif1, (0, 22))
	elif (pontuacaototal<=30000):
		dif1 = dificuldade_font.render('Dificuldade: Intermediário ', 1, (255,255,0))
		screen.blit(dif1, (0, 22))
	elif (pontuacaototal<=40000):
		dif1 = dificuldade_font.render('Dificuldade: Profissional', 1, (255,0,255))
		screen.blit(dif1, (0, 22))
	elif (pontuacaototal<=60000):
		dif1 = dificuldade_font.render('Dificuldade: Star Wars', 1, (255,0,0))
		screen.blit(dif1, (0, 22))

	screen.blit(nave['tela'], nave['posicao'])
	screen.blit(nave2['tela'],nave2['posicao'])

def raise_difficulty():
	global vel_dificul
	global pontuacaototal

	if(pontuacaototal >= 1000):
		vel_dificul = 1
	if (pontuacaototal >= 10000):
		vel_dificul = 2
	if (pontuacaototal >= 20000):
		vel_dificul = 3
	if (pontuacaototal >= 30000):
		vel_dificul = 4
	if (pontuacaototal >= 40000):
		vel_dificul = 5
	if (pontuacaototal >= 60000):
		vel_dificul = 6

def get_rect(obj):
	return Rect(obj['posicao'][0],obj['posicao'][1],obj['tela'].get_width(),obj['tela'].get_height())

def nave_collided():
	nave_rect = get_rect(nave)
	for asteroide in asteroides:
		if nave_rect.colliderect(get_rect(asteroide)):
			return True

	return False

def block_ship():
	global nave

	#POSIÇÃO PARA BARRAR A NAVE
	if(nave['posicao'][0] > 1150):
		nave['posicao'][0] = 1150
	if(nave['posicao'][0] < 0):
		nave['posicao'][0] = 0

def mov_ship():
	global nave

	if pygame.key.get_pressed()[K_a] :
		nave['posicao'][0] += -7

	elif pygame.key.get_pressed()[K_d] :
		nave['posicao'][0] +=  7

	block_ship()

def create_asteroide(vel_dificul,asteroidesNewPosition):
	global asteroidesNewPosition

	return {
		'tela'      : pygame.image.load('./Assets/asteroide1.png').convert_alpha(),
		'posicao'   : [asteroidesNewPosition, -64],                      # Coloca a posição passada pelo servidor
		'velocidade': vel_dificul
	}

def mover_asteroides():
	for asteroide in asteroides:
		asteroide['posicao'][1] += 8

# Main -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

# estabelece conexão
connect_to_server()

# prepara inicialização do jogo
pygame.init()
pygame.font.init()

explosion_sound  = pygame.mixer.Sound('./Assets/boom.wav')
musica_fundo	 = pygame.mixer.Sound('./Assets/boom.wav')

pygame.display.set_caption('Space Invaders')

font_name	 = pygame.font.get_default_font()
game_font	 = pygame.font.SysFont(font_name, 72)
score_font	 = pygame.font.SysFont(font_name, 35)
level_font	 = pygame.font.SysFont(font_name, 40)
dificuldade_font = pygame.font.SysFont(font_name, 30)
screen           = pygame.display.set_mode((1200, 700))
background       = pygame.image.load(background_filename).convert()

nave  = {
	'tela': pygame.image.load('./Assets/nave.png').convert_alpha(),
	'posicao': [1200/2, 700 - 60],
	'velocidade': {
		'x': 0,
	}
}

nave2 =  {
	'tela': pygame.image.load('./Assets/nave.png').convert_alpha(),
	'posicao': [1200/2, 700 - 60],
	'velocidade': {
		'x': 0,
	}
}

# cria as threads de enviar/receber/tratar mensagens
while True:
	try:
		r_mesg = threading.Thread(target=receive_messages)
		s_mesg = threading.Thread(target=send_message)
		att    = threading.Thread(target=atualiza_variaveis)
		r_mesg.start()
		s_mesg.start()
		att.start()
		break
	except:
		print("Error")
		pass

# jogo começa
while True:
	ini = time.process_time()

	# fechar o jogo caso clicar no 'x'
	for event in pygame.event.get():
		if event.type == QUIT:
			exit()

	render_scene()
	raise_difficulty()
	mover_asteroides()
	mov_ship()

	# se houve colisão jogo termina
	collided = nave_collided()
	if collided or collided_2:
		print("Collided : ",collided)
		r_mesg.join()
		s_mesg.join()
		att.join()
		break

	pontuacaototal += 1 #PONTUAÇÃO SENDO ADICIONADA DENTRO DE UMA VARIAVEL, ANINHADA COM WHILE
	pontos         += 1

	pygame.display.update()

	# Tenta fazer o jogo rodar a 60 FPS
	end = time.process_time()
	idle = 0.0167 - (end - ini)
	if idle > 0:
		time.sleep(idle)

# tela de vitoria/derrota
while True :
	for event in pygame.event.get():
		if event.type == QUIT:
			exit()

	render_scene()

	if not explodir_nave: #CONDIÇÃO DO SISTEMA SONORO CASO A NAVE EXPLODA
		musica_fundo.stop()
		explodir_nave = True
		explosion_sound.play()
		nave['posicao'][0] += nave['velocidade']['x']

		screen.blit(nave['tela'], nave['posicao'])
	else:
		if collided :
			text = game_font.render('VOCE PERDEU!!', 1, (255, 0, 0))
			screen.blit(text, (450, 350)) #TXT DO GAME OVER APOS O JOGO
		else :
			text = game_font.render('VOCE GANHOU!!', 1, (0, 255, 0))
			screen.blit(text, (450, 350)) #TXT DO GAME OVER APOS O JOGO

	pygame.display.update()


# -*- coding: utf-8 -*-
"""
Created on Wed May 19 19:11:10 2021

@author: cjove
"""
import matplotlib.pyplot as plt
import seaborn as sn
from pandas.plotting import scatter_matrix

names_emg_step = steps_emg_ordered.columns
names_imu_step = steps_imu_ordered.columns

shape_emg_step = steps_emg_ordered.shape
shape_imu_step = steps_imu_ordered.shape

steps_emg_ordered.info()
steps_imu_ordered.info()

"""Descripción del dataset"""

description_emg_step = steps_emg_ordered.describe()
description_imu_step = steps_imu_ordered.describe()
description_y_step = y_step.describe()

"""Correlaciones"""

corrMatrix_imu_step = steps_imu_ordered.corr()
corrMatrix_emg_step = steps_emg_ordered.corr()

"""Heatmaps"""
sn.heatmap(corrMatrix_imu_step, annot=False)
plt.show()

sn.heatmap(corrMatrix_emg_step, annot=False)
plt.show()

"""Examinamos la distribución de las variables"""

"""Boxplot"""
length_emg = np.arange(0,shape_emg_step[1],10)
length_imu = np.arange(0,shape_imu_step[1],10)

for i in length_emg:
    plt.boxplot(steps_emg_ordered[names_emg_step[i:i+10]], labels=names_emg_step[i:i+10])
    plt.show()
    plt.violinplot(steps_emg_ordered[names_emg_step[i:i+10]])
    plt.show()

for i in length_imu:
    plt.boxplot(steps_imu_ordered[names_imu_step[i:i+10]], labels=names_imu_step[i:i+10])
    plt.show()
    plt.violinplot(steps_imu_ordered[names_imu_step[i:i+10]])
    plt.show()

"""Exploramos el efecto de escalar los datos"""

scaler = StandardScaler()
scaler.fit(steps_imu_ordered)
steps_imu_scaled = scaler.transform(steps_imu_ordered)
steps_imu_scaled = pd.DataFrame(steps_imu_scaled, columns = names_imu_step)
scaler.fit(steps_emg_ordered)
steps_emg_scaled = scaler.transform(steps_emg_ordered)
steps_emg_scaled = pd.DataFrame(steps_emg_scaled, columns = names_emg_step)

for i in length_emg:
    plt.boxplot(steps_emg_scaled[names_emg_step[i:i+10]], labels=names_emg_step[i:i+10])
    plt.show()
    plt.violinplot(steps_emg_scaled[names_emg_step[i:i+10]])
    plt.show()

for i in length_imu:
    plt.boxplot(steps_imu_scaled[names_imu_step[i:i+10]], labels=names_imu_step[i:i+10])
    plt.show()
    plt.violinplot(steps_imu_scaled[names_imu_step[i:i+10]])
    plt.show()

"""
Fuente:
https://machinelearningmastery.com/model-based-outlier-detection-and-removal-in-python/
"""
#1. Isolation Forest

from sklearn.ensemble import IsolationForest

iso = IsolationForest(contamination=0.11)
yhat_emg = iso.fit_predict(steps_emg_scaled)
yhat_imu = iso.fit_predict(steps_imu_scaled)

mask_emg = yhat_emg != -1
mask_imu = yhat_imu != -1

print(yhat_emg[mask_emg].shape, yhat_emg.shape)
print(yhat_imu[mask_imu].shape, yhat_imu.shape)

#2.Minimum Covariance Determinant

from sklearn.covariance import EllipticEnvelope

ee = EllipticEnvelope(contamination=0.01)

yhat_emg_ee = ee.fit_predict(steps_emg_scaled)
yhat_imu_ee = ee.fit_predict(steps_imu_scaled)

mask_emg_ee = yhat_emg_ee != -1
mask_imu_ee = yhat_imu_ee != -1

print(yhat_emg_ee[mask_emg_ee].shape, yhat_emg_ee.shape)
print(yhat_imu_ee[mask_imu_ee].shape, yhat_imu_ee.shape)

#3.OneClassSVM

from sklearn.svm import OneClassSVM

oc = OneClassSVM(nu=0.01)
yhat_emg_oc = oc.fit_predict(steps_emg_scaled)
yhat_imu_oc = oc.fit_predict(steps_imu_scaled)

mask_emg_oc = yhat_emg_oc != -1
mask_imu_oc = yhat_imu_oc != -1

print(yhat_emg_ee[mask_emg_oc].shape, yhat_emg_oc.shape)
print(yhat_imu_ee[mask_imu_oc].shape, yhat_imu_oc.shape)

list_values = np.arange(0.01,1,0.1)

isof_step = []
ee_step = []
oc_step = []

for i in list_values:
    iso = IsolationForest(contamination= i)
    yhat_emg = iso.fit_predict(steps_emg_scaled)
    yhat_imu = iso.fit_predict(steps_imu_scaled)

    mask_emg = yhat_emg != -1
    mask_imu = yhat_imu != -1

    emg = yhat_emg[mask_emg].shape
    imu = yhat_imu[mask_imu].shape

    isof_step.append({'EMG':emg,'IMU':imu})

    ee = EllipticEnvelope(contamination=i)

    yhat_emg_ee = ee.fit_predict(steps_emg_scaled)
    yhat_imu_ee = ee.fit_predict(steps_imu_scaled)

    mask_emg_ee = yhat_emg_ee != -1
    mask_imu_ee = yhat_imu_ee != -1

    emg = yhat_emg_ee[mask_emg_ee].shape
    imu = yhat_imu_ee[mask_imu_ee].shape

    ee_step.append({'EMG':emg,'IMU':imu})

    oc = OneClassSVM(nu=i)
    yhat_emg_oc = oc.fit_predict(steps_emg_scaled)
    yhat_imu_oc = oc.fit_predict(steps_imu_scaled)

    mask_emg_oc = yhat_emg_oc != -1
    mask_imu_oc = yhat_imu_oc != -1

    emg = yhat_emg_ee[mask_emg_oc].shape
    imu = yhat_imu_ee[mask_imu_oc].shape

    oc_step.append({'EMG':emg,'IMU':imu})

#out_iso_imu_step = out_iso_imu_step.drop(['IsoForest'], axis=1)
#out_iso_emg_step = out_iso_emg_step.drop(['IsoForest'], axis=1)

import pandas as pd
outliers_step = pd.DataFrame({
    'Isolation Forest': isof_step,
    'Minimum Covariance Determinant': ee_step,
    'OneClassSVM': oc_step}, index = list_values
    )

"""
Generación del dataset sin outliers
"""
steps_imu_ordered_out = steps_imu_ordered.copy()
steps_emg_ordered_out = steps_emg_ordered.copy()

steps_emg_ordered_out['IsoForest'] = yhat_emg
steps_imu_ordered_out['IsoForest'] = yhat_emg
y_step_out = y_step.copy()
y_step_out['IsoForest'] = yhat_emg

out_iso_imu_step = steps_imu_ordered_out.loc[steps_imu_ordered_out['IsoForest'] ==1]
out_iso_emg_step = steps_emg_ordered_out.loc[steps_emg_ordered_out['IsoForest'] ==1]
y_step_out = y_step_out.loc[y_step_out['IsoForest'] ==1]
out_iso_imu_step = out_iso_imu_step.drop(['IsoForest'], axis=1)
out_iso_emg_step = out_iso_emg_step.drop(['IsoForest'], axis=1)
y_step_out = y_step_out.drop(['IsoForest'],axis =1)


#!/usr/bin/env python
# coding: utf-8

# # Forecasting exercise for SARIMA
# Feb 28th 2022

# In[1]:

import sys
print(sys.executable)
import numpy as np
import pandas as pd
import os
print(os.getcwd())
from IPython.display import display
import seaborn as sns
import matplotlib.pyplot as plt
import statsmodels.api as sm
from statsmodels.tsa.seasonal import seasonal_decompose
from statsmodels.graphics.tsaplots import plot_pacf, plot_acf
from statsmodels.tsa.statespace.sarimax import SARIMAX
from sklearn.metrics import mean_absolute_percentage_error as mape
import pmdarima as pm #help us to select p,d,q
import warnings
warnings.filterwarnings('ignore')

# In[2]:

# Custom functions:
import src.colorsetup
from src.timeseriesFunctions import plot_time_series, get_years, dickey_fuller_test, mape, cross_validate

# ## Input data

# In[3]:

data_path = 'https://vincentarelbundock.github.io/Rdatasets/csv/datasets/sunspot.year.csv'
data= pd.read_csv(data_path, usecols= ['time', 'value'], index_col= 'time', parse_dates= ['time'])
data

# In[4]:

plot_time_series(data.index, data.value, title= "Yearly Sunpots", ylabel= "Sunspots")

# In[5]:

get_years(data)

# In[6]:

cutoff_1= [str(i) for i in range(1700,1800)]
cutoff_2= [str(i) for i in range(1800,1900)]
cutoff_3= [str(i) for i in range(1900,1988)]

print(np.std(data.loc[cutoff_1]['value']))
print(np.std(data.loc[cutoff_2]['value']))
print(np.std(data.loc[cutoff_3]['value']))

# In[7]:

# Return the natural logarithm of one plus the input array, element-wise
data['log_transformation']= np.log1p(data.value)
print(np.std(data.loc[cutoff_1]['log_transformation']))
print(np.std(data.loc[cutoff_2]['log_transformation']))
print(np.std(data.loc[cutoff_3]['log_transformation']))

# In[8]:

plot_time_series(data.index, data.log_transformation, title= "Yearly Sunpots", ylabel= "log(Sunspots)")

# In[9]:

seasonal_plot= seasonal_decompose(data.log_transformation, period= 11).plot()

# In[10]:

acf= plot_acf(data.log_transformation, zero= False)

# In[11]:

pacf= plot_pacf(data.log_transformation, zero= False)

# Differencing

# In[12]:

data['lag_11']= data.log_transformation.shift(11)
data['seasonal_diff'] = data.log_transformation - data['lag_11']

# In[ ]:

# data.head(12)

# In[13]:

dickey_fuller_test(data['seasonal_diff'].dropna())

# In[14]:

acf_seasonal_diff= plot_acf(data['seasonal_diff'].dropna(), zero= False)
pacf_seasonal_diff= plot_pacf(data['seasonal_diff'].dropna(), zero= False)

# ## Start with a very simple model

# In[15]:

start_model= SARIMAX(data.log_transformation, order= (0,0,0), seasonal_order= (0,1,0,12), trend= 'c').fit()

# In[17]:

pacf_residuals= plot_pacf(start_model.resid[start_model.loglikelihood_burn:], zero= False)

# In[18]:

auto_model= pm.auto_arima(data.log_transformation,
                          start_p= 0, max_p= 3,
                          d= 0,
                          start_q= 0, max_q= 3,
                          m= 11,
                          seasonal= True,
                          start_P= 0,
                          D= 1,
                          trace= True, error_action= 'ignore', suppress_warnings= True, stepwise= True)

# In[19]:

print('Order: {} Seasonal order: {}'.format(auto_model.order, auto_model.seasonal_order))

# In[ ]:

eda_model= SARIMAX(data.log_transformation, order= (2,0,0), seasonal_order= (2,1,0,11), trend= 'c').fit()

# In[21]:

auto_diag= eda_model.plot_diagnostics(figsize= (18,8))

# In[ ]:

warnings.filterwarnings("ignore")
series = data['log_transformation']
horizon = 3
start = int(len(data.value)*.75) # 75% for training_set
step_size = 1
order = auto_model.order
seasonal_order = auto_model.seasonal_order

log_cv1 = cross_validate(series,horizon,start,step_size,
                    order = order,
                    seasonal_order = seasonal_order)

# In[23]:

exp_cv1= np.expm1(log_cv1)

# In[24]:

display(data.tail())
exp_cv1.tail()

# In[33]:

image_folder= "../../../plots/time_series_analysis/"
plot_name= "sarima.forecast.png"

print("MAPE: {:.4f}".format(mape(exp_cv1)))
exp_cv1.actual.plot(color= "black")
exp_cv1.forecast.plot(color= "orange")

plt.title("SARIMA model with MAPE: {:.4f}".format(mape(exp_cv1)))
plt.ylabel("Time series")
plt.xlabel("Time (Years)")
plt.legend()
plt.grid(alpha= 0.4)
plt.savefig(image_folder+plot_name, transparent= True)
plt.show()

# In[34]:

# Zoom in
exp_cv1['actual'].loc['1970':'1988'].plot(color= "black")
exp_cv1['forecast'].loc['1970':'1988'].plot(color= "orange")

plt.legend()
plt.show()

# In[35]:

import session_info
session_info.show()

# In[ ]:


from django.conf import settings
from celery_tasks.main import celery_app
# 导入Django内置发送邮件函数
from django.core.mail import send_mail

@celery_app.task(name='send_verify_email')
def send_verify_email(email, verify_url):
    # 设置邮件内容
    html_message = '
尊敬的用户您好！
' \
                   '
感谢您使用美多商城。
' \
                   '
您的邮箱为：%s 。请点击此链接激活您的邮箱：
' \
                   '
%s
' % (email, verify_url, verify_url)
    # 调用django.core.mail种的send_mail方法进行邮件发送
    send_mail(subject='注册激活', message='', from_email=settings.EMAIL_FROM, recipient_list=[email],
              html_message=html_message)


import numpy as np
import cv2
import time
c = cv2.VideoCapture(0)
time.sleep(3)
bg = 0

for i in range(60):

    ret, bg = c.read()

bg = np.flip(bg,axis=1)
while(c.isOpened()):

    ret, img = c.read()
    if not ret:
        break
    img = np.flip(img,axis=1)
    hsv = cv2.cvtColor(img,cv2.COLOR_BGR2HSV)

    lower_red = np.array([0,120,70])
    upper_red = np.array([10,255,255])

    m1 = cv2.inRange(hsv, lower_red,upper_red)

    lower_red = np.array([170,120,70])
    upper_red =  np.array([180,255,255])
    m2 = cv2.inRange(hsv,lower_red,upper_red)

    m1 = m1 +m2

    m1 = cv2.morphologyEx(m1,cv2.MORPH_OPEN,np.ones((3,3),np.uint8), iterations = 2)
    m1 = cv2.morphologyEx(m1, cv2.MORPH_DILATE,np.ones((3,3),np.uint8), iterations = 1)

    m2 =cv2.bitwise_not(m1)

    r1 = cv2.bitwise_and(bg,bg,mask=m1)
    r2 = cv2.bitwise_and(img,img,mask=m2)
    final_output = cv2.addWeighted(r1,1,r2,1,0)
    cv2.imshow('INVISIBLE BOT',final_output)
    k = cv2.waitKey(10)
    if k==27:
        break
c.release()
Gcv2.destroyAllWindows()


from PIL import Image
import os.path
from django.core.exceptions import ValidationError

def resolution_checker(image):
    '''check resolution of uploaded images, reject if too small or too big'''
    im = Image.open(image)
    width, height = im.size
    size_in_bytes = os.stat(image).st_size
    if width < 1200 or height < 800:
        return False
    if size_in_bytes > 2_621_400:
        return False
    return True

def file_size(value):
    '''check file size, this is important because of model field'''
    # file has to be less than 2.5MB, else
    # if gets handled by the TempFileUploader.
    # This code does not work with the TempFileUploader.
    limit = 2621400
    if value.size > limit:
        raise ValidationError('File too large. Size should not exceed 2 MiB.')

def image_resizer(image):
    '''image resizer utility to generate thumbnails'''
    # also creates the path to save thumbnails
    size = (480, 480)
    thumbnail_path = os.path.splitext(image)[0] + '_thumbnail'
    with Image.open(image) as im:
        im.thumbnail(size)
        im.save(thumbnail_path, 'PNG')
    return thumbnail_path


import matplotlib.pyplot as plt

from dival.util.plot import plot_image
from dival.measure import PSNR, SSIM
from dival import DataPairs

from dliplib.utils import Params
from dliplib.reconstructors.dip import DeepImagePriorReconstructor
from dliplib.utils.helper import load_standard_dataset, set_use_latex

set_use_latex()

# load data
dataset = load_standard_dataset('ellipses')
test_data = dataset.get_data_pairs('validation', 5)

index = 3
obs, gt = test_data[index]
test_data = DataPairs([obs], [gt], name='test: %d' % index)

# forward operator
ray_trafo = dataset.ray_trafo
params = Params.load('ellipses_dip')

results = []

# Architecture-hyper-parameters
channels = [1, 8, 32, 64, 128, 128, 128, 128]
scales = [5, 5, 5, 5, 5, 4, 3, 2]
iters = [5000] * 8

# Compute reconstructions with each different architectures
for ch, sc, it in zip(channels, scales, iters):
    print('Channels: %d, Scales: %d' % (ch, sc))
    params.channels = (ch,) * sc
    params.skip_channels = (0,) * sc
    params.scales = sc
    params.iterations = it
    reconstructor = DeepImagePriorReconstructor(ray_trafo=ray_trafo, hyper_params=params.dict, name='DIP')
    result = reconstructor.reconstruct(obs)
    results.append(result)

ch = 128
sc = 5
skip_channels = [[0, 0, 0, 0, 4],
                 [0, 0, 0, 4, 4],
                 [0, 0, 4, 4, 4],
                 [0, 4, 4, 4, 4]]

for skip in skip_channels:
    print('Skip Channels:')
    print(skip)
    params.channels = (ch,) * sc
    params.skip_channels = skip
    params.scales = sc
    reconstructor = DeepImagePriorReconstructor(ray_trafo=ray_trafo, hyper_params=params.dict, name='DIP')
    result = reconstructor.reconstruct(obs)
    results.append(result)

fig = plt.figure(figsize=(9.1, 8.3))
for i in range(len(results)):
    ax = fig.add_subplot(3, 4, i+1)
    psnr = PSNR(results[i], gt)
    ssim = SSIM(results[i], gt)
    plot_image(results[i], ax=ax, xticks=[], yticks=[], cmap='pink')
    if i < 8:
        ax.set_title('Channels: %d, Scales: %d' % (channels[i], scales[i]))
    else:
        ax.set_title('Skip: {}'.format(skip_channels[i-8]))
    ax.set_xlabel('PSNR: %.2f, SSIM: %.4f' % (psnr, ssim))

plt.tight_layout()
plt.tight_layout()

plt.savefig('ellipses_architectures.pdf')
plt.savefig('ellipses_architectures.pgf')
plt.show()


import argparse
import os
import random
import sys

import numpy as np
import torch

from unityagents import UnityEnvironment

import DDPG
import mDDPG
import OurDDPG
import TD3
import utils

class UnityWrapper:
    def __init__(self, env, train_mode=True):
        self.env = env
        self.brain_name = env.brain_names[0]
        self.train_mode = train_mode

        brain = env.brains[self.brain_name]
        action_size = brain.vector_action_space_size

        self.action_space = action_size
        self.observation_space = brain.vector_observation_space_size
        self.num_agents = len(self.reset())

        self.action_space_low = -1
        self.action_space_high = 1
        self._max_episode_steps = 1000
        self.num_agents = 1

    def reset(self, train_mode=None):
        if train_mode is None:
            train_mode = self.train_mode
        env_info = self.env.reset(train_mode=train_mode)[self.brain_name]
        return env_info.vector_observations[0]

    def step(self, actions):
        env_info = self.env.step(vector_action=actions)[self.brain_name]
        rewards = env_info.rewards  # get reward (for each agent)
        dones = [1 if x else 0 for x in env_info.local_done]  # see if episode finished
        next_states = env_info.vector_observations  # get next state (for each agent)
        return next_states[0], rewards[0], dones[0], None

    def sample(self):
        return [random.uniform(self.action_space_low, self.action_space_high) for _ in range(self.action_space)]

    def close(self):
        return self.env.close()

    def seed(self, seed):
        pass

def executable():
    if sys.platform == "linux":
        return "Reacher_Linux_NoVis/Reacher.x86_64"
    if sys.platform == "darwin":
        return "Reacher.app"

# Runs policy for X episodes and returns average reward
def evaluate_policy(env, policy, eval_episodes=10):
    rewards = []
    for _ in range(eval_episodes):
        episode_reward = 0
        obs = env.reset()
        done = False
        while not done:
            action = policy.select_action(np.array(obs))
            obs, reward, done, _ = env.step(action)
            episode_reward += reward
        rewards.append(episode_reward)

    rewards = np.array(rewards)
    print("---------------------------------------")
    print("Evaluation over %d episodes: %f" % (eval_episodes, rewards.mean()))
    print("---------------------------------------")
    return rewards

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--policy_name", default="TD3")  # Policy name
    parser.add_argument("--env_name", default="Reacher-v2")
    parser.add_argument("--seed", default=0, type=int)  # Sets Gym, PyTorch and Numpy seeds
    parser.add_argument("--start_timesteps", default=1e3, type=int)  # How many time steps purely random policy is run for
    parser.add_argument("--eval_freq", default=5e3, type=float)  # How often (time steps) we evaluate
    parser.add_argument("--max_timesteps", default=1e6, type=float)  # Max time steps to run environment for
    parser.add_argument("--save_models", action="store_true")  # Whether or not models are saved
    parser.add_argument("--expl_noise", default=0.1, type=float)  # Std of Gaussian exploration noise
    parser.add_argument("--batch_size", default=100, type=int)  # Batch size for both actor and critic
    parser.add_argument("--discount", default=0.99, type=float)  # Discount factor
    parser.add_argument("--tau", default=0.005, type=float)  # Target network update rate
    parser.add_argument("--policy_noise", default=0.2, type=float)  # Noise added to target policy during critic update
    parser.add_argument("--noise_clip", default=0.5, type=float)  # Range to clip target policy noise
    parser.add_argument("--policy_freq", default=2, type=int)  # Frequency of delayed policy updates
    args = parser.parse_args()

    file_name = "%s_%s_%s" % (args.policy_name, args.env_name, str(args.seed))
    print("---------------------------------------")
    print("Settings: %s" % (file_name))
    print("---------------------------------------")

    for dirname in ("./results", "./rewards"):
        os.makedirs(dirname, exist_ok=True)
    if args.save_models:
        os.makedirs("./pytorch_models", exist_ok=True)

    unity = UnityEnvironment(file_name=executable(), no_graphics=True)
    env = UnityWrapper(unity, train_mode=True)

    # Set seeds
    env.seed(args.seed)
    torch.manual_seed(args.seed)
    np.random.seed(args.seed)

    state_dim = env.observation_space
    action_dim = env.action_space
    max_action = 1

    # Initialize policy
    if args.policy_name == "TD3":
        policy = TD3.TD3(state_dim, action_dim, max_action)
    elif args.policy_name == "OurDDPG":
        policy = OurDDPG.DDPG(state_dim, action_dim, max_action)
    elif args.policy_name == "DDPG":
        policy = DDPG.DDPG(state_dim, action_dim, max_action)
    elif args.policy_name == "mDDPG":
        policy = mDDPG.DDPG(state_dim, action_dim, max_action)

    replay_buffer = utils.ReplayBuffer()

    # Evaluate untrained policy
    evaluations = [evaluate_policy(env, policy).mean()]

    total_timesteps = 0
    timesteps_since_eval = 0
    episode_num = 0
    episode_reward = 0
    episode_timesteps = 0
    done = True
    rewards = []

    while total_timesteps < args.max_timesteps:
        if done:
            if total_timesteps != 0:
                print(
                    ("Total T: %d Episode Num: %d Episode T: %d Reward: %f")
                    % (total_timesteps, episode_num, episode_timesteps, episode_reward)
                )
                rewards.append(episode_reward)
                if args.policy_name == "TD3":
                    policy.train(
                        replay_buffer,
                        episode_timesteps,
                        args.batch_size,
                        args.discount,
                        args.tau,
                        args.policy_noise,
                        args.noise_clip,
                        args.policy_freq,
                    )
                else:
                    policy.train(replay_buffer, episode_timesteps, args.batch_size, args.discount, args.tau)

            # Evaluate episode
            if timesteps_since_eval >= args.eval_freq:
                timesteps_since_eval %= args.eval_freq
                evaluations.append(evaluate_policy(env, policy).mean())

                if args.save_models:
                    policy.save(file_name, directory="./pytorch_models")
                np.save("./results/%s" % (file_name), evaluations)

            # Reset environment
            obs = env.reset()
            done = False
            episode_reward = 0
            episode_timesteps = 0
            episode_num += 1

        # Select action randomly or according to policy
        if total_timesteps < args.start_timesteps:
            action = env.sample()
        else:
            action = policy.select_action(np.array(obs))
            if args.expl_noise != 0:
                action = (action + np.random.normal(0, args.expl_noise, size=env.action_space)).clip(
                    env.action_space_low, env.action_space_high
                )

        # Perform action
        new_obs, reward, done, _ = env.step(action)
        done_bool = 0 if episode_timesteps + 1 == env._max_episode_steps else float(done)
        episode_reward += reward

        # Store data in replay buffer
        replay_buffer.add((obs, new_obs, action, reward, done_bool))

        obs = new_obs

        episode_timesteps += 1
        total_timesteps += 1
        timesteps_since_eval += 1

    # Final evaluation
    evaluations.append(evaluate_policy(env, policy, 100).mean())
    if args.save_models:
        policy.save("%s" % (file_name), directory="./pytorch_models")
    np.save("./results/%s" % (file_name), evaluations)
    np.save("./rewards/%s" % (file_name), rewards)

if __name__ == "__main__":
    main()


from xml.etree.ElementTree import Element, SubElement, ElementTree

class CharacterElement:
    def __init__ (self, player_name, image_path,x,y):
        self.player_name       = player_name
        self.image_path        = image_path
        self.x                 = x
        self.y                 = y

    # def __init__ (self):
    #     self.player_name    = 'player'
    #     self.image_path     = 'testCharacter_1_image'

    def make_character_snippet (self):
        e_character_root = Element("character")
        e_character_root.set('location.name','table')
        e_character_root.set('location.x',self.x)
        e_character_root.set('location.y',self.y)
        e_character_root.set('posZ','0')
        e_character_root.set('rotate','0')
        e_character_root.set('roll','0')
        e_character_root.append(self._make_character_element())
        e_character_root.append(self._make_chat_palette())
        return e_character_root

    def _make_character_element (self):
        e_character = Element("data")
        e_character.set('name','character')

        e_image = SubElement(e_character, "data")
        e_image.set('name','image')
        e_image_path = SubElement(e_image,'data')
        e_image_path.set('type','image')
        e_image_path.set('name','imageIdentifier')
        e_image_path.text = self.image_path

        e_common = SubElement(e_character, "data")
        e_common.set('name','common')
        e_player_name = SubElement(e_common, "data")
        e_player_name.set('name','name')
        e_player_name.text = self.player_name
        e_size = SubElement(e_common, "data")
        e_size.set('name','size')
        e_size.text = '1'

        e_detail = SubElement(e_character, "data")
        e_detail.set('name','detail')
        e_resource = SubElement(e_detail, "data")
        e_resource.set('name','resource')
        e_wealth = SubElement(e_resource, "data")
        e_wealth.set('type','numberResource')
        e_wealth.set('currentValue','0')
        e_wealth.set('name','wealth')
        e_wealth.text = '16'

        e_authority = SubElement(e_resource, "data")
        e_authority.set('type','numberResource')
        e_authority.set('currentValue','0')
        e_authority.set('name','authority')
        e_authority.text = '16'

        e_information = SubElement(e_detail, "data")
        e_information.set('name','information')
        e_explanation = SubElement(e_information, "data")
        e_explanation.set('type','note')
        e_explanation.set('name','explanation')

        e_memo = SubElement(e_information, "data")
        e_memo.set('type','note')
        e_memo.set('name','memo')

        return e_character

    def _make_chat_palette (self):
        e_chat_palette = Element("chat-palette")
        e_chat_palette.set('dicebot','')
        return e_chat_palette


from flask import request, jsonify
from mysqlAccess import mysqlConnection, mysqlCloseConnection, mysqlQuery, mysqlInsert, mysqlUpdate
from configApp import app

@app.route('/equipas', methods=['GET'])
def equipas():
    accessPoints = ["all", "att", "describe", "insert", "remove"]
    return jsonify(accessPoints)

@app.route('/equipas/all', methods=['GET'])
def equipasAll():
    conn = mysqlConnection()
    query = "SELECT * FROM equipas;"
    records = mysqlQuery(conn, query)
    mysqlCloseConnection(conn)
    return jsonify(records)

@app.route('/equipas/att', methods=['GET'])
def equipasAtt():
    conn = mysqlConnection()
    cursor = conn.cursor()
    cursor.execute(f'SELECT * FROM equipas;')
    column_names = [i[0] for i in cursor.description]
    data = cursor.fetchall()
    rows = []
    for row in data:
        row_dict = {}
        for i in range(len(column_names)):
            row_dict[column_names[i]] = row[i]
        rows.append(row_dict)
    mysqlCloseConnection(conn)
    return jsonify(rows)

@app.route('/equipas/describe', methods=['GET'])
def equipasDescribe():
    if 'idEquipa' not in request.args:
        return "Error: No id field provided. Please specify an id."

    idEquipa = request.args['idEquipa']
    conn = mysqlConnection()
    query = "SELECT * FROM equipas where idEquipa = '"+idEquipa+"';"
    records = mysqlQuery(conn, query)
    mysqlCloseConnection(conn)
    return jsonify(records)

@app.route('/equipas/insert', methods=['GET'])
def equipasInsert():
    if 'idEquipa' not in request.args:
        return "Error: No id field provided. Please specify an id."

    idEquipa = request.args['idEquipa']
    nome = request.args['nome']
    numMembros = request.args['numMembros']

    conn = mysqlConnection()
    query = "INSERT INTO equipas ( idEquipa, nome, numMembros ) VALUES( '"+idEquipa+"', '"+nome+"' , '"+numMembros+"' );"
    print(query)
    records = mysqlInsert(conn, query)
    mysqlCloseConnection(conn)
    print(records)
    return jsonify(records)

@app.route('/equipas/remove', methods=['GET'])
def equipasRemove():
    if 'idEquipa' not in request.args:
        return "Error: No id field provided. Please specify an id."

    idEquipa = request.args['idEquipa']

    conn = mysqlConnection()
    query = "DELETE FROM equipas WHERE idEquipa='"+idEquipa+"';"
    records = mysqlUpdate(conn, query)
    mysqlCloseConnection(conn)
    return jsonify(records)

@app.route('/equipas/update', methods=['GET'])
def equipasUpdate():
    if 'idEquipa' not in request.args:
        return "Error: No id field provided. Please specify an id."

    conn = mysqlConnection()
    idEquipa = request.args['idEquipa']

    if 'nome' not in request.args:
        query = "SELECT nome FROM equipas WHERE idEquipa='"+idEquipa+"';"
        records = mysqlQuery(conn, query)
        nome = str(records[0][0])
    else:
        nome = request.args['nome']

    if 'numMembros' not in request.args:
        query = "SELECT numMembros FROM equipas WHERE idEquipa='"+idEquipa+"';"
        records = mysqlQuery(conn, query)
        numMembros = str(records[0][0])
    else:
        numMembros = request.args['numMembros']

    query = "UPDATE equipas SET nome='"+nome+"', numMembros='"+numMembros+"' WHERE idEquipa='"+idEquipa+"';"

    records = mysqlUpdate(conn, query)
    mysqlCloseConnection(conn)
    return jsonify(records)


from django import forms
from django.contrib.auth import get_user_model

from orders.models import UserAddress

class GuestCheckoutForm(forms.Form):
    email = forms.EmailField()
    email2 = forms.EmailField(label="Verify email")

    def clean(self):
        email = self.cleaned_data.get('email')
        email2 = self.cleaned_data.get('email2')

        if email == email2:
            user = get_user_model()

            # Checking the mail already exist in the user database or not
            user_exist = user.objects.filter(email=email).count()
            if user_exist != 0:
                raise forms.ValidationError("This user already exist. Please login instead.")
        else:
            raise forms.ValidationError("Please confirm emails are the same.")

class AddressSelectForm(forms.Form):
    billing_address = forms.ModelChoiceField(
            queryset=UserAddress.objects.filter(type="billing"),
            widget=forms.RadioSelect,
            empty_label=None
            )
    shipping_address = forms.ModelChoiceField(
            queryset=UserAddress.objects.filter(type="shipping"),
            widget=forms.RadioSelect,
            empty_label=None
            )

# Address create form
class AddressCreateForm(forms.ModelForm):
    class Meta:
        model = UserAddress
        fields = ["street", "city", "state", "zipcode", "type"]


# -*- coding: utf-8 -*-

import os
from twisted.internet import task, reactor
from scrapy.crawler import CrawlerRunner
from scrapy.utils.project import get_project_settings
from scrapy.utils.log import configure_logging

configure_logging()

def run_crawl():
    runner = CrawlerRunner(get_project_settings())

    if os.environ.get('NAROU_ID', '') != "" and os.environ.get('NAROU_PW', '') != "":
        runner.crawl('narou')
    if os.environ.get('HAMELN_ID', '') != "" and os.environ.get('HAMELN_PW', '') != "":
        runner.crawl('hameln')
    if os.environ.get('KAKUYOMU_ID', '') != "":
        runner.crawl('kakuyomu')

def _main():
    interval = int(os.environ.get('INTERVAL_MINUTES', '30')) * 60

    instance = task.LoopingCall(run_crawl)
    instance.start(interval)

    reactor.run()

if __name__ == '__main__':
    _main()


#!/usr/bin/env python3

"""Tools to compare new molecules to the chemical space in the NP Atlas DB"""

import pandas as pd
from pathlib import Path

from rdkit import Chem
from rdkit.Chem import AllChem
from rdkit import DataStructs

def generate_fingerprint(smiles):
    """Calculate molecular fingerprint using Morgan fingerprint radius 2"""

    mol = Chem.MolFromSmiles(smiles)
    return AllChem.GetMorganFingerprint(mol, 2)

def calculate_similarity(atlas_fp, input_fp):
    """Calculate similarity score between two rdkit fingerprints using Dice similarity scoring"""

    return round(DataStructs.DiceSimilarity(atlas_fp, input_fp), 2)

if __name__ == "__main__":

    atlas_tsv_path = Path("/Users/roger/Git/rgl_utilities/data/NP_Atlas/NPAtlas_2022_09.tsv")
    new_compound_name = "Abyssomicin 2"
    new_compound_smiles = "C[C@H]1CCC(=O)C2=C3O[C@@]4(C)[C@@H](C)C[C@]3(OC2=O)[C@@H](/C=C/C1=O)[C@@H]4O"
    new_compound_fingerprint = generate_fingerprint(new_compound_smiles)

    input_df = pd.read_csv(atlas_tsv_path, delimiter='\t')
    input_df["molecule_fingerprint"] = input_df["compound_smiles"].map(generate_fingerprint)
    input_df["Dice_similarity_score"] = input_df["molecule_fingerprint"].apply(calculate_similarity,
                                                                               args=(new_compound_fingerprint, ))
    input_df = input_df.drop(["molecule_fingerprint"], axis=1)

    output_path = Path(atlas_tsv_path.parent, new_compound_name + "_atlas_structure_similarity.tsv")
    input_df.to_csv(output_path, index=False, sep='\t', line_terminator='\n')


import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import DBSCAN
from sklearn.ensemble import IsolationForest

def manage_outliers(data, method='StandardScaler', threshold=3, strategy='median'):
    """
    ---------------------------------------------------------------
    Goal:
    Detects and manages outliers in a dataset using various methods.
    ---------------------------------------------------------------
    Parameters:
    -----------
    data : pandas.DataFrame or numpy.ndarray
        The dataset to manage outliers for.
    method : str, optional (default='StandardScaler')
        The method to use for outlier detection. Can be one of:
            - 'StandardScaler': Uses the StandardScaler method to scale the data and
               then uses the z-score method based on the standard deviation.
            - 'DBSCAN': Uses the DBSCAN clustering algorithm to identify outliers.
            - 'IsolationForest': Uses the Isolation Forest algorithm to identify outliers.
    threshold : float, optional (default=3)
        The threshold used to determine outliers.
        Values above or below this threshold are considered outliers.
    strategy : str, optional (default='median')
        The strategy used to manage outliers. Can be one of:
            - 'median': Replaces outliers with the median value of the feature.
            - 'mean': Replaces outliers with the mean value of the feature.
    ----------------------------------------------------------------------
    Returns:
    --------
    pandas.DataFrame
        The cleaned dataset with outliers managed.
    """
    # Scaling the data if method is StandardScaler
    if method == 'StandardScaler':
        scaler = StandardScaler()
        data = scaler.fit_transform(data)

    # Detecting outliers using the selected method
    if method == 'StandardScaler':
        z_scores = np.abs(data)
        outliers = np.where(z_scores > threshold)
    elif method == 'DBSCAN':
        dbscan = DBSCAN(eps=threshold, min_samples=2)
        outliers = dbscan.fit_predict(data) == -1
    elif method == 'IsolationForest':
        iso_forest = IsolationForest(contamination=threshold)
        outliers = iso_forest.fit_predict(data) == -1

    # Managing outliers using the selected strategy
    if strategy == 'median':
        replacements = np.median(data, axis=0)
    elif strategy == 'mean':
        replacements = np.mean(data, axis=0)

    data[outliers] = replacements

    # Returning the cleaned data
    if method == 'StandardScaler':
        data = scaler.inverse_transform(data)
    return pd.DataFrame(data)

if __name__ == "__main__":

    # Creating a simple dataframe with some outliers
    df = pd.DataFrame({
        'A': [1, 2, 3, 4, 5, 100],
        'B': [2, 4, 6, 8, 10, 200],
        'C': [5, 10, 15, 20, 25, 1000]
    })

    # Using the manage_outliers() function to clean the dataframe
    clean_df = manage_outliers(df, method='IsolationForest', threshold=0.05, strategy='mean')

    # Printing the original dataframe and the cleaned dataframe
    print("Original Dataframe:\n", df)
    print("\nCleaned Dataframe:\n", clean_df)


import os
import subprocess
from setuptools import setup, find_packages

version = "master"
filename = "master"

setup(name="xdemo_client",

      version=filename,

      description="A generic and configurable software launcher",

      long_description="A generic and configurable software launcher",

      author="Florian Lier",

      author_email="flier[at]techfak.uni-bielefeld.de",

      url="https://github.com/warp1337/xdemo_client",

      download_url="https://github.com/warp1337/xdemo_client",

      packages=find_packages(exclude=["*.unittests", "*.unittests.*", "unittests.*", "unittests"]),

      scripts=['bin/xdemo_client', 'bin/xdemo_master'],

      include_package_data=True,

      keywords=['Test',
                'System Testing',
                'Component Based Testing',
                'Software Tests',
                'Automated System Execution'
                'Execution Engine',
                'Experiment Execution'],

      license="LGPLv3",

      classifiers=['Development Status :: Beta',
                   'Environment :: Console',
                   'Environment :: Robotics',
                   'Intended Audience :: Developers',
                   'License :: OSI Approved :: GNU Library or ' +
                   'Lesser General Public License (LGPL)',
                   'Operating System :: OS Independent',
                   'Programming Language :: Python',
                   'Topic :: Text Processing :: Markup :: XML'],

      install_requires=['colorlog==2.10.0',
                        'psutil==5.0.0',
                        'paramiko==2.1.2',
                        'sshtunnel==0.1.2'],

      # Workaround for: http://bugs.python.org/issue856103
      zip_safe=False

      )

# Make bin/scripts executable
if os.name.lower() == "linux":
    subprocess.call(["chmod -R ugo+x bin"], shell=True)


import os

import numpy as np
import tensorflow as tf

import mnist
from utils import dense_to_one_hot, sigmaprime, sigma

data_dir = 'data'
cifar10_path = os.path.join(data_dir, 'cifar10')
mnist_path = os.path.join(data_dir, 'mnist')

# check for existence
os.path.exists(data_dir)

# To stop potential randomness
seed = 128
rng = np.random.RandomState(seed)

# number of neurons in each layer
input_num_units = 28*28
hidden_num_units = 300
output_num_units = 10

# define placeholders
x = tf.placeholder(tf.float32, [None, input_num_units])
y = tf.placeholder(tf.float32, [None, output_num_units])

# set remaining variables
epochs = 5
batch_size = 128
learning_rate = 0.01

weights = {
    'hidden': tf.Variable(tf.random_normal([input_num_units, hidden_num_units], seed=seed)),
    'output': tf.Variable(tf.random_normal([hidden_num_units, output_num_units], seed=seed))
}

biases = {
    'hidden': tf.Variable(tf.random_normal([hidden_num_units], seed=seed)),
    'output': tf.Variable(tf.random_normal([output_num_units], seed=seed))
}

fully_connected_layer1 = tf.add(tf.matmul(x, weights['hidden']), biases['hidden'])
fully_connected_layer1_tanh = sigma(fully_connected_layer1)
# fully_connected_layer1_tanh = tf.nn.tanh(fully_connected_layer1)
# fully_connected_layer1_tanh = fully_connected_layer1

fully_connected_layer2 = tf.add(tf.matmul(fully_connected_layer1_tanh, weights['output']), biases['output'])
fully_connected_layer2_tanh = sigma(fully_connected_layer2)
# fully_connected_layer2_tanh = tf.nn.tanh(fully_connected_layer2)

loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=fully_connected_layer2_tanh, labels=y))
# optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

d_z_2 = tf.multiply(loss, sigmaprime(fully_connected_layer2))
d_b_2 = d_z_2
d_w_2 = tf.matmul(tf.transpose(fully_connected_layer1_tanh), d_z_2)

d_a_1 = tf.matmul(d_z_2, tf.transpose(weights['output']))
d_z_1 = tf.multiply(d_a_1, sigmaprime(fully_connected_layer1))
d_b_1 = d_z_1
d_w_1 = tf.matmul(tf.transpose(x), d_z_1)

eta = tf.constant(0.5)
step = [
    tf.assign(weights['hidden'], tf.subtract(weights['hidden'], tf.multiply(eta, d_w_1))),
    tf.assign(biases['hidden'], tf.subtract(biases['hidden'], tf.multiply(eta, tf.reduce_mean(d_b_1, axis=[0])))),
    tf.assign(weights['output'], tf.subtract(weights['output'], tf.multiply(eta, d_w_2))),
    tf.assign(biases['output'], tf.subtract(biases['output'], tf.multiply(eta, tf.reduce_mean(d_b_2, axis=[0]))))
]

acct_mat = tf.equal(tf.argmax(fully_connected_layer2_tanh, 1), tf.argmax(y, 1))
acct_res = tf.reduce_sum(tf.cast(acct_mat, tf.float32))

# load data
mnist = mnist.load_mnist(mnist_path)

# cifar10_train_images, \
# cifar10_train_labels, \
# cifar10_validation_images, \
# cifar10_validation_labels, \
# cifar10_test_images, \
# cifar10_test_labels = cifar10.maybe_download_and_extract(cifar10_path)

sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())

for i in range(10000):
    batch_xs, batch_ys = mnist.train.next_batch(10)
    batch_ys = dense_to_one_hot(batch_ys, 10)

    sess.run(step, feed_dict={x: batch_xs,
                                y: batch_ys})
    if i % 1000 == 0:
        mnist_labels = dense_to_one_hot(mnist.test.labels[:1000], 10)
        res = sess.run(acct_res, feed_dict =
                       {x: mnist.test.images[:1000],
                        y: mnist_labels})
        print (i, res)


from __future__ import absolute_import

import functools
import logging
import logging.config
import pprint
import traceback

from logging.handlers import SysLogHandler
from logging.handlers import WatchedFileHandler

# A list of things we want to replicate from logging.
# levels
CRITICAL = logging.CRITICAL
FATAL = logging.FATAL
ERROR = logging.ERROR
WARNING = logging.WARNING
WARN = logging.WARN
INFO = logging.INFO
DEBUG = logging.DEBUG
NOTSET = logging.NOTSET

# methods
getLogger = logging.getLogger
debug = logging.debug
info = logging.info
warning = logging.warning
warn = logging.warn
error = logging.error
exception = logging.exception
critical = logging.critical
log = logging.log

# classes
root = logging.root
config = logging.config
Formatter = logging.Formatter

# handlers
StreamHandler = logging.StreamHandler
WatchedFileHandler = WatchedFileHandler
SysLogHandler = SysLogHandler

def log_debug(f):
    @functools.wraps(f)
    def wrapper(*args, **kw):
        logging.debug('%s(%s, %s) ->', f.func_name, str(args), str(kw))
        rv = f(*args, **kw)
        logging.debug(pprint.pformat(rv, indent=2))
        logging.debug('')
        return rv
    return wrapper

def fail_gracefully(f):
    """Logs exceptions and aborts."""
    @functools.wraps(f)
    def wrapper(*args, **kw):
        try:
            return f(*args, **kw)
        except Exception as e:
            # tracebacks are kept in the debug log
            logging.debug(traceback.format_exc(e))

            # exception message is printed to all logs
            logging.critical(e)

            exit(1)
    return wrapper


import json
import urllib2

from django.shortcuts import render
from django.http.response import JsonResponse, HttpResponseForbidden, HttpResponse
from django.http.request import HttpRequest

from models import *

def require_login(func):
    @json_response
    def dec(request, *args, **kwargs):
        uuid = request.META.get('HTTP_UUID', '')
        if not uuid:
            return HttpResponseForbidden()

        user = User.create_or_get(uuid=uuid)
        if not user:
            return HttpResponseForbidden()
        request.current_user = user
        return func(request, *args, **kwargs)

    return dec

def json_response(func):
    def dec(request, *args, **kwargs):
        res = func(request, *args, **kwargs)
        if isinstance(res, HttpResponse):
            return res

        res = JsonResponse(res, safe=False)
        return res
    return dec

@json_response
def wx_login(request):
    code = request.GET.get('code')

    # app_id = ''
    # secret = ''
    # wx_url = 'https://api.weixin.qq.com/sns/jscode2session?appid=%s&secret=%s&js_code=%s&grant_type=authorization_code' % (
    #     app_id,
    #     secret,
    #     code,
    # )
    #
    # rep = urllib2.urlopen(wx_url)
    # data = json.loads(rep.read())
    # openid = data['openid']
    #
    openid = code
    user = User.create_or_get(openid)

    return {
        'uuid': user.uuid.hex,
    }

@require_login
def post_topic(request):
    data = json.loads(request.body)

    topic = Topic()
    topic.creator = request.current_user
    topic.content = data['content']

    if data['content']:
        topic.save()

    return {
        'rc': 0
    }

@require_login
def fetch_topics(request):
    order = request.GET.get('order', 'new')

    raw_topics = Topic.objects.all().order_by('-create_time')
    topics = []
    for topic in raw_topics:
        comments = Comment.all_comments(topic, request.current_user)
        topics.append({
            'id': topic.pk,
            'content': topic.content,
            'time': topic.create_time,
            'vote': {
                'number': VoteOnTopic.vote_of_topic(topic),
                'status': VoteOnTopic.vote_status(request.current_user, topic)
            },
            'comments': comments
        })

    return {
        'data': topics
    }

@require_login
def vote_topic(request):
    topic_id = request.GET.get('topic_id')
    topic = Topic.get_by_id(topic_id)
    vote_type = request.GET.get('type', '1')

    if not topic:
        return False

    vote = VoteOnTopic.create_or_get(request.current_user, topic)
    vote.vote_type = int(vote_type)

    vote.save()

    return True

@require_login
def post_comment_on_topic(request):
    topic_id = request.GET.get('topic_id')
    data = json.loads(request.body)
    topic = Topic.get_by_id(topic_id)

    if not topic:
        return False

    comment = Comment()
    comment.creator = request.current_user
    comment.content = data['content']
    comment.topic = topic

    comment.save()

    return True

def post_comment_on_comment(request):
    comment_id = request.GET.get('comment_id')
    reply_id = request.GET.get('reply_id', '-1')
    comment = Topic.get_by_id(comment_id)
    data = json.loads(request.body)

    if not comment:
        return False

    comment.reply(request.current_user, data['content'], reply_id)

    return True

@require_login
def vote_comment(request):
    comment_id = request.GET.get('comment_id')
    comment = Comment.get_by_id(comment_id)
    vote_type = request.GET.get('type', '1')

    if not comment:
        return False

    vote = VoteOnComment.create_or_get(request.current_user, comment)
    vote.vote_type = int(vote_type)

    vote.save()

    return True