import sys
import os
import signal
import time
from time import perf_counter

import numpy as np
import PIL.Image as im

import torch
import torch.nn as nn
import torch.nn.functional as nn_func
import torch.optim as optim
import torch.utils.data as data_utils
import torchvision.transforms.functional as xform

from skimage.metrics import peak_signal_noise_ratio as sk_psnr
from skimage.metrics import structural_similarity as sk_ssim

class NoiseUtils:
    def apply_noise(img, noise_type, noise_std):
        # TODO(ff-k): we may add other types of distributions here, such as Poisson or Bernoulli
        if noise_type == 'gaussian':
            noise = np.random.normal(0, noise_std, (img.size[1], img.size[0], len(img.getbands())))
        else:
            raise NotImplementedError('Unknown noise type: \'%s\'' % noise_type)

        return im.fromarray(np.uint8(np.clip((np.asarray(img, 'float') + (noise * 255.0)), 0, 255)))

class Noisier2NoiseDataset(data_utils.Dataset):
    def __init__(self, root_path, crop_dims, noise_type, noise_std, mode):
        super(Noisier2NoiseDataset, self).__init__()

        self.crop_dims = crop_dims
        self.noise_type = noise_type
        self.noise_std = noise_std

        self.root_path = root_path

        img_ext = ['jpg','jpeg', 'bmp', 'png', 'gif']
        self.img_path = [fp for fp in os.listdir(self.root_path) if any(fp.endswith(ext) for ext in img_ext)]

    def __len__(self):
        return len(self.img_path)

    def __getitem__(self, index):
        img_path = os.path.join(self.root_path, self.img_path[index])
        img =  im.open(img_path).convert('RGB') # NOTE(ff-k): make sure the image contains 3 channels (i.e not grayscale)

        if img.size[0] < self.crop_dims[0] or img.size[1] < self.crop_dims[1]:
            scale_factor_x = self.crop_dims[0]/img.size[0]
            scale_factor_y = self.crop_dims[1]/img.size[1]
            scale_factor = max(scale_factor_x, scale_factor_y)
            scaled_w = int(round(img.size[0]*scale_factor))
            scaled_h = int(round(img.size[1]*scale_factor))
            img = img.resize((scaled_w, scaled_h), resample=im.BICUBIC)

        crop_tl_x = np.random.randint(0, img.size[0] - self.crop_dims[0] + 1)
        crop_tl_y = np.random.randint(0, img.size[1] - self.crop_dims[1] + 1)
        img = img.crop((crop_tl_x, crop_tl_y, crop_tl_x + self.crop_dims[0], crop_tl_y + self.crop_dims[1]))

        img_noisy = NoiseUtils.apply_noise(img, self.noise_type, self.noise_std)
        img_noisier = NoiseUtils.apply_noise(img_noisy, self.noise_type, self.noise_std)

        device = torch.device('cpu')
        if torch.cuda.is_available():
            device = torch.device('cuda')

        tensor_noisy = xform.to_tensor(img_noisy).to(device=device, dtype=torch.float32)
        tensor_noisier = xform.to_tensor(img_noisier).to(device=device, dtype=torch.float32)

        return tensor_noisier, tensor_noisy

class UNet(nn.Module):
    def __init__(self, n, m):
        super(UNet, self).__init__()

        # NOTE(ff-k): As used in Noise2Noise, see Table 2

        leaky_relu_alpha = 0.1

        device = torch.device('cpu')
        if torch.cuda.is_available():
            device = torch.device('cuda')

        self.encoder_0 = nn.Sequential(
                # ENC_CONV0
                nn.Conv2d(n, 48, 3, stride=1, padding='same', device=device),
                nn.LeakyReLU(negative_slope=leaky_relu_alpha, inplace=True),
                )

        # NOTE(ff-k): where (1 <= i <= 5)
        self.encoder_i = nn.Sequential(
                # ENC_CONV_i
                nn.Conv2d(48, 48, 3, stride=1, padding='same', device=device),
                nn.LeakyReLU(negative_slope=leaky_relu_alpha, inplace=True),
                # ---------
                # POOL_i
                nn.MaxPool2d(kernel_size=2, stride=2, padding=0),
                )

        self.encoder_6 = nn.Sequential(
                # ENC_CONV6
                nn.Conv2d(48, 48, 3, stride=1, padding='same', device=device),
                nn.LeakyReLU(negative_slope=leaky_relu_alpha, inplace=True),
                )

        self.decoder_5 = nn.Sequential(
                # UPSAMPLE5
                nn.UpsamplingNearest2d(scale_factor=2),
                )

        self.decoder_4 = nn.Sequential(
                # DEC_CONV5A
                nn.Conv2d(96, 96, 3, stride=1, padding='same', device=device),
                nn.LeakyReLU(negative_slope=leaky_relu_alpha, inplace=True),
                # DEC_CONV5B
                nn.Conv2d(96, 96, 3, stride=1, padding='same', device=device),
                nn.LeakyReLU(negative_slope=leaky_relu_alpha, inplace=True),
                # UPSAMPLE4
                nn.UpsamplingNearest2d(scale_factor=2),
                )

        # NOTE(ff-k): where (3 >= i >= 1)
        self.decoder_i = nn.Sequential(
                # DEC_CONV_(i+1)_A
                nn.Conv2d(144, 96, 3, stride=1, padding='same', device=device),
                nn.LeakyReLU(negative_slope=leaky_relu_alpha, inplace=True),
                # DEC_CONV_(i+1)_B
                nn.Conv2d(96, 96, 3, stride=1, padding='same', device=device),
                nn.LeakyReLU(negative_slope=leaky_relu_alpha, inplace=True),
                # UPSAMPLE_i
                nn.UpsamplingNearest2d(scale_factor=2),
                )

        self.decoder_0 = nn.Sequential(
                # DEC_CONV1A
                nn.Conv2d(96+n, 64, 3, stride=1, padding='same', device=device),
                nn.LeakyReLU(negative_slope=leaky_relu_alpha, inplace=True),
                # DEC_CONV1B
                nn.Conv2d(64, 32, 3, stride=1, padding='same', device=device),
                nn.LeakyReLU(negative_slope=leaky_relu_alpha, inplace=True),
                # DEC_CONV1C
                nn.Conv2d(32, m, 3, stride=1, padding='same', device=device),
                )

    def forward(self, input):
        is_running = True
        while is_running == True:
            try:
                val = self.encoder_0(input)
                pool1 = self.encoder_i(val)
                pool2 = self.encoder_i(pool1)
                pool3 = self.encoder_i(pool2)
                pool4 = self.encoder_i(pool3)
                val = self.encoder_i(pool4)
                val = self.encoder_6(val)
                val = self.decoder_5(val)
                val = torch.cat((val, pool4), dim=1)
                val = self.decoder_4(val)
                val = torch.cat((val, pool3), dim=1)
                val = self.decoder_i(val)
                val = torch.cat((val, pool2), dim=1)
                val = self.decoder_i(val)
                val = torch.cat((val, pool1), dim=1)
                val = self.decoder_i(val)
                val = torch.cat((val, input), dim=1)
                val = self.decoder_0(val)
                is_running = False
            except:
                # NOTE(ff-k): try one more time after emptying the cache. it generally does not help, though
                torch.cuda.empty_cache()
                if is_running == False:
                    raise
                else:
                    is_running = False
        return val

def interrupt_handler(signum, frame):
    print('Received an interrupt signal and will terminate after completing the active epoch')
    globals()['interruped_by_user'] = True

def get_run_config(checkpoint=None):
    if checkpoint:
        run_config = checkpoint['run_config']
    else:
        run_config = {}
        run_config['dataset_id'] = 'coco'
        run_config['noise_type'] = 'gaussian'
        run_config['noise_std'] = 0.05

        # NOTE(ff-k): As used in Noise2Noise, see Section 3.1
        run_config['crop_width'] = 256
        run_config['crop_height'] = 256

        # NOTE(ff-k): In the original work, batch_size is 32 (Noisier2Noise, see Section 4).
        #             However, it is reduced to 16 in order to avoid 'out of memory' errors
        run_config['batch_size'] = 16

        # NOTE(ff-k): As used in Noisier2Noise, see Section 4
        run_config['adam_learning_rate_first_phase'] = 0.001
        run_config['adam_learning_rate_second_phase'] = 0.0001
        run_config['second_phase_start_progress'] = 0.9 # NOTE(ff-k): For example, if there are 165000 batches
                                                        #            in total, we will jump to second phase
                                                        #            after 150000 batches

        # NOTE(ff-k): As used in Noise2Noise, see Section A.2
        run_config['adam_beta_1'] = 0.9
        run_config['adam_beta_2'] = 0.99
        run_config['adam_eps'] = 1e-8

        # NOTE(ff-k): the number of epochs is not specified in none of the papers and our code does not
        #             use a max_epoch value. instead, we will keep running until the user interrupts
        run_config['epoch'] = 0

        # NOTE(ff-k): number of input and output channels, respectively. we may need to change
        #             number of input channels (i.e. 'n') in some cases (e.g. when we use Monte Carlo
        #             rendered images as input)
        run_config['n'] = 3
        run_config['m'] = 3

    return run_config

def main(mode, force_dataset, save_prob, checkpoint_path):

    if checkpoint_path == '':
        checkpoints_path = '../checkpoints/'
        checkpoints = sorted([fp for fp in os.listdir(checkpoints_path) if fp.endswith('pt')])
        if len(checkpoints) > 0:
            checkpoint_path = checkpoints_path + checkpoints[len(checkpoints)-1]
            if mode == 'train':
                print('Make sure to clean checkpoints directory if you want to start training from scratch')
        else:
            checkpoint = {}
            if mode == 'test':
                print('Could not find a pretrained network. Terminating...')
                return
    else:
        checkpoint = torch.load(checkpoint_path)

    if checkpoint_path != '':
        print('Using checkpoint: %s' % (checkpoint_path))
        checkpoint = torch.load(checkpoint_path)

    cfg = get_run_config(checkpoint)
    if force_dataset != '':
        cfg['dataset_id'] = force_dataset

    print('Running | mode: %s, dataset: %s' % (mode, cfg['dataset_id']));

    pytorch_model = UNet(cfg['n'], cfg['m'])
    pytorch_l2_loss = nn.MSELoss()

    if checkpoint:
        pytorch_model.load_state_dict(checkpoint['model_state_dict'])
        pytorch_l2_loss.load_state_dict(checkpoint['loss_fn_state_dict'])

    if mode == 'train':
        pytorch_dataset = Noisier2NoiseDataset('../data/' + cfg['dataset_id'] + '/train',
                                               [cfg['crop_width'], cfg['crop_height']],
                                               cfg['noise_type'], cfg['noise_std'], mode)
        pytorch_data_loader = data_utils.DataLoader(pytorch_dataset, batch_size=cfg['batch_size'], shuffle=True)
        # TODO(ff-k): Change learning rate based on batch index
        pytorch_optimizer = optim.Adam(pytorch_model.parameters(), lr=cfg['adam_learning_rate_first_phase'],
                                       betas=(cfg['adam_beta_1'], cfg['adam_beta_2']), eps=cfg['adam_eps'])
        if checkpoint:
            pytorch_optimizer.load_state_dict(checkpoint['optimizer_state_dict'])

        pytorch_model.train()

        checkpoint_save_interval = 1
        epoch = cfg['epoch']
        while globals()['interruped_by_user'] == False:
            # TODO(ff-k): Noisier2Noise does not use a validation set, but we may
            #             consider adding that

            time_epoch_begin = perf_counter()
            epoch_loss = 0
            for tensor_noisier, tensor_noisy in pytorch_data_loader:
                prediction = pytorch_model(tensor_noisier)
                loss = pytorch_l2_loss(prediction, tensor_noisy)
                epoch_loss = epoch_loss + loss.item()

                pytorch_optimizer.zero_grad()
                loss.backward()
                pytorch_optimizer.step()

            avg_epoch_loss = epoch_loss / len(pytorch_data_loader)
            recent_loss = loss.item()
            time_epoch_end = perf_counter()
            print('average loss: %f, recent_loss: %f, elapsed: %f' % (avg_epoch_loss, recent_loss, time_epoch_end-time_epoch_begin));
            if epoch % checkpoint_save_interval == 0 or globals()['interruped_by_user'] == True:
                cfg['epoch'] = epoch + 1
                torch.save({
                    'avg_epoch_loss': avg_epoch_loss,
                    'recent_loss': recent_loss,
                    'run_config': cfg,
                    'model_state_dict': pytorch_model.state_dict(),
                    'optimizer_state_dict': pytorch_optimizer.state_dict(),
                    'loss_fn_state_dict': pytorch_l2_loss.state_dict(),
                    }, checkpoints_path + 'checkpoint_%05d.pt' % (epoch))

            epoch = epoch + 1
    elif mode == 'test':
        pytorch_model.eval()

        device = torch.device('cpu')
        if torch.cuda.is_available():
            device = torch.device('cuda')

        denoised_path = '../denoised/'

        root_path = '../data/' + cfg['dataset_id'] + '/test/'
        img_ext = ['jpg','jpeg', 'bmp', 'png', 'gif']
        img_paths = [fp for fp in os.listdir(root_path) if any(fp.endswith(ext) for ext in img_ext)]

        max_loss = 0
        max_psnr = 0
        max_ssim = 0
        min_loss = float("inf")
        min_psnr = float("inf")
        min_ssim = float("inf")
        total_loss = 0
        total_psnr = 0
        total_ssim = 0
        for img_i in range(len(img_paths)):
            img_path = img_paths[img_i]
            img =  im.open(root_path + img_path).convert('RGB')

            # NOTE(ff-k): We have to use some padding to avoid unmatched tensor dimensions.
            #             Basically, we half the size of our input images (via max pool layers)
            #             5 times. Therefore, we have to make sure that image dimensions are
            #             divisible by 2^5=32.
            pad_w = int((img.size[0]+31)/32)*32-img.size[0]
            pad_h = int((img.size[1]+31)/32)*32-img.size[1]
            if pad_w > 0 or pad_h > 0:
                tmp = im.new(img.mode, (img.size[0]+pad_w, img.size[1]+pad_h))
                tmp.paste(img)
                img = tmp

            img_noisy = NoiseUtils.apply_noise(img, cfg['noise_type'], cfg['noise_std'])
            tensor_noisy = xform.to_tensor(img_noisy).to(device=device, dtype=torch.float32)
            tensor_noisy = tensor_noisy.unsqueeze(0)

            img_noisier = NoiseUtils.apply_noise(img_noisy, cfg['noise_type'], cfg['noise_std'])
            tensor_noisier = xform.to_tensor(img_noisier).to(device=device, dtype=torch.float32)
            tensor_noisier = tensor_noisier.unsqueeze(0)

            prediction = pytorch_model(tensor_noisier)

            if pad_w > 0:
                prediction[:, :, :, -pad_w:] = 0
                tensor_noisier[:, :, :, -pad_w:] = 0
                tensor_noisy[:, :, :, -pad_w:] = 0
            if pad_h > 0:
                prediction[:, :, -pad_h:, :] = 0
                tensor_noisier[:, :, -pad_h:, :] = 0
                tensor_noisy[:, :, -pad_h:, :] = 0

            loss = pytorch_l2_loss(prediction, tensor_noisy).item()
            if min_loss > loss:
                min_loss = loss
            if max_loss < loss:
                max_loss = loss
            total_loss = total_loss + loss

            tensor_residual = torch.sub(tensor_noisier[0], prediction[0])
            tensor_denoised = torch.sub(prediction[0], tensor_residual)
            tensor_denoised = torch.clamp(tensor_denoised, 0.0, 1.0)

            img_clean_01 = np.array(img, dtype=np.float32)/255
            img_predt_01 = np.transpose(tensor_denoised.detach().cpu().numpy(), (1, 2, 0))
            psnr = sk_psnr(img_clean_01, img_predt_01)
            if min_psnr > psnr:
                min_psnr = psnr
            if max_psnr < psnr:
                max_psnr = psnr
            total_psnr = total_psnr + psnr

            ssim = sk_ssim(img_clean_01, img_predt_01, multichannel=True)
            if min_ssim > ssim:
                min_ssim = ssim
            if max_ssim < ssim:
                max_ssim = ssim
            total_ssim = total_ssim + ssim

            if np.random.random() < save_prob:
                save_id = np.random.randint(2147483647)

                tensor_residual_min = torch.min(tensor_residual)
                tensor_residual_max = torch.max(tensor_residual)
                tensor_residual_rng = tensor_residual_max - tensor_residual_min
                tensor_residual_avg = (tensor_residual_max + tensor_residual_min)/2.0

                tensor_residual = torch.sub(tensor_residual, tensor_residual_avg)
                tensor_residual = torch.mul(tensor_residual, 1.0/tensor_residual_rng)
                tensor_residual = torch.add(tensor_residual, 0.5)
                tensor_residual = torch.clamp(tensor_residual, 0.0, 1.0)

                img_noisy = xform.to_pil_image(tensor_noisy[0])
                img_noisier = xform.to_pil_image(tensor_noisier[0])
                img_residual = xform.to_pil_image(tensor_residual)
                img_denoised = xform.to_pil_image(tensor_denoised)

                img_noisy.save(os.path.join(denoised_path, '%010d_noisy.png' % (save_id)))
                img_noisier.save(os.path.join(denoised_path, '%010d_noisier.png' % (save_id)))
                img_residual.save(os.path.join(denoised_path, '%010d_residual.png' % (save_id)))
                img_denoised.save(os.path.join(denoised_path, '%010d_denoised.png' % (save_id)))

        avg_loss = total_loss / len(img_paths)
        avg_psnr = total_psnr / len(img_paths)
        avg_ssim = total_ssim / len(img_paths)

        print('average loss: %f, min loss: %f, max loss: %f' % (avg_loss, min_loss, max_loss));
        print('average psnr: %f, min psnr: %f, max psnr: %f' % (avg_psnr, min_psnr, max_psnr));
        print('average ssim: %f, min ssim: %f, max ssim: %f' % (avg_ssim, min_ssim, max_ssim));

    else:
        raise NotImplementedError('Unknown running mode: \'%s\'' % mode)

if __name__ == "__main__":

    mode = 'test'
    force_dataset = ''
    save_prob = 0.0011
    checkpoint_path = ''

    argc = len(sys.argv)
    if argc > 4:
        mode = sys.argv[1]
        force_dataset = sys.argv[2]
        save_prob = float(sys.argv[3])
        checkpoint_path = sys.argv[4]
    elif argc > 3:
        mode = sys.argv[1]
        force_dataset = sys.argv[2]
        save_prob = float(sys.argv[3])
    elif argc > 2:
        mode = sys.argv[1]
        force_dataset = sys.argv[2]
    elif argc > 1:
        mode = sys.argv[1]

    if mode == 'train':
        globals()['interruped_by_user'] = False
        signal.signal(signal.SIGINT, interrupt_handler)

    # NOTE(ff-k): set random_seed to 0 or some other constant value to repeat results
    random_seed = int(time.time()*1000)%(2**32-1)
    np.random.seed(random_seed)

    main(mode, force_dataset, save_prob, checkpoint_path)


from kivy.app import App
from kivy.uix.label import Label
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.floatlayout import FloatLayout
from kivy.uix.button import Button
from kivy.properties import ListProperty, ObjectProperty
from kivy.uix.popup import Popup
from kivy.uix.progressbar import ProgressBar
from kivy.clock import Clock
from tensorflow.keras.applications import VGG16
from tensorflow.keras.layers import Dense, Flatten
from tensorflow.keras import Model
from tensorflow.keras.utils import to_categorical
from numpy import asarray
from sklearn import preprocessing
import os
import cv2
import numpy as np
import threading

from loadFile import PickerLayout

class TrainModel(FloatLayout):
    images = ListProperty([])
    labels = ListProperty([])

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        # orientation = 'horizontal'
        size_hint = (1,0.4)
        self.progress_value = 0
        self.thread_flag = False
        # self.image = image

        self.trainBox = BoxLayout(orientation = 'horizontal',
                                  size_hint = (None, None),
                                  width = 100,
                                  height = 40,
                                  pos_hint = {'center_x':0.5, 'center_y': 0.5})
        self.trainButton = Button(text = "Train Data",
                                  size = (100,40))
        self.trainBox.add_widget(self.trainButton)
        self.add_widget(self.trainBox)

        self.progressBar = ProgressBar()
        self.progressPop = Popup(title = 'Please wait while we train your images',
                                 content = self.progressBar,
                                 size_hint=(0.8, 0.2),
                                 auto_dismiss = False)

        self.trainButton.bind(on_press = self.progressUp)
        self.data = PickerLayout()

    def getImage(self, myPickerLayout, images):
        self.images = images

    def getLabel(self, myPickerLayout, labels):
        self.labels = labels

    def progressUp(self, instance):
        self.progressPop.open()
        self.thread_flag = True
        self.ev = threading.Event()
        self.thread = threading.Thread(target = self.train)
        self.thread.start()

    def dismiss(self, *args):
        self.ev.clear()
        self.thread_flag = False
        self.progressPop.dismiss()

    def clock(self, *args):
        #Stops the clock when the progress bar value reaches its maximum
        if self.progress_value >= 100:
            Clock.unschedule(self.clock)
        self.progressBar.value = self.progress_value

    def train(self):
        Clock.schedule_interval(self.clock, 1 / 60)
        image_input = []
        label_input = []
        for i in self.images:
            image_input.append(i)
        image_input = np.array(image_input)

        for i in self.labels:
            label_input.append(i)
        preprocess_label = preprocessing.LabelEncoder()
        label = preprocess_label.fit_transform(label_input)
        label = to_categorical(label)
        print(label.shape)
        num_label = len(label[0])
        model = VGG16(weights='imagenet',
                                  input_shape=(224, 224, 3),
                                  include_top=False)
        for layer in model.layers:
	        layer.trainable = False

        flat = Flatten()(model.layers[-1].output)
        flat2 = Dense(1024, activation='relu')(flat)
        output = Dense(num_label, activation='softmax')(flat2)
        # define new model
        model = Model(inputs=model.inputs, outputs=output)
        model.compile(optimizer='adam',
                      loss='binary_crossentropy',
                      metrics=['accuracy'])

        epochs = 3
        value = round(100 / epochs)
        for e in range(epochs):
            model.fit(image_input, label, epochs = 1, batch_size = 2)
            self.progress_value += value
        self.dismiss()
        # model.save("new model.h5")
        # print("model saved!")

        # y = model.predict(image_input)
        # print(y)
        # x = y.argmax(axis=-1)
        # print(x)
        # z = preprocess_label.inverse_transform(x)
        # print(z)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function, absolute_import, unicode_literals

import sys
import numpy as np

try:
    import talib
except:
    print('请安装TA-Lib库')
    sys.exit(-1)
from etasdk import *

'''
期货策略：海龟交易法
本策略通过计算FG801.CF和rb1801.CF的ATR.唐奇安通道和MA线,
-- 当价格上穿唐奇安通道且短MA在长MA上方时开多仓;
-- 当价格下穿唐奇安通道且短MA在长MA下方时开空仓(8手)
若有多仓则在价格跌破唐奇安平仓通道下轨的时候全平仓位,否则根据跌破
持仓均价 - x(x=0.5,1,1.5,2)倍ATR把仓位平至6/4/2/0手
若有空仓则在价格涨破唐奇安平仓通道上轨的时候全平仓位,否则根据涨破
持仓均价 + x(x=0.5,1,1.5,2)倍ATR把仓位平至6/4/2/0手
撮合周期为:1分钟
回测时间为:2017-09-15 到2017-10-01
'''

def onInitialize(api):
    print("analyzer initialize")
    # 设置响应模式
    api.setGroupMode(5000, False)
    # 设置关注的合约标的
    api.tickersCode = ["FG1801.CF", "rb1801.CF"]

    # api.parameter 分别为唐奇安开仓通道.唐奇安平仓通道.短ma.长ma.ATR的参数
    api.parameter = [55, 20, 10, 60, 20]
    api.tar = api.parameter[4]

    api.data_len = 600

    api.setSymbolPool(symbols=api.tickersCode)
    api.setRequireBars(ETimeSpan.MIN_1, api.data_len)

def onBeforeMarketOpen(api, tradeDate):
    # print(tradeDate)
    api.trading_day = tradeDate
    # 订阅行情
    api.setFocusSymbols(api.tickersCode)

def onBar(api, bar):
    bar_data = api.getBarsHistory(symbol=bar.symbol, timeSpan=ETimeSpan.MIN_1, count=api.data_len,
                                  priceMode=EPriceMode.FORMER, fields=None, df=True)
    close = bar_data["close"].values[-1]
    # 计算ATR
    atr = talib.ATR(bar_data["high"].values, bar_data["low"].values, bar_data["close"].values,
                    timeperiod=api.tar)[-1]

    # 计算唐奇安开仓和平仓通道
    api.don_open = api.parameter[0] + 1
    upper_band = talib.MAX(bar_data['close'].values[:-1], timeperiod=api.don_open)[-1]
    api.don_close = api.parameter[1] + 1
    lower_band = talib.MIN(bar_data['close'].values[:-1], timeperiod=api.don_close)[-1]

    # 若没有仓位则开仓
    position_long = api.getSymbolPosition(symbol=bar.symbol, positionSide=EPositionSide.LONG)
    position_short = api.getSymbolPosition(symbol=bar.symbol, positionSide=EPositionSide.SHORT)

    print(position_long)
    print(position_short)

    if not position_long.posQty and not position_short.posQty:
        # 计算长短ma线.DIF
        ma_short = talib.MA(bar_data['close'].values, timeperiod=(api.parameter[2] + 1))[-1]
        ma_long = talib.MA(bar_data['close'].values, timeperiod=(api.parameter[3] + 1))[-1]

        diff = ma_short - ma_long

        # 获取当前价格
        # 上穿唐奇安通道且短ma在长ma上方则开多仓
        if bar.close > upper_band and (diff > 0):
            api.targetPosition(symbol=bar.symbol, qty=8, positionSide=EPositionSide.LONG)
            print(bar.symbol, '市价单开多仓8手')
        # 下穿唐奇安通道且短ma在长ma下方则开空仓
        if bar.low < lower_band and (diff < 0):
            api.targetPosition(symbol=bar.symbol, qty=8, positionSide=EPositionSide.SHORT)
            print(bar.symbol, '市价单开空仓8手')

    elif position_long.posQty:
        # 价格跌破唐奇安平仓通道全平仓位止损
        if close < lower_band:
            api.targetPosition(symbol=bar.symbol, qty=0, positionSide=EPositionSide.LONG)
            print(bar.symbol, '市价单全平仓位')
        else:
            # 获取持仓均价
            vwap = position_long.posPrice
            # 根据持仓以来的最高价计算不同的止损价格带
            band = vwap - np.array([2, 1.5, 1, 0.5]) * atr
            # 计算最新应持仓位
            if bar.close <= band[0]:
                api.targetPosition(symbol=bar.symbol, qty=0, positionSide=EPositionSide.LONG)
                print(bar.symbol, '市价单平仓至0手')
            elif band[0] < bar.close <= band[1]:
                api.targetPosition(symbol=bar.symbol, qty=2, positionSide=EPositionSide.LONG)
                print(bar.symbol, '市价单平仓至2手')
            elif band[1] < bar.close <= band[2]:
                api.targetPosition(symbol=bar.symbol, qty=4, positionSide=EPositionSide.LONG)
                print(bar.symbol, '市价单平仓至4手')
            elif band[2] < bar.close <= band[3]:
                api.targetPosition(symbol=bar.symbol, qty=6, positionSide=EPositionSide.LONG)
                print(bar.symbol, '市价单平仓至6手')

    elif position_short.posQty:
        # 价格涨破唐奇安平仓通道或价格涨破持仓均价加两倍ATR平空仓
        if close > upper_band:
            api.targetPosition(symbol=bar.symbol, qty=0, positionSide=EPositionSide.SHORT)
            print(bar.symbol, '市价单全平仓位')
        else:
            # 获取持仓均价
            vwap = position_long.posPrice
            # 根据持仓以来的最高价计算不同的止损价格带
            band = vwap + np.array([2, 1.5, 1, 0.5]) * atr
            # 计算最新应持仓位
            if bar.close >= band[0]:
                api.targetPosition(symbol=bar.symbol, qty=0, positionSide=EPositionSide.SHORT)
                print(bar.symbol, '市价单平仓至0手')
            elif band[0] > bar.close >= band[1]:
                api.targetPosition(symbol=bar.symbol, qty=2, positionSide=EPositionSide.SHORT)
                print(bar.symbol, '市价单平仓至2手')
            elif band[1] > bar.close >= band[2]:
                api.targetPosition(symbol=bar.symbol, qty=4, positionSide=EPositionSide.SHORT)
                print(bar.symbol, '市价单平仓至4手')
            elif band[2] > bar.close >= band[3]:
                api.targetPosition(symbol=bar.symbol, qty=6, positionSide=EPositionSide.SHORT)
                print(bar.symbol, '市价单平仓至6手')

def onTerminate(api, exit_info):
    print("Finished test")
    pass


from ortools.sat.python import cp_model

class SolutionPrinter(cp_model.CpSolverSolutionCallback):

    def __init__(self, arcs, revs):
        cp_model.CpSolverSolutionCallback.__init__(self)
        self.arcs = arcs
        self.revs = revs
        self.solutions = 0

    def on_solution_callback(self):
        # Many thousands of solutions.
        self.solutions += 1
        return True
        result = []
        used = [arc for arc in self.arcs if self.Value(arc[2]) and arc[0] != arc[1]]
        arc = used[0]
        while arc:
            used.remove(arc)
            result.append(self.revs[arc[0]])
            arc = next((link for link in used if link[0] == arc[1]), None)
        if len(set(result)) != len(result):
            print('duplicates found!')
        if used:
            result.append(['unused:', used])
        print(f"{' '.join([str(x) for x in result])}")

class DiGraphSolver:

    def __init__(self, desc):
        self.model = cp_model.CpModel()
        self.status = cp_model.UNKNOWN
        self.timing = None
        self.score = None
        self.masters = None
        # Graph has 3 layers:
        #   1st is the node proper
        #   2nd is the set of node fixes (only one of which is allowed) which are presented as nodes of a digraph
        #   3rd is the list of potential tails for that fix.
        self.graph = desc.graph

        #   for setting up the circuit, we may as well extract the nodal fixes of each domino.
        self.nodal = {node: heads for m in self.graph.values() for node, heads in m.items()}

        # because AddCircuit requires numbers, we create a map of nodes->ints and it's reverse
        self.idx = {k: i for i, k in enumerate(self.nodal)}
        self.rev = {i: k for k, i in self.idx.items()}

        self.nodes = {self.idx[h]: [self.idx[t] for t in tails] for h, tails in self.nodal.items()}

        self.arcs = []
        self.result = []

    def setup(self):
        # Set the master pieces. they should all be used.
        # self.masters = {key: self.model.NewBoolVar(f'{key}') for key in self.pieces}

        # AddCircuit uses a list of directed arcs as tuples: each is (head, tail, boolVariable)
        # where the  boolVar truth value represents if the arc is used or not used.
        # For each head(node), create an arc for each of it's tails.
        self.arcs = [
            (self.idx[h], self.idx[t], self.model.NewBoolVar(f'{h}:{t}')) for h, ts in self.nodal.items() for t in ts
        ]
        self.model.Add(sum([a[2] for a in self.arcs]) == len(self.graph))

        # Lock out the graph from allowing both mirrors. This means that they have to be optionals.
        for m in self.graph.values():
            if len(m) > 1:
                heads = [a for (h, t, a) in self.arcs if self.rev[h] in m]
                self.model.Add(sum(heads) == 1)
                self.arcs += [(self.idx[n], self.idx[n], self.model.NewBoolVar(f'LP {n}')) for n in m]  # optional.

        # Now add the circuit as a constraint.
        self.model.AddCircuit(self.arcs)

    def solve(self) -> bool:
        cp_solver = cp_model.CpSolver()
        solution_printer = SolutionPrinter(self.arcs, self.rev)
        self.status = cp_solver.SearchForAllSolutions(self.model, solution_printer)
        return self.summarise(cp_solver, solution_printer.solutions)

    def summarise(self, cp_solver, solutions: int) -> bool:
        if self.status in (cp_model.OPTIMAL, cp_model.FEASIBLE):
            print(f"Challenge found {solutions} solutions.")
            return True
        else:
            if self.status == cp_model.INFEASIBLE:
                print(f"Challenge is infeasible after {cp_solver.WallTime()}s.")
            else:
                print(f"Solver ran out of time.")
            return False

class DominoDigraph:
    """
        28 Dominoes - 0:0 to 6:6
        56 Nodes = 1 for each end of each Domino as a head.
        Each end can be a head or a tail.
        Each end, as a tail, is only a head to the domino's own head.
        Each head only points to the tails (not heads) of other dominoes.
        It's own 3 or any other 1 <-[1:3]->It's own 1 or to any other 3.
        The loop is closed, no dominoes are optional.
    """

    def __init__(self, size: int):
        # the primary objects
        dominoes = set((x, y) for y in range(size + 1) for x in range(y, size + 1))
        # fix the rotations and store against each master
        fixed = {(a, b): {(a, b), (b, a)} for (a, b) in dominoes}
        # for each Domino; for each of it's fix; find those fixes from the Other dominoes which are potential tails.
        self.graph = {
            d: {
                d_fix: [  # connect the other(o) potential dominoes as tails to each fix
                    o_fix for o, o_fixes in fixed.items() if o != d for o_fix in o_fixes if o_fix[0] == d_fix[1]
                ] for d_fix in d_fixes
            } for d, d_fixes in fixed.items()
        }
        everything = True

def solve(problem):
    solver = DiGraphSolver(problem)
    solver.setup()
    solver.solve()

def solve_domino_digraph():
    #  2, 4, 6
    for n in [2, 4, 6, 7, 8, 9, 10, 11, 12]:
        print(f'trying {n}...')
        problem = DominoDigraph(n)
        solve(problem)

if __name__ == '__main__':
    solve_domino_digraph()


# coding: UTF-8
import requests
"""通用的工具类"""

class RequestsHelper:
    headers = {'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) "
                             "Chrome/72.0.3626.121 Safari/537.36",
               'Referer': 'http://y.qq.com'}

    @classmethod
    def set_domain(cls):
        """为之后更新不同音乐平台的api留接口"""
        pass

    @classmethod
    def url_is_error(cls, url):
        res = requests.get(url, headers=cls.headers)
        return res.raise_for_status()

    @staticmethod
    def splice_url(url, params: dict):
        """拼接url"""
        for k, v in params.items():
            url += (k + "=" + str(v))

        return url

    @classmethod
    def get_response_json(cls, url):
        response = requests.get(url, headers=cls.headers)
        return response.json()

    @classmethod
    def get_response_text(cls, url):
        response = requests.get(url, headers=cls.headers)
        return response.text


#!/usr/bin/env python3

import os
import sys
from operator import add
import shutil
from functools import reduce

def main():
    LOGDIR = "iterout"
    THRESOLD = 90

    dryrun = True
    if len(sys.argv) == 2 and sys.argv[1] == "1":
        dryrun = False
    else:
        print("dryrun mode")

    dirs = [p for p in (os.path.join(LOGDIR, d) for d in os.listdir(LOGDIR)) if os.path.isdir(p)]
    while len(dirs) != 0:
        path = dirs.pop()
        content = os.listdir(path)
        delete_flag = False

        # empty dir
        if len(content) == 0:
            delete_flag = True

        # log dir
        elif "iterlog.txt" in content:
            with open(os.path.join(path, "iterlog.txt")) as f:
                nlines = reduce(add, (1 for i in f), 0)
                if nlines < THRESOLD:
                    delete_flag = True

        # dig sub dir
        else:
            for c in content:
                subpath = os.path.join(path, c)
                if os.path.isdir(subpath):
                    dirs.append(subpath)

        if delete_flag:
            print("delete {}".format(path), content)
            if not dryrun:
                shutil.rmtree(path)

if __name__ == '__main__':
    main()


import os
import time
import pytest

from myutils.testhelpers import netcat_from_to, run_cmd

from myutils.remote_utils import PYTHON_CMD_PREFIX

path    = os.path.dirname(os.path.realpath(__file__))
srv_cmd = PYTHON_CMD_PREFIX+[os.path.join(path, 'server.py')]
cli_cmd = PYTHON_CMD_PREFIX+[os.path.join(path, 'client.py')]

def hostport(s):
    return s.split(':')

def test_addr_rewriting(p4run, controller, pools):
    netcat_from_to('h4', 'h1', '10.0.1.1', port=7000)

def run_server(port, host=None, wait_to_bind=True):
    server = run_cmd(srv_cmd + [port], host, background=True)
    if wait_to_bind: time.sleep(0.5)
    return server

def get_conns_and_die(server):
    out, _ = server.communicate('die\n')
    return int(out.strip())

def run_client(nconns, shost, port, host=None):
    return run_cmd(cli_cmd + [nconns, shost, port], host)

def test_server_client():
    """Checks that the server and client are working. Does not actually use loadbalancing."""
    server = run_server(4700)
    assert run_client(10, 'localhost', 4700) == 0
    assert get_conns_and_die(server) == 10

@pytest.mark.skip(reason='flaky and weird')
def test_equal_balancing(p4run, controller, pools):
    NUM_CONNS = 1000
    TOLERANCE = 0.9
    mypool = '10.0.1.1:8000'
    pool_ip, pool_port = hostport(mypool)

    servers = []
    for s in pools[mypool]:
        host, port = hostport(s)
        servers.append(run_server(host=host, port=port, wait_to_bind=False))
    time.sleep(0.5)  # give them time to bind

    assert run_client(NUM_CONNS, pool_ip, pool_port, host='h4') == 0

    for s in servers:
        assert get_conns_and_die(s) >= TOLERANCE*NUM_CONNS/len(pools[mypool])


from PyQt5 import QtWidgets
from PyQt5.QtGui import *
from PyQt5.QtCore import *
import random
import time
class MainPage:
    def __init__ (self,Ui,Db,Con):
        self.Dialog=QtWidgets.QDialog()
        self.ui=Ui
        self.db=Db
        self.con=Con
        self.qPixmapVar = QPixmap()
        self.rspPicture=["image/paper.PNG","image/Scissors.PNG","image/rock.PNG"]
        self.nexonImage=["image/nexon1.PNG","image/nexon2.PNG","image/nexon3.PNG","image/nexon4.PNG","image/nexon5.PNG","image/nexon6.png"]
        self.planeGame=["image/plane.PNG","image/attack.jpg","image/monster.PNG","image/attack2.jpg"]
        self.pictureSet(1800,300,2,self.nexonImage[0])
        self.monTrue=1
        for index in range(0,3):
            number=6+index
            if number==7:
                self.pictureSet(80,30,number,self.planeGame[3])
            else:self.pictureSet(150,120,number,self.planeGame[index])
        self.locate=0
        self.num=0
        self.playerNum=0
        self.score=0
    def mainFlow(self,value):
        if value==4:
            pass
        elif value==5:
            pass
        elif value==3:
            self.ui.stackedWidget.setCurrentIndex(0)
            self.gameReset()
            self.con.reset(self.ui.joinCheckList,self.ui.widEtc4List,self.ui.loginLineList,self.ui.findIdpageList,self.ui.joinEnterList)
        else:
            self.playerRSP(value)

    def pictureSet(self,wid,high,num,image):
        self.qPixmapVar.load(image)
        self.qPixmapVar = self.qPixmapVar.scaledToWidth(wid)
        self.qPixmapVar = self.qPixmapVar.scaledToHeight(high)
        self.ui.mainPic3List[num].setPixmap(self.qPixmapVar)

    def playerRSP(self,value):
        self.playerNum=value
        self.pictureSet(120,80,4,self.rspPicture[value])
        self.enemy(value)

    def rspAiPrint(self,random): # computer 와 같이 변환 가능
        self.pictureSet(120,80,5,self.rspPicture[random])

    def enemy(self,player):
        randAi=random.randint(0,2)
        self.pictureSet(120,80,5,self.rspPicture[randAi])
        if player==randAi:
            self.result="DRAW"
        elif player==2 and randAi==1:
            self.result="WIN"
        elif player==1 and randAi==0:
            self.result="WIN"
        elif player==0 and randAi==2:
            self.result="WIN"
        else:
            self.result="ROSE"
        self.ui.mainPic3List[3].setText(self.result)

    def mainPictureChange(self,num):
        self.pictureSet(1800,300,2,self.nexonImage[num])
        self.ui.mainPic3List[1].setText(str(num+1)+"/6")

    def scrollChange(self,value):
        X,Y,high2=value[0],value[1],value[2]
        X+=11
        Y+=45
        if self.ui.verticalScrollBar2.value()!=0:
            high=self.ui.verticalScrollBar2.value()
            high-=high2
            self.ui.MainWindow.setGeometry(X,Y-high*8,892,1985)
            print(Y-high)

    def location(self):
        X = self.ui.MainWindow.x()
        Y = self.ui.MainWindow.y()
        high=self.ui.verticalScrollBar2.value()
        self.locate=[X,Y,high]

    def attack(self,X):
        self.pictureSet(80,30,7,self.planeGame[1])
        self.ui.mainPic3List[7].setGeometry(220+X,1850,80,30)

        if X>=350:
            self.pictureSet(150,120,8,self.planeGame[3])
            self.pictureSet(80,30,7,self.planeGame[3])
            self.ui.mainPic3List[7].setGeometry(220,1850,80,30)
            self.score+=1
            self.ui.mainLogo3List[8].setText("score :"+str(self.score))
            self.monTrue=0

    def gameReset(self):
        self.monTrue=1
        self.pictureSet(150,120,8,self.planeGame[2])


import math
import numpy as np
import pandas as pd
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from collections import Counter
from sklearn.neighbors import KNeighborsClassifier
from random import random
import time
def L(x, y, p=2):
    if len(x) == len(y) and len(x) > 1:
        sum = 0
        for i in range(len(x)):
            sum += math.pow(x[i] - y[i], p)
        return math.pow(sum, 1 / p)
    else:
        return 0

class knn:
    def __init__(self, x_train, y_train, k=3, p=2):
        self.k = k
        self.p = p
        self.x_train = x_train
        self.y_train = y_train

    def predict(self, x):
        knn_list = []
        for i in range(self.k):
            dist = np.linalg.norm(x - x_train[i], ord=self.p)
            knn_list.append((dist, self.y_train[i]))

        for i in range(self.k, len(self.x_train)):
            max_index = knn_list.index(max(knn_list, key=lambda x: x[0]))
            dist = np.linalg.norm(x - x_train[i], ord=self.p)
            if knn_list[max_index][0] > dist:
                knn_list[max_index] = (dist, self.y_train[i])
        knn = [k[-1] for k in knn_list]
        count_pairs = Counter(knn)
        max_count = sorted(count_pairs.items(), key=lambda x: x[1])[-1][0]
        return max_count

    def score(self, x_test, y_test):
        right = 0
        for x, y in zip(x_test, y_test):
            label = self.predict(x)
            if label == y:
                right += 1
        return right / len(x_test)

# kd_tree
class KdNode(object):
    def __init__(self,dom_elt,split,left,right):
        self.dom_elt=dom_elt
        self.split=split
        self.left=left
        self.right=right

class KdTree(object):
    def __init__(self,data):
        k=len(data[0])
        def CreateNode(splite,data_set):
            if not data_set:
                return None
            data_set.sort(key=lambda x:x[splite])
            splite_position=len(data_set)//2
            median=data_set[splite_position]
            splite_next=(splite+1)%k
            return KdNode(median,splite,
                          CreateNode(splite_next,data_set[:splite_position]),
                          CreateNode(splite_next,data_set[splite_position+1:]))
        self.root=CreateNode(0,data)

def preorder(root):
    print(root.dom_elt)
    if root.left:
        preorder(root.left)
    if root.right:
        preorder(root.right)

from math import sqrt
from collections import namedtuple

result=namedtuple("result_tuple","nearest_point nearest_dist nodes_visited")

def find_nearest(tree,point):
    k=len(point)
    def travel(kd_node,target,max_dist):
        if kd_node is None:
            return result([0]*k,float("inf"),0)
        nodes_visited=1
        s=kd_node.split
        pivot=kd_node.dom_elt
        if target[s]<=pivot[s]:
            near_node=kd_node.left
            further_node=kd_node.right
        else:
            near_node=kd_node.right
            further_node=kd_node.left
        temp1=travel(near_node,target,max_dist)
        nearest=temp1.nearest_point
        dist=temp1.nearest_dist
        nodes_visited+=temp1.nodes_visited

        if dist<max_dist:
            max_dist=dist
        temp_dist=abs(pivot[s]-target[s])
        if max_dist<temp_dist:
            return result(nearest,dist,nodes_visited)

        temp_dist=sqrt(sum((p1-p2)**2 for p1,p2 in zip(pivot,target)))
        if temp_dist<dist:
            nearest=pivot
            dist=temp_dist
            max_dist=dist
        temp2=travel(further_node,target,max_dist)
        nodes_visited+=temp2.nodes_visited
        if temp2.nearest_dist<dist:
            nearest=temp2.nearest_point
            dist=temp2.nearest_dist
        return result(nearest,dist,nodes_visited)
    return travel(tree.root,point,float("inf"))

def random_point(k):
    return [random() for _ in range(k)]

def random_points(k,n):
    return [random_point(k) for _ in range(n)]

if __name__ == '__main__':
    iris = load_iris()
    df = pd.DataFrame(iris.data, columns=iris.feature_names)
    df['label'] = iris.target
    df.columns = ['sepal length', 'sepal width', 'petal length', 'petal width', 'label']
    print(df)

    data = np.array(df.iloc[:, :])
    x, y = data[:, :-1], data[:, -1]
    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
    print(x, y)

    clk = knn(x_train, y_train)
    score = clk.score(x_test, y_test)
    print(score)

    clk_sk = KNeighborsClassifier()
    clk_sk.fit(x_train, y_train)
    score_sk=clk.score(x_test, y_test)
    print(score_sk)

    N=400000
    t0=time.process_time()
    kd=KdTree(random_points(3,N))
    result=find_nearest(kd,[0.1,0.5,0.8])
    t1=time.process_time()
    print(t1-t0)
    print(result)


from typing import Dict, List

import matplotlib.pyplot as plt
import numpy as np

def movingaverage(interval: List, window_size: int) -> np.ndarray:
    window = np.ones(int(window_size)) / float(window_size)
    return np.convolve(interval, window, "same")

def plot_results(
    result: Dict[str, Dict],
    ymin: int = 0,
    ymax: int = None,
    yscale: str = "linear",
    moving: int = 5,
    alpha: float = 0.5,
) -> None:
    """workaround for tensorboard

    Args:
        result (Dict[str, Dict]): dictionary of history.history dicts
        ymin (int, optional): minimum y axis
        ymax (int, optional): maximum y axis. Defaults to None.
        yscale (str, optional): Linear of log scaling of y-axis. Defaults to "linear".
        moving (int, optional): Overlay a moving average. Defaults to 5.
        alpha (float, optional): transparancy. Defaults to 0.5.
    """

    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 10))
    move = type(moving) == int

    for key in result.keys():
        loss = result[key]["loss"]
        if move:
            z = movingaverage(loss, moving)
            z = np.concatenate([[np.nan] * moving, z[moving:-moving]])
            color = next(ax1._get_lines.prop_cycler)["color"]
            ax1.plot(z, label=key, color=color)
            ax1.plot(loss, label=key, alpha=alpha, color=color)
        else:
            ax1.plot(loss, label=key)

        ax1.set_yscale(yscale)
        ax1.set_ylim(ymin, ymax)
        ax1.set_title("train")

        valloss = result[key]["val_loss"]

        if move:
            z = movingaverage(valloss, moving)
            z = np.concatenate([[np.nan] * moving, z[moving:-moving]])
            color = next(ax2._get_lines.prop_cycler)["color"]
            ax2.plot(z, label=key, color=color)
            ax2.plot(valloss, label=key, alpha=alpha, color=color)
        else:
            ax2.plot(valloss, label=key)

        ax2.set_yscale(yscale)
        ax2.set_ylim(ymin, ymax)
        ax2.set_title("test")

    plt.legend()


import sys
sys.path.append('../src/')
import os
os.environ['KMP_DUPLICATE_LIB_OK']='True'
import numpy as np
from time import time, localtime, strftime

from nn.DQN import DQN_R
from nn.DoubleDQN import DoubleDQN

# np.random.seed(2020)

def train(conf, config, sofa):
    if conf['mode'] == 'DQN':
        rl = DQN_R(config)
    elif conf['mode'] == 'DoubleDQN':
        rl = DoubleDQN(config)

    num_users = sofa.num_users
    # num_episodes = num_users * 1000
    cumu_rewards = []
    learnstep_cr = []
    load_time, learn_time = 0, 0
    begin_time = time()
    # filename = conf["data.input.path"] + conf["data.input.dataset"] + '_' + \
    #     conf["data.gen_model"] + '_' + conf["data.debiasing"] + '_plot.pdf'
    # plot_data_file = conf["data.input.path"] + conf["data.input.dataset"] + '_' + \
    #     conf["data.gen_model"] + '_' + conf["data.debiasing"] + '_plot.data'
    filename = conf["data.input.path"] + config['SAVE_MODEL_FILE'] + 'plot.pdf'
    plot_data_file = conf["data.input.path"] + config['SAVE_MODEL_FILE'] + 'plot.data'

    users = np.arange(num_users, dtype=int)
    testuser_only = False
    if testuser_only and conf["data.input.dataset"].lower() == 'yahoo':
        test_file = conf["data.input.path"] + conf["data.input.dataset"] + "_test.ascii"
        RatingM = np.loadtxt(test_file).astype(int)
        users = users[np.where(np.sum(RatingM, axis=1)==0, False, True)]
        print("When learning a policy, we only use the user appeared in the testset. The number is %d" % users.shape[0], flush=True)
        num_users = users.shape[0]

    # ## for remove the non-click users
    # print("*********** Please note we remove the non-click users")
    # clicks = np.where(sofa.ratings > 3, 1, 0)
    # clicks_peruser = np.sum(clicks, axis=1)
    # keepusers = np.where(clicks_peruser<=4, False, True)
    # users = np.arange(num_users, dtype=int)[keepusers]
    # print("********** After removing clicks < 4, we have %d users" % (users.shape[0]))

    step = 0
    batch_size = 10
    epoch = 3000
    trick1_happen = 0
    for epoch in range(epoch):
        np.random.shuffle(users)

        ## single user interaction
        # for idx in range(num_users):
        #     # user_id = np.random.randint(num_users)
        #     user_id = users[idx]
        #     sofa.reset(user_id)
        #     state = sofa.state
        #     history = []
        #     cumu_reward = 0
        #     while True:
        #         action = rl.choose_action(state)
        #         history.append(action)
        #         (state_, reward, done) = sofa.step(action)
        #         cumu_reward += reward
        #         state = state_
        #         rl.store_memory(state, action, reward, state_)
        #         if done:
        #             break
        #     cumu_rewards.append(cumu_reward)
        #     rl.learn()
        #     step += 1

        # multi-user interaction
        losses = []
        start_idx = 0
        while start_idx < num_users:
            user_ids = users[start_idx:(start_idx+batch_size)]
            user_batch = user_ids.shape[0]
            start_idx += batch_size
            sofa.reset(user_ids)
            states = sofa.state.copy()
            cumu_reward = np.zeros(user_ids.shape[0])
            states_actions_batch = []
            while True:
                actions = rl.choose_actions(states, sofa.step_count)
                (states_, rewards, done) = sofa.step(actions)
                # print(states.shape, states_.shape)
                cumu_reward += rewards
                states_actions_batch.append((states.copy(), actions, rewards, states_.copy()))
                rl.store_memorys(states, actions, np.where(rewards==0, -1, 2), states_, sofa.step_count-1, sofa.step_count)
                # rl.store_memorys(states, actions, rewards, states_, sofa.step_count-1, sofa.step_count)
                states = states_.copy()
                if done:
                    break
            # if rl.epsilon == rl.epsilon_min:
            #     keep = np.where(cumu_reward<1, False, True)
            #     for turn in range(len(states_actions_batch)):
            #         (s, a, r, s_) = states_actions_batch[turn]
            #         trick1_happen += (user_batch - np.sum(keep))
            #         if np.sum(keep) == 0:
            #             continue
            #         rl.store_memorys(s[keep], a[keep], np.where(r==0, -1, 2)[keep], s_[keep], turn, turn+1)
            # else:
            #     for turn in range(len(states_actions_batch)):
            #         (s, a, r, s_) = states_actions_batch[turn]
            #         rl.store_memorys(s, a, np.where(r==0, -1, 2), s_, turn, turn+1)
            cumu_rewards += list(cumu_reward)
            for _ in range(4):
                loss = rl.learn()
                if loss is None:
                    break
                # print(loss)
                losses.append(loss)

            if rl.learn_step_counter == 0:
                continue
            if rl.learn_step_counter % 1000 == 0:
                # print(cumu_rewards)
                avg_cumulativeReward = np.mean(cumu_rewards)
                cumu_rewards = []
                learnstep_cr.append([rl.learn_step_counter, avg_cumulativeReward])
                if (len(learnstep_cr) % 10 == 0):
                    plot_cumuReward(learnstep_cr, filename, plot_data_file)
            ## save model
            if rl.learn_step_counter % 5000 == 0:
                print("\nlearning step : %d, time consuming %4e s" % (rl.learn_step_counter, time()-begin_time), flush=True)
                print("learning step: %d, lr : %.5e, epsilon: %2f" % (rl.learn_step_counter, rl.get_lr(), rl.epsilon))
                print("loss:%.3e" % (np.mean(losses)), flush=True)
                begin_time = time()
                rl.save_model()
    if (epoch + 1) % 100 == 0:
        print("Till epoch %d, the trick-1 works for %d times" % (epoch+1, trick1_happen))

    plot_cumuReward(learnstep_cr, filename, plot_data_file)
    print("plot save into file:%s" % filename)

def plot_cumuReward(learnstep_cr, filename='line_plot.pdf', plot_data_file='line_plot.data'):
    np.savetxt(plot_data_file, np.array(learnstep_cr))
    # print("plot save into file:%s" % filename )
    import matplotlib.pyplot as plt
    learnstep_cr = np.array(learnstep_cr)
    plt.plot(learnstep_cr[:, 0].astype('int'), learnstep_cr[:, 1].astype('float'))
    plt.ylabel('Cumulative_Click')
    plt.xlabel('training steps')
    plt.savefig(filename)
    plt.show()
    plt.close()


from PyQt5.QtWidgets import QMainWindow,QWidget,QApplication
import sys
class Pencere (QMainWindow):
    def __init__(self):
        super().__init__()
        self.program()
    def program(self):
        self.pencere=QWidget()
        self.pencere.setWindowTitle("Deneme")
        self.pencere.setMinimumSize(100,100)

        self.pencere.show()
app=QApplication(sys.argv)
pencere=Pencere()
sys.exit(app.exec())


#!/usr/bin/env python
import time
import sys
import cPickle
import gzip
import csv
import os
import argparse
import numpy as np
from scipy.spatial import distance
from scipy.spatial.distance import pdist
from scipy.cluster.hierarchy import dendrogram, linkage
from scipy.cluster.hierarchy import fclusterdata
from itertools import dropwhile
import timeit
import multiprocessing as mp, sys
from tempfile import NamedTemporaryFile
try:
        from Bio import SeqIO
        from Bio.Seq  import Seq
        from Bio.SeqRecord import SeqRecord
        from Bio.SeqIO.QualityIO import PairedFastaQualIterator
except ImportError:
        print("Biopython is not installed or is not in the path of python")
        sys.exit(1)

''' -------------------------------------------- '''
''' Function comes from Jordan Callicoat's post '''
''' http://code.activestate.com/recipes/363051/ '''
''' -------------------------------------------- '''
def flatten(l, ltypes=(list, tuple)):
    ltype = type(l)
    l = list(l)
    i = 0
    while i < len(l):
        while isinstance(l[i], ltypes):
            if not l[i]:
                l.pop(i)
                i -= 1
                break
            else:
                l[i:i + 1] = l[i]
        i += 1
    return ltype(l)

class ClusterCdr3():

    def __init__(self, Commands):
         self.commands=Commands
         self.Cdr3Sequences=[]
         self.Cdr3Labels=[]
         self.distancematrix=[]
         self.distancetable=[]
         self.linkagematrix=[]
         self.SequenceClusters=[]
         if isinstance(self.commands['FastaSequences'],dict):
             for k,v in self.commands['FastaSequences'].items():
                 self.Cdr3Sequences.append(v)
                 self.Cdr3Labels.append(k)
             # self.distancematrix=np.array(self._distancematrix())
             #print('Before constructing distance matrix')
             #t0 = time.time()
             self.distancematrix = self._distancematrix_rss()
             #t1 = time.time()
             #print('After constructing distance matrix')
             #print('Distance matrix time = ', t1-t0)

             #print('Before constructing linkage matrix')
             #t0 = time.time()
             self.linkagematrix = linkage(self.distancematrix,method=self.commands['method'])
             #t1 = time.time()
             #print('After constructing linkage matrix')
             #print('Linkage matrix time = ', t1-t0)

             self._dumpLinkageTable()
             self._distancetable()
         elif isinstance(self.commands['FastaSequences'],list):
            for item in self.commands['FastaSequences']:
                 self.Cdr3Sequences.append(str(item.seq))
                 self.Cdr3Labels.append(item.name)
            self.distancematrix=np.array(self._distancematrix())
            self.linkagematrix = linkage(self.distancematrix,method=self.commands['method'])
            self._dumpLinkageTable()
            self._distancetable()
         else:
            pass

    def hamming(self,u, v):
        string=''
        if len(u) == len(v):
            string=u
        else:
            string=min(u,v, key=len)
        Sum=0
        try:
            for i,value in enumerate(string):
                if u[i]!=v[i]:
                    Sum=Sum+1
                else:
                    Sum=Sum+0
        except IndexError:
            print("CDR3s are not the same length")
        return float(Sum)

    def hamming_normalized(self,u, v):
        string=''
        if len(u) == len(v):
            string=u
        else:
            string=min(u,v, key=len)
        Sum=0
        Length=len(u)
        try:
            for i,value in enumerate(string):
                if u[i]!=v[i]:
                    Sum=Sum+1
                else:
                    Sum=Sum+0
        except IndexError:
            print("CDR3s are not the same length")
        return float(Sum)/float(Length)

    def _distancematrix(self):
        Temp=[]
        for i,cdr3a in enumerate(self.Cdr3Sequences):
            for j,cdr3b in enumerate(self.Cdr3Sequences):
                if j>i:
                    Temp.append(self.hamming_normalized(cdr3a,cdr3b))
        return Temp

    def _distancematrix_rss(self):
	    nseq = len(self.Cdr3Sequences)
	    nelem = (nseq * (nseq - 1)) / 2
	    Temp = np.zeros(int(nelem))
	    count = 0
	    for i,cdr3a in enumerate(self.Cdr3Sequences):
		    for j,cdr3b in enumerate(self.Cdr3Sequences):
			    if j>i:
				    Temp[count] = self.hamming_normalized(cdr3a,cdr3b)
				    count += 1
	    return Temp

    def _dumpLinkageTable(self):
        otu_table = open(self.commands['otu-table'],'w')
        N=len(self.Cdr3Sequences)
        M=N-1
        for i,entry in enumerate(self.linkagematrix):
            if self.linkagematrix[i][0] >= N:
                A=M-self.linkagematrix[i][0]
            else:
                A=self.linkagematrix[i][0]

            if self.linkagematrix[i][1]>=N:
                B=M-self.linkagematrix[i][1]
            else:
                B=self.linkagematrix[i][1]
            otu_table.write('%d,%d,%d,%.4f,%d\n'%(-1*i-1,A,B,self.linkagematrix[i][2],self.linkagematrix[i][3]))
        otu_table.close()

    def _distancetable(self):
        Temp=[]
        Nodes=[]
        Distances=[]
        Clusters=[]
        ClustersFlattened=[]
        Size=len(self.linkagematrix)
        N=Size
        '''---------------------------------------------------'''
        ''' Generate three arrays for indexing clustered data '''
        '''---------------------------------------------------'''
        for entry in self.linkagematrix:
            Size=Size+1
            if entry[0] > N and entry[1] > N:
                #print('entry[1] and entry[2] are larger than N')
                Index0=Nodes.index(entry[0])
                Index1=Nodes.index(entry[1])
                Clusters.append( (Clusters[Index0],Clusters[Index1]) )
            elif entry[0] <=N and entry[1] > N:
                #print('entry[1] is larger than N')
                Index1=Nodes.index(entry[1])
                Clusters.append( (entry[0],Clusters[Index1]) )
            elif entry[0] > N and entry[1] <=N:
                #print('entry[0] is larger than N')
                Index0=Nodes.index(entry[0])
                Clusters.append( (Clusters[Index0],entry[1]) )
            else:
                Clusters.append( (entry[0],entry[1]) )
            Nodes.append(Size)
            Distances.append(entry[2])

        for entry in Clusters:
            ClustersFlattened.append( flatten(entry) )

        for i,j in zip(ClustersFlattened,Distances):
            self.distancetable.append((i,j))

    def _clustering(self):
        Temp=[]
        Clusters=[]
        Exclude=[]
        SequenceClusters=[]
        '''---------------------------------------------'''
        '''Determine all disances below some threshold  '''
        '''---------------------------------------------'''
        for i,value in enumerate(self.distancetable):
            if value[1] <=self.commands['cutoff']:
                Temp.append(value[1])
                Clusters.append(value[0])
        """
        Create an exlusion list to remove redundant clustering
        """
        Exclude=[]
        for i,v1 in enumerate(Clusters):
            if self.distancetable[i][1] <=self.commands['cutoff']:
                for j,v2 in enumerate(Clusters):
                    if j>i:
                        if len(set(v1).intersection(v2)):
                            if len(v1)>len(v2):
                                #print('Exclude redundant cluster',v2,j)
                                Exclude.append(j)
                            else:
                                #print('Exclude redundant cluster',v1,i)
                                Exclude.append(i)

        '''-------------------------------'''
        ''' This is the end of our search '''
        '''-------------------------------'''
        ClusterNumber=0
        for i,val in enumerate(Clusters):
            if i not in Exclude:
                try:
                    ''' I incremented the self.distancetable from [i] [i+1]'''
                    #print('Cluster',ClusterNumber,len(val), self.distancetable[i+1][1])
                    Sequences={}
                    for j in val:
                        idx=int(j)
                        #print(">%s\n%s"%(self.Cdr3Labels[idx],self.Cdr3Sequences[idx]))
                        Sequences[self.Cdr3Labels[idx]]=self.Cdr3Sequences[idx]
                    self.SequenceClusters.append(Sequences)
                    ClusterNumber=ClusterNumber+1
                except IndexError:
                    print('No clusters below threshold')

    def GetClustering(self):
        if self.SequenceClusters:
            return self.SequenceClusters
        else:
            self._clustering()
            return self.SequenceClusters

    def GetNonClustering(self):
        Unclustered={}
        if self.SequenceClusters:
            pass
        else:
            self._clustering

        for i,item in enumerate(self.Cdr3Labels):
            if any(item in k for k in self.SequenceClusters):
                pass
            else:
                Unclustered[self.Cdr3Labels[i]]=self.Cdr3Sequences[i]
        return Unclustered

    def getMatrix(self):
        return self.distancematrix

    def getDistanceTable(self):
        return self.distancetable

    def getLabels(self):
        return self.Cdr3Labels

    def getSequences(self):
        return self.Cdr3Sequences

    def getLinkageMatrix(self):
        return self.linkagematrix

#               0           1    2      3       4        5                6
#5cc8b743513b80c04c05206b year1 day0 IGHV3-23 IGHJ1 TNLKNGAYPH ACTAACCTGAAGAATGGGGCCTACCCCCAC
def cluster2(filename,chunk,sequence_identity,clustering_method):
    '''---------------------------------------'''
    ''' Function should operate on large file '''
    '''---------------------------------------'''
    if  os.path.splitext(filename)[-1] in [ '.gz', '.gzip']:
      file_handle = gzip.open(filename, "r")
    else:
      file_handle = open(filename, "r")
    otu_table=[]
    outfile=""
    RunningDict={}
    file_handle.seek(chunk[0])
    while(file_handle.tell() < chunk[1]):
        linestring=file_handle.readline().rstrip()
        data=linestring.split(' ')
        cdr3_length=len(data[3])
        RunningDict[data[0]]=data[3]
    ''' I do not check to ensure that each CDR3 is the same length '''
    if len(RunningDict)>1:
       normalized_hamming=(cdr3_length-int(round(sequence_identity*(cdr3_length))))/(float (cdr3_length))
       cdr3=ClusterCdr3({'FastaSequences':RunningDict,
                         'method':clustering_method,
                         'cutoff':normalized_hamming,
                         'otu-table':'erase.dat'})
       ''' ---------------------------------'''
       ''' Dump out table showing  clusters '''
       ''' ---------------------------------'''
       for cluster in cdr3.GetClustering():
          otu_table.append(cluster.keys())
       #otu_table.append(cdr3.GetNonClustering().keys())
    return  otu_table

def cluster(filename,chunk,sequence_identity,clustering_method):
    '''---------------------------------------'''
    ''' Function should operate on large file '''
    '''---------------------------------------'''
    if  os.path.splitext(filename)[-1] in [ '.gz', '.gzip']:
      file_handle = gzip.open(filename, "r")
    else:
      file_handle = open(filename, "r")

    '''-----------------------------------------'''
    ''' Store sequences in lists containing UMS '''
    '''-----------------------------------------'''
    otu_table=[]
    outfile=""
    RunningDict={}
    file_handle.seek(chunk[0])
    cdr3_length=0
    while(file_handle.tell() < chunk[1]):
        linestring=file_handle.readline().rstrip()
        data=linestring.split(' ')
        ''' -------------------------------------------------------------------'''
        ''' Store the sequences to avoid making routine dependent on Biopython '''
        '''--------------------------------------------------------------------'''
        cdr3_length=len(data[4])
        RunningDict[data[0]]=data[4]
    normalized_hamming=(cdr3_length-int(round(sequence_identity*(cdr3_length))))/(float (cdr3_length))
    cdr3=ClusterCdr3({'FastaSequences':RunningDict,
                      'method':clustering_method,
                      'cutoff':normalized_hamming,
                      'otu-table':'erase.dat'})
    ''' ---------------------------------'''
    ''' Dump out table showing  clusters '''
    ''' ---------------------------------'''
    for cluster in cdr3.GetClustering():
       otu_table.append(cluster.keys())
    #otu_table.append(cdr3.GetNonClustering().keys())
    file_handle.close()
    return otu_table

'''====================================================='''
''' Functions specific for handling the cluster files   '''
'''====================================================='''

def readin_lineage_file(table,filename):
   '''
    This function will read in a probability file. I have put in a numerical
    precision check to ensure that the probabilities sum to 1.0 for each label.
    I did not include a header file, but this is something I can add later on.
   '''

   #5cc8b747513b80c04c052e54 year1 day0 IGHV3-15 IGHJ1 ATSLIQGHTLATD
   counter=0
   temp=[]
   if  os.path.splitext(filename)[-1] in [ '.gz', '.gzip']:
        with gzip.open(filename,'rb') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=' ')
            for row in csv_reader:
                if row[0] == 'END':
                     if len(temp)>1:
                        table['cluster-%10.10d'%(counter)]=[]
                        table['cluster-%10.10d'%(counter)]=temp
                     else:
                        table['singleton-%10.10d'%(counter)]=[]
                        table['singleton-%10.10d'%(counter)]=temp
                     counter=counter+1
                     del temp
                     temp=[]
                else:
                     temp.append(' '.join(map(str,row)))
            csv_file.close()
   else:
        with open(filename) as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=' ')
            for row in csv_reader:
                if row[0] == 'END':
                     if len(temp)>1:
                        table['cluster-%10.10d'%(counter)]=[]
                        table['cluster-%10.10d'%(counter)]=temp
                     else:
                        table['singleton-%10.10d'%(counter)]=[]
                        table['singleton-%10.10d'%(counter)]=temp
                     counter=counter+1
                     del temp
                     temp=[]
                else:
                     temp.append(' '.join(map(str,row)))
            csv_file.close()
   return table

def populate_table(table,filename):
   if  os.path.splitext(filename)[-1] in [ '.gz', '.gzip']:
        with gzip.open(filename,'rb') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=' ')
            for row in csv_reader:
                if row[0] == 'END':
                     pass
                else:
                     label='%s %s %s'%(row[3],row[4],row[6])
                     if label in table:
                         table[label].append(row[0])
                     else:
                         table[label]=[]
                         table[label].append(row[0])
            csv_file.close()
   else:
        with open(filename) as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=' ')
            for row in csv_reader:
                if row[0] == 'END':
                     pass
                else:
                     label='%s %s %s'%(row[3],row[4],row[6])
                     if label in table:
                         table[label].append(row[0])
                     else:
                         table[label]=[]
                         table[label].append(row[0])
            csv_file.close()
   return table

def table_of_mongoids(table,filename):
   if  os.path.splitext(filename)[-1] in [ '.gz', '.gzip']:
        with gzip.open(filename,'rb') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=' ')
            for row in csv_reader:
                if row[0] == 'END':
                     pass
                else:
                     label='%s'%(row[0])
                     if label in table:
                         #table[label].append(' '.join(map(str,row)))
                         print('You must have unique MongoID')
                         pass
                     else:
                         #table[label]=[]
                         #table[label].append(' '.join(map(str,row)))
                         table[label]=' '.join(map(str,row))
            csv_file.close()
   else:
        with open(filename) as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=' ')
            for row in csv_reader:
                if row[0] == 'END':
                     pass
                else:
                     label='%s'%(row[0])
                     if label in table:
                         #table[label].append(' '.join(map(str,row)))
                         pass
                     else:
                         #table[label]=[]
                         #table[label].append(' '.join(map(str,row)))
                         table[label]=' '.join(map(str,row))
            csv_file.close()
   return table

def read_in_formatted_file(filename):
   '''
    This function will read in a probability file. I have put in a numerical
    precision check to ensure that the probabilities sum to 1.0 for each label.
    I did not include a header file, but this is something I can add later on.
   '''
   table={}
   if  os.path.splitext(filename)[-1] in [ '.gz', '.gzip']:
        with gzip.open(filename,'rb') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=' ')
            for row in csv_reader:
                if row[0] != 'END':
                    if row[0] in table:
                        pass
                    else:
                        table[row[0]]=row[1:]
            csv_file.close()
   else:
        with open(filename) as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=' ')
            for row in csv_reader:
                if row[0] != 'END':
                    if row[0] in table:
                        pass
                    else:
                        table[row[0]]=row[1:]
            csv_file.close()
   #print table
   #sys.exit(1)
   return table

def create_mongo_id_lookup_table(table):
   '''
    This function stores all the MongoID for each V3J and is tied to the starting table
   '''
   lookup_table={}
   for key in table.keys():
        label='%s %s %s'%(table[key][2],table[key][3],table[key][5])
        if label in lookup_table:
            lookup_table[label].append(key)
        else:
            lookup_table[label]=[]
            lookup_table[label].append(key)
   for key in lookup_table.keys():
       lookup_table[key]=list(set(lookup_table[key]))
   return lookup_table

def combine_formatted_files(table,filename):
   '''
    This function will read in a probability file. I have put in a numerical
    precision check to ensure that the probabilities sum to 1.0 for each label.
    I did not include a header file, but this is something I can add later on.
   '''
   #5cc8b747513b80c04c052e54 year1 day0 IGHV3-15 IGHJ1 ATSLIQGHTLATD
   if  os.path.splitext(filename)[-1] in [ '.gz', '.gzip']:
        with gzip.open(filename,'rb') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=' ')
            for row in csv_reader:
                if row[0] == 'END':
                     pass
                else:
                     label='%s_%s_%d'%(row[3],row[4],len(row[5]))
                     if label in table:
                         table[label].append(row)
                     else:
                         table[label]=[]
                         table[label].append(row)
            csv_file.close()
   else:
        with open(filename) as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=' ')
            for row in csv_reader:
                if row[0] == 'END':
                     pass
                else:
                     label='%s_%s_%d'%(row[3],row[4],len(row[5]))
                     if label in table:
                         table[label].append(row)
                     else:
                         table[label]=[]
                         table[label].append(row)
            csv_file.close()
   return table

def deduplicate_table(table,integer_keys):
    reduced=NamedTemporaryFile(suffix='.cluster',dir='./',delete=False)
    deduplicated={}
    reduce_keys=integer_keys[1:]
    '''------------------------------------'''
    ''' You now have a table wich V-J-CDR3 '''
    '''------------------------------------'''
    for key in table.keys():
        '''--------------------------------'''
        ''' You now have a list of entries '''
        '''--------------------------------'''
        deduplicated[key]=[]
        for entry in table[key]:
            label='%s'%(' '.join(map(str,[entry[num] for num in reduce_keys])))
            if label not in deduplicated[key]:
                string=' '.join(map(str,entry))
                deduplicated[key].append(string)
            else:
                pass
    for key in deduplicated.keys():
        for line in deduplicated[key]:
            reduced.write('%s\n'%line)
        reduced.write('END\n')
    reduced.close()
    return reduced.name

def dump_out_cluster_file(filename,integer_keys):
   '''
    This function will read in a probability file. I have put in a numerical
    precision check to ensure that the probabilities sum to 1.0 for each label.
    I did not include a header file, but this is something I can add later on.
   '''
   clustered_outfile=NamedTemporaryFile(suffix='.cluster',dir='./',delete=False)
   #5cc8b747513b80c04c052e54 year1 day0 IGHV3-15 IGHJ1 ATSLIQGHTLATD
   if  os.path.splitext(filename)[-1] in [ '.gz', '.gzip']:
        with gzip.open(filename,'rb') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=' ')
            for row in csv_reader:
                if row[0] == 'END':
                     clustered_outfile.write('END\n')
                else:
                     clustered_outfile.write('%s\n'%(' '.join(map(str,[row[num] for num in integer_keys]))))
            csv_file.close()
   else:
        with open(filename) as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=' ')
            for row in csv_reader:
                if row[0] == 'END':
                     clustered_outfile.write('END\n')
                else:
                     clustered_outfile.write('%s\n'%(' '.join(map(str,[row[num] for num in integer_keys]))))
            csv_file.close()
   clustered_outfile.close()
   return clustered_outfile.name

def main():
    t0 = time.time()
    cluster={}
    usage = "usage: %prog [options] arg\n./cluster-cdr3.py --fasta-sequences=<file> --cdr3-length=<int> --sequence-identity-threshold=<float>"
    cluster = argparse.ArgumentParser(description="Will cluster cdr3 sequences of the same length")
    cluster.add_argument("--full-formatted-file",
                        action="store",
                        dest="full-formatted-file",
                        type=str,
                        help='FASTA formatted file containing either DNA or AA sequences')
    cluster.add_argument("--fasta-file",
                        action="store",
                        dest="fasta-file",
                        type=str,
                        help='FASTA formatted file containing either DNA or AA sequences')
    cluster.add_argument("--user-fasta-file",
                        action='store_true',
                        dest="user-fasta-file",
                        help='user defined their own formatted file',
                        default=False)
    cluster.add_argument("--user-formatted-file",
                        action='store_true',
                        dest="user-formatted-file",
                        help='user defined their own formatted file',
                        default=False)
    cluster.add_argument("--formatted-file",
                        action="store",
                        dest="formatted-file",
                        type=str,
                        help='FASTA formatted file containing either DNA or AA sequences')
    cluster.add_argument("--clustered-formatted-outfile",
                        action="store",
                        dest="clustered-formatted-outfile",
                        type=str,
                        help='FASTA formatted file containing either DNA or AA sequences')
    cluster.add_argument("--cdr3-length",
                        action="store",
                        type=int,
                        dest="cdr3-length",
                        help='length of sequence (all sequences must be of the same length)')
    cluster.add_argument("--sequence-identity",
                        action="store",
                        dest="sequence-identity",
                        type=float,
                        help='sequence identity threshold',
                        default=0.90)
    cluster.add_argument("--linkage-file",
                        action="store",
                        dest="linkage-file",
                        type=str,
                        help='linkage table',
                        default='linkage.table')
    cluster.add_argument("--number-processors",
                        action="store",
                        dest="number-processors",
                        type=int,
                        help='total number of processors to use',
                        default=40)
    cluster.add_argument('--keys-for-clustering',
                         action='store',
                         dest='keys-for-clustering',
                         nargs="*",
                         type=int,
                         default=[0,3,4,6])
    cluster.add_argument("--use-otu-file",
                  action='store_true',
                  help='do not dump out clustered files in FASTA file',
                  default=False)
    cluster.add_argument("--clustering-type",
                        action="store",
                        dest="clustering-type",
                        type=str,
                        help='type of clustering (single,complete or average)',
                        choices=['single', 'complete', 'average'],
                        default='complete')

    options = vars(cluster.parse_args())
    '''-----------------------------------'''
    ''' Read in sequences from FASTA file '''
    '''-----------------------------------'''
    #FastaSequences = SeqIO.index(options['fasta-file'], "fasta")

    #if len(FastaSequences)<2:
    #    sys.exit(1)

    '''--------------------------'''
    '''Read in the length of CDR '''
    '''--------------------------'''
    #Cdr3Length=options['cdr3-length']

    '''--------------------------------------------------'''
    ''' Identity will be converted to a Hamming Distance '''
    '''--------------------------------------------------'''
    SequenceIdentity=options['sequence-identity']

    '''----------------------------'''
    '''Normalized Hamming Distance '''
    '''----------------------------'''
    #NormalizedHamming=(Cdr3Length-int(round(SequenceIdentity*(Cdr3Length))))/(float (Cdr3Length))

    if options['user-formatted-file']==True:
        '''deuplicate the file '''
        cluster_file=options['formatted-file']
        sys.exit(1)
    elif options['user-fasta-file']==True:
        print('Currently not supported')
        sys.exit(1)
        FastaSequences = SeqIO.index(options['fasta-file'], "fasta")
        '''Convert these into the proper format'''
    else:
        table={}
        combine_formatted_files(table,options['full-formatted-file'])
        reduced_table=deduplicate_table(table,options['keys-for-clustering'])
        cluster_file=dump_out_cluster_file(reduced_table, options['keys-for-clustering'])
    '''-------------------------------------'''
    ''' Read in a correctly formatted table '''
    '''-------------------------------------'''
    global_position=0
    #fh=open(options['formatted-file'])
    fh=open(cluster_file)
    locations=[]
    anchor=0
    #filesize=os.path.getsize(options['formatted-file'])
    filesize=os.path.getsize(cluster_file)
    while( global_position <= filesize+1 ):
        row = fh.readline().rstrip()
        if row == 'END':
           locations.append((anchor,fh.tell()-4))
           anchor=fh.tell()
        else:
           global_position = global_position + len(row) + 1
    fh.close()

    '''-----------------------------------'''
    ''' Multiprocess and break up the job '''
    '''-----------------------------------'''
    jobs=[]
    pool = mp.Pool(options['number-processors'])
    for chunk in locations:
        job = pool.apply_async(cluster2,(cluster_file,chunk,options['sequence-identity'],options['clustering-type']))
        jobs.append(job)

    '''------------------'''
    ''' Finish things up '''
    '''------------------'''
    output = []
    for job in jobs:
        try:
           output.append( job.get() )
        except ValueError:
           pass
    pool.close()
    '''------------------------------------------'''
    ''' Store clusters in a list of dictionaries '''
    '''------------------------------------------'''
    otu_table=[]
    for line in output:
        for entry in line:
            otu_table.append(entry)

    '''-------------------------------------------------------------------------------'''
    ''' Dump out DNA data into a flat file that is similar to the one used originally '''
    '''             The DNA sequences should be from each cluster (buggy if no clusters)                     '''
    '''-------------------------------------------------------------------------------'''
    if len(otu_table) > 0:
       clustered_outfile=gzip.open(options['clustered-formatted-outfile'],'wb')
       table=read_in_formatted_file(options['full-formatted-file'])
       lookup_table=create_mongo_id_lookup_table(table)
       clusteredlabels=[]
       for cluster in otu_table:
           for mongoid in cluster:
                clustered_outfile.write('%s %s\n'%(mongoid,' '.join(table[mongoid])))
                clusteredlabels.append(mongoid)
           clustered_outfile.write('END\n')
       clustered_outfile.close()
       print('====> Done finding all clusters\n')
       '''================================================'''
       ''' This next section is very messy and needs help '''
       '''================================================'''
       table={}
       #{'IGHV4-39 IGHJ3 GCGAGGCGCGGGAACTTCATGCCCCTTGATGCTTTTGATTTT': ['STAU-469']
       populate_table(table,options['full-formatted-file'])

       table_mongoids={}
       #'STAU-466': 'STAU-466 new new IGHV4-39 IGHJ4 ARRGNFMPLDAFDF
       table_of_mongoids(table_mongoids,options['full-formatted-file'])

       lineage={}
       readin_lineage_file(lineage,options['clustered-formatted-outfile'])

       '''=========================================================='''
       ''' Add identical v3j clonotypes to clusters (undeduplicate) '''
       '''=========================================================='''
       appear=[]
       outfile = gzip.open(options['clustered-formatted-outfile'], 'wb')
       for key in lineage.keys():
           duplicate_entry=[]
           for entry in lineage[key]:
               temp=entry.split(' ')
               v3j='%s'%' '.join(map(str,[temp[num] for num in options['keys-for-clustering'][1:]]))
               del temp
               for mongoid in table[v3j]:
                   appear.append(mongoid)
                   duplicate_entry.append(table_mongoids[mongoid])
               #remove v3j from table
               #del my_dict['key']
           '''--------------------------------------'''
           ''' Deduplciate to avoid double counting '''
           '''--------------------------------------'''
           for unique_record in list(set(duplicate_entry)):
                 outfile.write('%s\n'%(unique_record))
           outfile.write('%s\n'%'END')
           del duplicate_entry
       outfile.close()
       appear=list(set(appear))
       print('====> Done Adding identical v3j clonotypes to  all clusters\n')

       '''====================================================================='''
       ''' Now determine all labels that don't appear in the clustered file and '''
       ''' and create a list of lists for all labels not appearing in clusters '''
       '''====================================================================='''
       list_of_lists=[]
       for key in list(set(table_mongoids.keys()).difference(set(appear))):
           temp=table_mongoids[key].split(' ')
           temp2=[]
           v3j='%s'%' '.join(map(str,[temp[num] for num in options['keys-for-clustering'][1:]]))
           if key in table[v3j]:
               for entry in table[v3j]:
                   #print table['%s %s %s'%(temp[3],temp[4],temp[6])]  #provides list of identical v3js
                   temp2.append(table_mongoids[entry])
               list_of_lists.append(temp2)
               del temp
               del temp2

       '''===================================='''
       ''' Remove redundant copies of entries '''
       '''===================================='''
       new_k = []
       for elem in list_of_lists:
           if elem not in new_k:
               new_k.append(elem)

       '''============================='''
       ''' Dump out singleton clusters '''
       '''============================='''
       outfile = gzip.open(options['clustered-formatted-outfile'], 'ab')
       k = new_k
       for elements in k:
           for element in elements:
               outfile.write('%s\n'%element)
           outfile.write('%s\n'%'END')
       outfile.close()
       print('====> Done Adding identical v3j clonotypes to  all singleton clusters\n')
       sys.exit(1)

       ''' Try and place these into a lists of lists based on their identity with V3J clonotype'''
       #print(table)
       #print('Total number of lineages: %d'%(len(otu_table)))
       #print('Total number of sequences: %d'%(len(table)))
       #print('Total number of sequences in clusters: %d'%(len(clusteredlabels)))
       #print('Total number of singletons: %d'%(len(singletons)))
    else:
       print('No clusters!')
       pass
    if os.path.exists(cluster_file):
       pass
       #os.remove(cluster_file)

if __name__ == "__main__":
     main()


# Задача 1. В каждой группе учится от 20 до 30 студентов.
# По итогам экзамена все оценки заносятся в таблицу.
# Каждой группе отведена своя строка. Определите группу с наилучшим средним баллом.
from random import randint
from statistics import mean

group = [[randint(2,5) for _ in range(0, randint(20, 30))] for _ in range(0, randint(2,11))]
gpa, group_number = mean(group[0]) , 1

def Compare_gpa(row, count):
    global gpa
    global group_number
    if mean(row) > gpa:
        gpa, group_number = mean(row), count

def Print_list(group):
    global gpa
    global group_number
    count = 1
    for row in group:
        Compare_gpa(row, count)
        print(f'{count}: {row}')
        count +=1
    print(f'\nГруппа:  {group_number}\nСредний балл: {round(gpa, 2)}')

Print_list(group)


from sympy import symbols
import numpy as np
from sklearn import datasets
import math
from sympy import plot_implicit as plt_eq
from sympy import lambdify
import matplotlib.pyplot as plt

#Reading Datasets
iris = datasets.load_iris()

#apriori probabilities
P_setosa = 1/3
P_virginica = 1/3
P_versicolor = 1/3

X_setosa_temp = iris.data[:50,:]
X_versicolor_temp = iris.data[50:100,:]
X_virginica_temp = iris.data[100:150,:]

np.random.shuffle(X_setosa_temp)
np.random.shuffle(X_versicolor_temp)
np.random.shuffle(X_virginica_temp)

X_setosa = X_setosa_temp[:40,:]
X_versicolor = X_versicolor_temp[:40,:]
X_virginica = X_virginica_temp[:40,:]

X_setosa_test = X_setosa_temp[40:,:]
X_versicolor_test = X_versicolor_temp[40:,:]
X_virginica_test = X_virginica_temp[40:,:]

#mean calculated
mean_X_setosa = np.mean(X_setosa,axis = 0)
mean_X_versicolor = np.mean(X_versicolor,axis = 0)
mean_X_virginica = np.mean(X_virginica,axis = 0)

#calculate covariance matrices
covariance_X_setosa = np.cov(X_setosa.T)
covariance_X_versicolor = np.cov(X_versicolor.T)
covariance_X_virginica = np.cov(X_virginica.T)

#inverses calcualtion
inv_covariance_X_setosa = np.linalg.inv(covariance_X_setosa)
inv_covariance_X_versicolor = np.linalg.inv(covariance_X_versicolor)
inv_covariance_X_virginica =np.linalg.inv(covariance_X_virginica)

#determinant calculation
det_covariance_X_setosa = np.linalg.det(covariance_X_setosa)
det_covariance_X_versicolor = np.linalg.det(covariance_X_versicolor)
det_covariance_X_virginica =np.linalg.det(covariance_X_virginica)

x1,x2,x3,x4 = symbols("x1 x2 x3 x4")

X_new_point = np.array([[x1],[x2],[x3],[x4]])

A_setosa=-(.5)*inv_covariance_X_setosa
B_setosa=np.matmul(inv_covariance_X_setosa,mean_X_setosa)
C_setosa= math.log(P_setosa) - (.5*math.log(det_covariance_X_setosa))- (.5*(np.matmul(mean_X_setosa.T,np.matmul(inv_covariance_X_setosa,mean_X_setosa))))

A_versicolor=-(.5)*inv_covariance_X_versicolor
B_versicolor=np.matmul(inv_covariance_X_versicolor,mean_X_versicolor)
C_versicolor=math.log(P_versicolor) - (.5*math.log(det_covariance_X_versicolor))- (.5*(np.matmul(mean_X_versicolor.T,np.matmul(inv_covariance_X_versicolor,mean_X_versicolor))))

A_virginica=-(.5)*inv_covariance_X_virginica
B_virginica=np.matmul(inv_covariance_X_virginica,mean_X_virginica)
C_virginica=math.log(P_virginica) - .5*math.log(det_covariance_X_virginica)- .5*(np.matmul(mean_X_virginica.T,np.matmul(inv_covariance_X_virginica,mean_X_virginica)))

#curve g1 g2
#curve g1 g3
#curve g2 g3

#Generating Discriminant Functions
G1 = np.matmul(X_new_point.T,np.matmul(A_setosa,X_new_point)) + np.matmul(B_setosa,X_new_point)+C_setosa
G2 = np.matmul(X_new_point.T,np.matmul(A_versicolor,X_new_point)) + np.matmul(B_versicolor,X_new_point)+C_versicolor
G3 = np.matmul(X_new_point.T,np.matmul(A_virginica,X_new_point)) + np.matmul(B_virginica,X_new_point)+C_virginica

G1 = G1[0][0]
G2 = G2[0][0]
G3 = G3[0][0]

error_count = 0

print("\nGiving 10 Setosa inputs - \n")
for i,each in enumerate(X_setosa_test):
    r1,r2,r3,r4 = each
    g1_val=G1.subs([(x1, r1), (x2, r2), (x3, r3),(x4,r4)])
    g2_val=G2.subs([(x1, r1), (x2, r2), (x3, r3),(x4,r4)])
    g3_val=G3.subs([(x1, r1), (x2, r2), (x3, r3),(x4,r4)])
    if(g1_val>g2_val and g1_val>g3_val):
        print(i,"Setosa")
    if(g2_val>g1_val and g2_val>g3_val):
        print(i,"Versicolor")
        error_count+=1
    if(g3_val>g2_val and g3_val>g1_val):
        error_count+=1
        print(i,"Virginica")

print("\nGiving 10 Versicolor inputs - \n")
for i,each in enumerate(X_versicolor_test):
    r1,r2,r3,r4 = each
    g1_val=G1.subs([(x1, r1), (x2, r2), (x3, r3),(x4,r4)])
    g2_val=G2.subs([(x1, r1), (x2, r2), (x3, r3),(x4,r4)])
    g3_val=G3.subs([(x1, r1), (x2, r2), (x3, r3),(x4,r4)])
    if(g2_val>g1_val and g2_val>g3_val):
        print(i,"Versicolor")
    if(g1_val>g2_val and g1_val>g3_val):
        error_count+=1
        print(i,"Setosa")
    if(g3_val>g2_val and g3_val>g1_val):
        error_count+=1
        print(i,"Virginica")

print("\nGiving 10 Virginica inputs - \n")
for i,each in enumerate(X_virginica_test):
    r1,r2,r3,r4 = each
    g1_val=G1.subs([(x1, r1), (x2, r2), (x3, r3),(x4,r4)])
    g2_val=G2.subs([(x1, r1), (x2, r2), (x3, r3),(x4,r4)])
    g3_val=G3.subs([(x1, r1), (x2, r2), (x3, r3),(x4,r4)])
    if(g1_val>g2_val and g1_val>g3_val):
        error_count+=1
        print(i,"Setosa")
    if(g2_val>g1_val and g2_val>g3_val):
        error_count+=1
        print(i,"Versicolor")
    if(g3_val>g2_val and g3_val>g1_val):
        print(i,"Virginica")

#Accuracy Calculation
print("accuracy = "+str(1-(error_count/150)))


import csv
from collections import defaultdict

word_counts = defaultdict(int)

with open('test.txt', 'r') as in_file:
    for line in in_file:
        words = line.split()
        for word in words:
            word_counts[word] += 1

with open('demo copy.csv', 'w', newline='') as out_file:
    writer = csv.writer(out_file)
    writer.writerow(['Word', 'Count'])
    for word, count in word_counts.items():
        writer.writerow([word, count])
        print(word, count)


#!/usr/bin/env python3

import argparse

from migen import *
from flipsyfat.cores.sd_emulator import SDEmulator
from flipsyfat.cores.sd_trigger import SDTrigger
from flipsyfat.cores.sd_timer import SDTimer
from flipsyfat.cores.clock import ClockOutput
from misoc.targets.papilio_pro import BaseSoC
from misoc.cores.gpio import GPIOTristate
from migen.build.generic_platform import *
from misoc.integration.soc_sdram import *
from misoc.integration.builder import *

io = [
    ("sdemu", 0,
        Subsignal("clk", Pins("C:8")),
        Subsignal("cmd", Pins("C:9")),
        Subsignal("d", Pins("C:10 C:11 C:12 C:13")),
        IOStandard("LVCMOS33")
    ),
    ("trigger", 0,
        Pins("C:0 C:1 C:2 C:3 C:4 C:5 C:6 C:7"),
        IOStandard("LVCMOS33")
    ),
    ("gpio", 0,
        Pins("A:0 A:1 A:2 A:3 A:4 A:5 A:6 A:7 " +
             "A:8 A:9 A:10 A:11 A:12 A:13 A:14 A:15"),
        IOStandard("LVCMOS33")
    ),
    ("debug", 0,
        Pins("B:12 B:13 B:14 B:15"),
        IOStandard("LVCMOS33")
    ),
    ("clkout", 0,
        Pins("C:14 C:15"),
        IOStandard("LVCMOS33")
    ),
]

class Flipsyfat(BaseSoC):
    mem_map = {
        "sdemu": 0x30000000,
    }
    mem_map.update(BaseSoC.mem_map)

    def __init__(self, **kwargs):
        BaseSoC.__init__(self, uart_baudrate=500000, **kwargs)
        self.platform.add_extension(io)

        self.submodules.sdemu = SDEmulator(self.platform, self.platform.request("sdemu"))
        self.register_mem("sdemu", self.mem_map["sdemu"], self.sdemu.bus, self.sdemu.mem_size)
        self.csr_devices += ["sdemu"]
        self.interrupt_devices += ["sdemu"]

        self.submodules.sdtimer = SDTimer(self.sdemu.ll)
        self.csr_devices += ["sdtimer"]

        self.submodules.sdtrig = SDTrigger(self.sdemu.ll, self.platform.request("trigger"))
        self.csr_devices += ["sdtrig"]

        self.submodules.gpio = GPIOTristate(self.platform.request("gpio"))
        self.csr_devices += ["gpio"]

        self.submodules.clkout = ClockOutput(self.platform.request("clkout"))
        self.csr_devices += ["clkout"]

        # Activity LED
        self.io_activity = self.sdemu.ll.block_read_act | self.sdemu.ll.block_write_act
        self.sync += self.platform.request("user_led").eq(self.io_activity)

        # Debug signals
        self.comb += self.platform.request("debug").eq(Cat(
            self.sdemu.ll.block_read_act,
            self.sdemu.ll.data_out_done,
            self.sdtimer._capture.re,
        ))

def main():
    parser = argparse.ArgumentParser(description="Flipsyfat port to the Papilio Pro")
    builder_args(parser)
    soc_sdram_args(parser)
    args = parser.parse_args()

    soc = Flipsyfat(**soc_sdram_argdict(args))
    builder = Builder(soc, **builder_argdict(args))
    builder.build()

if __name__ == "__main__":
    main()