#!/home/samhaug/anaconda2/bin/python

'''
==============================================================================

File Name : grid_to_thickness.py
Purpose : get thickness/topo of transition zone from grid file
Creation Date : 21-08-2018
Last Modified : Thu 23 Aug 2018 12:33:46 PM EDT
Created By : Samuel M. Haugland

==============================================================================
'''

import numpy as np
from matplotlib import pyplot as plt
from subprocess import call
from os import listdir
import h5py
import argparse

def main():
    parser = argparse.ArgumentParser(description='make GMT file for plot')
    parser.add_argument('-f','--file', metavar='H5_FILE',type=str,
                        help='h5 grid file')
    parser.add_argument('-n','--name', type=str,
                       help='filename',default='tz_thickness.dat')
    args = parser.parse_args()

    f = h5py.File(args.file,'r')
    gc = f['grid_count'][:]+1
    gcmask = np.mean(gc,axis=2)
    gcmask[gcmask < 800] = 0.
    gcmask[gcmask >= 800] = 1.
    grid = f['grid'][:]*(1./gc)
    h = f['h'][:]

    idx_400 = np.where(h==400)[0][0]
    m_400 = np.argmax(grid[:,:,idx_400-50:idx_400+50],axis=2)
    m_400 = ((400-50)+(m_400))

    idx_670 = np.where(h==670)[0][0]
    m_670 = np.argmax(grid[:,:,idx_670-50:idx_670+50],axis=2)
    m_670 = ((670-50)+(m_670))

    #m_670 = m_670*gcmask
    #m_400 = m_400*gcmask
    m = (m_670-m_400)*gcmask

    lat = f['lat'][:]
    lon = f['lon'][:]
    yy,xx = np.meshgrid(lat,lon)
    yy = np.reshape(yy,yy.size)
    xx = np.reshape(xx,xx.size)
    m = np.reshape(m,m.size)
    #plt.imshow(gcmask,aspect='auto')
    #plt.show()
    gcmask = np.reshape(gcmask,gcmask.size)
    np.savetxt(args.name,np.c_[xx,yy,m])
    np.savetxt('mask_'+args.name,np.c_[xx,yy,gcmask])

main()


from console_tools import show_dirs, show_files, main_dialog, write_results_to_file
import os

number_ = 'первый'
first_file = None
first_keys = ()
second_file = None
second_keys = ()

while True:
    current_dir = os.getcwd()
    files = os.listdir(current_dir)
    files.insert(0, '..')

    try:
        answer = input(f'Текущая директория: {current_dir}\n'
                        f'chdir - Сменить директорию\n'
                        f'select - Выбрать {number_} файл\n')

        if answer == 'chdir':
            show_dirs(files)
            path_text = 'Выберите папку'
        elif answer == 'select':
            show_files(files)
            path_text = 'Выберите файл'

        path_ = int(input(path_text))

        if answer == 'chdir':
            if not main_dialog(answer, files[path_]):
                print('Ошибка перехода в директорию')
            continue

        elif answer == 'select':
            try:
                if not first_keys:
                    first_keys = main_dialog(answer, files[path_])
                    if first_keys:
                        print('Первый файл обработан')
                        number_ = 'второй'
                    else:
                        print('Не смог получить ключи из первого файла')
                else:
                    second_keys = main_dialog(answer, files[path_])
                    if first_keys:
                        print('Второй файл обработан')
                        break
                    else:
                        print('Не смог получить ключи из второго файла')
            except:
                print('Ошибка при выборе файла/папки')
                continue
    except:
        print('Неизвестная команда')
        continue

unique_elements = second_keys - first_keys
if unique_elements:
    write_results_to_file(unique_elements)
else:
    write_results_to_file()
    print('Уникальные элементы не найдены')


import torchvision.models as models

from models.fcn import *
from models.unet import *

def get_model(name, n_classes):
    model = _get_model_instance(name)

    if name in ['frrnA', 'frrnB']:
        model = model(n_classes, model_type=name[-1])

    elif name in ['fcn32s', 'fcn16s', 'fcn8s']:
        model = model(n_classes=n_classes)
        vgg16 = models.vgg16(pretrained=True)
        model.init_vgg16_params(vgg16)

    elif name == 'segnet':
        model = model(n_classes=n_classes,
                      is_unpooling=True)
        vgg16 = models.vgg16(pretrained=True)
        model.init_vgg16_params(vgg16)

    elif name == 'unet':
        model = model(n_classes=n_classes, is_batchnorm=True, in_channels=1, is_deconv=True)

    else:
        model = model(n_classes=n_classes)

    return model

def _get_model_instance(name):
    try:
        return {
            'fcn32s': fcn32s,
            'fcn8s': fcn8s,
            'fcn16s': fcn16s,
            'unet': unet
        }[name]
    except:
        print('Model {} not available'.format(name))


import sys
import os
import numpy as np

class ConwayGame():

    #Initializes the game with given inputs
    def __init__(self, rows=5, cols=5, gen=3, grid_init_file=None, alive_prob=0.5, modified_rule=False):
        self.gen = gen
        self.modified_rule = modified_rule
        if(grid_init_file != None):
            self.grid = np.loadtxt(grid_init_file, dtype=np.int8)
            self.rows = self.grid.shape[0]
            self.cols = self.grid.shape[1]
        else:
            self.rows = rows
            self.cols = cols
            temp = np.random.binomial(rows*cols, alive_prob, (rows,cols)) / (rows*cols)
            temp[temp>=0.5] = 1
            temp[temp<0.5] = 0
            self.grid = temp.astype(int)
            np.savetxt('initial_state.txt', self.grid, fmt='%i')

    def check_alive(self, row, col):
        #To count how many neighbour cells are alive
        alive_neighbors = 0

        if row>0 and col>0 and self.grid[row-1][col-1] == 1:
            alive_neighbors = alive_neighbors + 1
        if row>0 and self.grid[row-1][col] == 1:
            alive_neighbors = alive_neighbors + 1
        if row>0 and (col+1)<self.cols and self.grid[row-1][col+1] == 1:
            alive_neighbors = alive_neighbors + 1
        if col>0 and self.grid[row][col-1] == 1:
            alive_neighbors = alive_neighbors + 1
        if (col+1)<self.cols and self.grid[row][col+1] == 1:
            alive_neighbors = alive_neighbors + 1
        if (row+1)<self.rows and col>0 and self.grid[row+1][col-1] == 1:
            alive_neighbors = alive_neighbors + 1
        if (row+1)<self.rows and self.grid[row+1][col] == 1:
            alive_neighbors = alive_neighbors + 1
        if (row+1)<self.rows and (col+1)<self.cols and self.grid[row+1][col+1] == 1:
            alive_neighbors = alive_neighbors + 1
        if(modified_rule):
            alive_neighbors = alive_neighbors + 1

        #Checking if the cell is alive
        if self.grid[row][col] == 1:
            if alive_neighbors < 2:
                return False
            if alive_neighbors == 2 or alive_neighbors == 3:
                return True
            return False
        else:
            if alive_neighbors == 3:
                return True
            return False

    #Starts the game and runs till the provided generations and
    def start_game(self, save_gen_no=-1):
        till_gen = self.gen
        #Checking if the user has specified a particular generation to run the game till
        if(save_gen_no != -1):
            till_gen = save_gen_no

        #Looping through generations
        for i in range(till_gen):
            #Creating a copy to hold the next generation values
            grid_copy = np.zeros((self.rows,self.cols), dtype=np.int8)

            for row in range(self.rows):
                for col in range(self.cols):
                    is_alive = self.check_alive(row, col)
                    if is_alive:
                        grid_copy[row][col] = 1
                    else:
                        grid_copy[row][col] = 0
            self.grid = grid_copy
        np.savetxt('final_state.txt', self.grid, fmt='%i')

#For getting the index positions of the options (-d, -g and -f), if provided by the user
def get_index_options():
    options = {'-d':-1, '-g':-1, '-f':-1, '-p':-1, '-r':1}
    l = len(sys.argv)
    for i in range(1, l):
        if sys.argv[i] in options:
            options[sys.argv[i]] = i
    return options

if __name__ == '__main__':
    #Default values for the specifications
    rows, cols, gen, grid_init_file, alive_prob, modified_rule = 5, 5, 3, None, 0.5, False
    n_args = len(sys.argv)

    #Checking if the user asked for help option
    if(n_args>12 or (n_args == 2 and (sys.argv[1] == '-h' or sys.argv[1] == '--help'))):
        print('Options (optional) for running this program:')
        print('-d 10 10: For providing rows and columns for the grid')
        print('-g 3: For providing number of generations to run the game')
        print('-f \'initial_state.txt\': For providing input file for the initial state of the game')
        print('-p 0.3: For providing probability value for the cells to be alive in the initial state')
        print('-r True: For modifying the rule of neighbor relation')
        sys.exit()
    options = get_index_options()

    #Checking if the user selected -d option for specifying dimensions of the grid
    if(options['-d'] != -1):
        index = options['-d']
        if((index+1) >= n_args or (index+2) >= n_args or sys.argv[index+1].isdigit() == False or sys.argv[index+2].isdigit() == False):
            print('If -d option is selected then you must provide rows and columns for the grid as integers')
            sys.exit()
        else:
            rows = int(sys.argv[index+1])
            cols = int(sys.argv[index+2])

    #Checking if the user selected -g option for specifying number of generations
    if(options['-g'] != -1):
        index = options['-g']
        if((index+1) >= n_args or sys.argv[index+1].isdigit() == False):
            print('If -g option is selected then you must provide number of generations as integer')
            sys.exit()
        else:
            gen = int(sys.argv[index+1])

    #Checking if the user selected -f option for specifying initial state for the game
    if(options['-f'] != -1):
        index = options['-f']
        if((index+1) >= n_args or os.path.isfile(sys.argv[index+1]) == False):
            print('If -h option is selected then you must provide a valid input file')
            sys.exit()
        else:
            grid_init_file = sys.argv[index+1]

    #Checking if the user selected -p option for specifying probability value
    if(options['-p'] != -1):
        index = options['-f']
        if((index+1) >= n_args or sys.argv[index+1].isdigit() == False):
            print('If -h option is selected then you must provide a valid input file')
            sys.exit()
        else:
            grid_init_file = sys.argv[index+1]
    game = ConwayGame(rows, cols, gen, grid_init_file)
    game.start_game()


import random

from app.models import *

models = [Test, Question, Answer, Journal, Discipline]

def create(cls, **kwargs):
    obj = cls(**kwargs)
    obj.save()
    return obj

for model in models:
    model.objects.all().delete()
print("Data deleted")

d = []
for i in range(1, 4):
    d.append(create(Discipline, name="Discipline №{}".format(i)))
    print("Discipline №{} created".format(i))

for i in range(1, 3):
    test = create(Test, name="Test №{}".format(i), discipline=random.choice(d))
    print("Test №{} created".format(i))
    for k in range(1, 11):
        question_text = "Question №{}".format(k)
        question = create(Question, text=question_text, test=test)
        print("\tQuestion №{} created".format(k))
        answers = []
        for j in range(1, 5):
            answer_text = "Answer №{}".format(j)
            answer = create(Answer, text=answer_text, question=question)
            print("\t\tAnswer №{} created".format(j))
            answers.append(answer)
        right_answer = random.choice(answers)
        right_answer.right = True
        right_answer.save(update_fields=['right'])


import random, glob, sys
import os.path as osp
import tensorflow as tf
import numpy as np

from helpers_05 import grouper, flatten, fully_connected_layer
from helpers_06 import maybe_download, maybe_extract

def shuffle_arrays(*args):
    "Shuffles list of NumPy arrays in unison"
    state = np.random.get_state()
    for array in args:
        np.random.set_state(state)
        np.random.shuffle(array)

def shuffle_lists(*args):
    curr_state = random.getstate()
    for a in args:
        random.setstate(curr_state)
        random.shuffle(a)

def batches(data, labels, batch_size, shuffler=None):
    '''old batcher was array only'''
    if len(data) != len(labels):
        raise ValueError('Image data and label data must be same size')
    if batch_size > len(data):
        raise ValueError('Batch size cannot be larger than size of datasets')
    if shuffler is not None:
        assert shuffler in [shuffle_arrays, shuffle_lists]
        shuffler(data, labels)
    for res in zip(grouper(data, batch_size),
                   grouper(labels, batch_size)):
        yield res

def array_batches(*args, **kwargs):
    kwargs.setdefault('shuffler', shuffle_arrays)
    return batches(*args, **kwargs)

def list_batches(*args, **kwargs):
    kwargs.setdefault('shuffler', shuffle_lists)
    return batches(*args, **kwargs)

def download_hh(local_dir="data/hh", noisy=True):
    url = 'https://www.dropbox.com/s/c452t2dpaq8nm2y/' # hamsterhare.tar.gz?dl=1'
    fn = "hamsterhare.tar.gz"
    maybe_download(fn, url, local_dir, add_url_suffix="?dl=1", noisy=noisy)
    maybe_extract(osp.join(local_dir, fn), noisy=noisy)

def train_test_split_hh_filenames(test_pct = .2, parent_dir="data/hh"):
    hamster_dir = osp.join(parent_dir, "hamsterhare", "hamster")
    hare_dir = osp.join(parent_dir, "hamsterhare", "hare")

    jpeg = "*.JPEG"

    hamster_files = glob.glob(osp.join(hamster_dir, jpeg))
    hare_files    = glob.glob(osp.join(hare_dir, jpeg))

    all_files = list(hamster_files) + list(hare_files)
    all_labels = [0] * len(hamster_files) + [1] * len(hare_files)

    shuffle_lists(all_files, all_labels)
    train_idx = int(len(all_files) * (1.0-test_pct))

    train_files,  test_files  = all_files[ :train_idx], all_files[ train_idx:]
    train_labels, test_labels = all_labels[:train_idx], all_labels[train_idx:]
    return train_files, train_labels, test_files, test_labels

def fully_connected_xavier_relu_layer(incoming_layer, num_nodes,
                                      b_val=0.01,
                                      keep_prob=None, name=None):
    ''' from wk 05 nb: pass through for fully_connected_layer with xavier init '''
    incoming_layer = tf.convert_to_tensor(incoming_layer)
    prev_num_nodes = incoming_layer.shape.dims[-1].value

    w_stddev = np.sqrt(2.0 / prev_num_nodes)

    # from helpers_05
    return fully_connected_layer(incoming_layer, num_nodes,
                                 w_stddev = w_stddev, b_val=b_val,
                                 activation_fn = tf.nn.relu,
                                 keep_prob=keep_prob,
                                 name=name)

if __name__ == "__main__":
    download_hh()


class Allergies():
    score_key = {
        'eggs': 1,
        'peanuts': 2,
        'shellfish': 4,
        'strawberries': 8,
        'tomatoes': 16,
        'chocolate': 32,
        'pollen': 64,
        'cats': 128
    }

    def __init__(self, score):
        self.score = score

        # populate list of all allergies
        self.list = [allergy for allergy, value in sorted(self.score_key.items(), key=lambda x: x[1])
                     if self.score & self.score_key[allergy]]

    def is_allergic_to(self, food):
        # use bit operative to parse our base2 allergy dict
        return self.score & self.score_key[food]


import ply.yacc as yacc
import sys, os, os.path
import traceback

# Get the token map from the lexer.  This is required.
from js_global import glob

from js_ast import *
from js_lex import tokens, StringLit, HexInt
from ply.lex import LexToken, Lexer

#AST nodes that are used in intermediate stages of parsing,
#but are NEVER EVER in the final AST tree.
from js_parser_only_ast import *
from js_process_ast_parser_only import *

from js_parse import *

"""
This is a special "mode" that changes
the syntax to a statically-typed language,
optimized and checked for writing high-performance
code, but can still compile into JavaScript
"""

def p_statementlist(p):
  ''' statementlist : statement
                    | statement_nonctrl
                    | statementlist statement
                    | statementlist statement_nonctrl
                    |
  '''
  set_parse_globals(p);
  if len(p) == 1:
    p[0] = StatementList()
  elif len(p) == 2:
    n = StatementList()
    n.add(p[1])
    p[0] = n
  elif len(p) == 3:
    if type(p[1]) != StatementList:
      p[0] = StatementList()
      p[0].add(p[1])
      p[0].add(p[2])
    else:
      p[0] = p[1]
      if p[2] != None:
        p[0].add(p[2])

def p_class(p):
  '''class : CLASS ID template_opt class_tail'''
  set_parse_globals(p)

  tail = p[4]
  heritage = tail[0]
  cls = ClassNode(p[2], heritage)

  for n in tail[1]:
    cls.add(n)

  p[0] = cls
  if p[3] != None:
    p[0].template = p[3]

def p_exprclass(p):
  '''exprclass : CLASS id_opt class_tail'''
  set_parse_globals(p)

  tail = p[3]
  heritage = tail[0]

  if p[2] == None:
    p[2] = "(anonymous)"

  cls = ClassNode(p[2], heritage)

  for n in tail[1]:
    cls.add(n)

  p[0] = expand_harmony_class(cls)

def p_class_tail(p):
  '''class_tail : class_heritage_opt LBRACKET class_body_opt RBRACKET'''
  set_parse_globals(p)

  p[0] = [p[1], p[3]]

  for i in range(2):
    if p[0][i] == None:
      p[0][i] = []

def p_class_list(p):
  '''class_list : var_type
                | class_list COMMA var_type
  '''
  set_parse_globals(p)

  if len(p) == 2:
    p[0] = [p[1]];
  else:
    p[0] = p[1];
    if type(p[0]) != list:
      p[0] = [p[0]]
    p[0].append(p[3])

def p_class_heritage(p):
  '''class_heritage : EXTENDS class_list'''

  set_parse_globals(p)
  p[0] = p[2]

def p_class_heritage_opt(p):
  '''class_heritage_opt : class_heritage
                        |
  '''
  set_parse_globals(p)

  if len(p) == 2:
    p[0] = p[1]

def p_class_body_opt(p):
  '''class_body_opt : class_element_list
                    |
  '''
  set_parse_globals(p)

  if len(p) == 1:
    p[0] = []
  else:
    p[0] = p[1]

  if p[0] == None:
    p[0] = []

def p_class_element_list(p):
  '''class_element_list : class_element
                        | class_element_list class_element
  '''
  set_parse_globals(p)

  if len(p) == 2:
    p[0] = [p[1]]
  else:
    p[0] = p[1]
    p[0].append(p[2])

def p_class_element(p):
  '''class_element : method_def
                   | STATIC method_def
                   | class_var
  '''
  set_parse_globals(p)

  if len(p) == 2:
    p[0] = p[1]
  else:
    p[0] = p[2]
    p[0].is_static = True

def p_class_var(p):
  '''class_var : class_vartype ID SEMI
               | class_vartype ID ASSIGN expr SEMI
  '''
  set_parse_globals(p)

  p[0] =  ClassMember(p[2])
  if len(p) == 6:
    p[0].add(p[4])

def p_basic_var_type(p):
  '''
  basic_var_type : BYTE
                 | INT
                 | SHORT
                 | FLOAT
                 | DOUBLE
                 | CHAR
  '''
  p[0] = BuiltinTypeNode(p[1])

def p_var_type2(p):
  ''' var_type2 : basic_var_type
                | ID
                | ID template_ref
  '''
  if len(p) == 2:
    if type(p[1]) == str:
      p[0] = TypeRefNode(p[1])
    else:
      p[0] = p[1]
  else:
    p[0] = TypeRefNode(p[1])
    p[0].template = p[2]

def p_class_vartype(p):
  '''class_vartype : var_type2
                   | prop_modifiers var_type2
  '''
  set_parse_globals(p)

  if len(p) == 2:
    p[0] = p[1]
  else:
    p[0] = p[2]
    p[0].modifiers = p[1]

def p_prop_modifiers(p):
  '''prop_modifiers : type_modifiers UNSIGNED
                    | type_modifiers SIGNED
                    | type_modifiers CONST
                    | STATIC
                    | UNSIGNED
                    | CONST
                    |
  '''
  set_parse_globals(p)

  if len(p) == 2:
    p[0] = set([p[1]])
  else:
    p[0] = p[1]
    p[0].add(p[2])

def p_method(p):
  '''method : ID LPAREN funcdeflist RPAREN func_type_opt LBRACKET statementlist_opt RBRACKET'''
  set_parse_globals(p)

  name = p[1]
  params = p[3]
  statementlist = p[7]

  if statementlist == None:
    statementlist = StatementList()

  p[0] = MethodNode(name)
  p[0].add(params)
  p[0].add(statementlist)
  if p[5] != None:
    p[0].type = p[5]

def p_method_def(p):
  #I don't want to make get/set exclusive parse tokens,
  #so I'm going to enforce that here in the production function.

  '''method_def : method
                | ID ID LPAREN RPAREN func_type_opt LBRACKET statementlist_opt RBRACKET
                | ID ID LPAREN setter_param_list RPAREN func_type_opt LBRACKET statementlist_opt RBRACKET
  '''
  set_parse_globals(p)

  if len(p) == 2:
    p[0] = p[1]
  elif p[1] == "get" and len(p) == 9:
    name = p[2]
    p[0] = MethodGetter(name)
    if p[7] == None: p[7] = StatementList()
    p[0].add(p[7])
    if p[5] != None:
      p[0].type = p[5]
  elif p[1] == "set" and len(p) == 10:
    name = p[2]
    p[0] = MethodSetter(name)
    p[0].add(p[4])
    if p[8] == None: p[8] = StatementList()
    p[0].add(p[8])
    if p[6] != None:
      p[0].type = p[6]
  else:
    glob.g_error = True
    glob.g_error_pre = p
    print_err(p, True)
    raise SyntaxError("Expected 'get' or 'set'");

def p_setter_param_list(p):
  '''
    setter_param_list : ID
  '''
  set_parse_globals(p)

  p[0] = ExprListNode([VarDeclNode(ExprNode([]), name=p[1])])
  return

_parser = yacc.yacc()
parser = Parser(_parser);


#=========================================================================
# RegIncr
#=========================================================================
# This is a simple model for a registered incrementer. An eight-bit value
# is read from the input port, registered, incremented by one, and
# finally written to the output port.

from pymtl3 import *

class RegIncr( Component ):

  # Constructor

  def construct( s ):

    # Port-based interface

    s.in_ = InPort ( Bits8 )
    s.out = OutPort( Bits8 )

    # Sequential logic

    s.reg_out = Wire( 8 ) #

    @update_ff
    def block1():
      if s.reset:
        s.reg_out <<= 0
      else:
        s.reg_out <<= s.in_

    # ''' TUTORIAL TASK ''''''''''''''''''''''''''''''''''''''''''''''''''
    # This model is incomplete. As part of the tutorial you will insert a
    # combinational concurrent block here to model the incrementer logic,
    # and later you will insert a line tracing function to compactly
    # output the input, register, and output values.
    # ''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''


import chainer
import cv2 as cv
import numpy as np

from pathlib import Path
from chainer import cuda

from sklearn.model_selection import train_test_split

xp = cuda.cupy
cuda.get_device(0).use()

class DatasetLoader:
    def __init__(self,
                 src_path: Path,
                 tgt_path: Path,
                 extension=".png",
                 size=128):

        self.src_path = src_path
        self.tgt_path = tgt_path

        self.extension = extension
        self.size = size

        self.src_list, self.src_len = self._glob_and_length(src_path)
        self.tgt_list, self.tgt_len = self._glob_and_length(tgt_path)

        self.src_train_list, self.src_val_list = self._train_val_split()
        self.src_len = len(self.src_train_list)

    def __repr__(self):
        return f"source dataset length: {self.src_len} target dataset length: {self.tgt_len}"

    def _glob_and_length(self, path: Path):
        pathlist = list(path.glob(f"*{self.extension}"))
        pathlen = len(pathlist)

        return pathlist, pathlen

    def _train_val_split(self):
        split_point = int(len(self.src_list) * 0.95)
        x_train = self.src_list[:split_point]
        x_val = self.src_list[split_point:]

        return x_train, x_val

    @staticmethod
    def _vairble(array_list):
        return chainer.as_variable(xp.array(array_list).astype(xp.float32))

    def _prepare_img(self, path: Path):
        img = cv.imread(str(path))
        img = cv.resize(img, (self.size, self.size), interpolation=cv.INTER_CUBIC)

        # horizontal flip
        if np.random.randint(2):
            img = img[:, ::-1, :]

        img = img[:, :, ::-1]
        img = (img.transpose(2, 0, 1) - 127.5) / 127.5

        return img

    def _get_img(self, mode="source"):
        if mode == "source":
            rnd = np.random.randint(self.src_len)
            img_path = self.src_train_list[rnd]

        elif mode == "target":
            rnd = np.random.randint(self.tgt_len)
            img_path = self.tgt_list[rnd]

        img = self._prepare_img(img_path)

        return img

    def train(self, batchsize):
        src_box = []
        tgt_box = []

        for _ in range(batchsize):
            src = self._get_img(mode="source")
            tgt = self._get_img(mode="target")

            src_box.append(src)
            tgt_box.append(tgt)

        src = self._vairble(src_box)
        tgt = self._vairble(tgt_box)

        return src, tgt

    def valid(self, validsize):
        src_box = []

        for index in range(validsize):
            img_path = self.src_val_list[index]
            img = self._prepare_img(img_path)
            src_box.append(img)

        src = self._vairble(src_box)

        return src

class TestDatasetLoader:
    def __init__(self,
                 src_path: Path,
                 extension=".png",
                 size=128):

        self.src_path = src_path

        self.extension = extension
        self.size = size

        self.src_list, self.src_len = self._glob_and_length(src_path)

    def __repr__(self):
        return f"source dataset length: {self.src_len}"

    def _glob_and_length(self, path: Path):
        pathlist = list(path.glob(f"*{self.extension}"))
        pathlen = len(pathlist)

        return pathlist, pathlen

    @staticmethod
    def _vairble(array_list):
        return chainer.as_variable(xp.array(array_list).astype(xp.float32))

    def _prepare_img(self, path: Path):
        img = cv.imread(str(path))
        img = cv.resize(img, (self.size, self.size), interpolation=cv.INTER_CUBIC)

        # horizontal flip
        if np.random.randint(2):
            img = img[:, ::-1, :]

        img = img[:, :, ::-1]
        img = (img.transpose(2, 0, 1) - 127.5) / 127.5

        return img

    def test(self):
        src_box = []

        for index in range(self.src_len):
            img_path = self.src_list[index]
            img = self._prepare_img(img_path)
            src_box.append(img)

        src = self._vairble(src_box)

        return src


import random
import pickle

class Doctor():

    QUALIFIERS = ['Why do you say that ', 'You seem to think that ',
                  'Did I just hear you say that ', 'Why do you believe that ' ]

    REPLACEMENTS = {'I': 'you', 'me': 'you', 'my': 'your',
                    'we': 'you', 'us': 'you', 'am': 'are',
                    'you': 'I', 'You': 'I'}

    HEDGES = ['Go on.', 'I would like to hear more about that.',
              'And what do you think about this?', 'Please continue.']

    def __init__(self):
        self.history = []

        self.person = None

        self.firstTalk = True

        #dat file object
        #self.datFile = None

        #if dataisexisted already

    def greeting(self, client):

        return 'hello please state your name <>'
    def farewell(self):
        return 'Have a nice day!'

    def reply(self, sentence):
        '''
            yes all desrves a firstTalk
            search through data files
            if old , say welcome back
                open it
                say + the last history
            else greetings
                create one
            if not firstTalk
            chat chat

            all convo reserves to history
            if disconnect save it
        '''
        answer = ''
        if self.firstTalk:
            self.person = str(sentence)
            #print(self.person)
            try:
                with open('users/'+self.person+'.dat', 'rb') as f:
                    self.history = pickle.load(f)
                    answer = 'Welcome Back ' +  \
                    self.person + \
                    ', Last Discussion: ' \
                    + str(self.history[-1])

            except Exception:
                #create one
                file = open('users/'+self.person+'.dat', 'x')
                file.close()
                answer = 'Greetings! ' + self.person

            finally:
                self.firstTalk = False
            #just dump the person
            return answer
        else:
            choice = random.randint (1, 10)
            if choice == 1:
                if len(self.history) > 3:
                    answer = 'Earlier you said that ' + \
                    self.change_person(random.choice(self.history))
                else:
                    answer = random.choice(Doctor.HEDGES)
            elif choice in (2,3,4,5):
                answer = random.choice(Doctor.QUALIFIERS) + \
                self.change_person(sentence)
            else:
                answer = random.choice(Doctor.HEDGES)
        self.history.append(answer)
        return answer

    def updateFile(self):
        '''it should update the open file then close. Should add more codes than close itself'''

        with open('users/'+self.person+'.dat', 'wb') as f:
            pickle.dump(self.history, f)

    def change_person(self, sentence):
        oldlist = sentence.split()
        newlist = []
        for word in oldlist:
            if word in Doctor.REPLACEMENTS:
                newlist.append(Doctor.REPLACEMENTS[word])
            else:
                newlist.append(word)
        return " ".join(newlist)


import abjad

class AttachmentHandler:
    def __init__(
        self,
        starting_dynamic=None,
        ending_dynamic=None,
        hairpin=None,
        articulation_list=None,
        text_list=None,
        line_style=None,
        clef=None,
    ):
        def cyc(lst):
            count = 0
            while True:
                yield lst[count % len(lst)]
                count += 1

        self.starting_dynamic = starting_dynamic
        self.ending_dynamic = ending_dynamic
        self.hairpin = hairpin
        self.articulation_list = articulation_list
        self.text_list = text_list
        self.line_style = line_style
        self.clef = clef
        self._cyc_articulations = cyc(articulation_list)
        self._cyc_dynamics = cyc([starting_dynamic, ending_dynamic])
        self._cyc_text = cyc(text_list)

    def __call__(self, selections):
        return self.add_attachments(selections)

    def _apply_text_and_span_lr(self, selections):
        text = self._cyc_text
        for run in abjad.Selection(selections).runs():
            leaves = abjad.Selection(run).leaves()
            start_span = abjad.StartTextSpan(
                command=r"\startTextSpanOne",
                left_text=abjad.Markup(
                    fr"\markup \upright {{ {next(text)} }}",
                ),
                style=self.line_style,
            )
            stop_span = abjad.StartTextSpan(
                command=r"\startTextSpanOne",
                right_padding=2.5,
                left_text=abjad.Markup(
                    fr"\markup \upright {{ {next(text)} }}",
                ),
                style="solid-line-with-hook",
            )
            abjad.text_spanner(leaves[0], start_text_span=start_span)
            abjad.attach(abjad.StopTextSpan(command=r"\stopTextSpanOne"), leaves[-1])
            abjad.text_spanner(leaves[-1], start_text_span=stop_span)

    def _apply_text_and_span_l_only(self, selections):
        text = self._cyc_text
        for run in abjad.Selection(selections).runs():
            leaves = abjad.Selection(run).leaves()
            span = abjad.StartTextSpan(
                command=r"\startTextSpanOne",
                right_padding=2.5,
                left_text=abjad.Markup(
                    fr"\markup \upright {{ {next(text)} }}",
                ),
                style="solid-line-with-hook",
            )
            abjad.text_spanner(leaves[0], start_text_span=span)

    def add_attachments(self, selections):
        runs = abjad.Selection(selections).runs()
        ties = abjad.Selection(selections).logical_ties(pitched=True)
        for run in runs:
            if self.clef is not None:
                abjad.attach(
                    abjad.Clef(self.clef), run[0]
                )  # doesn't seem to always work, redo entire attachment handler?
            if len(run) > 1:
                leaves = abjad.Selection(run).leaves()
                if self.starting_dynamic is not None:
                    abjad.attach(abjad.Dynamic(self.starting_dynamic), leaves[0])
                if self.hairpin is not None:
                    abjad.attach(abjad.StartHairpin(self.hairpin), leaves[0])
                if self.ending_dynamic is not None:
                    abjad.attach(abjad.Dynamic(self.ending_dynamic), leaves[-1])
                    abjad.attach(
                        abjad.StartHairpin("--"), leaves[-1]
                    )  # makes ending with a logical tie weird. If problematic: reduce indentation by 1
                if self.text_list is not None:
                    if len(self.text_list) > 1:
                        self._apply_text_and_span_lr(run)
                    else:
                        self._apply_text_and_span_l_only(run)
            else:
                leaves = abjad.Selection(run).leaves()
                dynamic = next(self._cyc_dynamics)
                if self.starting_dynamic is not None:
                    if self.ending_dynamic is not None:
                        abjad.attach(abjad.Dynamic(dynamic), leaves[0])
                    else:
                        abjad.attach(abjad.Dynamic(self.starting_dynamic), leaves[0])
                if self.starting_dynamic is None:
                    if self.ending_dynamic is not None:
                        abjad.attach(abjad.Dynamic(self.ending_dynamic), leaves[0])
                abjad.attach(abjad.StartHairpin("--"), leaves[0])
                if self.text_list is not None:
                    self._apply_text_and_span_l_only(run)
        for tie in ties:
            if len(tie) == 1:
                if self.articulation_list is not None:
                    articulation = self._cyc_articulations
                    abjad.attach(abjad.Articulation(next(articulation)), tie[0])
        return selections


# -*- coding: cp1252 -*-
# -*- coding: utf-8 -*-
""" docopt.py

    Usage:
        docopt_example.py -h
        docopt_example.py <required> [-f | -g | -o ]
        docopt_example.py <repeating>...
    Options:
        -h,--help       : show this help message

"""
import urllib
import json
import pprint
from docopt import docopt
def main(docopt_args):
    if docopt_args["--Check Weather"]:
        weather()
        print ("You have used the required argument: " + docopt_args["<required>"])

        # Get flags used
        if docopt_args["-- Check Weather"]:
            weather()

        else:
            print ("   with no flags.\n")
    # User passed 1 or more repeating arguments
    elif docopt_args["<repeating>"]:
        print ("You have used the repeating args:")
        print ('   ' + '\n   '.join(docopt_args["<repeating>"]) + '\n')

def weather():
    #Python 2.7
    print('***WEATHER FORECASTING***\n')

    city = str(raw_input('Enter city(lower case)\n'))          #user input city
    country= str(raw_input('Enter country(initials eg ke)\n'))   #user input country
    print('Please waiting as data is being fetched...')

    #API provided by the yahoo weather forecast
    url = ("https://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20weather.forecast%20where%20woeid%20in%20(select%20woeid%20from%20geo.places(1)%20where%20text%3D%22"+country+"%2C%20"+city+"%22)&format=json&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys")
    #urllib provides a method that get the data from their server
    feeds = urllib.urlopen(url)

    feed_data = feeds.read()  #reads the data which in json format
    feed_json = json.loads(feed_data)  #the json methods loads the data to be displayed

    values = (feed_json['query'],['location'],['wind'],['atmosphere'])
    pprint.pprint(values) #the data is printed using the pprint module for better display

    print("\nReload if data displayed is not clear!!!")
    return pprint.pprint(values)

if __name__ == "__main__":
    args = docopt(__doc__)
    main(args)


# GraphQL
from graphene_sqlalchemy import SQLAlchemyConnectionField
from graphql import ResolveInfo

MYPY = False
if MYPY:
    from .filters import FilterSet  # noqa: F401; pragma: no cover

class FilterableConnectionField(SQLAlchemyConnectionField):
    filter_arg = "filters"

    @classmethod
    def get_query(cls, model, info: ResolveInfo, sort=None, **args):
        """Standard get_query with filtering."""
        query = super().get_query(model, info, sort, **args)

        request_filters = args.get(cls.filter_arg)
        if request_filters:
            filter_set = cls.get_filter_set(info)
            query = filter_set.filter(info, query, request_filters)

        return query

    @classmethod
    def get_filter_set(cls, info):
        """
        Get field filter set.

        Args:
            info: Graphene resolve info object.

        Returns:
            FilterSet class from field args.

        """
        field_name = info.field_asts[0].name.value
        schema_field = info.parent_type.fields.get(field_name)
        filters_type = schema_field.args[cls.filter_arg].type
        filters = filters_type.graphene_type  # type: FilterSet
        return filters


class Solution:
    """
    @param str: a string
    @return: return a string
    """
    def reverseWords(self, str):
        # write your code here
        if len(str) == 0:
            return str

        start = 0
        end = 0
        str_arr = list(str)
        self.reverse(str_arr, 0, len(str_arr) - 1)
        for char in str_arr:
            if char == " ":
                self.reverse(str_arr, start, end - 1)
                start = end + 1
            end += 1

        self.reverse(str_arr, start, end - 1)
        return "".join(str_arr)

    def reverse(self, arr, start, end):
        if start >= end:
            return
        while start < end:
            arr[start], arr[end] = arr[end], arr[start]
            start += 1
            end -= 1


import serial
import time
import re
import RPi.GPIO as GPIO
from gsmmodem import pdu as PduDecoder
from datetime import datetime as dt
from datetime import timedelta as td
import configparser
from pprint import pprint
import sys
import utils.error_logger as err_log
import traceback

class UserSMS:
    def __init__(self, gsm_mod, db, gsm_id):
        print("<< Imported User SMS")
        today = dt.today()
        self.gsm_mod = gsm_mod
        self.db = db
        self.gsm_id = gsm_id
        self.error_logger = err_log.ErrorLogger(gsm_id, 'User')
        csq = gsm_mod.get_csq()
        db.write_csq(gsm_id, today, csq)
        print(">> CSQ:", csq)

    def start_server(self):
        print(">> Starting Users SMS server.")
        while(True):
            today = dt.today()
            if (today.minute % 10 == 0):
                csq = self.gsm_mod.get_csq()
                self.db.write_csq(self.gsm_id, today, csq)
                print(">> CSQ:", csq)
            sms_count = self.gsm_mod.count_sms()
            print(">> Message count(s): ",sms_count)
            if sms_count > 0:
                sms = self.fetch_inbox()
                if len(sms) > 0:
                    print("<< Storing inbox to database.")
                    for x in sms:
                        store_status = self.db.insert_inbox_sms(x.simnum, x.data, x.dt )
                        if not store_status:
                            print(">> Unknown mobile number. Storing mobile number for future reference.")
                            store_number = self.insert_unknown_number(x.simnum, x.data, x.dt)
                else:
                    print(">> No new message. Sleeping for 10 seconds.\n\n")
                    time.sleep(10)
                self.gsm_mod.delete_sms()

            pending_sms = self.fetch_pending_user_outbox()

            if len(pending_sms) > 0:
                print(">> Pending message(s): ",len(pending_sms),"")
                self.send_pending_sms(pending_sms)
            else:
                print(">> No pending message. Sleeping for 10 seconds.\n\n")
                time.sleep(10)

    def fetch_inbox(self):
        print("<< Fetching inbox.")
        sms = []
        sms = self.gsm_mod.get_all_sms()
        return sms

    def fetch_pending_user_outbox(self):
        print("<< Fetching outbox.")
        pending = self.db.get_all_outbox_sms_users_from_db(5, self.gsm_id)
        return pending

    def send_pending_sms(self, sms):
        for (user_id, mobile_id, outbox_id, stat_id,
         sim_num, firstname, lastname, sms_msg, send_status) in sms:
            print("<< Sending SMS to: ", firstname, lastname, " (",sim_num,")")
            print("<< Message: ", sms_msg)
            stat = self.gsm_mod.send_sms(sms_msg, sim_num)
            if stat is 0:
                print(">> SMS ID #:",outbox_id," has successfully sent!")
            else:
                print(">> SMS ID #:",outbox_id," failed! Will retry later...")
            update_status = self.update_outbox_status(stat, outbox_id, stat_id, send_status)
            if update_status != 1:
                print(">> Error occurred. Failed to update send status.")

    def update_outbox_status(self, stat, outbox_id , stat_id, send_status, ts_send = ""):
        ts_sent = dt.today().strftime("%Y-%m-%d %H:%M:%S")
        if stat == 0:
            ret_val = self.db.update_send_status(stat_id , 5, ts_sent)
        else:
            if send_status == 4:
                send_status = send_status + 2
            else:
                send_status = send_status + 1
            ret_val = self.db.update_send_status(stat_id , send_status)
        return ret_val

    def insert_unknown_number(self, sim_num, msg, ts):
        print("<< Inserting unknown number.")
        store_number = self.db.save_unknown_number(sim_num, self.gsm_id)
        store_status = self.db.insert_inbox_sms(sim_num, msg, ts)


##currently using default beta-profile params from vikhlinin, change if well-motivated

from formulas import sNFW_density_profile,     vikhlinin_density_profile,     rescale_profile_by_mass
import yt.units as u
from cluster_generator import HydrostaticEquilibrium, VirialEquilibrium, ClusterModel
from astropy import cosmology
import numpy as np

f = cosmology.default_cosmology
f = f.get()
rho_crit = f.critical_density0.to('Msun kpc**-3')

# The `formulas` package contains a number of astrophysically relevant formulas, including radial profiles.
# For DM, we'll use the "super-NFW" profile, and for the gas, we'll use Alexey's modifications to the $\beta$-model.
# What these lines do is create the profile formulas only, but doesn't set any values.

def rv(M):
    rv = (4*np.pi*200*rho_crit/(3*M))**(-1./3)
    return rv

def rs(rv, c):
    rs = rv/c
    return rs

def rhos(M, c, rs):
    return M/(4*np.pi*(rs**3)*(np.log(1+c) - c/(1+c)))

def csNFW(cNFW):
    return 0.76*cNFW + 1.36

def a(rv, c):
    rh = rv*(0.6082 - 0.1843*np.log10(c) - 0.1011*np.log10(c)**2 + 0.03918*np.log(c)**3)
    #Lokas+ 2001
    return rh/7.3#5.478 #Lilley+ 2015a(M1, c1)

def main(M1, c1, fgas = 0.17, npartfrac = 1, alpha1 = 1.0):
	npart1 = int(20000000*npartfrac)

	# dm1 = sNFW_density_profile(M="M_1", cs="c_s", rv="r_v") # for the primary cluster
	dm1 = tNFW_density_profile(rho_s="rho_s", r_s="r_s", r_t="r_t")
	"""
	Now we'll decide on some parameters for the DM profiles. We use the yt units package
	to set the actual units. We should always use units of Msun and kpc, since that's what
	cluster_generator understands. I've chosen more or less arbitrary values for now which
	are still physically sensible, but they'll need to be adjusted by you to try to match
	the conditions for A2146.
	"""
	rvir = rv((1-fgas)*M1) #the dark matter mass
	rvir1 = rvir.value * u.kpc
	cs = csNFW(c1)
	rs1 = rs(rvir, c1)
	# a1 = a(rs1, c1)
	rho0 = rhos((1-fgas)*M1, c1, rs1).to('g/cm**3')

	M1 *= u.Msun
	# Now we tell the profiles what the parameter values are
	# dm1.set_param_values(M_1=(1-fgas)*M1, c_s=cs, r_v=rvir)
	dm1.set_param_values(rho_s=rho0, r_s=rs1, r_t=2*rvir1)

	# Now we do all the same stuff for the gas profiles.
	gas1 = vikhlinin_density_profile(rho_0="rho_c1", r_c="r_c1", r_s="r_s1",
	                                 alpha='alpha1', beta="beta1", epsilon='eps1')

	"""
	We won't set the central densities yet, as you'll see why later. Again, picking
	more or less arbitrary values for these parameters...
	"""
	rhoc1=1*u.Msun/(u.kpc**3)
	rc1 = 0.1*rs1
	beta1 = 2./3.
	eps1 = 2.0

	# Setting all parameter values except central densities...
	gas1.set_param_values(r_c1=rc1, r_s1=rs1, alpha1=alpha1, beta1=beta1, eps1=eps1, rho_c1=rhoc1)

	"""
	We didn't set the value of the central densities yet. That's because it makes
	more physical sense to set it by a gas mass fraction of the dark matter. There's
	a function for this:
	"""
	Mgas1 = fgas*M1
	rescale_profile_by_mass(gas1, "rho_c1", Mgas1, rvir1)

	# We get the total density profiles by adding these two together
	tot1 = dm1+gas1

	# cluster_generator doesn't take profiles with units, so we give it the unitless versions
	profiles1 = {"density": gas1.unitless, "total_density": tot1.unitless}

	# now we set up HSE objects, with profiles from 0.1 kpc to 20 Mpc. This may take a while.
	hse1 = HydrostaticEquilibrium.from_scratch("dens_tden", 0.1, 30000.0, profiles1)

	# We now write the hydrostatic profiles to HDF5
	# hse1.write_model_to_h5("ebeling_profile_m%0.1e_c%0.1f_a%0.1f.h5" % (M1, c1, alpha1), in_cgs=True, overwrite=True)
	hse1.write_model_to_h5("tNFW_profile_m%0.1e_c%0.1f_a%0.1f.h5" % (M1, c1, alpha1), in_cgs=True, overwrite=True)
	vir1 = VirialEquilibrium.from_hse_model(hse1)

	# Generate the particles. I've chosen a 3.8/1 ratio in particle number to match the mass ratio
	# This will also take a while.
	parts1 = vir1.generate_particles(npart1)

	# Write the particles
	parts1.write_particles_to_h5("tNFW_%0.1e_particles_c%0.1f_a%0.1f.h5" % (M1, c1, alpha1), in_cgs=True, overwrite=True)

	##vary M1, M2, a1 (=r200/2), a2

if __name__=="__main__":
	M1 = np.array([1.1e15, 9.3e14, 8.1e14])
	M2 = np.array([2.6e14, 2.4e14, 2.1e14])
	ratio = (M2/M1).astype(int)
	for ind in range(3):
		main(M1[ind], 4.1)
		main(M2[ind], 5.2, npartfrac = ratio[ind])

		if ind == 1:
			main(M1[ind], 3.1)
			main(M2[ind], 6.3, npartfrac = ratio[ind])

		# main(M2[ind], 6.3, npart1 = 20000000*ratio[ind])


"""
librarian.models

Shipyard data models pertaining to the lookup of the past: ExecRecord,
Dataset, etc.
"""
import csv
import logging
import os
import os.path
import re
import time
import io

from django.contrib.humanize.templatetags.humanize import naturaltime
from django.db import models, transaction
from django.core.exceptions import ValidationError
from django.core.validators import RegexValidator
from django.core.files import File
from django.db.models.functions import Now
from django.utils import timezone
from django.conf import settings
from django.template.defaultfilters import filesizeformat, pluralize
from django.db.models.signals import post_delete
from django.urls import reverse

import metadata.models
import archive.models
import librarian.signals
from constants import maxlengths
from container.models import ContainerDataset
import six

import file_access_utils

LOGGER = logging.getLogger(__name__)

def get_upload_path(instance, filename):
    """
    Helper method for uploading dataset_files for Dataset.
    This is outside of the Dataset class, since @staticmethod and other method decorators were used instead of the
    method pointer when this method was inside Dataset class.

    :param instance:  Dataset instance
    :param filename: Dataset.dataset_file.name
    :return:  The upload directory for Dataset files.
    """
    # noinspection PyTypeChecker
    return instance.UPLOAD_DIR + os.sep + time.strftime('%Y_%m') + os.sep + filename

class ExternalFileDirectory(models.Model):
    """
    A database table storing directories whose contents we can make Datasets out of.
    """
    name = models.CharField(
        help_text="Human-readable name for this external file directory",
        unique=True,
        max_length=maxlengths.MAX_EXTERNAL_PATH_LENGTH
    )
    path = models.CharField(
        help_text="Absolute path",
        max_length=maxlengths.MAX_EXTERNAL_PATH_LENGTH
    )

    def __str__(self):
        return self.name

    def list_files(self):
        """
        Return a list of tuples representing files under this directory.

        The tuple looks like:
        ([absolute file path], [file path with external file directory name substituted])
        """
        path_with_slash = self.path if self.path.endswith("/") else "{}/".format(self.path)
        all_files = []
        for root, _dirs, files in sorted(os.walk(self.path)):
            for f in files:
                f = os.path.join(root, f)
                all_files.append((f, f.replace(path_with_slash, "[{}]/".format(self.name), 1)))
        return all_files

    def save(self, *args, **kwargs):
        """
        Normalize the path before saving.
        """
        self.path = os.path.normpath(self.path)
        super(ExternalFileDirectory, self).save(*args, **kwargs)

class Dataset(metadata.models.AccessControl):
    """
    A (possibly temporary) data file.

    That is to say, at some point, there was a data file uploaded to/
    generated by Shipyard, which was coherent with its
    specified/generating CDT and its producing
    TransformationOutput/cable (if it was generated), and this
    represents it, whether or not it was saved to the database.

    PRE: the actual file that the Dataset represents (whether
    it still exists or not) is/was coherent (e.g. checked using
    CDT.summarize_csv()).
    """
    UPLOAD_DIR = "Datasets"  # This is relative to kive.settings.MEDIA_ROOT

    name = models.CharField(max_length=maxlengths.MAX_FILENAME_LENGTH)
    description = models.TextField(help_text="Description of this Dataset.",
                                   max_length=maxlengths.MAX_DESCRIPTION_LENGTH,
                                   blank=True)
    date_created = models.DateTimeField(default=timezone.now,
                                        help_text="Date of Dataset creation.",
                                        db_index=True)

    # Datasets are stored in the "Datasets" folder
    dataset_file = models.FileField(upload_to=get_upload_path,
                                    help_text="Physical path where datasets are stored",
                                    blank=True,
                                    default='',
                                    db_index=True,
                                    max_length=maxlengths.MAX_FILENAME_LENGTH)

    externalfiledirectory = models.ForeignKey(
        ExternalFileDirectory,
        verbose_name="External file directory",
        help_text="External file directory containing the data file",
        null=True,
        blank=True,
        on_delete=models.CASCADE
    )
    external_path = models.CharField(
        help_text="Relative path of the file within the specified external file directory",
        blank=True,
        max_length=maxlengths.MAX_EXTERNAL_PATH_LENGTH
    )

    logger = logging.getLogger('librarian.Dataset')

    # For validation of Datasets when being reused, or when being
    # regenerated.  A blank MD5_checksum means that the file was
    # missing (not created when it was supposed to be created).
    MD5_checksum = models.CharField(
        max_length=64,
        validators=[RegexValidator(
            regex=re.compile("(^[0-9A-Fa-f]{32}$)|(^$)"),
            message="MD5 checksum is not either 32 hex characters or blank")],
        blank=True,
        default="",
        help_text="Validates file integrity")

    _redacted = models.BooleanField(default=False)

    # The last time a check was performed on this external file, to see whether
    # the external file referenced was still there.
    # See external_file_check() for details.
    last_time_checked = models.DateTimeField(default=timezone.now,
                                             help_text="Date-time of last (external) dataset existence check.",
                                             null=True)

    dataset_size = models.BigIntegerField(
        blank=True,
        null=True,
        help_text="Size of the dataset file in bytes.  If null, this has not been computed yet or there is no "
                  "internally stored file."
    )
    is_external_missing = models.BooleanField(
        default=False,
        help_text='True if the external file was missing when last checked.')
    is_uploaded = models.BooleanField(
        default=False,
        help_text='True if the file was uploaded, not an output.')

    class Meta:
        ordering = ["-date_created", "name"]

    def __init__(self, *args, **kwargs):
        super(Dataset, self).__init__(*args, **kwargs)
        self.logger = logging.getLogger(self.__class__.__name__)

    def __repr__(self):
        return 'Dataset(name={!r})'.format(self.name)

    def __str__(self):
        """
        Unicode representation of a Dataset.

        This is simply S[pk] if it has no data.
        """
        if not self.has_data():
            return "S{}".format(self.pk)

        display_name = self.name if self.name != "" else "[no name specified]"

        return "{} (created by {} on {})".format(display_name, self.user, self.date_created)

    @property
    def is_purged(self):
        return not (self.dataset_file or self.external_path)

    def external_absolute_path(self):
        if not self.external_path:
            return None
        return os.path.normpath(os.path.join(self.externalfiledirectory.path, self.external_path))

    def get_open_file_handle(self, mode="rb", raise_errors=False):
        """
        Retrieves an open Django file with which to access the data.

        This is self.dataset_file if possible, falls back to the external file if possible,
        and otherwise returns None.

        NOTE: for python3 there is a significant difference in opening a file in binary or
        text mode.
        Use binary when calculating hashes.
        Use text when read the CSV contents.
        """
        if self.dataset_file:
            try:
                self.dataset_file.open(mode)
            except IOError as e:
                if raise_errors:
                    raise
                self.logger.warning('error accessing dataset file: %s', e)
                return None
            return self.dataset_file
        elif self.external_path:
            abs_path = self.external_absolute_path()
            try:
                fhandle = open(abs_path, mode)
            except IOError as e:
                if raise_errors:
                    raise
                self.logger.warning('error accessing external file: %s', e)
                return None
            return File(fhandle, name=abs_path)
        if raise_errors:
            raise ValueError('Dataset has no dataset_file or external_path.')
        return None

    def all_rows(self, data_check=False, insert_at=None, limit=None, extra_errors=None):
        """ Returns an iterator over all rows of this Dataset.

        :param bool data_check: each field becomes a tuple: (value, [error])
        :param list insert_at: [column_index] add blank columns at each
        zero-based index
        :param int limit: maximum row number returned (header is not counted)
        :param list extra_errors: this will have extra rows added to it that
        contain the first error in each column, if they appear after the row
        limit. [(row_num, [(field_value, [error])])] row_num is 1-based.
        :return: an iterator over the rows, each row is either [field_value] or
        [(field_value, [error])], depending on data_check.
        """
        data_handle = self.get_open_file_handle("r")
        if data_handle is None:
            raise RuntimeError('Dataset file has been removed.')

        with data_handle:
            reader = csv.reader(data_handle)
            if data_check:
                row_errors = {}
            for row_num, row in enumerate(reader):
                if not data_check:
                    if limit is not None and row_num > limit:
                        break
                else:
                    row_error = row_errors.pop(row_num, {})
                    if limit is not None and row_num > limit and not row_error:
                        if row_errors:
                            # Still have errors on later rows
                            continue
                        # No more errors
                        break
                    new_row = []
                    for column_num, value in enumerate(row, 1):
                        failed_column = row_error.get(column_num)
                        if failed_column is None:
                            new_errors = []
                        else:
                            new_errors = failed_column.check_basic_constraints(value)
                        new_row.append((value, new_errors))

                    row = new_row
                if insert_at is not None:
                    dummy = ('', []) if data_check else ''
                    [row.insert(pos, dummy) for pos in insert_at]
                if limit is None or row_num <= limit:
                    yield row
                else:
                    extra_errors.append((row_num, row))

    def header(self):
        rows = self.all_rows()
        return next(rows)

    def rows(self, data_check=False, insert_at=None, limit=None, extra_errors=None):
        rows = self.all_rows(data_check,
                             insert_at,
                             limit=limit,
                             extra_errors=extra_errors)
        for i, row in enumerate(rows):
            if i == 0:
                pass  # skip header
            else:
                yield row

    def clean(self):
        """
        Checks coherence of this Dataset.

        If it has data (i.e. an associated Dataset), it cleans that
        Dataset.  Then, if there is an associated DatasetStructure,
        clean that.

        Note that the MD5 checksum is already checked via a validator.
        """
        if not (self.externalfiledirectory and self.external_path or
                not self.externalfiledirectory and not self.external_path):
            raise ValidationError(
                {
                    "external_path": "Both externalfiledirectory and external_path should be set or "
                                     "neither should be set"
                }
            )

        if self.has_data() and not self.check_md5():
            error_str = ('File integrity of "{}" lost. Current checksum "{}" does not equal expected checksum ' +
                         '"{}"').format(self, self.compute_md5(), self.MD5_checksum)
            raise ValidationError(
                {
                    "dataset_file": error_str
                }
            )

    # noinspection PyUnusedLocal
    def validate_uniqueness_on_upload(self, *args, **kwargs):
        """
        Validates that the name and MD5 of the Dataset are unique.

        This isn't at the model level because we do want to allow
        these duplicates (e.g. empty files generated by the same
        Pipeline), but we want to check files on upload.
        """
        query = Dataset.objects.filter(MD5_checksum=self.MD5_checksum,
                                       name=self.name)

        if query.exclude(pk=self.pk).exists():
            error_str = "A Dataset with that name and MD5 already exists."
            raise ValidationError(
                {
                    "dataset_file": error_str
                }
            )

    def get_access_limits(self, access_limits=None):
        if access_limits is None:
            access_limits = []

        # Is this an output from a container run?
        for container_dataset in self.containers.filter(argument__type='O'):
            access_limits.append(container_dataset.run)

        return access_limits

    @property
    def absolute_url(self):
        """
        :return str: URL to access the dataset_file
        """
        return reverse('dataset_download', kwargs={"dataset_id": self.id})

    def get_view_url(self):
        return reverse('dataset_view', kwargs={"dataset_id": self.id})

    def get_filesize(self):
        """
        :return int: size of dataset_file in bytes or None if the file handle
        cannot be accessed.
        """
        data_handle = None
        try:
            data_handle = self.get_open_file_handle("rb")
            if data_handle is None:
                return None
            return data_handle.size
        finally:
            if data_handle is not None:
                data_handle.close()

    def get_formatted_filesize(self):
        unformatted_size = self.get_filesize()
        if unformatted_size is None:
            return 'missing'
        return filesizeformat(unformatted_size)

    def compute_md5(self):
        """Computes the MD5 checksum of the Dataset.
        Return None if the file could not be accessed.
        """
        data_handle = self.get_open_file_handle("rb")
        if data_handle is None:
            self.logger.warning('cannot access file handle')
            return None
        with data_handle:
            return file_access_utils.compute_md5(data_handle.file)

    def check_md5(self):
        """
        Checks the MD5 checksum of the Dataset against its stored value.

        The stored value is used when regenerating data
        that once existed, as a coherence check.

        Return True if the check passed, otherwise False.
        """
        # Recompute the MD5, see if it equals what is already stored
        new_md5 = self.compute_md5()
        if self.MD5_checksum != new_md5:
            if self.dataset_file:
                filename = self.dataset_file.name
            else:
                filename = self.external_absolute_path()
            self.logger.warning('MD5 mismatch for %s: expected %s, but was %s.',
                                filename,
                                self.MD5_checksum,
                                new_md5)
            return False
        return True

    def has_data(self, raise_errors=False):
        try:
            data_handle = self.get_open_file_handle("rb", raise_errors=True)
            data_handle.close()
            return True
        except ValueError:
            # No file recorded for this dataset.
            return False
        except IOError:
            # Recorded file is not found or not readable.
            if raise_errors:
                raise
            return False

    def has_structure(self):
        """ Compound datatypes were removed, so all datasets are now raw. """
        return False

    def is_raw(self):
        """ Compound datatypes were removed, so all datasets are now raw. """
        return True

    def num_rows(self):
        """Returns number of rows in the associated Dataset.

        This returns None if the Dataset is raw.
        """
        return None if self.is_raw() else self.structure.num_rows

    def get_cdt(self):
        """ Compound datatypes were removed, so all datasets are now raw. """
        return None

    def set_md5(self, file_path=None, file_handle=None):
        """Set the MD5 hash from a file.

        Closes the file after the MD5 is computed.
        :param str file_path:  Path to file to calculate MD5 for.
            Defaults to dataset_file.path, and not used if file_handle supplied.
        :param file file_handle: file handle of file to calculate MD5.  File
            must be seeked to the beginning.
            If file_handle empty, then uses file_path.
        """
        opened_file_ourselves = False
        if file_handle is None:
            if file_path is None:
                file_path = self.dataset_file.path
            file_handle = io.open(file_path, "rb")
            opened_file_ourselves = True

        try:
            self.MD5_checksum = file_access_utils.compute_md5(file_handle)
        finally:
            if opened_file_ourselves:
                file_handle.close()

    @transaction.atomic
    def register_file(self, file_path, file_handle=None):
        """
        Save and register a new file for this Dataset.

        Compute and set the MD5.

        Closes the file afterwards if the file source is a string file path.
        Does not close the file afterwards if file source is a file handle.

        INPUTS
        file_path           file to upload as the new contents
        file_handle         file handle of the file to upload as the new contents.
                            If supplied, then does not reopen the file in file_path.
                            Moves handle to beginning of file before calculating MD5.
                            If None, then opens the file in file_path.

        PRE
        self must not have a file already associated
        """
        assert not bool(self.dataset_file)

        opened_file_ourselves = False
        if file_handle is None:
            file_handle = io.open(file_path, mode="rb")
            opened_file_ourselves = True

        try:
            full_name = file_path
            assert isinstance(full_name, six.string_types), "fname '{}' is not a string {}".format(
                file_handle.name,
                type(file_handle.name)
            )
            fname = os.path.basename(full_name)
            self.dataset_file.save(fname, File(file_handle))
        finally:
            if opened_file_ourselves:
                file_handle.close()

        self.clean()
        self.save()

    @classmethod
    def create_empty(cls, user=None, cdt=None, users_allowed=None, groups_allowed=None,
                     file_source=None, instance=None):
        """Create an empty Dataset.

        INPUTS
        cdt   CompoundDatatype for the new Dataset
                            (None indicates a raw Dataset)
        instance            None or a Dataset to fill in (e.g. if we get a dummy one from DatasetForm)

        OUTPUTS
        empty_dataset            Dataset with a blank MD5
        """
        users_allowed = users_allowed or []
        groups_allowed = groups_allowed or []

        if user is None:
            assert file_source is not None
            user = file_source.top_level_run.user
            users_allowed = file_source.top_level_run.users_allowed.all()
            groups_allowed = file_source.top_level_run.groups_allowed.all()
        elif file_source is not None:
            assert user == file_source.top_level_run.user
            assert set(users_allowed) == set(file_source.top_level_run.users_allowed.all())
            assert set(groups_allowed) == set(file_source.top_level_run.groups_allowed.all())

        with transaction.atomic():

            empty_dataset = instance or cls()
            empty_dataset.user = user
            empty_dataset.MD5_checksum = ""
            empty_dataset.dataset_file = None
            empty_dataset.file_source = file_source
            empty_dataset.last_time_checked = None
            # Save so we can add permissions.
            empty_dataset.save()

            if cdt:
                raise NotImplementedError(
                    'Compound data types are no longer supported.')

            for user in users_allowed:
                empty_dataset.users_allowed.add(user)
            for group in groups_allowed:
                empty_dataset.groups_allowed.add(group)
            empty_dataset.clean()

        return empty_dataset

    # noinspection PyUnusedLocal
    @classmethod
    def create_dataset(cls,
                       file_path,
                       user=None,
                       users_allowed=None,
                       groups_allowed=None,
                       cdt=None,
                       keep_file=True,
                       name=None,
                       description=None,
                       file_source=None,
                       check=True,
                       file_handle=None,
                       instance=None,
                       externalfiledirectory=None,
                       precomputed_md5=None,
                       is_uploaded=False):
        """
        Helper function to make defining SDs and Datasets faster.

        user and name must both be set if make_dataset=True.
        make_dataset creates a Dataset from the given file path to go
        with the SD. file_source can be a RunAtomic to register the
        Dataset with, or None if it was uploaded by the user (or if
        make_dataset=False). If check is True, do a ContentCheck on the
        file.  file_path is an absolute path; if externalfiledirectory
        is specified, file_path will be checked to ensure that it's
        inside the specified directory.

        Returns the Dataset created.
        """
        users_allowed = users_allowed or []
        groups_allowed = groups_allowed or []

        if user is None:
            assert file_source is not None
            user = file_source.top_level_run.user
            users_allowed = file_source.top_level_run.users_allowed.all()
            groups_allowed = file_source.top_level_run.groups_allowed.all()
        elif file_source is not None:
            assert user == file_source.top_level_run.user
            assert set(users_allowed) == set(file_source.top_level_run.users_allowed.all())
            assert set(groups_allowed) == set(file_source.top_level_run.groups_allowed.all())

        if file_path:
            LOGGER.debug("Creating Dataset from file {}".format(file_path))
            file_name = file_path
        elif file_handle:
            LOGGER.debug("Creating Dataset from file {}".format(file_handle.name))
            file_name = str(file_handle.name)
        else:
            raise ValueError("Must supply either the file path or file handle")

        if not isinstance(file_name, six.string_types):
            raise ValueError("file_name '{}' is not a string '{}'".format(file_name, type(file_name)))
        with transaction.atomic():
            external_path = ""
            # We do this in the transaction because we're accessing ExternalFileDirectory.
            if externalfiledirectory:
                # Check that file_path is in the specified ExternalFileDirectory.
                normalized_path = os.path.normpath(file_name)
                normalized_efd_with_slash = "{}/".format(os.path.normpath(externalfiledirectory.path))
                assert normalized_path.startswith(normalized_efd_with_slash)
                external_path = normalized_path.replace(normalized_efd_with_slash, "", 1)

            new_dataset = cls.create_empty(user, cdt=cdt,
                                           users_allowed=users_allowed, groups_allowed=groups_allowed,
                                           instance=instance, file_source=file_source)

            new_dataset.name = name or ""
            new_dataset.description = description or ""
            new_dataset.externalfiledirectory = externalfiledirectory
            new_dataset.external_path = external_path
            new_dataset.last_time_checked = timezone.now()
            new_dataset.is_uploaded = is_uploaded

            if precomputed_md5 is not None:
                new_dataset.MD5_checksum = precomputed_md5
            else:
                new_dataset.set_md5(file_name, file_handle)
            if file_handle is not None:
                file_handle.seek(0)

            if keep_file:
                new_dataset.register_file(file_path=file_name, file_handle=file_handle)

            new_dataset.clean()
            new_dataset.save()
        return new_dataset

    @transaction.atomic
    def build_redaction_plan(self, redaction_accumulator=None):
        """
        Create a list of what will be affected when redacting this Dataset.
        """
        redaction_plan = redaction_accumulator or archive.models.empty_redaction_plan()
        assert self not in redaction_plan["Datasets"]
        if self.is_redacted():
            return redaction_plan
        redaction_plan["Datasets"].add(self)

        # Make a special note if this Dataset is associated with an external file.
        if self.external_path:
            redaction_plan["ExternalFiles"].add(self)

        return redaction_plan

    @transaction.atomic
    def redact_this(self):
        """
        Helper function that only redacts this Dataset and does not handle any recursion.
        """
        if self.is_redacted():
            return

        self._redacted = True
        self.MD5_checksum = ""
        self.externalfiledirectory = None
        if self.external_path:
            self.external_path = ""
        self.save(update_fields=["_redacted", "MD5_checksum", "externalfiledirectory", "external_path"])

        if bool(self.dataset_file):
            self.dataset_file.delete(save=True)
        if self.has_structure():
            self.structure.delete()

    @transaction.atomic
    def redact(self):
        redaction_plan = self.build_redaction_plan()
        archive.models.redact_helper(redaction_plan)

    def is_redacted(self):
        return self._redacted

    @property
    def uploaded(self):
        """ Kept for backward compatibility in serializer. """
        return self.is_uploaded

    @transaction.atomic
    def build_removal_plan(self, removal_accumulator=None):
        """
        Make a manifest of objects to remove when removing this Dataset.
        """
        removal_plan = removal_accumulator or metadata.models.empty_removal_plan()
        assert self not in removal_plan["Datasets"]
        removal_plan["Datasets"].add(self)

        for run_dataset in self.containers.all():
            run = run_dataset.run
            if (run_dataset.argument.type == 'I' and
                    run not in removal_plan['ContainerRuns']):
                run.build_removal_plan(removal_plan)

        # Make a special note if this Dataset is associated with an external file.
        if self.external_path:
            removal_plan["ExternalFiles"].add(self)

        return removal_plan

    @transaction.atomic
    def remove(self):
        removal_plan = self.build_removal_plan()
        metadata.models.remove_helper(removal_plan)

    @classmethod
    def find_unneeded(cls):
        """ Finds datasets that could be purged.

        Excludes datasets that were uploaded or from an active run.
        :return: a QuerySet
        """
        # Exclude Datasets that are currently in use
        active_dataset_ids = ContainerDataset.objects.filter(
            run__end_time=None).values_list('dataset_id').order_by()

        unneeded = cls.objects.filter(
            is_uploaded=False).exclude(  # Only purge outputs.
            pk__in=active_dataset_ids).exclude(  # Don't purge while it's in use.
            dataset_file=None).exclude(  # External file.
            dataset_file='').exclude(  # Already purged.
            dataset_size=None)  # New dataset.
        return unneeded

    @classmethod
    def scan_file_names(cls):
        """ Yield all file names, relative to MEDIA_ROOT. """
        dataset_root = os.path.join(settings.MEDIA_ROOT, cls.UPLOAD_DIR)
        for dirpath, dirnames, filenames in os.walk(dataset_root):
            for filename in filenames:
                file_path = os.path.join(dirpath, filename)
                yield os.path.relpath(file_path, settings.MEDIA_ROOT)
            if not (dirnames or filenames):
                # Empty folder can be purged.
                yield os.path.relpath(dirpath, settings.MEDIA_ROOT)

    @classmethod
    def external_file_check(cls, batch_size=1000):
        """ Perform a consistency check of external files. """
        last_id = None
        missing_count = 0
        last_missing_date = None
        last_missing_path = None
        while True:
            batch = Dataset.objects.filter(
                externalfiledirectory__isnull=False).order_by(
                'id').prefetch_related('externalfiledirectory')
            if last_id is not None:
                batch = batch.filter(id__gt=last_id)
            batch = batch[:batch_size]
            batch_count = 0
            found_ids = []
            for dataset in batch:
                batch_count += 1
                last_id = dataset.id
                path_name = dataset.external_absolute_path()
                if path_name is None:
                    raise RuntimeError("Unexpected None for external dataset path!")
                if os.path.exists(path_name):
                    found_ids.append(dataset.id)
                else:
                    missing_count += 1
                    if not dataset.is_external_missing:
                        dataset.is_external_missing = True
                        if (last_missing_date is None or
                                dataset.last_time_checked > last_missing_date):
                            last_missing_date = dataset.last_time_checked
                            last_missing_path = path_name
                        dataset.save()
            Dataset.objects.filter(id__in=found_ids).update(
                last_time_checked=Now(),
                is_external_missing=False)
            if batch_count < batch_size:
                break
        if last_missing_date is not None:
            cls.logger.error(
                "Missing %d external dataset%s. Most recent from %s, last checked %s.",
                missing_count,
                pluralize(missing_count),
                last_missing_path,
                naturaltime(last_missing_date))

    def increase_permissions_from_json(self, permissions_json):
        """
        Grant permission to all users and groups specified in the parameter.

        The permissions_json parameter should be a JSON string formatted as it would
        be by the permissions widget used in the UI.
        """
        self.grant_from_json(permissions_json)

    def unique_filename(self) -> str:
        "Create a unique filename based on this dataset's name and ID."
        unique_id = self.id
        name, extension = os.path.splitext(
            self.name)  # Splitext retains a '.' if it's present
        return "{}_{}{}".format(name, unique_id, extension)

# Register signals.
post_delete.connect(librarian.signals.dataset_post_delete, sender=Dataset)


import pandas as pd
import numpy as np
import numbers
import re
import random
import math
from sklearn.utils import shuffle
from sklearn import decomposition

def extract_full_dataset():
    training_features = pd.read_csv("spam_polluted_no_missing/train_feature.txt", header=None, sep='\s+').values
    training_label = pd.read_csv("spam_polluted_no_missing/train_label.txt", header=None, sep='\s+').values
    testing_features = pd.read_csv("spam_polluted_no_missing/test_feature.txt", header=None, sep='\s+').values
    testing_label = pd.read_csv("spam_polluted_no_missing/test_label.txt", header=None, sep='\s+').values

    training_dataset = np.column_stack((training_features, training_label))
    testing_dataset = np.column_stack((testing_features, testing_label))

    return training_dataset, testing_dataset

def calculate_accuracy(pred, true):
    correct_predictions = [i for i, j in zip(pred, true) if i == j]
    return len(correct_predictions) / len(true) * 100

def summarize(dataset):
    summaries = [(np.mean(attribute), np.var(attribute)) for attribute in zip(*dataset)]
    del summaries[-1]
    return summaries

def build_dictionary(dataset):
    dictionary = {}
    for j in range(len(dataset)):
        data_point = dataset[j]
        class_value = data_point[-1]
        if class_value not in dictionary:
            dictionary[class_value] = []
        dictionary[class_value].append(data_point)
    return dictionary

def count(number, y):
    total = 0
    for val in y:
        if val == number:
            total += 1
    return total

def calculate_prob(x, mean, variance):
    if variance <= 0.001:
        variance = 0.001
    loss = (np.power(x - mean, 2)) / variance

    exp_val = math.exp(-0.5 * loss)

    output = (1 / (math.sqrt(2 * math.pi * variance))) * exp_val
    # print(output)
    return output

def predict(x, statistics, prob_y_zero, prob_y_one):
    predictions = []
    predictions_probability = []
    for i in range(len(x)):
        result = calculateClassProbabilities(statistics, x[i])
        result[0.0] *= prob_y_zero
        result[1.0] *= prob_y_one
        bestLabel, bestProb = None, -1
        for classValue, probability in result.items():
            if bestLabel is None or probability > bestProb:
                bestProb = probability
                bestLabel = classValue
        predictions_probability.append(bestProb)
        predictions.append(bestLabel)
    return predictions, predictions_probability

def main():
    trainingSet, testingSet = extract_full_dataset()

    trainingSet = shuffle(trainingSet)

    training_x = np.copy(trainingSet[:, 0:-1])
    training_y = np.copy(trainingSet[:, -1])

    testing_x = np.copy(testingSet[:, 0:-1])
    testing_y = np.copy(testingSet[:, -1])

    # PCA

    pca = decomposition.PCA(n_components=100)
    pca.fit(training_x)
    training_x = pca.transform(training_x)

    testing_x = pca.transform(testing_x)

    print(training_x.shape, testing_x.shape)

    # Updating training set and testing set after PCA
    trainingSet = np.column_stack((training_x, training_y))
    testingSet = np.column_stack((testing_x, testing_y))
    # Class priors - Training

    count_zero = count(0.0, training_y)
    count_one = count(1.0, training_y)

    prob_zero_y = count_zero / len(training_y)
    prob_one_y = count_one / len(training_y)

    # print(prob_zero_y, prob_one_y)

    training_collection = build_dictionary(trainingSet)

    statistics = {}

    for classValue, instances in training_collection.items():
        statistics[classValue] = summarize(instances)

    # Calculating training and testing error
    train_prediction, train_prediction_prob = predict(training_x, statistics, prob_zero_y, prob_one_y)
    test_prediction, test_prediction_prob = predict(testing_x, statistics, prob_zero_y, prob_one_y)

    train_accuracy = calculate_accuracy(train_prediction, training_y)
    test_accuracy = calculate_accuracy(test_prediction, testing_y)

    print("Training accuracy:", train_accuracy)

    print("Testing accuracy is:", test_accuracy)

def calculateClassProbabilities(summaries, inputVector):
    probabilities = {}
    for classValue, classSummaries in summaries.items():
        probabilities[classValue] = 1
        for i in range(len(classSummaries)):
            mean, variance = classSummaries[i]
            x = inputVector[i]
            # print(x,mean,variance)
            probabilities[classValue] *= calculate_prob(x, mean, variance)
    return probabilities

if __name__ == '__main__':
    main()


"""
自己查成绩实在是麻烦，一次次的查，一个爬虫解决
Author: Aber Sheeran
Time: 2017-12-24
"""
from Base import Base

class GetTheFuckTranscripts(Base):
    """获取期末成绩"""
    def get_transcripts(self, year, semester):
        """获取特定学年特定学期的成绩"""
        html = self.get_page("http://jwgl.ahnu.edu.cn/query/cjquery/index?action=ok&xkxn=%s&xkxq=%s" % (year, semester))
        # 一波切割字符串骚操作,不用re也不用xpath或者bs
        table = html.split("table")[2]
        return "{0}{1}{2}{3}{4}".format(
            '\n\n<link rel="stylesheet" href="https://agent.cathaysian.cn/cloudflare?url=ajax/libs/pure/1.0.0/pure-min.css">\n\n',
            '\n
\n",
            '\n',
        )

    def check_transcripts_integrity(self, html):
        """检查成绩的完整性"""
        from bs4 import BeautifulSoup
        page = BeautifulSoup(html, "lxml")
        for each in page.find_all("tr")[1:-1]:
            if each.find_all("td")[7].text == "":
                return False
        return True

if __name__ == "__main__":
    main = GetTheFuckTranscripts({
        "username":"",
        "password":""
    })
    data = main.get_transcripts("2017-2018", "2")
    print(data)


from djangae.forms.fields import ListWidget

def get_widgets_for_fields(model_fields):
    widgets = None
    for field in model_fields:
        field_type_name = field.get_internal_type()
        if field_type_name in ('SetField', 'ListField'):
            widgets = {field.name: ListWidget}
    return widgets


from myhdl import *

def dp_block_gen(dp_bus, clock, reset, g_count_max):
    """
    """
    @always_seq(clock.posedge, reset=reset)
    def incLogic():
        if dp_bus.ready==True:
            dp_bus.data.next = (dp_bus.data + 1) % g_count_max
            dp_bus.valid.next = True
            dp_bus.sop.next = False
            dp_bus.eop.next = False

            if dp_bus.data == 0:
                dp_bus.sop.next = True
            elif dp_bus.data == g_count_max-1:
                dp_bus.eop.next = True

    return incLogic


import concurrent.futures
from time import sleep
from random import randint

def do_job(num):
    sleep_sec = randint(1, 3)
    print('value: %d, sleep: %d sec.' % (num, sleep_sec))
    sleep(sleep_sec)

with concurrent.futures.ProcessPoolExecutor(max_workers=5) as worker:
    for num in range(100):
        print('#%d Worker initialization' % num)
        worker.submit(do_job, num)


import context
import torch
import argparse
import config as cfg
from ipeps.ipeps_c4v import *
from groups.pg import *
from ctm.one_site_c4v.env_c4v import *
from ctm.one_site_c4v import ctmrg_c4v
from ctm.one_site_c4v.rdm_c4v import rdm2x1_sl
from ctm.one_site_c4v import transferops_c4v
from models import j1j2
# from optim.ad_optim_sgd_mod import optimize_state
from optim.ad_optim_lbfgs_mod import optimize_state
# from optim.ad_optim import optimize_state
import json
import unittest
import logging
log = logging.getLogger(__name__)

# parse command line args and build necessary configuration objects
parser= cfg.get_args_parser()
# additional model-dependent arguments
parser.add_argument("--j1", type=float, default=1., help="nearest-neighbour coupling")
parser.add_argument("--j2", type=float, default=0., help="next nearest-neighbour coupling")
parser.add_argument("--top_freq", type=int, default=-1, help="freuqency of transfer operator spectrum evaluation")
parser.add_argument("--top_n", type=int, default=2, help="number of leading eigenvalues"+
    "of transfer operator to compute")
parser.add_argument("--force_cpu", action='store_true', help="evaluate energy on cpu")
args, unknown_args = parser.parse_known_args()

def main():
    cfg.configure(args)
    cfg.print_config()
    torch.set_num_threads(args.omp_cores)
    torch.manual_seed(args.seed)

    model= j1j2.J1J2_C4V_BIPARTITE(j1=args.j1, j2=args.j2)
    energy_f= model.energy_1x1_lowmem

    # initialize the ipeps
    if args.instate!=None:
        state= read_ipeps_c4v(args.instate)
        if args.bond_dim > max(state.get_aux_bond_dims()):
            # extend the auxiliary dimensions
            state= extend_bond_dim(state, args.bond_dim)
        state.add_noise(args.instate_noise)
        # state.sites[(0,0)]= state.sites[(0,0)]/torch.max(torch.abs(state.sites[(0,0)]))
        state.sites[(0,0)]= state.site()/state.site().norm()
    elif args.opt_resume is not None:
        state= IPEPS_C4V(torch.tensor(0.))
        state.load_checkpoint(args.opt_resume)
    elif args.ipeps_init_type=='RANDOM':
        bond_dim = args.bond_dim
        A= torch.rand((model.phys_dim, bond_dim, bond_dim, bond_dim, bond_dim),\
            dtype=cfg.global_args.dtype, device=cfg.global_args.device)
        # A= make_c4v_symm(A)
        # A= A/torch.max(torch.abs(A))
        A= A/A.norm()
        state = IPEPS_C4V(A)
    else:
        raise ValueError("Missing trial state: -instate=None and -ipeps_init_type= "\
            +str(args.ipeps_init_type)+" is not supported")

    print(state)

    @torch.no_grad()
    def ctmrg_conv_f(state, env, history, ctm_args=cfg.ctm_args):
        if not history:
            history=dict({"log": []})
        rdm2x1= rdm2x1_sl(state, env, force_cpu=ctm_args.conv_check_cpu)
        dist= float('inf')
        if len(history["log"]) > 0:
            dist= torch.dist(rdm2x1, history["rdm"], p=2).item()
        history["rdm"]=rdm2x1
        history["log"].append(dist)
        if dist<ctm_args.ctm_conv_tol or len(history["log"]) >= ctm_args.ctm_max_iter:
            log.info({"history_length": len(history['log']), "history": history['log']})
            return True, history
        return False, history

    state_sym= to_ipeps_c4v(state)
    ctm_env= ENV_C4V(args.chi, state_sym)
    init_env(state_sym, ctm_env)

    ctm_env, *ctm_log = ctmrg_c4v.run(state_sym, ctm_env, conv_check=ctmrg_conv_f)
    loss= energy_f(state_sym, ctm_env)
    obs_values, obs_labels= model.eval_obs(state_sym,ctm_env)
    print(", ".join(["epoch","energy"]+obs_labels))
    print(", ".join([f"{-1}",f"{loss}"]+[f"{v}" for v in obs_values]))

    def loss_fn(state, ctm_env_in, opt_context):
        ctm_args= opt_context["ctm_args"]
        opt_args= opt_context["opt_args"]
        # 0) preprocess
        # create a copy of state, symmetrize and normalize making all operations
        # tracked. This does not "overwrite" the parameters tensors, living outside
        # the scope of loss_fn
        state_sym= to_ipeps_c4v(state, normalize=True)

        # possibly re-initialize the environment
        if opt_args.opt_ctm_reinit:
            init_env(state_sym, ctm_env_in)

        # 1) compute environment by CTMRG
        ctm_env_out, *ctm_log= ctmrg_c4v.run(state_sym, ctm_env_in,
            conv_check=ctmrg_conv_f, ctm_args=ctm_args)

        # 2) evaluate loss with converged environment
        loss = energy_f(state_sym, ctm_env_out, force_cpu=args.force_cpu)

        return (loss, ctm_env_out, *ctm_log)

    def _to_json(l):
        re=[l[i,0].item() for i in range(l.size()[0])]
        im=[l[i,1].item() for i in range(l.size()[0])]
        return dict({"re": re, "im": im})

    @torch.no_grad()
    def obs_fn(state, ctm_env, opt_context):
        if ("line_search" in opt_context.keys() and not opt_context["line_search"]) \
            or not "line_search" in opt_context.keys():
            state_sym= to_ipeps_c4v(state, normalize=True)
            epoch= len(opt_context["loss_history"]["loss"])
            loss= opt_context["loss_history"]["loss"][-1]
            obs_values, obs_labels = model.eval_obs(state_sym, ctm_env)
            print(", ".join([f"{epoch}",f"{loss}"]+[f"{v}" for v in obs_values]+\
                [f"{torch.max(torch.abs(state.site((0,0))))}"]))

            if args.top_freq>0 and epoch%args.top_freq==0:
                coord_dir_pairs=[((0,0), (1,0))]
                for c,d in coord_dir_pairs:
                    # transfer operator spectrum
                    print(f"TOP spectrum(T)[{c},{d}] ",end="")
                    l= transferops_c4v.get_Top_spec_c4v(args.top_n, state_sym, ctm_env)
                    print("TOP "+json.dumps(_to_json(l)))

    def post_proc(state, ctm_env, opt_context):
        symm, max_err= verify_c4v_symm_A1(state.site())
        # print(f"post_proc {symm} {max_err}")
        if not symm:
            # force symmetrization outside of autograd
            with torch.no_grad():
                symm_site= make_c4v_symm(state.site())
                # we **cannot** simply normalize the on-site tensors, as the LBFGS
                # takes into account the scale
                # symm_site= symm_site/torch.max(torch.abs(symm_site))
                state.sites[(0,0)].copy_(symm_site)

    # optimize
    # optimize_state(state, ctm_env, loss_fn, obs_fn=obs_fn, post_proc=post_proc)
    optimize_state(state, ctm_env, loss_fn, obs_fn=obs_fn)

    # compute final observables for the best variational state
    outputstatefile= args.out_prefix+"_state.json"
    state= read_ipeps_c4v(outputstatefile)
    ctm_env = ENV_C4V(args.chi, state)
    init_env(state, ctm_env)
    ctm_env, *ctm_log = ctmrg_c4v.run(state, ctm_env, conv_check=ctmrg_conv_f)
    opt_energy = energy_f(state,ctm_env)
    obs_values, obs_labels = model.eval_obs(state,ctm_env)
    print(", ".join([f"{args.opt_max_iter}",f"{opt_energy}"]+[f"{v}" for v in obs_values]))

if __name__=='__main__':
    if len(unknown_args)>0:
        print("args not recognized: "+str(unknown_args))
        raise Exception("Unknown command line arguments")
    main()

class TestOpt(unittest.TestCase):
    def setUp(self):
        args.j2=0.0
        args.bond_dim=2
        args.chi=16
        args.opt_max_iter=3
        try:
            import scipy.sparse.linalg
            self.SCIPY= True
        except:
            print("Warning: Missing scipy. Arnoldi methods not available.")
            self.SCIPY= False

    # basic tests
    def test_opt_SYMEIG(self):
        args.CTMARGS_projector_svd_method="SYMEIG"
        main()

    def test_opt_SYMEIG_LS_strong_wolfe(self):
        args.CTMARGS_projector_svd_method="SYMEIG"
        args.OPTARGS_line_search="strong_wolfe"
        main()

    def test_opt_SYMEIG_LS_backtracking(self):
        args.CTMARGS_projector_svd_method="SYMEIG"
        args.OPTARGS_line_search="backtracking"
        main()

    def test_opt_SYMEIG_LS_backtracking_SYMARP(self):
        if not self.SCIPY: self.skipTest("test skipped: missing scipy")
        args.CTMARGS_projector_svd_method="SYMEIG"
        args.OPTARGS_line_search="backtracking"
        args.OPTARGS_line_search_svd_method="SYMARP"
        main()

    @unittest.skipIf(not torch.cuda.is_available(), "CUDA not available")
    def test_opt_SYMEIG_gpu(self):
        args.GLOBALARGS_device="cuda:0"
        args.CTMARGS_projector_svd_method="SYMEIG"
        main()

    @unittest.skipIf(not torch.cuda.is_available(), "CUDA not available")
    def test_opt_SYMEIG_LS_backtracking_SYMARP_gpu(self):
        if not self.SCIPY: self.skipTest("test skipped: missing scipy")
        args.GLOBALARGS_device="cuda:0"
        args.CTMARGS_projector_svd_method="SYMEIG"
        args.OPTARGS_line_search="backtracking"
        args.OPTARGS_line_search_svd_method="SYMARP"
        main()


from networkit import *
client = gephi.streaming.GephiStreamingClient(
        url="http://localhost:8080/workspace1")

def exportgraph(G):
    client.exportGraph(G)
    plp = community.PLP(G).run()
    plm = community.PLM(G).run()
    client.exportNodeValues(G, plp.getPartition(), "PLP")
    client.exportNodeValues(G, plm.getPartition(), "PLM")

G = readGraph('36.graph', Format.EdgeList,
		commentPrefix='%', separator=' ',
                firstNode=1,
                continuous=False, directed=False)
exportgraph(G)


"""
Problem :
You have an array of n element.
How do you find 2 elements from n such as sum of them gives S?
"""
import random
arr = []
n = int(input("enter no of terms: "))
for i in range(n):
    arr.append(int(input("enter no: ")))
arr.sort()
S = int(input("enter sum to be found: "))

def listSearchSum(arr, S):
    """
    arr :  array with all numbers
    S : sum to be found
    """
    dicto = {}
    for i in arr:
        dicto[(S - i)] = i
    for i in arr:
        if i in dicto:
            return [i, dicto[i]]
    else:
        return -1

print(listSearchSum(arr, S))