from aita.aita_dataset import AitaDataset
import datetime
from keras.preprocessing.sequence import pad_sequences
import numpy as np
import os
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
from transformers import BertTokenizer, BertForSequenceClassification, get_linear_schedule_with_warmup, AdamW
import random
from sklearn.model_selection import train_test_split
import tensorflow as tf
import time
import torch
from torch.utils.data import DataLoader, random_split, SequentialSampler, RandomSampler

def format_time(elapsed):
    '''
    Takes a time in seconds and returns a string hh:mm:ss
    '''
    # Round to the nearest second.
    elapsed_rounded = int(round((elapsed)))

    # Format as hh:mm:ss
    return str(datetime.timedelta(seconds=elapsed_rounded))

def flat_accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return np.sum(pred_flat == labels_flat) / len(labels_flat)

def pad_data(tokenizer, input_ids):
    # We'll borrow the `pad_sequences` utility function to do this.

    MAX_LEN = 512

    print('\nPadding/truncating all sentences to %d values...' % MAX_LEN)

    print('\nPadding token: "{:}", ID: {:}'.format(tokenizer.pad_token, tokenizer.pad_token_id))

    # Pad our input tokens with value 0.
    # "post" indicates that we want to pad and truncate at the end of the sequence,
    # as opposed to the beginning.
    input_ids = pad_sequences(
        input_ids,
        maxlen=MAX_LEN,
        dtype="long",
        value=0,
        truncating="post",
        padding="post"
    )
    print('\nDone')
    return input_ids

def set_up_datasets():
     # Load the BERT tokenizer.
    print('Loading BERT tokenizer...')
    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

    aita_dataset = AitaDataset(tokenizer, max_length=512)

    train_size = int(np.floor(len(aita_dataset) * 0.9))
    test_size = len(aita_dataset) - train_size
    batch_size = 32

    # Create the DataLoader for our training set.
    train_data, validation_data = random_split(aita_dataset, [train_size, test_size])
    train_sampler = RandomSampler(train_data)
    train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size, num_workers=4)

    # Create the DataLoader for our validation set.
    validation_sampler = SequentialSampler(validation_data)
    validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)

    return train_dataloader, validation_dataloader

def train(train_dataloader, epochs=4):
    device = torch.device('cpu')

    print("Loading model")
    model = BertForSequenceClassification.from_pretrained(
        "bert-base-uncased", # Use the 12-layer BERT model, with an uncased vocab.
        num_labels = 1, # The number of output labels--2 for binary classification.
                    # You can increase this for multi-class tasks.
        output_attentions = False, # Whether the model returns attentions weights.
        output_hidden_states = False, # Whether the model returns all hidden-states.
        max_position_embeddings=512
    )

    # Total number of training steps is number of batches * number of epochs.
    total_steps = len(train_dataloader) * epochs

    # Note: AdamW is a class from the huggingface library (as opposed to pytorch)
    # I believe the 'W' stands for 'Weight Decay fix"
    optimizer = AdamW(
        model.parameters(),
        lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5
        eps = 1e-8 # args.adam_epsilon  - default is 1e-8.
    )

    # Create the learning rate scheduler.
    scheduler = get_linear_schedule_with_warmup(
        optimizer,
        num_warmup_steps=0, # Default value in run_glue.py
        num_training_steps=total_steps
    )

    # This training code is based on the `run_glue.py` script here:
    # https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128

    # Set the seed value all over the place to make this reproducible.
    # seed_val = 42

    # random.seed(seed_val)
    # np.random.seed(seed_val)
    # torch.manual_seed(seed_val)

    # Store the average loss after each epoch so we can plot them.
    loss_values = []

    # For each epoch...
    for epoch_i in range(0, epochs):

        # ========================================
        #               Training
        # ========================================

        # Perform one full pass over the training set.

        print("")
        print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
        print('Training...')

        # Measure how long the training epoch takes.
        t0 = time.time()

        # Reset the total loss for this epoch.
        total_loss = 0

        # Put the model into training mode. Don't be mislead--the call to
        # `train` just changes the *mode*, it doesn't *perform* the training.
        # `dropout` and `batchnorm` layers behave differently during training
        # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)
        model.train()

        # For each batch of training data...
        for step, batch in enumerate(train_dataloader):

            # Progress update every 40 batches.
            if step % 40 == 0 and not step == 0:
                # Calculate elapsed time in minutes.
                elapsed = format_time(time.time() - t0)

                # Report progress.
                print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))

            if step > 1:
                break

            # Unpack this training batch from our dataloader.
            #
            # As we unpack the batch, we'll also copy each tensor to the GPU using the
            # `to` method.
            #
            # `batch` contains three pytorch tensors:
            #   [0]: input ids
            #   [1]: attention masks
            #   [2]: labels

            b_input_ids = batch[0].to(device)
            b_input_mask = batch[1].to(device)
            b_labels = batch[2].to(device)

            # Always clear any previously calculated gradients before performing a
            # backward pass. PyTorch doesn't do this automatically because
            # accumulating the gradients is "convenient while training RNNs".
            # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)
            model.zero_grad()

            # Perform a forward pass (evaluate the model on this training batch).
            # This will return the loss (rather than the model output) because we
            # have provided the `labels`.
            # The documentation for this `model` function is here:
            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification
            outputs = model(
                b_input_ids,
                # token_type_ids=None,
                attention_mask=b_input_mask,
                labels=b_labels
            )

            # The call to `model` always returns a tuple, so we need to pull the
            # loss value out of the tuple.
            loss = outputs[0]

            # Accumulate the training loss over all of the batches so that we can
            # calculate the average loss at the end. `loss` is a Tensor containing a
            # single value; the `.item()` function just returns the Python value
            # from the tensor.
            total_loss += loss.item()

            # Perform a backward pass to calculate the gradients.
            loss.backward()

            # Clip the norm of the gradients to 1.0.
            # This is to help prevent the "exploding gradients" problem.
            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

            # Update parameters and take a step using the computed gradient.
            # The optimizer dictates the "update rule"--how the parameters are
            # modified based on their gradients, the learning rate, etc.
            optimizer.step()

            # Update the learning rate.
            scheduler.step()

            # Calculate the average loss over the training data.
            avg_train_loss = total_loss / len(train_dataloader)

            # Store the loss value for plotting the learning curve.
            loss_values.append(avg_train_loss)

        print("")
        print("\tAverage training loss: {0:.2f}".format(avg_train_loss))
        print("\tTraining epoch took: {:}".format(format_time(time.time() - t0)))
    print("")
    print("Training complete!")

    return model

def validate(validation_dataloader, model):
    # ========================================
        #               Validation
        # ========================================
        # After the completion of each training epoch, measure our performance on
        # our validation set.

        print("")
        print("Running Validation...")

        t0 = time.time()

        # Put the model in evaluation mode--the dropout layers behave differently
        # during evaluation.
        model.eval()

        # Tracking variables
        eval_loss, eval_accuracy = 0, 0
        nb_eval_steps, nb_eval_examples = 0, 0

        # Evaluate data for one epoch
        for i, batch in enumerate(validation_dataloader):

            if i > 2:
                break
            # Unpack the inputs from our dataloader
            b_input_ids, b_input_mask, b_labels = batch

            # Telling the model not to compute or store gradients, saving memory and
            # speeding up validation
            with torch.no_grad():

                # Forward pass, calculate logit predictions.
                # This will return the logits rather than the loss because we have
                # not provided labels.
                # token_type_ids is the same as the "segment ids", which
                # differentiates sentence 1 and 2 in 2-sentence tasks.
                # The documentation for this `model` function is here:
                # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification
                outputs = model(
                    b_input_ids,
                    token_type_ids=None,
                    attention_mask=b_input_mask
                )

            # Get the "logits" output by the model. The "logits" are the output
            # values prior to applying an activation function like the softmax.
            logits = outputs[0]

            # Move logits and labels to CPU
            logits = logits.numpy()
            label_ids = b_labels.numpy()

            # Calculate the accuracy for this batch of test sentences.
            tmp_eval_accuracy = flat_accuracy(logits, label_ids)

            # Accumulate the total accuracy.
            eval_accuracy += tmp_eval_accuracy

            # Track the number of batches
            nb_eval_steps += 1

        # Report the final accuracy for this validation run.
        print("\tAccuracy: {0:.2f}".format(eval_accuracy/nb_eval_steps))
        print("\tValidation took: {:}".format(format_time(time.time() - t0)))

def save_model(model, path):
    model.save_pretrained(path)

def main():
    train_loader, valid_loader = set_up_datasets()

    model = train(train_loader, epochs=1)

    validate(valid_loader, model)

    save_model(model, '/Users/dashiell/workspace/aita/models')

if __name__ == '__main__':
    main()


import torch
from ....torchio import DATA
from ....utils import is_image_dict
from .. import RandomTransform

class RandomFlip(RandomTransform):
    def __init__(
            self,
            axes=(0,),
            flip_probability=0.5,
            seed=None,
            verbose=False,
            ):
        super().__init__(seed=seed, verbose=verbose)
        self.axes = axes
        assert flip_probability > 0
        assert flip_probability <= 1
        self.flip_probability = flip_probability

    def apply_transform(self, sample):
        axes_to_flip_hot = self.get_params(self.axes, self.flip_probability)
        sample['random_flip'] = axes_to_flip_hot
        for image_dict in sample.values():
            if not is_image_dict(image_dict):
                continue
            tensor = image_dict[DATA]
            dims = []
            for dim, flip_this in enumerate(axes_to_flip_hot):
                if not flip_this:
                    continue
                actual_dim = dim + 1  # images are 4D
                dims.append(actual_dim)
            tensor = torch.flip(tensor, dims=dims)
            image_dict[DATA] = tensor
        return sample

    @staticmethod
    def get_params(axes, probability):
        axes_hot = [False, False, False]
        for axis in axes:
            random_number = torch.rand(1)
            flip_this = bool(probability > random_number)
            axes_hot[axis] = flip_this
        return axes_hot


from gui.PieceImages import PieceImages
from dialogs.DialogPromotion import DialogPromotion

# python chess
from chess.polyglot import *
import chess
import os

# PyQt and python system functions
from  PyQt4.QtGui import *
from  PyQt4.QtCore import *
import sys, random, time

from logic.gamestate import MODE_ENTER_MOVES,MODE_GAME_ANALYSIS,MODE_PLAY_BLACK,MODE_ANALYSIS,MODE_PLAY_WHITE,MODE_PLAYOUT_POS

def idx_to_str(x):
    return chr(97 + x % 8)

class Point():
    def __init__(self,x,y):
        self.x = x
        self.y = y

    def to_str(self):
        return idx_to_str(self.x) + str(self.y+1)

    def __eq__(self, other):
        return self.x == other.x and self.y == other.y

    def __ne__(self, other):
        return not self.__eq__(other)

class ChessboardView(QWidget):

    def __init__(self,gamestate,engine):
        super(ChessboardView, self).__init__()
        #super(QWidget, self).__init__()
        policy = QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Preferred)
        self.setSizePolicy(policy)

        self.gs = gamestate
        self.engine = engine

        self.pieceImages = PieceImages()

        self.borderWidth = 12

        self.moveSrc = None
        self.grabbedPiece = None
        self.grabbedX = None
        self.grabbedY = None
        self.drawGrabbedPiece = False

        self.flippedBoard = False

        #self.cache = BoardCache()

        self.initUI()

    def initUI(self):
        self.show()

    def heightForWidth(self, width):
        return width

    def resizeEvent(self, e):
        self.setMinimumWidth(self.height())

    def paintEvent(self, event):

        qp = QPainter()
        qp.begin(self)
        self.drawBoard(event, qp)
        qp.end()

    def getBoardPosition(self,x,y):
        (boardSize,squareSize) = self.calculateBoardSize()
        # check if x,y are actually on the board
        if(x > self.borderWidth and y > self.borderWidth
           and x < (boardSize - self.borderWidth)
           and y < (boardSize - self.borderWidth)):
            x = x - self.borderWidth
            y = y - self.borderWidth
            x = x // squareSize
            y = 7 - (y // squareSize)
            if(self.flippedBoard):
                return Point(7-x,7-y)
            else:
                return Point(x,y)
        return None

    @pyqtSlot()
    def flip_board(self):
        if(self.flippedBoard):
            self.flippedBoard = False
        else:
            self.flippedBoard = True
        self.update()

    def touchPiece(self, x, y, mouse_x, mouse_y):
        self.moveSrc = Point(x,y)
        piece = self.gs.current.board().piece_at(y*8+x).symbol()
        self.grabbedPiece = piece
        self.grabbedX = mouse_x
        self.grabbedY = mouse_y
        self.drawGrabbedPiece = True

    def debug_msg(self,s):
        msgBox = QMessageBox()
        msgBox.setText(s)
        msgBox.exec_()

        #self.gt.current.board.set_at(x,y,'e')

    def _make_uci(self,point_src,point_dst,promote_to=None):
        uci = point_src.to_str() + point_dst.to_str()
        if(promote_to != None):
            uci += promote_to
        return uci

    def executeMove(self, uci):
        temp = self.gs.current
        move = chess.Move.from_uci(uci)
        # check if move already exists
        variation_moves = [ x.move for x in self.gs.current.variations ]
        if(move in variation_moves):
            for node in self.gs.current.variations:
                if(node.move == move):
                    self.gs.current = node
        # otherwise create a new node
        else:
            self.gs.current.add_variation(move)
            self.gs.current = self.gs.current.variation(move)
        #"+str(( self.gs.board().turn + 1)%2))
        #new_node = chess.pgn.GameNode()
        #new_node.parent = temp
        #new_node.move = chess.Move.from_uci(uci)
        self.moveSrc = None
        self.grabbedPiece = None
        self.drawGrabbedPiece = False
        #self.movesEdit = MovesEdit(self)
        #self.movesEdit.update_san()
        # check if game is drawn, or checkmate due to various conditions
        if(self.gs.current.board().is_checkmate()): # due to checkmate
            msgBox = QMessageBox()
            msgBox.setText("Checkmate.")
            msgBox.setInformativeText("The game is over!")
            msgBox.exec_()
            self.emit(SIGNAL("checkmate"))
        elif(self.gs.current.board().is_stalemate()): # due to stalemate
            msgBox = QMessageBox()
            msgBox.setText("Stalemate.")
            msgBox.setInformativeText("The game is drawn!")
            msgBox.exec_()
            self.emit(SIGNAL("drawn"))
        elif(self.gs.current.board().is_insufficient_material()): # due to insufficient material
            msgBox = QMessageBox()
            msgBox.setText("Insufficient material to win.")
            msgBox.setInformativeText("The game is drawn!")
            msgBox.exec_()
            self.emit(SIGNAL("drawn"))
        elif(self.gs.current.board().can_claim_threefold_repitition()): # due to threefold repetition
            msgBox = QMessageBox()
            msgBox.setText("Threefold repetition.")
            msgBox.setInformativeText("The game is drawn!")
            msgBox.exec_()
            self.emit(SIGNAL("drawn"))

        if(self.gs.mode == MODE_ANALYSIS):
            uci_string = self.gs.printer.to_uci(self.gs.current)
            self.engine.uci_send_position(uci_string)
            self.engine.uci_go_infinite()
        #self.update()
        if((self.gs.mode == MODE_PLAY_WHITE and self.gs.current.board().turn == chess.BLACK) or
            (self.gs.mode == MODE_PLAY_BLACK and self.gs.current.board().turn == chess.WHITE)):
            book_move = None
            s = (str(os.listdir(".")))
            #msgBox = QMessageBox()
            #msgBox.setText(s)
            #msgBox.exec_()
            #print("LIST OF FILES"+s)
            with open_reader("./books/varied.bin") as reader:
                #self.debug_msg("ok, openend file")
                entries = reader.get_entries_for_position(self.gs.current.board())
                moves = []
                for entry in entries:
                    move = entry.move().uci()
                    moves.append(move)
                    #self.emit(SIGNAL("bestmove(QString)"),move)
                l = len(moves)
                if(l > 0):
                    book_move = True
                    n = random.randint(0,l-1)
                    book_move = moves[n]
            if(book_move != None):
                #self.debug_msg("ok, sending book move")
                self.emit(SIGNAL("bestmove(QString)"),book_move)
            else:
                uci_string = self.gs.printer.to_uci(self.gs.current)
                self.engine.uci_send_position(uci_string)
                self.engine.uci_go_movetime(self.gs.computer_think_time)
        elif(self.gs.mode == MODE_PLAY_WHITE or self.gs.mode == MODE_PLAY_BLACK): pass
            #self.engine.uci_go_infinite()
        elif(self.gs.mode == MODE_PLAYOUT_POS):
            uci_string = self.gs.printer.to_uci(self.gs.current)
            self.engine.uci_send_position(uci_string)
            self.engine.uci_go_movetime(self.gs.computer_think_time)
        self.emit(SIGNAL("statechanged()"))

    def resetMove(self):
        #self.gt.current.board.set_at(self.moveSrc.x,self.moveSrc.y,self.grabbedPiece)
        self.moveSrc = None
        self.grabbedPiece = None
        self.drawGrabbedPiece = False

    def __players_turn(self):
        if(self.gs.mode == MODE_PLAYOUT_POS):
            return True
        if(self.gs.mode == MODE_PLAY_BLACK and self.gs.current.board().turn == chess.WHITE):
            return False
        if(self.gs.mode == MODE_PLAY_WHITE and self.gs.current.board().turn == chess.BLACK):
            return False
        return True

    def _is_valid_and_promotes(self,uci):
        if self.gs.mode == MODE_GAME_ANALYSIS:
            return False
        if(not self.__players_turn()):
            return False
        legal_moves = self.gs.current.board().legal_moves
        for lm in legal_moves:
            if(uci == lm.uci()[0:4] and len(lm.uci())==5):
                return True
        return False

    def _is_valid(self,uci):
        if self.gs.mode == MODE_GAME_ANALYSIS:
            return False
        if(not self.__players_turn()):
            return False
        legal_moves = self.gs.current.board().legal_moves
        return (len([x for x in legal_moves if x.uci() == uci]) > 0)

    def mousePressEvent(self, mouseEvent):
        pos = self.getBoardPosition(mouseEvent.x(), mouseEvent.y())
        if(pos):
            i = pos.x
            j = pos.y
            if(self.grabbedPiece):
                #m = Point(i,j)
                uci = self._make_uci(self.moveSrc,pos)
                if(self._is_valid_and_promotes(uci)):
                    promDialog = DialogPromotion(self.gs.current.board().turn == chess.WHITE)
                    answer = promDialog.exec_()
                    if(answer):
                        uci += promDialog.final_piece.lower()
                        self.executeMove(uci)
                elif(self._is_valid(uci)):
                    self.executeMove(uci)
                else:
                    self.resetMove()
                    if(self.gs.current.board().piece_at(j*8+i) != None):
                        self.touchPiece(i,j,mouseEvent.x(),mouseEvent.y())
            else:
                if(self.gs.current.board().piece_at(j*8+i) != None):
                    self.touchPiece(i,j,mouseEvent.x(),mouseEvent.y())
        self.update()

    def mouseMoveEvent(self, mouseEvent):
        button = mouseEvent.button()
        if(button == 0 and (not self.grabbedPiece == None)):
            self.grabbedX = mouseEvent.x()
            self.grabbedY = mouseEvent.y()
            self.drawGrabbedPiece = True
            self.update()

    def mouseReleaseEvent(self, mouseEvent):
        self.drawGrabbedPiece = False
        pos = self.getBoardPosition(mouseEvent.x(), mouseEvent.y())
        if(pos and self.grabbedPiece != None):
            if(pos != self.moveSrc):
                uci = self._make_uci(self.moveSrc, pos)
                if(self._is_valid_and_promotes(uci)):
                    promDialog = DialogPromotion(self.gs.current.board().turn == chess.WHITE)
                    answer = promDialog.exec_()
                    if(answer):
                        uci += promDialog.final_piece.lower()
                        self.executeMove(uci)
                elif(self._is_valid(uci)):
                    self.executeMove(uci)
                else:
                    self.resetMove()
        self.update()

    def calculateBoardSize(self):
        size = self.size()
        boardSize = min(size.width(), size.height())
        squareSize = (boardSize-(2*self.borderWidth))//8
        boardSize = 8 * squareSize + 2 * self.borderWidth
        return (boardSize,squareSize)

    def on_statechanged(self):
        self.update()

    def drawBoard(self, event, qp):
        penZero = QPen(Qt.black, 1, Qt.NoPen)
        qp.setPen(penZero)

        darkBlue = QColor(56,66,91)
        #Fritz 13
        #lightBlue = QtGui.QColor(111,132,181)
        #darkWhite = QtGui.QColor(239,239,239)
        #Fritz 6
        lightBlue = QColor(90,106,173)
        lightBlue2 = QColor(166,188,231)
        darkWhite = QColor(239,239,239)

        qp.setBrush(darkBlue)

        (boardSize,squareSize) = self.calculateBoardSize()

        qp.drawRect(1,1,boardSize,boardSize)

        boardOffsetX = self.borderWidth;
        boardOffsetY = self.borderWidth;

        # if the board is flipped, also
        # flip colors so that the board
        # loop works without any change/additional
        # if-checks
        dark = lightBlue2
        light = lightBlue
        if(self.flippedBoard):
            dark = lightBlue
            light = lightBlue2

        # draw Board
        #board = self.cache.get(self.gs.current)
        board = self.gs.current.board()
        for i in range(0,8):
            for j in range(0,8):
                if((j%2 == 0 and i%2 ==1) or (j%2 == 1 and i%2 ==0)):
                    qp.setBrush(dark)
                else:
                    qp.setBrush(light)
                #draw Square
                if(self.flippedBoard):
                    x = boardOffsetX+((7-i)*squareSize)
                else:
                    x = boardOffsetX+(i*squareSize)
                # drawing coordinates are from top left
                # whereas chess coords are from bottom left
                y = boardOffsetY+((7-j)*squareSize)
                qp.drawRect(x,y,squareSize,squareSize)
                #draw Piece
                piece = None
                if(self.flippedBoard):
                    piece = board.piece_at((7-j)*8+i)
                else:
                    piece = board.piece_at(j*8+i)
                if(piece != None and piece.symbol() in ('P','R','N','B','Q','K','p','r','n','b','q','k')):
                    # skip piece that is currently picked up
                    if(not self.flippedBoard):
                        if(not (self.drawGrabbedPiece and i == self.moveSrc.x and j == self.moveSrc.y)):
                            qp.drawImage(x,y,self.pieceImages.getWp(piece.symbol(), squareSize))
                    else:
                        if(not (self.drawGrabbedPiece and i == self.moveSrc.x and (7-j) == self.moveSrc.y)):
                            qp.drawImage(x,y,self.pieceImages.getWp(piece.symbol(), squareSize))

        if(self.drawGrabbedPiece):
            offset = squareSize // 2
            qp.drawImage(self.grabbedX-offset,self.grabbedY-offset,self.pieceImages.getWp(self.grabbedPiece, squareSize))

        qp.setPen(darkWhite)
        qp.setFont(QFont('Decorative',8))

        for i in range(0,8):
            if(self.flippedBoard):
                idx = str(chr(65+(7-i)))
                qp.drawText(boardOffsetX+(i*squareSize)+(squareSize/2)-4,
                            boardOffsetY+(8*squareSize)+(self.borderWidth-3),idx)
                qp.drawText(4,boardOffsetY+(i*squareSize)+(squareSize/2)+4,str(i+1))
            else:
                idx = str(chr(65+i))
                qp.drawText(boardOffsetX+(i*squareSize)+(squareSize/2)-4,
                            boardOffsetY+(8*squareSize)+(self.borderWidth-3),idx)
                qp.drawText(4,boardOffsetY+(i*squareSize)+(squareSize/2)+4,str(8-i))


import socket
import sys
import os
import ssl
import asyncore

def main():
    HOST = '127.0.0.1'    # The remote host
    PORT = 1234              # The same port as used by the server
    s = None
    for res in socket.getaddrinfo(HOST, PORT, socket.AF_UNSPEC, socket.SOCK_STREAM):
        af, socktype, proto, canonname, sa = res
        try:
            s = socket.socket(af, socktype, proto)
        except OSError as msg:
            s = None
            continue
        try:
            s.connect(sa)
        except OSError as msg:
            s.close()
            s = None
            continue
        break
    if s is None:
        print('could not open socket')
        sys.exit(1)
    with s:
        s.sendall(b'Hello, world')
        data = s.recv(1024)
    print('Received', repr(data))

#ssl
#Client socket example with default context and IPv4/IPv6 dual stack
def main1():
    hostname = 'www.baidu.com'
    context = ssl.create_default_context()

    with socket.create_connection((hostname, 443)) as sock:
        with context.wrap_socket(sock, server_hostname=hostname) as ssock:
            print(ssock.version())

#Client socket example with custom context and IPv4:
def main2():
    hostname = 'www.python.org'
    # PROTOCOL_TLS_CLIENT requires valid cert chain and hostname
    context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
    context.load_verify_locations('path/to/cabundle.pem')

    with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock:
        with context.wrap_socket(sock, server_hostname=hostname) as ssock:
            print(ssock.version())

class HTTPClient(asyncore.dispatcher):
    def __init__(self, host, path):
        asyncore.dispatcher.__init__(self)
        self.create_socket()
        self.connect( (host, 80) )
        self.buffer = bytes('GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' %
                            (path, host), 'ascii')

    def handle_connect(self):
        pass

    def handle_close(self):
        self.close()

    def handle_read(self):
        print(self.recv(8192))

    def writable(self):
        return (len(self.buffer) > 0)

    def handle_write(self):
        sent = self.send(self.buffer)
        self.buffer = self.buffer[sent:]

def main3():
    client = HTTPClient('www.python.org', '/')
    asyncore.loop()

if (__name__=="__main__"):
    main3()


#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
#from __future__ import unicode_literals

import os.path
from unittest import SkipTest

import numpy as np
from numpy.testing import assert_allclose, assert_equal

import pytest
import itertools

from freediscovery.text import FeatureVectorizer
from freediscovery.categorization import Categorizer
from freediscovery.dupdet import DuplicateDetection
from freediscovery.cluster import Clustering
from freediscovery.lsi import LSI
from freediscovery.io import parse_ground_truth_file
from freediscovery.utils import categorization_score
from freediscovery.exceptions import OptionalDependencyMissing
from .run_suite import check_cache

basename = os.path.dirname(__file__)
data_dir = os.path.join(basename, "..", "data", "ds_001", "raw")

@pytest.mark.parametrize('use_hashing, method', itertools.product([False, True],
                                                ['Categorization', 'LSI',
                                                 'DuplicateDetection', 'Clustering']))
def test_features_hashing(use_hashing, method):
    # check that models work both with and without hashing

    cache_dir = check_cache()

    n_features = 20000

    fe = FeatureVectorizer(cache_dir=cache_dir)
    uuid = fe.preprocess(data_dir, file_pattern='.*\d.txt', n_features=n_features,
                         use_hashing=use_hashing)
    uuid, filenames  = fe.transform()

    ground_truth = parse_ground_truth_file(
                            os.path.join(data_dir, "..", "ground_truth_file.txt"))

    if method == 'Categorization':
        cat = Categorizer(cache_dir=cache_dir, dsid=uuid, cv_n_folds=2)
        index = cat.fe.search(ground_truth.index.values)

        try:
            coefs, Y_train = cat.train(
                                    index,
                                    ground_truth.is_relevant.values,
                                    )
        except OptionalDependencyMissing:
            raise SkipTest

        Y_pred = cat.predict()
        X_pred = np.arange(cat.fe.n_samples_, dtype='int')
        idx_gt = cat.fe.search(ground_truth.index.values)

        scores = categorization_score(idx_gt,
                            ground_truth.is_relevant.values,
                            X_pred, Y_pred)
        assert_allclose(scores['precision'], 1, rtol=0.5)
        assert_allclose(scores['recall'], 1, rtol=0.5)
        cat.delete()
    elif method == 'LSI':
        lsi = LSI(cache_dir=cache_dir, dsid=uuid)
        lsi_res, exp_var = lsi.transform(n_components=100)  # TODO unused variables
        lsi_id = lsi.mid
        assert lsi.get_dsid(fe.cache_dir, lsi_id) == uuid
        assert lsi.get_path(lsi_id) is not None
        assert lsi._load_pars(lsi_id) is not None
        lsi.load(lsi_id)

        idx_gt = lsi.fe.search(ground_truth.index.values)
        idx_all = np.arange(lsi.fe.n_samples_, dtype='int')

        for accumulate in ['nearest-max', 'centroid-max']:
                            #'nearest-diff', 'nearest-combine', 'stacking']:
            _, Y_train, Y_pred, ND_train = lsi.predict(
                                    idx_gt,
                                    ground_truth.is_relevant.values,
                                    accumulate=accumulate)
            scores = categorization_score(idx_gt,
                                ground_truth.is_relevant.values,
                                idx_all, Y_pred)
            assert_allclose(scores['precision'], 1, rtol=0.5)
            assert_allclose(scores['recall'], 1, rtol=0.3)
    elif method == 'DuplicateDetection':
        dd = DuplicateDetection(cache_dir=cache_dir, dsid=uuid)
        try:
            dd.fit()
        except ImportError:
            raise SkipTest
        cluster_id = dd.query(distance=10)
    elif method =='Clustering':
        if not use_hashing:
            cat = Clustering(cache_dir=cache_dir, dsid=uuid)
            cm = getattr(cat,'k_means')
            labels, htree = cm(2, lsi_components=20)

            terms = cat.compute_labels(n_top_words=10)
        else:
            with pytest.raises(NotImplementedError):
                Clustering(cache_dir=cache_dir, dsid=uuid)

    else:
        raise ValueError


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Oct 11 22:16:20 2020

@author: sahebsingh
"""

from util import *
import numpy as np
from scipy.linalg import block_diag

def buildGadget(l, n):
    # the secret vector [s] is an (n-1)-dimensional vector,
    # the secret key [t] is -s‖1, an n-dimensional vector
    #
    # the error vector [e] is an m-dimensional vector
    #
    # the matrix [A] is an (n-1)×m matrix (n-1 rows, m = n×l columns) #
    # the public key [B] is ( A ) which is an n×m matrix
    # ( sA+e )
    #
    g = 2**np.arange(l)
    return block_diag(*[g for null in range(n)])

def encrypt(keys, message):
    stat("Encrypting message")
    #
    # the gadget matrix [G] is an n×m matrix (n rows, m = n×l columns)
    #
    # the matrix R is an m×m matrix (n×l rows, n×l columns)
    #
    # the ciphertext is (n×m)⋅(m×m) *> an n×m matrix
    #
    R = np.random.randint(2, size=(keys.m, keys.m), dtype=np.int64).astype(keys.datatype)
    G = buildGadget(keys.l, keys.n)
    return (np.dot(keys.PK, R) + message*G)%keys.q


import numpy as np
from six import BytesIO
from PIL import Image

import tensorflow as tf
import json
from object_detection.utils import ops as utils_ops
from object_detection.utils import label_map_util
from object_detection.utils import visualization_utils as vis_util
import os
import pathlib

def load_image_into_numpy_array(path):
    """Load an image from file into a numpy array.

    Puts image into numpy array to feed into tensorflow graph.
    Note that by convention we put it into a numpy array with shape
    (height, width, channels), where channels=3 for RGB.

    Args:
      path: the file path to the image

    Returns:
      uint8 numpy array with shape (img_height, img_width, 3)
    """
    img_data = tf.io.gfile.GFile(path, 'rb').read()
    image = Image.open(BytesIO(img_data))
    (im_width, im_height) = image.size
    return np.array(image.getdata()).reshape(
        (im_height, im_width, 3)).astype(np.uint8)

def get_keypoint_tuples(eval_config):
    """Return a tuple list of keypoint edges from the eval config.

    Args:
      eval_config: an eval config containing the keypoint edges

    Returns:
      a list of edge tuples, each in the format (start, end)
    """
    tuple_list = []
    kp_list = eval_config.keypoint_edge
    for edge in kp_list:
        tuple_list.append((edge.start, edge.end))
    return tuple_list

def get_model_detection_function(model):
    """Get a tf.function for detection."""

    @tf.function
    def detect_fn(image):
        """Detect objects in image."""

        image, shapes = model.preprocess(image)
        prediction_dict = model.predict(image, shapes)
        detections = model.postprocess(prediction_dict, shapes)

        return detections, prediction_dict, tf.reshape(shapes, [-1])

    return detect_fn

def run_inference_for_single_image(model, image):
    image = np.asarray(image)
    # The input needs to be a tensor, convert it using `tf.convert_to_tensor`.
    input_tensor = tf.convert_to_tensor(image)
    # The model expects a batch of images, so add an axis with `tf.newaxis`.
    input_tensor = input_tensor[tf.newaxis,...]

    # Run inference
    model_fn = model.signatures['serving_default']
    output_dict = model_fn(input_tensor)

    # All outputs are batches tensors.
    # Convert to numpy arrays, and take index [0] to remove the batch dimension.
    # We're only interested in the first num_detections.
    num_detections = int(output_dict.pop('num_detections'))
    output_dict = {key:value[0, :num_detections].numpy()
                   for key,value in output_dict.items()}
    output_dict['num_detections'] = num_detections

    # detection_classes should be ints.
    output_dict['detection_classes'] = output_dict['detection_classes'].astype(np.int64)

    # Handle models with masks:
    if 'detection_masks' in output_dict:
        # Reframe the the bbox mask to the image size.
        detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks(
            output_dict['detection_masks'], output_dict['detection_boxes'],
            image.shape[0], image.shape[1])
        detection_masks_reframed = tf.cast(detection_masks_reframed > 0.5,
                                           tf.uint8)
        output_dict['detection_masks_reframed'] = detection_masks_reframed.numpy()

    return output_dict

def show_inference(model, image_path, category_index, output_path):
    # the array based representation of the image will be used later in order to prepare the
    # result image with boxes and labels on it.
    image_np = np.array(Image.open(image_path))
    # Actual detection.
    file_name = os.path.basename(image_path)
    output_dict = run_inference_for_single_image(model, image_np)
    # Visualization of the results of a detection.
    vis_util.visualize_boxes_and_labels_on_image_array(
        image_np,
        output_dict['detection_boxes'],
        output_dict['detection_classes'],
        output_dict['detection_scores'],
        category_index,
        instance_masks=output_dict.get('detection_masks_reframed', None),
        use_normalized_coordinates=True,
        line_thickness=4,
        max_boxes_to_draw=1000,)

    image = Image.fromarray(image_np)
    image.save(os.path.join(output_path, file_name))

def show_gt(image_path, category_index, coors, tags, output_path):
    # the array based representation of the image will be used later in order to prepare the
    # result image with boxes and labels on it.
    image_np = np.array(Image.open(image_path))

    # Actual detection.
    file_name = os.path.basename(image_path)
    # Visualization of the results of a detection.
    vis_util.visualize_boxes_and_labels_on_image_array(
        image_np,
        coors,
        tags,
        np.ones_like(tags),
        category_index,
        instance_masks=None,
        use_normalized_coordinates=True,
        line_thickness=4,
        max_boxes_to_draw=1000,)

    image = Image.fromarray(image_np)
    image.save(os.path.join(output_path, file_name))

def main():
    assets_dict = {}
    with open('/home/kimberly/projects/ui_data_v13/ui/test_vott/overlapped.vott', "r") as fp:
        data = json.load(fp)
        assets = data['assets']
        for a in assets:
            assets_dict[assets[a]['name']] = a
    base_dir = '/home/kimberly/projects/ui_data_v13/ui/'
    pbtxt = os.path.join(base_dir, 'ui.pbtxt')

    model_dir = '/home/kimberly/projects/ui_data_v13/ui/model'

    detection_model = tf.saved_model.load(model_dir)
    PATH_TO_LABELS = pbtxt
    category_index = label_map_util.create_category_index_from_labelmap(PATH_TO_LABELS, use_display_name=True)
    category_dict = {}
    for cat in category_index.values():
        category_dict[cat['name']] = cat['id']
    PATH_TO_TEST_IMAGES_DIR = pathlib.Path(base_dir+'/test')
    TEST_IMAGE_PATHS = sorted(list(PATH_TO_TEST_IMAGES_DIR.glob("*")))
    for image_path in TEST_IMAGE_PATHS:
        # show_inference(detection_model, image_path, category_index)
        image_name = os.path.basename(image_path)
        asset_key = assets_dict[image_name]
        with open('/home/kimberly/projects/ui_data_v13/ui/test_vott/'+asset_key+ '-asset.json', "r") as fp:
            asset_data = json.load(fp)
            regions = asset_data['regions']
            im_width = asset_data['asset']['size']['width']
            im_height = asset_data['asset']['size']['height']
            coors = []
            tags = []
            for r in regions:
                height = r['boundingBox']['height']
                width = r['boundingBox']['width']
                left = r['boundingBox']['left']
                top = r['boundingBox']['top']
                # coor = [left / width, top / height, (left + width) / width, (top + height) / height]
                coor = [top / im_height, left / im_width, (top + height) / im_height, (left + width) / im_width]

                coors.append(coor)
                tag = r['tags'][0]
                tag = tag.lower()
                tag_id = category_dict[tag]
                tags.append(tag_id)
            coors = np.asarray(coors)
            tags = np.asarray(tags)
        show_inference(detection_model, image_path, category_index, '/home/kimberly/projects/ui_data_v13/ui/test_detected')
        show_gt(image_path, category_index, coors, tags, '/home/kimberly/projects/ui_data_v13/ui/gt')

if __name__ == '__main__':
    main()


"""add_timestamp_related column in ProductReleaseInfo

Revision ID: bab26a6e0ae2
Revises: 3057c19a2f4f
Create Date: 2021-04-22 05:45:14.676499

"""
import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision = 'bab26a6e0ae2'
down_revision = '3057c19a2f4f'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('product_release_info', sa.Column('created_at', sa.TIMESTAMP(),
                  server_default=sa.func.current_timestamp(), nullable=False))
    op.add_column('product_release_info', sa.Column('updated_at', sa.TIMESTAMP(),
                  server_default=sa.func.current_timestamp(), nullable=False))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('product_release_info', 'updated_at')
    op.drop_column('product_release_info', 'created_at')
    # ### end Alembic commands ###


import pygame
import configparser
import json
import time
from helpers import load_sound, play_sound, get_length, stop_sound, load_image

config = configparser.ConfigParser()
config.read("settings.ini")

hero_speed = config.getfloat('PLAYER', 'hero_speed')
jump_force = config.getfloat('PLAYER', 'jump_force')
dash_speed = config.getint('PLAYER', 'dash_speed')
wall_jump = json.loads(config.get("PLAYER", "wall_jump"))

down_timer = config.getfloat("CAMERA", "down_timer")
width, height = json.loads(config.get("MAIN", "res"))
h_width, h_height = json.loads(config.get("PLAYER", "size"))
fps = config.getint("MAIN", "fps")

# Загрузка звуков
walk_sound = load_sound("data/sounds/walk.wav")
walk_sound.set_volume(0.5)
walk_time = get_length(walk_sound)
jump_sound = load_sound("data/sounds/jump.wav")
jump_sound.set_volume(0.4)
dash_sound = load_sound("data/sounds/dash.wav")
dash_sound.set_volume(0.4)
sliding_sound = load_sound("data/sounds/wall_slide.wav")
sliding_sound.set_volume(0.4)
slide_time = get_length(sliding_sound)

idle_right = [load_image('data/PassiveReaper_Right/PassiveIdleReaper-Sheet.png', h_width, h_height),
                 load_image('data/PassiveReaper_Right/PassiveIdleReaper-Sheet2.png', h_width, h_height),
                 load_image('data/PassiveReaper_Right/PassiveIdleReaper-Sheet3.png', h_width, h_height),
                 load_image('data/PassiveReaper_Right/PassiveIdleReaper-Sheet4.png', h_width, h_height),
                 load_image('data/PassiveReaper_Right/PassiveIdleReaper-Sheet5.png', h_width, h_height)]

idle_left = [load_image('data/PassiveReaper_Left/PassiveIdleReaper-Sheet.png', h_width, h_height),
                 load_image('data/PassiveReaper_Left/PassiveIdleReaper-Sheet2.png', h_width, h_height),
                 load_image('data/PassiveReaper_Left/PassiveIdleReaper-Sheet3.png', h_width, h_height),
                 load_image('data/PassiveReaper_Left/PassiveIdleReaper-Sheet4.png', h_width, h_height),
                 load_image('data/PassiveReaper_Left/PassiveIdleReaper-Sheet5.png', h_width, h_height)]

go_right = [load_image('data/Reaper_go_right/RunningReaper.png', h_width, h_height),
            load_image('data/Reaper_go_right/RunningReaper2.png', h_width, h_height),
            load_image('data/Reaper_go_right/RunningReaper3.png', h_width, h_height),
            load_image('data/Reaper_go_right/RunningReaper4.png', h_width, h_height),
            load_image('data/Reaper_go_right/RunningReaper6.png', h_width, h_height),
            load_image('data/Reaper_go_right/RunningReaper7.png', h_width, h_height)]

go_left = [load_image('data/Reaper_go_left/RunningReaper.png', h_width, h_height),
           load_image('data/Reaper_go_left/RunningReaper2.png', h_width, h_height),
           load_image('data/Reaper_go_left/RunningReaper3.png', h_width, h_height),
           load_image('data/Reaper_go_left/RunningReaper4.png', h_width, h_height),
           load_image('data/Reaper_go_left/RunningReaper6.png', h_width, h_height),
           load_image('data/Reaper_go_left/RunningReaper7.png', h_width, h_height)]

# Инициализация классов состояния персонажа
class DashState:
    def __init__(self, velocity_x, next_state, player):
        self.dash_timer = .15 # Длительность рывка
        self.velocity_x = velocity_x
        self.next_state = next_state # Следующее состояние

        if player.sounds and not player.dash_done:   # Воспроизводим звук
            play_sound(dash_sound)

    def handle_event(self, player, event):
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_LEFT:
                player.last_dir = "left"
                self.next_state = MoveState(-hero_speed)
            elif event.key == pygame.K_RIGHT:
                player.last_dir = "right"
                self.next_state = MoveState(hero_speed)

        if event.type == pygame.KEYUP:
            if event.key == pygame.K_LEFT:
                self.next_state = IdleState()
            elif event.key == pygame.K_RIGHT:
                self.next_state = IdleState()

    def update(self, player, dt, platforms, camera=None):
        global walk_time, slide_time

        if not player.dash_done: # В воздухе может быть выполнен только один рывок
            walk_time = get_length(walk_sound)
            slide_time = get_length(sliding_sound)
            self.dash_timer -= dt # Таймер
            self.velocity_x = dash_speed

            # В зависимости от последнего направления персонажа вектор скорости рывка меняется
            if player.last_dir == "left":
                self.velocity_x = -self.velocity_x

            player.rect.x += self.velocity_x
            player.collide(self.velocity_x, 0, platforms)

            # Если таймер истек
            if self.dash_timer <= 0:
                player.vel.y = 0
                player.dash_done = True
                player.state = self.next_state
        else:
            player.state = self.next_state

class JumpState:
    def __init__(self, velocity_y, next_state):
        self.velocity_y = velocity_y
        self.next_state = next_state

    def handle_event(self, player, event):
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_LEFT:
                player.last_dir = "left"
                self.next_state = MoveState(-hero_speed)
            elif event.key == pygame.K_RIGHT:
                player.last_dir = "right"
                self.next_state = MoveState(hero_speed)

        if event.type == pygame.KEYUP:
            if event.key == pygame.K_LEFT:
                self.next_state = IdleState()
            elif event.key == pygame.K_RIGHT:
                self.next_state = IdleState()

    def update(self, player, dt, platforms, camera=None):
        global walk_time, slide_time

        walk_time = get_length(walk_sound)
        slide_time = get_length(sliding_sound)

        if player.onGround:
            if player.sounds:   # Воспроизводим звук
                play_sound(jump_sound)

            player.vel.y = -self.velocity_y
            player.onGround = False
            player.double_jump = True

        player.gravitation()
        player.collide(0, player.vel.y, platforms)
        player.state = self.next_state

class DoubleJumpState(JumpState):
    def __init__(self, velocity_y, next_state):
        super().__init__(velocity_y, next_state)

    def update(self, player, dt, platforms, camera=None):
        global walk_time, slide_time

        walk_time = get_length(walk_sound)
        slide_time = get_length(sliding_sound)

        if player.sounds:   # Воспроизводим звук
            play_sound(jump_sound)

        player.vel.y = -self.velocity_y

        player.gravitation()
        player.collide(0, player.vel.y, platforms)

        player.double_jump = False
        player.state = self.next_state

class WallJumpState:
    def __init__(self, velocity_x, velocity_y, next_state):
        self.velocity_y = velocity_y
        self.velocity_x = velocity_x
        self.next_state = next_state

    def handle_event(self, player, event):
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_LEFT:
                player.last_dir = "left"
                self.next_state = MoveState(-hero_speed)
            elif event.key == pygame.K_RIGHT:
                player.last_dir = "right"
                self.next_state = MoveState(hero_speed)

        if event.type == pygame.KEYUP:
            if event.key == pygame.K_LEFT:
                self.next_state = IdleState()
            elif event.key == pygame.K_RIGHT:
                self.next_state = IdleState()
            elif event.key == pygame.K_a:
                self.next_state = IdleState()
            elif event.key == pygame.K_d:
                self.next_state = IdleState()

    def update(self, player, dt, platforms, camera=None):
        global walk_time, slide_time

        # Игрок не может прыгать с большой скоростью
        if not player.wall_jump_done:
            walk_time = get_length(walk_sound)
            slide_time = get_length(sliding_sound)
            if player.sounds:   # Воспроизводим звук
                play_sound(jump_sound)

            self.velocity_x = self.velocity_x if player.last_dir == "left" else -self.velocity_x

            player.vel.x = self.velocity_x
            player.rect.x += self.velocity_x
            player.collide(player.vel.x, 0, platforms)

            player.vel.y = self.velocity_y
            player.gravitation()
            player.collide(0, player.vel.y, platforms)

            player.sliding_timer = .7
            player.timer_cd = .1

            player.wall_jump_done = True
            player.double_jump = True
            player.dash_done = False
            player.falling = True
            player.touched_wall = False
            player.onWall = False
            player.wall_first_touch = True

            player.state = self.next_state
        else:
            player.state = self.next_state

class MoveState:
    def __init__(self, velocity_x):
        global walk_time

        self.velocity_x = velocity_x
        self.anim_count = 0
        walk_time = get_length(walk_sound)

    def handle_event(self, player, event):
        global walk_time

        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_LSHIFT and player.last_dir:
                return DashState(dash_speed, player.state, player)
            if event.key == pygame.K_LEFT:
                walk_time = get_length(walk_sound)
                player.go_left = True
                player.last_dir = "left"
                self.velocity_x = -hero_speed
            if event.key == pygame.K_RIGHT:
                walk_time = get_length(walk_sound)
                player.go_right = True
                player.last_dir = "right"
                self.velocity_x = hero_speed
            elif event.key == pygame.K_SPACE:
                if player.double_jump and not player.onWall:
                    return DoubleJumpState(jump_force, player.state)
                elif player.isSliding and not player.wall_jump_done:
                    return WallJumpState(wall_jump[0], wall_jump[1], player.state)

                return JumpState(jump_force, player.state)

        if event.type == pygame.KEYUP:
            if event.key == pygame.K_LEFT:
                player.go_left = False
            if event.key == pygame.K_RIGHT:
                player.go_right = False
            if event.key == pygame.K_LEFT and self.velocity_x < 0:
                player.left = False
                return IdleState()
            elif event.key == pygame.K_RIGHT and self.velocity_x > 0:
                player.right = False
                return IdleState()
            if event.key == pygame.K_a and self.velocity_x < 0:
                return IdleState()
            elif event.key == pygame.K_d and self.velocity_x > 0:
                return IdleState()

    def update(self, player, dt, platforms, camera=None):
        global walk_time, slide_time

        # Воспроизводим звуки ходьбы
        if player.onGround and player.sounds:
            if walk_time >= get_length(walk_sound):
                play_sound(walk_sound)
                walk_time = 0
            else:
                walk_time += dt

        # Воспроизводим анимацию движения
        if self.anim_count >= fps:
            self.anim_count = 0

        if player.last_dir == 'right':
            player.image = go_right[self.anim_count // 10]

        if player.last_dir == 'left':
            player.image = go_left[self.anim_count // 10]
        self.anim_count += 1

        # Если персонаж находится на стене и двигается в ее сторону, он начинает скользить
        if player.onWall and not player.onGround:
            if player.wall_pos == player.last_dir:
                player.isSliding = True

                if player.sounds:
                    if slide_time >= get_length(sliding_sound):
                        play_sound(sliding_sound)
                        slide_time = 0
                    else:
                        slide_time += dt
            else:
                stop_sound()
                player.isSliding = True

                if not player.falling:
                    player.timer(dt)

                player.sliding_timer = .7

        # Обновляем координаты персонажа и проверям на столкновения
        player.vel.x = self.velocity_x
        player.rect.x += player.vel.x
        player.collide(player.vel.x, 0, platforms)

        player.gravitation()
        player.collide(0, player.vel.y, platforms)

class IdleState:
    def __init__(self):
        self.timer = down_timer
        self.anim_count = 0
        self.camera_down = False

    def handle_event(self, player, event):
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_LEFT:
                player.last_dir = "left"
                return MoveState(-hero_speed)
            elif event.key == pygame.K_RIGHT:
                player.last_dir = "right"
                return MoveState(hero_speed)
            elif event.key == pygame.K_LSHIFT and player.last_dir:
                return DashState(dash_speed, player.state, player)
            elif event.key == pygame.K_SPACE:
                if player.double_jump and not player.onWall:
                    return DoubleJumpState(jump_force, player.state)
                elif player.isSliding and not player.wall_jump_done:
                    return WallJumpState(wall_jump[0], wall_jump[1], player.state)

                return JumpState(jump_force, player.state)

    def update(self, player, dt, platforms, camera=None):
        # Воспроизводим анимацию покоя
        if self.anim_count >= fps:
            self.anim_count = 0

        if player.last_dir == 'right':
            player.image = idle_right[self.anim_count // 15]
        if player.last_dir == 'left':
            player.image = idle_left[self.anim_count // 15]

        self.anim_count += 1

        # Опускаем камеру при длительном нажатии клавиши
        keys = pygame.key.get_pressed()

        if keys[pygame.K_DOWN]:
            self.timer -= dt

            if self.timer <= 0 and not self.camera_down:
                camera.y_offset *= 0.4
                self.camera_down = True
        else:
            self.timer = down_timer
            self.camera_down = False
            camera.y_offset = height // 2 + h_height

        if player.onGround:
            stop_sound()

        player.gravitation()
        player.collide(0, player.vel.y, platforms)


#!/usr/bin/env python

import numpy as np
import pylab as plt
import sys
from astropy.table import Table

rt = sys.argv[1]
t = Table.read('monte_carlo/monte_carlo_results{}.fits'.format(rt))
n = t['nrealisations'][0]
var = t['var1'][0]

colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd',
          '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']
plt.figure()
plt.title('rt = {}, nreal = {}, variance = {}'.format(rt, n, var))
#rp = np.unique(t['rp'])
rp = [10,20,30,40,50,60,70]
for i, j in enumerate(rp):
    w = j == t['rp']
    kin = t['kappa_in'][w]
    kout = t['kappa_out'][w]
    plt.plot(kin, kout, color=colors[i], label='rp = {}'.format(j))
plt.legend()
plt.xlabel('kappa input')
plt.ylabel('kappa output')
plt.show()
plt.savefig('plots/mc/mc_rt{}.png'.format(rt))


'''
Eigensolver SSMFE.solve() demo.

Usage:
  try_solve [--help | -h | options] <size>

Arguments:
  size  problem size

Options:
  -o <op>, --operator=<op>     operator (laplace | diag) [default: laplace]
  -w <which>, --which=<which>  which eigenvalues [default: right]
  -t <tol> , --vtol=<tol>      eigenvector error tolerance [default: 1e-2]
  -p, --with-preconditioning   use preconditioning
'''

__version__ = '0.1.0'
from docopt import docopt
args = docopt(__doc__, version=__version__)

import math
import numpy
import pylab
import scipy.linalg as la
import sys
import time

from pSSMFE import *

#operators
def apply_lap(n, x):
    y = 2*x
    y[1:n,:] -= x[:n-1,:]
    y[:n-1,:] -= x[1:n,:]
    return y
def apply_diag(n, x):
    y = x.copy()
    for i in range(n):
        y[i,:] = (i + 1)*x[i,:]
    return y

#preconditioners
def apply_jacobi(n, x):
    y = 2*x
    y[1:n,:] += x[:n-1,:]
    y[:n-1,:] += x[1:n,:]
    return y
def apply_inv_diag(n, x):
    y = x.copy()
    for i in range(n):
        y[i,:] = x[i,:]/(i + 1)
    return y

def stop(x, nep):
    return x.shape[0] >= nep
##    return (input('more? ') is not 'y')

def new_stop(solver, nep):
    return solver.lam.shape[0] >= nep

n = int(args['<size>'])
operator = args['--operator']
which = args['--which']
tol = float(args['--vtol'])
with_prec = args['--with-preconditioning']

pc = None
if operator == 'diag':
    op = lambda x: apply_diag(n, x)
    if with_prec:
        pc = lambda x: apply_inv_diag(n, x)
else:
    op = lambda x: apply_lap(n, x)
    if with_prec:
        pc = lambda x: apply_jacobi(n, x)

solver = SSMFE(n)
solver.op = op
solver.pc = pc
solver.which = which
solver.tolerance = tol
solver.verbosity = 2

solver.solve(lambda x: new_stop(x, 10))
##lmd, v = solver.solve(n, op, lambda x: stop(x, 10), pc)
##lmd, v = solver.solve(n, op, stop, pc)
print('lambda:')
print(solver.lam)
if which[0] != 'b':
    print('%d eigenpairs converged in %d iterations in %f sec'\
      % (solver.converged, solver.iteration, solver.total_time))
else:
    print('%d + %d eigenpairs converged in %d iterations in %f sec'\
      % (solver.lcon, solver.rcon, solver.iteration, solver.total_time))

##solver.reset()
##lmd, v = solver.solve(n, op, stop, pc)
##print('lambda:')
##print(lmd)
##if which[0] != 'b':
##    print('%d eigenpairs converged in %d iterations in %f sec'\
##      % (solver.converged, solver.iteration, solver.total_time))
##else:
##    print('%d + %d eigenpairs converged in %d iterations in %f sec'\
##      % (solver.lcon, solver.rcon, solver.iteration, solver.total_time))

ni = solver.converged
v = solver.v
solver = SSMFE(n, ni, v, data_type = 'complex')
solver.op = op
solver.pc = pc
solver.verbosity = 2
solver.solve(lambda x: new_stop(x, 10))
##lmd, v = solver.solve(n, op, lambda x: stop(x, 10), pc)
##lmd, v = solver.solve(n, op, stop, pc)
print('lambda:')
print(solver.lam)
if which[0] != 'b':
    print('%d eigenpairs converged in %d iterations in %f sec'\
      % (solver.converged, solver.iteration, solver.total_time))
else:
    print('%d + %d eigenpairs converged in %d iterations in %f sec'\
      % (solver.lcon, solver.rcon, solver.iteration, solver.total_time))


import re
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from nltk.tokenize import word_tokenize, WhitespaceTokenizer, TweetTokenizer
np.random.seed(seed=234)
from io import StringIO

# The below reads in N lines of text from the 40-million-word news corpus I used (provided by SwiftKey for educational purposes).
N = 10000
with open("ptb.train.txt") as myfile:
    articles = [next(myfile) for x in range(N)]
joined_articles = [" ".join(articles)]

# The below takes out anything that's not a letter, replacing it with a space, as well as any single letter that is not the pronoun "I" or the article "a."
def clean_article(article):
    art1 = re.sub("[^A-Za-z]", ' ', article)
    art2 = re.sub("\s[B-HJ-Zb-hj-z]\s", ' ', art1)
    art3 = re.sub("^[B-HJ-Zb-hj-z]\s", ' ', art2)
    art4 = re.sub("\s[B-HJ-Zb-hj-z]$", ' ', art3)
    return art4.lower()

# The below breaks up the words into n-grams of length 1 to 5 and puts their counts into a Pandas dataframe with the n-grams as column names.  The maximum number of n-grams can be specified if a large corpus is being used.
ngram_bow = CountVectorizer(stop_words = None, preprocessor = clean_article, tokenizer = WhitespaceTokenizer().tokenize, ngram_range=(1,5), max_features = None, max_df = 1.0, min_df = 1, binary = False)
ngram_count_sparse = ngram_bow.fit_transform(joined_articles)
ngram_count = pd.DataFrame(ngram_count_sparse.toarray())
ngram_count.columns = ngram_bow.get_feature_names()

# The below turns the n-gram-count dataframe into a Pandas series with the n-grams as indices for ease of working with the counts.  The second line can be used to limit the n-grams used to those with a count over a cutoff value.
sums = ngram_count.sum(axis = 0)
sums = sums[sums > 0]
ngrams = list(sums.index.values)

# The function below gives the total number of occurrences of 1-grams in order to calculate 1-gram frequencies
def number_of_onegrams(sums):
    onegrams = 0
    for ng in ngrams:
        ng_split = ng.split(" ")
        if len(ng_split) == 1:
            onegrams += sums[ng]
    return onegrams

# The function below makes a series of 1-gram frequencies.  This is the last resort of the back-off algorithm if the n-gram completion does not occur in the corpus with any of the prefix words.
def base_freq(og):
    freqs = pd.Series()
    for ng in ngrams:
        ng_split = ng.split(" ")
        if len(ng_split) == 1:
            freqs[ng] = sums[ng] / og
    return freqs

# For use in later functions so as not to re-calculate multiple times:
bf = base_freq(number_of_onegrams(sums))

# The function below finds any n-grams that are completions of a given prefix phrase with a specified number (could be zero) of words 'chopped' off the beginning.  For each, it calculates the count ratio of the completion to the (chopped) prefix, tabulating them in a series to be returned by the function.  If the number of chops equals the number of words in the prefix (i.e. all prefix words are chopped), the 1-gram base frequencies are returned.
def find_completion_scores(prefix, chops, factor = 0.4):
    cs = pd.Series()
    prefix_split = prefix.split(" ")
    l = len(prefix_split)
    prefix_split_chopped = prefix_split[chops:l]
    new_l = l - chops
    if new_l == 0:
        return factor**chops * bf
    prefix_chopped = ' '.join(prefix_split_chopped)
    for ng in ngrams:
        ng_split = ng.split(" ")
        if (len(ng_split) == new_l + 1) and (ng_split[0:new_l] == prefix_split_chopped):
            cs[ng_split[-1]] = factor**chops * sums[ng] / sums[prefix_chopped]
    return cs

# The below tries different numbers of 'chops' up to the length of the prefix to come up with a (still unordered) combined list of scores for potential completions of the prefix.
def score_given(given, fact = 0.4):
    sg = pd.Series()
    given_split = given.split(" ")
    given_length = len(given_split)
    for i in range(given_length+1):
        fcs = find_completion_scores(given, i, fact)
        for i in fcs.index:
            if i not in sg.index:
                sg[i] = fcs[i]
    return sg

#The below takes the potential completion scores, puts them in descending order and re-normalizes them as a percentage (pseudo-probability).
def score_output(given, fact = 0.4):
    sg = score_given(given, fact)
    ss = sg.sum()
    sg = 100 * sg / ss
    sg.sort_values(axis=0, ascending=False, inplace=True)
    sg = round(sg,1)[0:15]
    return sg.to_json()


"""
This script is used to train and validate recording-level classifiers
"""

import os
import argparse
import yaml
from recording_level_feature_extraction import RecordingLevelFeatureExtraction
from utils import check_balance,train_recording_level_classifier,save_model,plot_feature_histograms

script_dir = os.path.dirname(__file__)
if not script_dir:
    with open(r'./config.yaml') as file:
        conf = yaml.load(file, Loader=yaml.FullLoader)
else:
    with open(script_dir + '/config.yaml') as file:
        conf = yaml.load(file, Loader=yaml.FullLoader)

config = conf['recording_level_classifier']
seed = 500

def recording_level_classifier(inputs_path,model_name):
    '''
    Train recording level classifier using audio and text features
    :param inputs_path: the directory where samples are divided into
    class-folders
    :param model_name: the name of the model that we are gonna train
    :return: None. The classifier will be saved at the directory
    which is declared at the config.yaml file and with the input name.
    '''
    basic_features_params = config
    feature_extractor = RecordingLevelFeatureExtraction(basic_features_params)
    features, labels, class_mapping, feature_list, feature_names, class_names = feature_extractor.transform(inputs_path)

    plot_feature_histograms(feature_list, feature_names, class_names)

    is_imbalanced = check_balance(labels)

    clf = train_recording_level_classifier(features, labels, is_imbalanced, config, seed)
    model_dict = {}
    model_dict['classifier_type'] = config['classifier_type']
    model_dict['class_mapping'] = class_mapping
    model_dict['classifier'] = clf
    model_dict['features_type'] = config['features_type']
    model_dict['reference_text'] = config['reference_text']
    model_dict['text_segmentation_params'] = config['text_segmentation_params']
    model_dict['audio_features'] = config['audio_features']
    model_dict['pyaudio_params'] = config['pyaudio_params']
    out_folder = config['out_folder']
    if model_name is None:
        save_model(model_dict,out_folder, name="basic_classifier")
    else:
        save_model(model_dict,out_folder, out_model=model_name)
    return None

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("-i", "--input",required=True,
                        help="the directory where samples "
                             "are devided into class folders")
    parser.add_argument("-mn","--model_name", required=True,
                        help="the name of the model "
                             "that we are gonna train")
    args = parser.parse_args()
    if os.path.exists(args.input) is False:
        raise FileNotFoundError()
    recording_level_classifier(args.input, args.model_name)


#!/usr/bin/env python
# -*- coding:utf-8 -*-

from django.test import TestCase
from book.models import Classfication, Author, Publisher, Book
from django.core.exceptions import ValidationError

# Create your tests here.

class BookModelTest(TestCase):

#     def test_saving_and_retrieving_books(self):
#         author = Author.objects.create(
#             first_name='Zhiyong', last_name='Xu', email='13980668829@163.com')
#         publisher = Publisher.objects.create(
#             name='Pub', country='China', website='http://www.pub.com')
#         classfication = Classfication.objects.create()
#
#         first_book = Book(classfication=classfication,
#                           publisher=publisher)
#         first_book.title = 'The first (ever) book'
#         first_book.save()
#         first_book.authors = (author,)
#         first_book.save()
#
#         second_book = Book(classfication=classfication,
#                            publisher=publisher)
#         second_book.title = 'The second book'
#         second_book.save()
#         second_book.authors = (author,)
#         second_book.save()
#
#         saved_book = Book.objects.first()
#         self.assertEqual(saved_book, first_book)
#
#         saved_books = Book.objects.all()
#         self.assertEqual(saved_books.count(), 2)
#
#         first_saved_book = saved_books[0]
#         second_saved_book = saved_books[1]
#         self.assertEqual(first_saved_book.title, 'The first (ever) book')
# #        self.assertContains(first_saved_book.authors, author)
#         self.assertEqual(first_saved_book.classfication, classfication)
#         self.assertEqual(first_saved_book.publisher, publisher)
#         self.assertEqual(second_saved_book.title, 'The second book')
#         self.assertEqual(second_saved_book.classfication, classfication)
#         self.assertEqual(second_saved_book.publisher, publisher)

    def test_cannot_save_empty_title_books(self):
        publisher = Publisher.objects.create(
            name='Pub', country='China', website='http://www.pub.com')
        classfication = Classfication.objects.create()

        book = Book(classfication=classfication,
                    publisher=publisher)
        book.title = ''
        with self.assertRaises(ValidationError):
            book.save()
            book.full_clean()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
#############################################################################
Package script, testing who python works

#############################################################################
Sylvain @ GIS / Biopolis / Singapore
Sylvain Jun-Zhe RIONDET <Riondet_Sylvain_from.tp@gis.a-star.edu.sg>
Started on 2019-12-11
Reads Binning Project
#############################################################################
"""

from datetime import datetime
from pathlib import Path

__version__ = "0.8.3"
PLOT_ME_ROOT = Path.home().joinpath("PLoT-ME")
LOGS = PLOT_ME_ROOT.joinpath(f"logs/{datetime.now():%Y-%m-%d_%H-%M}.log")
LOGS.parent.mkdir(parents=True, exist_ok=True)
RECORDS = PLOT_ME_ROOT.joinpath(f"logs/classify_timings.tsv")

from plot_me import parse_DB, classify, tools, bio

if __name__ == '__main__':
    print(f"PLoT-ME version {__version__}. "
          f"Information in the readme file and on https://github.com/sylvain-ri/PLoT-ME. ")


"""This module implements a simple spindown model for an isolated pulsar.
"""
# spindown.py
# Defines Spindown timing model class
import numpy
import astropy.units as u
try:
    from astropy.erfa import DAYSEC as SECS_PER_DAY
except ImportError:
    from astropy._erfa import DAYSEC as SECS_PER_DAY
import parameter as p
from .timing_model import TimingModel, MissingParameter
from ..phase import *
from ..utils import time_from_mjd_string, time_to_longdouble, str2longdouble, taylor_horner,\
                    time_from_longdouble

class Spindown(TimingModel):
    """This class provides a simple timing model for an isolated pulsar."""
    def __init__(self):
        super(Spindown, self).__init__()

        # The number of terms in the taylor exapansion of spin freq (F0...FN)
        #self.num_spin_terms = maxderivs

        self.add_param(p.floatParameter(name="F0", value=0.0, units="Hz",
                       description="Spin-frequency", long_double=True))

        self.add_param(p.prefixParameter(name="F1", value=0.0, units='Hz/s^1',
                       description="Spindown-rate",
                       unitTplt=self.F_unit,
                       descriptionTplt=self.F_description,
                       type_match='float',long_double=True))

        self.add_param(p.MJDParameter(name="TZRMJD",
                       description="Reference epoch for phase = 0.0",
                       time_scale='tdb'))

        self.add_param(p.MJDParameter(name="PEPOCH",
                       description="Reference epoch for spin-down",
                       time_scale='tdb'))

        self.phase_funcs += [self.spindown_phase,]

    def setup(self):
        super(Spindown, self).setup()
        # Check for required params
        for p in ("F0",):
            if getattr(self, p).value is None:
                raise MissingParameter("Spindown", p)

        # Check continuity
        F_terms = self.get_prefix_mapping('F').keys()
        F_terms.sort()
        F_in_order = range(1,max(F_terms)+1)
        if not F_terms == F_in_order:
            diff = list(set(F_in_order) - set(F_terms))
            raise MissingParameter("Spindown", "F%d"%diff[0])
        # If F1 is set, we need PEPOCH
        if self.F1.value != 0.0:
            if self.PEPOCH.value is None:
                raise MissingParameter("Spindown", "PEPOCH",
                        "PEPOCH is required if F1 or higher are set")

        self.num_spin_terms = len(self.get_prefix_mapping('F')) + 1

    def F_description(self, x):
        """Template function for description"""
        if x <1:
            return "Spin-frequency"
        else:
            return "Spin-frequency %d derivative"%x

    def F_unit(self,x):
        """Template function for unit"""
        if x <1:
            return "Hz"
        else:
            return "Hz/s^%d"%x

    def get_spin_terms(self):
        """Return a list of the spin term values in the model: [F0, F1, ..., FN]
        """
        return [getattr(self, "F%d"%ii).value for ii in range(self.num_spin_terms)]

    def spindown_phase(self, toas, delay):
        """Spindown phase function.

        delay is the time delay from the TOA to time of pulse emission
          at the pulsar, in seconds.

        This routine should implement Eq 120 of the Tempo2 Paper II (2006, MNRAS 372, 1549)

        returns an array of phases in long double
        """
        # If TZRMJD is not defined, use the first time as phase reference
        # NOTE, all of this ignores TZRSITE and TZRFRQ for the time being.
        # TODO: TZRMJD should be set by default somewhere in a standard place,
        #       after the TOAs are loaded (RvH -- June 2, 2015)
        # NOTE: Should we be using barycentric arrival times, instead of TDB?
        if self.TZRMJD.value is None:
            self.TZRMJD.value = toas['tdb'][0] - delay[0]*u.s
        # Warning(paulr): This looks wrong.  You need to use the
        # TZRFREQ and TZRSITE to compute a proper TDB reference time.
        if not hasattr(self, "TZRMJDld"):
            self.TZRMJDld = time_to_longdouble(self.TZRMJD.value)

        # Add the [0.0] because that is the constant phase term
        fterms = [0.0] + self.get_spin_terms()

        dt_tzrmjd = (toas['tdbld'] - self.TZRMJDld) * SECS_PER_DAY - delay
        # TODO: what timescale should we use for pepoch calculation? Does this even matter?
        dt_pepoch = (time_to_longdouble(self.PEPOCH.value) - self.TZRMJDld) * SECS_PER_DAY

        phs_tzrmjd = taylor_horner(dt_tzrmjd-dt_pepoch, fterms)
        phs_pepoch = taylor_horner(-dt_pepoch, fterms)
        return phs_tzrmjd - phs_pepoch

    def d_phase_d_F0(self, toas):
        """Calculate the derivative wrt F0"""
        # NOTE: Should we be using barycentric arrival times, instead of TDB?
        # TODO: toas should have units from the table
        tdb = toas['tdbld'].quantity * u.day
        dt_pepoch = time_to_longdouble(self.PEPOCH.value) * u.day
        delay = self.delay(toas) * u.s
        dpdF0 = -(tdb - dt_pepoch) - delay
        return dpdF0.decompose()

    def d_phase_d_F1(self, toas):
        """Calculate the derivative wrt F1"""
        # NOTE: Should we be using barycentric arrival times, instead of TDB?
        # TODO: what timescale should we use for pepoch calculation? Does this even matter?
        tdb = toas['tdbld'] * u.day
        delay = self.delay(toas) * u.s
        dt_pepoch = time_to_longdouble(self.PEPOCH.value) * u.day
        dt = tdb - dt_pepoch - delay
        dpdF1 = -0.5 * dt ** 2
        return dpdF1.decompose()


#!/usr/bin/python3.4
# coding: utf-8
__author__ = 'eirikstavelin'
__version__ = '0.0.1a'

import sys
import os
import networkx as nx
from time import time
from collections import Counter
import configparser
conf = configparser.ConfigParser()
conf.optionxform = str
conf.read('config.ini')
from pymongo import MongoClient

# db
client = MongoClient(conf['database']['path'], int(conf['database']['port']))
db = client[conf['database']['mongo_db']]

def load_lat_lons():
    docs = db.newsNetActors.find({}, {
        "url": 1,
        "is_comp_site": 1,
        "registered_domain": 1,
        "lat": 1,
        "lon": 1
    })
    # sql = """SELECT  url, is_comp_site, registered_domain, lat, lon
    #          FROM newsNetActors"""
    # cur.execute(sql)
    # rows = cur.fetchall()
    d = {}
    for row in docs:
        if row["is_comp_site"] != 'no':
            # advanced case
            d[row["url"]] = (float(row["lat"]), float(row["lon"]))
        else:
            # normal case
            d[row["registered_domain"]] = (float(row["lat"]), float(row["lon"]))  # noqa
    # print("Loaded lat/lons")
    return d

def get_lat_lon_from_dict(url, d):
    if url in d:
        return d[url]
    else:
        return (0.0, 0.0)

def get_visits(base_domain):
    """ return the number of uniqe internal links we have visited to collect
    external links for a give base_domain (bt.no for simple and
    http://nrk.no/hordaland) for composite """

    # this is select distinct count - lots harder with mongo..
    n = db.internal_links.aggregate(
        [
            {"$match": {"link_base_domain": base_domain}},
            {"$group": {"_id": "$link"}}
        ]
    )
    n = len(list(n))
    return int(n)

def db2gexf3(filname):
    d = load_lat_lons()
    # sql = '''SELECT url, is_comp_site , registered_domain, country FROM
    #            newsNetActors
    #          '''  # WHERE url LIKE "%nrk%"
    # cur.execute(sql)
    # rows = cur.fetchall()
    rows = db.newsNetActors.find({}, {
        "url": 1,
        "is_comp_site": 1,
        "registered_domain": 1,
        "country": 1,
        "country_tld": 1,
        "level": 1,
        "name": 1
    })
    G = nx.DiGraph()  # Graph or DiGraph()
    scope_of_study = []  # just to keep track of number of newsoutlets
    # first lets add all newssites
    for r in rows:
        # I also want to know how many urls we crawled
        # in order to get the number of out lainks (added later)
        if r["is_comp_site"].strip() != "no":
            # a composite site - use url
            n = get_visits(r["url"])
            G.add_node(r["url"],
                       latitude=get_lat_lon_from_dict(r["url"], d)[0],   # lat
                       longitude=get_lat_lon_from_dict(r["url"], d)[1],  # lng
                       country=r["country"],
                       tld=r['country_tld'],
                       n_uniqe_internal=n,
                       level=r['level'],
                       name=r['name']
                       )
            scope_of_study.append((r["url"], True))  # is special
        else:
            # a "normal site" - use registered domain
            n = get_visits(r["registered_domain"])
            G.add_node(r["registered_domain"],
                       latitude=get_lat_lon_from_dict(r["registered_domain"], d)[0],   # lat
                       longitude=get_lat_lon_from_dict(r["registered_domain"], d)[1],  # lng
                       country=r["country"],
                       tld=r['country_tld'],
                       n_uniqe_internal=n,
                       level=r['level'],
                       name=r['name']
                       )
            scope_of_study.append((r["registered_domain"], False))  # !special
    print(G)

    print(G.edges(), "\t n edges: ", G.number_of_edges())
    print(G.nodes(), "\t\t n nodes: ", G.number_of_nodes())
    print("n scope_of_study", len(scope_of_study),
          "(set)", len(set(scope_of_study)))
    print("*"*50)
    print()

    # then add relations
    # as this is only done to create gephi graph
    # and the special and normal cases are now mixed
    # we can afford to to this the "expencive way"
    # and query in a loop..
    scope_list = [nn[0] for nn in scope_of_study]

    for t in scope_of_study:
        print("this entity: ", t)  # ('http://www.nrk.no/buskerud/', True)
        # create a counter from this utl (t[0])
        cnt = Counter()

        # this query is slow. Why?
        # sql = """SELECT * FROM external_links WHERE from_base_url = ? """
        # cur.execute(sql, (t[0],))
        # rows = cur.fetchall()
        rows = db.external.find({
            "from_base_url": t[0]
        })
        for ro in rows:
            if ro["to_base_url"] in scope_list:
                cnt[ro["to_base_url"]] += 1
        print()

        # now convert counter to weight in egde
        for k, v in cnt.most_common():
            print(t[0], "-->", k, "n=", v)
            # compute distance in km and add as attr to edge
            # print(v, G.node[t[0]]['n_uniqe_internal'], str(t[0]+" - "+k))
            v_over_internal = 0
            if not v == 0 and not G.node[t[0]]['n_uniqe_internal'] == 0:
                v_over_internal = v/float(G.node[t[0]]['n_uniqe_internal'])
            G.add_edge(t[0], k, weight=v, w_over_internal=v_over_internal,
                       Label=str(t[0]+" - "+k))
    # create folder for grphi files, if not exists
    print()
    folder = "./graph_files/"
    if not os.path.exists(folder):
        os.makedirs(folder)
    nx.write_gexf(G, folder+filname)
    print("Wrote gephi file: ", filname)

if __name__ == '__main__':
    try:
        filename = sys.argv[1]
    except:
        print("createGephiGraph.py filename.gexf")
        sys.exit(0)
    if len(filename) > 6 and filename.endswith(".gexf"):
        start = time()
        db2gexf3(filename)
        print("Done in %s" % (time()-start))
    else:
        print("createGephiGraph.py filename.gexf")


#!/usr/bin/python3
import argparse
import requests
import hashlib
import apikey
import re

def doitbydomain(connection_details,dominio):
    ''' function that give you the auth_info passing domain_name '''

    xml = '''
    <?xml version='1.0' encoding='UTF-8' standalone='no' ?>
    <!DOCTYPE OPS_envelope SYSTEM 'ops.dtd'>
    <OPS_envelope>
    <header>
        <version>0.9</version>
    </header>
    <body>
    <data_block>
        <dt_assoc>
            <item key="protocol">XCP</item>
            <item key="action">GET</item>
            <item key="object">DOMAIN</item>
            <item key="attributes">
                <dt_assoc>
                    <item key="domain">'''+dominio+'''</item>
                    <item key="type">domain_auth_info</item>
                </dt_assoc>
            </item>
        </dt_assoc>
    </data_block>
    </body>
    </OPS_envelope>
    '''

    md5_obj = hashlib.md5()
    md5_obj.update((xml + connection_details['api_key']).encode('utf-8'))
    signature = md5_obj.hexdigest()

    md5_obj = hashlib.md5()
    md5_obj.update((signature + connection_details['api_key']).encode('utf-8'))
    signature = md5_obj.hexdigest()

    headers = {
            'Content-Type':'text/xml',
            'X-Username': connection_details['reseller_username'],
            'X-Signature':signature,
    }

    #debug points
    #print ('***********************************',xml,'***********************************')
    r = requests.post(connection_details['api_host_port'], data = xml, headers=headers )

    if r.status_code == requests.codes.ok:
        print(r.text)
        return (re.search(r"auth_info\">(.*)<",r.text)).group(1)
    else:
        return (r.status_code)
        #_filewr.write(r.status_code)
        #print (r.text)
        #_filewr.write(r.text)

parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-d", metavar='<domain name>', help="domain name to check <e.g. btitalia.it> ", type=str )
group.add_argument("-f", metavar='<file name>', help="text file containing domains name to check one for line <e.g. domain_lists.txt> ")
parser.add_argument("-mode", help="0 or 1 - 0 for real operation and 1 just for test  ", type=int, default=0 )
parser.add_argument("-v", "--verbosity", help="output operation on screen", action = "store_true")
args = parser.parse_args()
#debug points
#print(args)
#print(apikey.api_key)

dominio = args.d
''' get domain name from first parameter in the command string '''
TEST_MODE = args.mode
''' get Test Mode from the second parameter in the command string '''

connection_options = {
        'live' : {
         # IP whitelisting required
             'reseller_username': apikey.username,
             'api_key': apikey.api_key,
             'api_host_port': 'https://rr-n1-tor.opensrs.net:55443',
        },
        'test' : {
             # IP whitelisting not required
             'reseller_username': apikey.username,
             'api_key': apikey.api_key,
             'api_host_port': 'https://horizon.opensrs.net:55443',
            }
}

if TEST_MODE == 1:
    connection_details = connection_options['test']
else:
    connection_details = connection_options['live']

if (args.f != None):
    try:
        with open(args.f) as filein :
            print (filein)
            for dominio in filein:
                dominio=  dominio.rstrip("\n")
                auth_code = doitbydomain(connection_details, dominio)
                print(dominio , "|" , auth_code)
            filein.close()
    except IOError as e:
        print ("Unable to open file , Does not exist or no read permissions")
else:
    auth_code = doitbydomain(connection_details, dominio)
    print(dominio , "|" , auth_code)


from urllib.request import Request, urlopen
import requests, zipfile, io
import pandas as pd
from pvlib import solarposition
from datetime import time, datetime, timedelta, timezone

default_url = "https://energyplus.net/weather-download/north_and_central_america_wmo_region_4/USA/CA/USA_CA_Oakland.Intl.AP.724930_TMY/USA_CA_Oakland.Intl.AP.724930_TMY.epw"

def get_data(url):
    """ Return a list of the data from api call.
    """
    if url[-3:] == "zip" or url[-3:] == "all":
        request = requests.get(url)
        zf = zipfile.ZipFile(io.BytesIO(request.content))
        for i in zf.namelist():
            if i[-3:] == 'epw':
                epw_name = i
        data = zf.read(epw_name)
        data = repr(data).split("\\n")
        return data
    headers = {'User-Agent': 'Mozilla/5.0'}
    req = Request(url, headers = headers)
    epw = urlopen(req).read().decode()
    lines = epw.split("\n")
    return lines

def create_df(default_url):
    """ Extract and clean the data. Return a pandas data from a url.
    """
    lst = get_data(default_url)
    meta = lst[0].strip().split(',')
    city = meta[1]
    country = meta[3]
    latitude = float(meta[-4])
    longitude = float(meta[-3])
    time_zone = float(meta[-2])
    site_elevation = meta[-1]
    location_name = (city + ", " + country)

    lst = lst[8:len(lst) - 1]
    lst = [line.strip().split(',') for line in lst]

    # Each data row exlude index 4 and 5, and everything afterdayssnow
    for line in lst:
        del line[4:6]
        del line[9]
        del line[-1]
        del line[-1]
        del line[-1]

    col_names = ['year', 'month', 'day', 'hour', 'DBT', 'DPT', 'RH',
                'Apressure', 'EHrad', 'HIRrad', 'GHrad',
                'DNrad', 'DifHrad', 'GHillum', 'DNillum', 'DifHillum',
                'Zlumi', 'Wdir', 'Wspeed', 'Tskycover', 'Oskycover',
                'Vis', 'Cheight', 'PWobs', 'PWcodes', 'Pwater',
                'AsolOptD', 'SnowD', 'DaySSnow']

    epw_df = pd.DataFrame(columns = col_names, data = lst)

    # Add fake_year
    epw_df['fake_year'] = 'year'

    # Add in month names
    month_look_up = {'1': 'Jan', '2': 'Feb', '3': 'Mar', '4': 'Apr', '5': 'May',
            '6': 'Jun', '7': 'Jul', '8': 'Aug', '9': 'Sep', '10': 'Oct', '11': 'Nov', '12': 'Dec'}
    epw_df['month_names'] = epw_df['month'].apply(lambda x: month_look_up[x])

    # Add in DOY
    def doy_helper(row):
        """ Helper function for the DOY column.
        """
        month = int(row['month'])
        year = int(row['year'])
        day = int(row['day'])
        period = pd.Period(day = day, month = month, year = year, freq = 'D')
        return period.dayofyear

    epw_df['DOY'] = epw_df.apply(lambda row: doy_helper(row), axis = 1)

    # Change to int type
    change_to_int = ['year', 'day', 'month', 'hour']
    for col in change_to_int:
        epw_df[col] = epw_df[col].astype(int)

    # Change to float type
    change_to_float = ['DBT', 'DPT', 'RH', 'Apressure', 'EHrad', 'HIRrad', 'GHrad',
    'DNrad', 'DifHrad', 'GHillum', 'DNillum', 'DifHillum', 'Zlumi', 'Wdir', 'Wspeed', 'Tskycover',
    'Oskycover', 'Vis', 'Cheight', 'PWobs', 'PWcodes', 'Pwater', 'AsolOptD', 'SnowD', 'DaySSnow']
    for col in change_to_float:
        epw_df[col] = epw_df[col].astype(float)

    # Add in times df
    date = datetime(2000, 6, 21, 12 - 1, 0, 0, 0, tzinfo = timezone.utc)
    tz = timedelta(days = 0, hours = time_zone - 1, minutes = 0)
    date = date - tz
    tz = 'UTC'
    times = pd.date_range(
                '2019-01-01 00:00:00',
                '2020-01-01',
                closed = 'left',
                freq = 'H',
                tz = tz
            )
    delta = timedelta(days = 0, hours = time_zone - 1, minutes = 0)
    times = times - delta
    times_df = pd.DataFrame(times, columns = ['times'])
    epw_df = pd.concat([epw_df, times_df], axis = 1)
    epw_df.set_index("times", drop = False, append = False, inplace = True, verify_integrity = False)

    # Add in solpos df
    solpos = solarposition.get_solarposition(times, latitude, longitude)
    epw_df = pd.concat([epw_df, solpos], axis = 1)

    return epw_df, meta


# -*- coding: utf-8 -*-

#  format of "/proc/net/tcpprobe"
#
#  11.30814 192.168.123.42:35120 192.168.123.99:80 1472 0x259a6f5 0x259a6f5 10 2147483647 19840 36604 150784
#  ^        ^                    ^                ^    ^         ^         ^  ^          ^     ^     ^
#  |        |                    |                |    |         |         |  |          |     |     +-------- [10] RWND
#  |        |                    |                |    |         |         |  |          |     +--------------  [9] sRTT (!) µs
#  |        |                    |                |    |         |         |  |          +--------------------  [8] Send window
#  |        |                    |                |    |         |         |  +-------------------------------  [7] Slow start threshold
#  |        |                    |                |    |         |         +----------------------------------  [6] Congestion window
#  |        |                    |                |    |         +--------------------------------------------  [5] Unacknowledged sequence #
#  |        |                    |                |    +------------------------------------------------------  [4] Next send sequence #
#  |        |                    |                +-----------------------------------------------------------  [3] Bytes in packet
#  |        |                    +----------------------------------------------------------------------------  [2] Receiver address:port
#  |        +-------------------------------------------------------------------------------------------------  [1] Sender address:port
#  +----------------------------------------------------------------------------------------------------------  [0] Time seconds

import sys
import os
import select
import subprocess
from ipaddress import ip_address

from ...utils.utilty import Utility
from .input_base import InputBase
from ...sample import Sample

PATH_TO_TCP_PROBE = "/proc/net/tcpprobe"
TCP_PROBE_FORMAT = ['time', 'src', 'dst', 'packetSize', 'seq', 'unackSeq', 'cwnd', 'sst', 'swnd', 'rtt', 'rwnd']
TCP_PROBE_POLL_TIMEOUT = 0.01

class TcpProbeInput(InputBase):
    def __init__(self, options, infoRegistry):
        self.options = options
        self.infoRegistry = infoRegistry
        self.__linesProccessed = 0
        self.__matchedLinesProccessed = 0
        self.closed = False

    def startupCheck(self):
        readable = os.access(PATH_TO_TCP_PROBE, os.R_OK)
        if(not readable):
            Utility.eprint(\
                "Error: TCP-Probe file \"" + PATH_TO_TCP_PROBE + "\" not readable. Exiting..."\
                "\n\n"\
                "Kernel module loaded? Permissions set? Try:\n"\
                "sudo modprobe tcp_probe full=1 port=0 && sudo chmod 444 /proc/net/tcpprobe"\
                "\n\n"\
                "OR use \"tcpinfo\" as input-backend instead"\
                )
            sys.exit(1)

        # count stdout of lsof and do some dirty casting...
        command = "lsof -t " + PATH_TO_TCP_PROBE + "|wc -l"
        process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=None, shell=True)
        numberOfProcsAsString = bytes(process.communicate()[0]).decode('utf-8')
        numberOfProcs = int(numberOfProcsAsString)
        if(numberOfProcs > 0):
            Utility.eprint("Error: Input file already in use by another process. Exiting...")
            sys.exit(1)

    def startUp(self):
        self.__tcpProbeFileHandler = open(PATH_TO_TCP_PROBE, "r")

    def tearDown(self):
        self.closed = True
        self.__tcpProbeFileHandler.close()

    def retrieveNewSamples(self):
        newSamples = []
        if(self.closed):
            return newSamples

        inputReady,outputReady,exceptReady = select.select([self.__tcpProbeFileHandler.fileno()], [], [], TCP_PROBE_POLL_TIMEOUT)
        if(inputReady):
            for s in inputReady:
                if(self.closed):
                    return newSamples
                elif(s == self.__tcpProbeFileHandler.fileno()):
                    try:
                        line = self.__tcpProbeFileHandler.readline()
                        proccessSample = self.processLogLine(line)
                        newSamples.append(proccessSample)

                    except IOError:
                        pass

                    else:
                        return newSamples
        return newSamples

    def processLogLine(self, line):
        self.__linesProccessed += 1
        self.__matchedLinesProccessed += 1
        log = False
        tmpData = line.strip().split(" ")
        data = dict(zip(TCP_PROBE_FORMAT, tmpData))

        srcIp, separator, srcPort = data['src'].rpartition(':')
        assert separator
        srcPort = int(srcPort)
        srcIp = ip_address(srcIp.strip("[]"))

        dstIp, separator, dstPort = data['dst'].rpartition(':')
        assert separator
        dstPort = int(dstPort)
        dstIp = ip_address(dstIp.strip("[]"))

        if(self.options.filterWhiteListing):
            log = False
            if((srcPort in self.options.filterSrcPorts) or (dstPort in self.options.filterDstPorts)):
                log = True

        else: # blacklisting
            log = True
            if(srcPort in self.options.filterSrcPorts or dstPort in self.options.filterDstPorts):
                log = False

        # line matches - continue processing
        if(log):
            self.__matchedLinesProccessed += 1

            # time since opening of tcp-probe file
            relativeTimestamp = float(data['time'])
            rtt = float(data['rtt']) / 1000 # now rtt is in ms
            seq = int(data['seq'], 0)

            # create ne sample object and return it
            sample = Sample(srcIp, srcPort, dstIp, dstPort)
            sample.relativeTimestamp = relativeTimestamp
            sample.cwnd = int(data['cwnd'])
            sample.rwnd = int(data['rwnd'])
            sample.sst = int(data['sst'])
            sample.seq = seq
            sample.rtt = rtt

            return sample
        else:
            return None