import math
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from utils.kfolds import KFolds, read_csv
from utils.prediction import compute_prediction, compute_error
from utils.feature_scaling import scaling, scaling_by_id
from sklearn.linear_model import LogisticRegression
import argparse

def main(path, delimiter, verbose):
	data_frame, all_headers = read_csv(path, delimiter, verbose)

	# selecting headers of interest
	headers = ['id',
		#'hash_email',
		#'hash_email_conversion',
		#'hash_userid',
		'rank',
		'occurrences',
		'lifetime',
		'nb_days',
		'nb_idtags',
		'nb_idtags_site',
		'nb_idtags_media',
		#'click_rate',
		'nb_purchases',
		'last_time',
		'nb_ips']
	headers_to_drop = list(set(all_headers) - set(headers))
	headers_to_scale = headers[:]
	headers_to_scale.remove('id')
	headers_to_scale.remove('rank')

	# K-Fold cross-validation
	nb_folds = 10
	fold = KFolds(data_frame, nb_folds)
	missranked_scores_train = []
	missranked_scores_test = []

	for k in range(nb_folds):
		train, test = fold.get_fold(k)
		train = train.sort_values(by='id')
		test = test.sort_values(by='id')

		# dropping not usefull columns
		for drop in headers_to_drop:
		    train = train.drop(drop, 1)
		    test = test.drop(drop, 1)

		# train set
		train, mean, std = scaling(train, headers_to_scale)
		train = train.reset_index(drop=True)
		X = train[headers_to_scale].values
		Y = train['rank']==1

		# training Logistic regression model
		regularization = 1e10
		logreg = LogisticRegression(C=regularization)
		logreg.fit(X, Y)

		# computing score on train set
		Y_score_train = logreg.predict_proba(X)[:,1]
		Y_predicted_train = compute_prediction(train, Y_score_train, verbose)
		missranked_train, wellranked_train, total_train = compute_error(Y, Y_predicted_train)
		missranked_scores_train.append(missranked_train/total_train)

		# test set
		test = scaling(test, headers_to_scale, mean, std)
		test = test.reset_index(drop=True)
		X_test = test[headers_to_scale].values
		Y_test = test['rank'].values==1

		# computing score on test set
		Y_score_test = logreg.predict_proba(X_test)[:,1]
		Y_predicted_test = compute_prediction(test, Y_score_test, verbose)
		missranked_test, wellranked_test, total_test = compute_error(Y_test, Y_predicted_test)
		missranked_scores_test.append(missranked_test/total_test)

		# printing intermediate results
		if verbose:
			print('\n**** fold ', k, '****')
			print('train set:')
			print('   missranked =', round(missranked_train/total_train, 3))
			print('   wellranked =', round(wellranked_train/total_train, 3))
			print('test set:')
			print('   missranked =', round(missranked_test/total_test, 3))
			print('   wellranked =', round(wellranked_test/total_test, 3))

	# printing final result
	if verbose:
		print('\n******** MEAN over all folds ********')
		print('Train missranked = ', np.mean(missranked_scores_train))
		print(' Test missranked = ', np.mean(missranked_scores_test))
		print('  Train accuracy = ', 1 - np.mean(missranked_scores_train))
		print('   Test accuracy = ', 1 - np.mean(missranked_scores_test))

if __name__ == '__main__':
    # Parse the different arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('--dataset', help='path to the dataset', required=True)
    parser.add_argument('--delimiter', help='delimiter used in the dataset', required=True)
    parser.add_argument('--verbose', help='increase output verbosity', action='store_true', required=False)

    # Recover the arguments
    opts = parser.parse_args()

    # Execute the main function
    main(opts.dataset, opts.delimiter, opts.verbose)


"""
Goldair GECO WiFi Heater device.
"""
try:
    from homeassistant.components.climate import ClimateEntity
except ImportError:
    from homeassistant.components.climate import ClimateDevice as ClimateEntity

from homeassistant.components.climate.const import (
    ATTR_HVAC_MODE,
    HVAC_MODE_HEAT,
    SUPPORT_TARGET_TEMPERATURE,
)
from homeassistant.const import ATTR_TEMPERATURE, STATE_UNAVAILABLE

from ..device import GoldairTuyaDevice
from .const import (
    ATTR_ERROR,
    ATTR_TARGET_TEMPERATURE,
    HVAC_MODE_TO_DPS_MODE,
    PROPERTY_TO_DPS_ID,
)

SUPPORT_FLAGS = SUPPORT_TARGET_TEMPERATURE

class GoldairGECOHeater(ClimateEntity):
    """Representation of a Goldair GECO WiFi heater."""

    def __init__(self, device):
        """Initialize the heater.
        Args:
            device (GoldairTuyaDevice): The device API instance."""
        self._device = device

        self._support_flags = SUPPORT_FLAGS

        self._TEMPERATURE_STEP = 1
        self._TEMPERATURE_LIMITS = {"min": 15, "max": 35}

    @property
    def supported_features(self):
        """Return the list of supported features."""
        return self._support_flags

    @property
    def should_poll(self):
        """Return the polling state."""
        return True

    @property
    def name(self):
        """Return the name of the climate device."""
        return self._device.name

    @property
    def unique_id(self):
        """Return the unique id for this heater."""
        return self._device.unique_id

    @property
    def device_info(self):
        """Return device information about this heater."""
        return self._device.device_info

    @property
    def icon(self):
        """Return the icon to use in the frontend for this device."""
        hvac_mode = self.hvac_mode

        if hvac_mode == HVAC_MODE_HEAT:
            return "mdi:radiator"
        else:
            return "mdi:radiator-disabled"

    @property
    def temperature_unit(self):
        """Return the unit of measurement."""
        return self._device.temperature_unit

    @property
    def target_temperature(self):
        """Return the temperature we try to reach."""
        return self._device.get_property(PROPERTY_TO_DPS_ID[ATTR_TARGET_TEMPERATURE])

    @property
    def target_temperature_step(self):
        """Return the supported step of target temperature."""
        return self._TEMPERATURE_STEP

    @property
    def min_temp(self):
        """Return the minimum temperature."""
        return self._TEMPERATURE_LIMITS["min"]

    @property
    def max_temp(self):
        """Return the maximum temperature."""
        return self._TEMPERATURE_LIMITS["max"]

    async def async_set_temperature(self, **kwargs):
        """Set new target temperatures."""
        if kwargs.get(ATTR_TEMPERATURE) is not None:
            await self.async_set_target_temperature(kwargs.get(ATTR_TEMPERATURE))

    async def async_set_target_temperature(self, target_temperature):
        target_temperature = int(round(target_temperature))

        limits = self._TEMPERATURE_LIMITS
        if not limits["min"] <= target_temperature <= limits["max"]:
            raise ValueError(
                f"Target temperature ({target_temperature}) must be between "
                f'{limits["min"]} and {limits["max"]}.'
            )

        await self._device.async_set_property(
            PROPERTY_TO_DPS_ID[ATTR_TARGET_TEMPERATURE], target_temperature
        )

    @property
    def current_temperature(self):
        """Return the current temperature."""
        return self._device.get_property(PROPERTY_TO_DPS_ID[ATTR_TEMPERATURE])

    @property
    def hvac_mode(self):
        """Return current HVAC mode, ie Heat or Off."""
        dps_mode = self._device.get_property(PROPERTY_TO_DPS_ID[ATTR_HVAC_MODE])

        if dps_mode is not None:
            return GoldairTuyaDevice.get_key_for_value(HVAC_MODE_TO_DPS_MODE, dps_mode)
        else:
            return STATE_UNAVAILABLE

    @property
    def hvac_modes(self):
        """Return the list of available HVAC modes."""
        return list(HVAC_MODE_TO_DPS_MODE.keys())

    async def async_set_hvac_mode(self, hvac_mode):
        """Set new HVAC mode."""
        dps_mode = HVAC_MODE_TO_DPS_MODE[hvac_mode]
        await self._device.async_set_property(
            PROPERTY_TO_DPS_ID[ATTR_HVAC_MODE], dps_mode
        )

    @property
    def device_state_attributes(self):
        """Get additional attributes that HA doesn't naturally support."""
        error = self._device.get_property(PROPERTY_TO_DPS_ID[ATTR_ERROR])

        return {ATTR_ERROR: error or None}

    async def async_update(self):
        await self._device.async_refresh()


import argparse
import tensorflow as tf
import os
import sys
import time
import yaml
import np_methods
from tensorflow.keras.optimizers.schedules import PiecewiseConstantDecay
from voc_data import VOCDataset
from datasets.COCOdataset import COCOdataset
from anchor import generate_default_boxes, get_anchors, get_default_params
from network_new import create_ssd
from losses import create_losses, ssd_losses
import tensorflow_model_optimization as tfmot
quantize_model = tfmot.quantization.keras.quantize_model
import numpy as np
from box_utils import decode, compute_nms
import cv2
import time
#https://github.com/VisDrone/VisDrone-Dataset
parser = argparse.ArgumentParser()
parser.add_argument('--data-dir', default='/media/essys/bd21e577-e5db-47a3-8845-f27be3f083a4/dataset/voc/VOCdevkit/')
parser.add_argument('--data-year', default='2012')
parser.add_argument('--arch', default='ssd300')
parser.add_argument('--batch-size', default=16, type=int)
parser.add_argument('--num-batches', default=100, type=int)
parser.add_argument('--neg-ratio', default=3, type=int)
parser.add_argument('--initial-lr', default=7e-4, type=float)
parser.add_argument('--momentum', default=0.9, type=float)
parser.add_argument('--weight-decay', default=1e-5, type=float)
parser.add_argument('--num-epochs', default=120, type=int)
parser.add_argument('--checkpoint-dir', default='checkpoints')
parser.add_argument('--pretrained-type', default='base')
parser.add_argument('--gpu-id', default='2,3')
parser.add_argument('--freeze', default=1, type=int)
parser.add_argument('--quant', default=0, type=int)

args = parser.parse_args()
print(tf.executing_eagerly())
print("running gpus :" , args.gpu_id)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id

NUM_CLASSES = 2
def post_process(confs, locs,scores, default_boxes, mode=1):
    # confs = tf.squeeze(confs, 0)
    # locs = tf.squeeze(locs, 0)
    # i have o return the locs back
    # print(scores)
    # print(confs)
    # print(locs)
    print(tf.math.reduce_max(locs), tf.math.reduce_min(locs))
    newres = decode(default_boxes, locs).numpy()
    if(mode == 2):
        confs = tf.math.softmax(confs, axis=-1)
        classes = tf.math.argmax(confs, axis=-1)
        scores = tf.math.reduce_max(confs, axis=-1)

    out_boxes = []
    out_labels = []
    out_scores = []
    # print(confs.shape,classes.shape, scores.shape, boxes.shape)
    for c in range(1, NUM_CLASSES):
        if(mode == 1):
            cls_scores = np.zeros(np.shape(confs))
            cls_scores[confs  == c]  = confs[confs  == c]
        else:
            cls_scores = confs[:, c]

        score_idx = cls_scores > 0.5
        # cls_boxes = tf.boolean_mask(boxes, score_idx)
        # cls_scores = tf.boolean_mask(cls_scores, score_idx)
        cls_boxes = newres[score_idx]
        cls_scores = cls_scores[score_idx]

        nms_idx = compute_nms(cls_boxes, cls_scores, 0.35, 200)
        cls_boxes = tf.gather(cls_boxes, nms_idx)
        cls_scores = tf.gather(cls_scores, nms_idx)
        cls_labels = [c] * cls_boxes.shape[0]

        out_boxes.append(cls_boxes)
        out_labels.extend(cls_labels)
        out_scores.append(cls_scores)

    out_boxes = tf.concat(out_boxes, axis=0)
    out_scores = tf.concat(out_scores, axis=0)

    boxes = tf.clip_by_value(out_boxes, 0.0, 1.0).numpy()
    classes = np.array(out_labels)
    scores = out_scores.numpy()

    return boxes, classes, scores
def draw_image(img, bboxes, classes, mode = 'gt'):
    h,w,c = np.shape(img)
    for i in range(len(bboxes)):
            bbox = bboxes[i]
            x1,y1, x2,y2 = bbox
            # x1,y1, x2,y2 = x1, y1-y2/2,x1 + x2/2,y1 +  y2/2
            x1,y1,x2,y2 = int(x1 * w), int(y1 * h), int(x2 * w), int(y2 * h)
            print(x1, x2, y1, y2, mode)
            cv2.rectangle(img, (x1, y1), (x2, y2), (255,0,0), 2)
            cv2.rectangle(img, (x1, y1), (x1+10, y1+15), (0,0,0), -1)
            if(type(classes[i]) in [np.int64, np.int32, np.int8, int]):
                cv2.putText(img,str(classes[i]), (x1,y1+10),1, 1.0, (255,255,255))
            else:
                cv2.putText(img,str(classes[i].numpy()), (x1,y1+10),1, 1.0, (255,255,255))

@tf.function
def train_step(imgs, gt_confs, gt_locs,gt_scores, ssd, criterion, optimizer):
    with tf.GradientTape() as tape:
        # start = time.time()
        out = ssd(imgs)
        # print(time.time() - start, " s spent for model out")
        confs_, locs_ = out[:6], out[6:]

        confs  = []
        locs = []
        for i in range(len(confs_)):
            confs.append(tf.reshape(confs_[i], [confs_[i].shape[0], -1, NUM_CLASSES]))
            locs.append(tf.reshape(locs_[i], [locs_[i].shape[0], -1, 4]))
        confs = tf.concat(confs, axis=1)
        locs = tf.concat(locs, axis=1)
        # print(confs.shape, locs.shape)
        # start = time.time()
        conf_loss, loc_loss = criterion(
            confs, locs, gt_confs, gt_locs, gt_scores)
        # print(time.time() - start, " s spent for loss calculation")
        loss = conf_loss + loc_loss
        l2_loss = [tf.nn.l2_loss(t) for t in ssd.trainable_variables]
        l2_loss = args.weight_decay * tf.math.reduce_sum(l2_loss)
        loss += l2_loss

    gradients = tape.gradient(loss, ssd.trainable_variables)
    optimizer.apply_gradients(zip(gradients, ssd.trainable_variables))

    return loss, conf_loss, loc_loss, confs, locs
# @tf.function
# def distributed_train_step(imgs, gt_confs, gt_locs, q_aware_model, criterion, optimize, strategy):
#   loss, conf_loss, loc_loss, l2_loss,confs_pred,locs_pred = strategy.run(train_step, args=(imgs, gt_confs, gt_locs, q_aware_model, criterion, optimize))
#   print(conf_loss.values[0].shape, '-------------')
#   return strategy.reduce(tf.distribute.ReduceOp.SUM, loss, axis=None), conf_loss.values[0], loc_loss.values[0], l2_loss,confs_pred.values[0],locs_pred.values[0]
# @tf.function
# def distributed_test_step(imgs, gt_confs, gt_locs, q_aware_model, criterion, optimize, strategy):
#   return strategy.run(train_step, args=(imgs, gt_confs, gt_locs, q_aware_model, criterion, optimize))

if __name__ == '__main__':
    os.makedirs(args.checkpoint_dir, exist_ok=True)

    with open('./config.yml') as f:
        cfg = yaml.load(f)

    try:
        config = cfg['SSD300']#[args.arch.upper()]
    except AttributeError:
        raise ValueError('Unknown architecture: {}'.format(args.arch))
    default_boxes = generate_default_boxes(config)
    # config = get_default_params()
    # config._replace(num_classes=NUM_CLASSES)
    # default_boxes =  get_anchors(config)
    # batch_generator, val_generator, info
    # voc = VOCDataset(
    #     args.data_dir, args.data_year, default_boxes,
    #     config['image_size'],
    #     batch_size=args.batch_size,
    #     augmentation=['flip'])  # the patching algorithm is currently causing bottleneck sometimes
    # batch_generator = voc.generate(subset='train',num=2000)
    # val_generator = voc.generate(subset='val',num=2)

    # mirrored_strategy = tf.distribute.MirroredStrategy()
    # mirrored_strategy = tf.distribute.MirroredStrategy(devices=["/gpu:0", "/gpu:1", "/gpu:2"])
    # GLOBAL_BATCH_SIZE = args.batch_size * 3
    voc = COCOdataset('/media/essys/bd21e577-e5db-47a3-8845-f27be3f083a4/dataset/coco',default_boxes,batch_size=args.batch_size,datatype='train2017',
                      input_shape=[config['image_size'], config['image_size']], augmentation=['flip'])
    voc_val = COCOdataset('/media/essys/bd21e577-e5db-47a3-8845-f27be3f083a4/dataset/coco',default_boxes,datatype='val2017',
                      input_shape=[config['image_size'], config['image_size']], augmentation=['none'])
    # batch_generator= voc.generate(20000)
    # val_generator= voc_val.generate(20000)

    try:
        ssd = create_ssd(NUM_CLASSES, args.arch,args.batch_size,
                        args.pretrained_type,
                        checkpoint_dir=args.checkpoint_dir,scope=None, freeze=args.freeze, quant=False)
    except Exception as e:
        print(e)
        print('The program is exiting...')
        sys.exit()

    print('quantizing enabled : ', args.quant)
    print('freezing enabled : ', args.freeze)
    if(args.quant > 0):
        q_aware_model = quantize_model(ssd)
    else:
        q_aware_model = ssd
    # criterion = create_losses(args.neg_ratio, NUM_CLASSES)
    criterion = ssd_losses

    steps_per_epoch = len(voc) // args.batch_size
    lr_fn = PiecewiseConstantDecay(
        boundaries=[int(steps_per_epoch * args.num_epochs * 2 / 3),
                    int(steps_per_epoch * args.num_epochs * 5 / 6)],
        values=[args.initial_lr, args.initial_lr * 0.1, args.initial_lr * 0.01])

    optimizer = tf.keras.optimizers.SGD(
        learning_rate=lr_fn,
        momentum=args.momentum)

    train_log_dir = 'logs/train'
    val_log_dir = 'logs/val'
    train_summary_writer = tf.summary.create_file_writer(train_log_dir)
    val_summary_writer = tf.summary.create_file_writer(val_log_dir)
    global_step = 0
    for epoch in range(args.num_epochs):
        avg_loss = 0.0
        avg_conf_loss = 0.0
        avg_loc_loss = 0.0
        start = time.time()
        batch_generator= voc.generate(2000)
        val_generator= voc_val.generate(2)
        for i, (imgs, gt_confs, gt_locs,gt_scores, o_imgs, o_boxes, o_labels) in enumerate(batch_generator):
            global_step = global_step + 1
            # print(imgs.shape, gt_confs.shape, gt_locs.shape)
            loss, conf_loss, loc_loss,confs_pred,locs_pred = train_step(
                imgs, gt_confs, gt_locs,gt_scores, q_aware_model, criterion, optimizer)
            avg_loss = (avg_loss * i + loss.numpy()) / (i + 1)
            avg_conf_loss = (avg_conf_loss * i + conf_loss.numpy()) / (i + 1)
            avg_loc_loss = (avg_loc_loss * i + loc_loss.numpy()) / (i + 1)
            if (i + 1) % 20 == 0:
                print('Epoch: {} Batch {} Time: {:.2}s | Loss: {:.4f} Conf: {:.4f} Loc: {:.4f}'.format(
                    epoch + 1, i + 1, time.time() - start, avg_loss, avg_conf_loss, avg_loc_loss))
                with train_summary_writer.as_default():
                    tf.summary.scalar('loss', avg_loss, step=global_step)
                    tf.summary.scalar('conf_loss', avg_conf_loss, step=global_step)
                    tf.summary.scalar('loc_loss', avg_loc_loss, step=global_step)

                    # here we draw bounding boxes and print summary
                    gboxes, gclasses, gscores = post_process(gt_confs.numpy()[0], gt_locs.numpy()[0], gt_scores.numpy()[0], default_boxes)
                    boxes, classes, scores = post_process(confs_pred.numpy()[0], locs_pred.numpy()[0],None, default_boxes, mode=2)
                    img_gt = cv2.cvtColor(np.uint8(o_imgs[0]).copy(), cv2.COLOR_RGB2BGR)
                    img_pred = cv2.cvtColor(np.uint8(o_imgs[0]).copy(), cv2.COLOR_RGB2BGR)
                    draw_image(img_gt, gboxes, gclasses)
                    draw_image(img_pred, boxes ,classes, mode='pred')
                    tf.summary.image("train/gt", np.expand_dims(img_gt, 0),step=global_step)
                    tf.summary.image("train/pred", np.expand_dims(img_pred, 0),step=global_step)

            if((i+1) % 500 ==0):
                ssd.save_weights(os.path.join(args.checkpoint_dir, 'ssd_epoch_{}_step_{}.h5'.format(epoch + 1, i+1)))
        avg_val_loss = 0.0
        avg_val_conf_loss = 0.0
        avg_val_loc_loss = 0.0
        for i, (imgs, gt_confs, gt_locs,gt_scores, o_imgs, o_boxes, o_labels) in enumerate(val_generator):
            out = ssd(imgs)
            val_confs_, val_locs_ = out[:6], out[6:]

            val_confs  = []
            val_locs = []
            for j in range(len(val_confs_)):
                val_confs.append(tf.reshape(val_confs_[j], [val_confs_[j].shape[0], -1, NUM_CLASSES]))
                val_locs.append(tf.reshape(val_locs_[j], [val_locs_[j].shape[0], -1, 4]))
            val_confs = tf.concat(val_confs, axis=1)
            val_locs = tf.concat(val_locs, axis=1)
            val_conf_loss, val_loc_loss = criterion(
                val_confs, val_locs, gt_confs, gt_locs,gt_scores)
            val_loss = val_conf_loss + val_loc_loss
            avg_val_loss = (avg_val_loss * i + val_loss.numpy()) / (i + 1)
            avg_val_conf_loss = (avg_val_conf_loss * i + val_conf_loss.numpy()) / (i + 1)
            avg_val_loc_loss = (avg_val_loc_loss * i + val_loc_loss.numpy()) / (i + 1)

        with val_summary_writer.as_default():
            tf.summary.scalar('loss', avg_val_loss, step=global_step)
            tf.summary.scalar('conf_loss', avg_val_conf_loss, step=global_step)
            tf.summary.scalar('loc_loss', avg_val_loc_loss, step=global_step)
            gboxes, gclasses, gscores = post_process(gt_confs.numpy()[0], gt_locs.numpy()[0], gt_scores.numpy()[0], default_boxes)
            boxes, classes, scores = post_process(val_confs.numpy()[0], val_locs.numpy()[0],None, default_boxes, mode=2)
            img_gt = cv2.cvtColor(np.uint8(o_imgs[0]), cv2.COLOR_RGB2BGR)
            img_pred = cv2.cvtColor(np.uint8(o_imgs[0]), cv2.COLOR_RGB2BGR)
            draw_image(img_gt, gboxes, gclasses)
            draw_image(img_pred, boxes ,classes, mode='pred')
            tf.summary.image("val/gt", np.expand_dims(img_gt, 0),step=global_step)
            tf.summary.image("val/pred", np.expand_dims(img_pred, 0),step=global_step)

        ssd.save_weights(
            os.path.join(args.checkpoint_dir, 'ssd_epoch_{}.h5'.format(epoch + 1)))


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as seabornInstance
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsRegressor
from sklearn import metrics

def readCSV():
    dataSet = pd.read_csv("LinearMultivariateInput.csv")
    return dataSet

def LinearModel(historicWindow, predictionWindow, dataSet):

    returnList = []

    print(dataSet.describe())

    X = dataSet[['year', 'month', 'day', 'hour', 'minute', 'PAH', historicWindow]].values
    y = dataSet[predictionWindow].values

    selFeat = ['year', 'month', 'day', 'hour', 'minute', 'PAH', historicWindow]

    # plt.figure(figsize=(15, 10))
    # plt.tight_layout()
    # seabornInstance.distplot(dataSet['PAH'])
    # plt.show()

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

    regressor = KNeighborsRegressor(n_neighbors=7)
    regressor.fit(X_train, y_train)

    y_pred = regressor.predict(X_test)

    df = pd.DataFrame({'Actual': y_test, 'Predicted': y_pred})
    df1 = df.head(25)
    print(df1)

    df1.plot(kind='bar', figsize=(10, 8))
    plt.grid(which='major', linestyle='-', linewidth='0.5', color='green')
    plt.grid(which='minor', linestyle=':', linewidth='0.5', color='black')
    #plt.show()

    # print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))
    # print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))
    # print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))
    #returnList.append(movingAverage)
    returnList.append(historicWindow)
    returnList.append(predictionWindow)
    returnList.append(metrics.mean_absolute_error(y_test, y_pred))
    returnList.append(metrics.mean_squared_error(y_test, y_pred))
    returnList.append(np.sqrt(metrics.mean_squared_error(y_test, y_pred)))
    return returnList

def main():
    dataSet = readCSV()
    df = pd.DataFrame(columns=['HistoricWindow', 'PredictionWindow', 'MAE', 'MSE', 'RMSE'])
    HistoricList = ['PAH-24', 'PAH-12', 'PAH-6', 'PAH-3', 'PAH-1']
    PredictionList = ['quarter', 'half', 'PAH+1', 'PAH+2', 'PAH+3', 'PAH+4', 'PAH+6', 'PAH+12', 'PAH+24']
    movingAverage = ['MPAH-6', 'MPAH-3']

    for i in HistoricList:
        for j in PredictionList:
            output = LinearModel(i, j, dataSet)
            print(output)
            df.loc[len(df)] = output

    df.to_csv("LinearImputationkNNOutput.csv", index=False, header=['HistoricWindow', 'PredictionWindow', 'MAE', 'MSE', 'RMSE'])
    print("done")

if __name__ == "__main__":
    main()


from obspy import read

from seisnn.data.io import write_pkl
from seisnn.utils import get_dir_list
from seisnn.plot import plot_dataset

pick_pkl = "/mnt/tf_data/catalog/new_pick.dataset"
predict_pkl_dir = "/mnt/tf_data/dataset/2018_02_18_predict"
predict_pkl_list = get_dir_list(predict_pkl_dir)

new_pick = []
try:
    for i, pkl in enumerate(predict_pkl_list):
        trace = read(pkl).traces[0]
        start_time = trace.stats.starttime
        for p in trace.picks:
            if p.evaluation_mode == "automatic":
                if not p.evaluation_status == "confirmed" and not p.evaluation_status == "rejected":
                    pick_time = p.time - start_time
                    plot_dataset(trace, xlim=(pick_time - 2, pick_time + 4))
                    key_in = input("Confirmed pick? [y/N]")
                    if key_in == "y" or key_in == "Y":
                        print("Current file... %d out of %d" % (i, len(predict_pkl_list)))
                        p.evaluation_mode = "manual"
                        p.evaluation_status = "reviewed"
                        new_pick.append(p)

        if i % 100 == 0:
            print("Current file... %d out of %d" % (i, len(predict_pkl_list)))

except KeyboardInterrupt:
    print("Keyboard interrupt.")
finally:
    write_pkl(new_pick, pick_pkl)
    print("Pick saved.")


from django.shortcuts import render
from django.contrib.auth import authenticate, login

def main_page(request):

    '''
    The site is first loaded, or if JS is disabled and full page loads are required.
    '''

    if not "focused_content" in context.keys():
        context["focused_content"] = "primary"

    # Render primary content.
    if not "primary_content" in context.keys():
        if request.user.is_authenticated:
            context["primary_content"] = "find_friends/find_friends.html"
            context["find_friend_form"] = FindFriendForm()
        else:
            context["primary_content"] = "accounts/user_login.html"
            context["login_form"] = LoginForm()

    # Render secondary content.
    if not "secondary_content" in context.keys():
        try:
            context["secondary_content"] = request.session["secondary_content"]
        except:
            context["secondary_content"] = "content/about.html"
    if not "secondary_content_title" in context.keys():
        try:
            context["secondary_content_title"] = request.session["secondary_content_title"]
        except:
            context["secondary_content_title"] = "About SJMS"

    return render(request, "main_page/main_page.html", context)


#!/usr/bin/python

"""файл для асинхронного моделирования с имзмененными исходными данными(много цветов)"""
"""Другой подсчет штрафа в цикле"""
from random import choice
import random
import json
import asyncio
from time import time, sleep

#Метод считывания графа из файла .json
with open("data(color)4.json", 'r', encoding='utf-8') as fh: #открываем файл на чтение
    graph = json.load(fh) #загружаем из файла данные в словарь graph
print("GRAPH=", graph)

resulting_var = 0
class Agent():
    """Класс для создания агента(каждого узла модели)"""
    def __init__(self, name): #name = цифры
        """Конструктор - изначальные аттрибуты агента при создании:"""
        self.name = name
        self.neighbors = []  # Список соседей агента
        self.val = graph["vars"][self.name]["initial"]   # Определение изначального цвета агента(задан в исходных данных)
        self.domen = graph["vars"][self.name]["domain"]  # Домен значений
        self.current_cycle = {}  # Состояние текущего цикла агента
        self.next_cycle = {}    # Состояние следующего цикла агента
        self.current_cost  = 0   # Текущая стоимость(сумма штрафов) агента
        self.res = 0   #Результат работы всех циклов агента
        print(f"Агент {self.name} - создан")
        print(f"Значение агента ({self.name}) = ", self.val)

    def neighbor_search(self):
        """Определение списка соседей для текущего агента по исходным данным:"""
        for name in graph["pairs"]:
            if self.name == name[0]:
                self.neighbors.append(name[1])
            elif self.name == name[1]:
                self.neighbors.append(name[0])
        self.neighbors = list(set(self.neighbors))  # Оставляем только уникальные значения в списке
        print(f"Соседи агента {self.name} = ", self.neighbors)
        print()

    async def send(self, agent):
        """Запись значения текущего агента переданному соседу(запись в его состояния циклов)"""
        if self.name in dict1[agent].neighbors: # Выполняем функцию только если соседи:
            print("------------")
            print(f"Агент {self.name} - сосед агента {agent}")
            print("Выполняем метод send")
            if self.name not in dict1[agent].current_cycle: #Есть ли значение текущего агента в текущем цикле соседа(i)
                print(f"Текущий Агент({self.name}) еще не отсылал свое значение агенту({dict1[agent].name})")
                dict1[agent].current_cycle[self.name] = self.val #Запись в словарь с ключом - имя текущего агента
                print(f"Текущий цикл агента({dict1[agent].name}) равен", dict1[agent].current_cycle) #Состояние текущего цикла соседа(i)
                if len(dict1[agent].neighbors) == len(dict1[agent].current_cycle): #Проверка на законченность цикла соседа
                    print(f"Цикл агента ({dict1[agent].name}) - завершен!")
                    # Расчет лучшего значения для переменной из цикла:
                    await dict1[agent].evaluate_cycle(agent)

            else:   #Если текущий агент уже есть в текущем цикле соседа:
                print(f"Текущий Агент{self.name} уже отправлял значение агенту ({dict1[agent].name})")
                dict1[agent].next_cycle[self.name] = self.val #Запись значения текущего агента в следующий цикл соседа
                print(f"Поэтому записано в следующий цикла агента({dict1[agent].name}) = ", dict1[agent].next_cycle)
            print("------------")
            print()

    async def func(self, val1, val2, name_neighbor):
        """Метод расчета стоимости между двумя значениями переменных агентов"""
        #global res1
        print("--------------")
        print(f"Выполняем метод func между ({self.name}) и ({name_neighbor})")
        res2 = 0
        for function in graph["functions"]:
            if (val1 == function[0] and val2 == function[1]) or (val1 == function[1] and val2 == function[2]):
                res2 = function[2]
        print(f"Значения текущего агента ({self.name})=({val1}), а значение агента ({name_neighbor})=({val2})")
        print(f"Поэтому штраф равен ", res2)
        print("--------------")
        print()
        return res2

    async def evaluate_cycle(self, name):
        """Метод выбора наилучшего значения для переменной агента исходя из минимальной стоимости(выполняется в конце цикла)"""
        #dict1[name].current_cycle[dict1[name].name] = dict1[name].val
        print(f"Текущий цикл агента ({dict1[name].name}) = ", dict1[name].current_cycle)
        print("----------------")
        print(f"Выполняем метод расчета стоимости агента {name}")

        cost_otnosit = {}
        cost_current = 0
        for key, value in dict1[name].current_cycle.items():
            # Расчет стоимости агентом при текущем значении, суммируя стоимости со всеми соседями:
            print("Расчет стоимости при текущем значении")
            cost_current = cost_current + await dict1[name].func(self.val, value, key)
        cost_otnosit[dict1[name].val] = cost_current
        print(f"Стоимость агента() при текущем значении ({dict1[name].val}) равна", cost_otnosit)

        # Создание словаря со стоимостью относительно состояния соседей, подставляя все значения из домена и суммируя их
        for args in dict1[name].domen:
            res1 = 0
            for key, value in dict1[name].current_cycle.items():
                res1 = res1 + await dict1[name].func(args, value, key)
                cost_otnosit[args] = res1
            print("cost_otnosit=", cost_otnosit)

        # Выбор значения для переменной агента исходя из стоимостей
        val_min = 0
        for key, value in cost_otnosit.items():
            if value == min(cost_otnosit.values()):
                print(f"Минимальная стоимость агентом({dict1[name].name}) достигается при цвете", key)
                print(f"Значение агента({dict1[name].name}) меняем на", key)
                val_min = value
                dict1[name].val = key
                print(f"Текущий цвет агента({dict1[name].name})=", dict1[name].val)
                dict1[name].res = dict1[name].res + value # Подсчет общей стоимости агента за всю работу модели
        global resulting_var
        resulting_var = resulting_var + val_min
        print(f"Прибавляем к финальной стоимости штраф цикла агента({dict1[name].name}) = ", val_min)
        print("Финальная стоимость на данный момент равна = ", resulting_var)
        cost_otnosit = {}
        dict1[name].current_cycle = {}  #Обнуляем текущий цикл:
        print("----------------")
        print()

def results():
    """Подсчет общей суммы штрафов всех агентов"""
    summa = 0
    for agent in graph["vars"].keys():
        summa = summa + dict1[agent].res
    for name in graph["vars"].keys():
        print(f"Цвет агента ({dict1[name].name}) - {dict1[name].val}")
    print("----------------------------------------------------")
    print("Общая стоимость модели(сумма всех штрафов) равна " + str(summa))
    print("Общий штраф по новому расчету равен ", resulting_var)

def init_agents():
    """Создание агентов по ключам графа"""
    global dict1
    for i in graph["vars"].keys():
        dict1[i] = Agent(i)
        dict1[i].neighbor_search()

#=======================MAIN==================#

start = time()

#Создание агентов по ключам графа
dict1 = {}
init_agents()

#agent1 = random.choice(list(graph["vars"].keys())) # Случайный агент из всех возможных

tasks = []
c = 0
event_loop = asyncio.get_event_loop()
for i in range(0, 1500):
    for agent in graph["vars"].keys():
        tasks.append(event_loop.create_task(dict1[agent].send(random.choice(list(graph["vars"].keys())))))
        c = c +1
        #print("Действие номер - ", c)
print("Длина очереди асинхронных действий=", len(tasks))
wait_tasks = asyncio.wait(tasks)
event_loop.run_until_complete(wait_tasks)
event_loop.close()

results()

#print(list(dict1.keys()))

#########Варианты для обычных тестов##########
# c = 0
# c1 = 10
# for i in range(1500):
#     for agent in graph["vars"].keys():
#         #c1 = c1 - 1
#         dict1[agent].send(random.choice(list(graph["vars"].keys())))
#         c = c +1
#
# results()

#print(choice(list(dict1.keys())))

print("Кол-во действий = ", c)
print(f"Script work time is {time() - start} s.")


# Discord bot code!
# Quick info
# Use: Runs a Discord bot
# Libraries: JSON, Requests, Discord, Asyncio
# Indent type: 4 spaces

# Links
# http://discordpy.readthedocs.io/en/latest/api.html - Discord.py API reference, lots of commands here.
# https://discordapp.com/developers/applications/me - My apps page on Discord.
# https://leovoel.github.io/embed-visualizer/ - Lets you generate code for posting Embeds. Click generate and select Discord.py from the dropdown.
# https://discordapi.com/permissions.html - Generates bot invites, straightforward.

#Intro code, no need to modify unless for librarys (Eg. Time)

print('Importing JSON')        #JSON library, allows you to get variables from JSON files online. NOT NEEDED
import json
print('Importing REQUESTS')    #Lets you send stuff to the internet for response, used for random cats. NOT NEEDED
import requests
print('Importing DISCORD')     #Discord.py library, allows you to interface with Discord API easily. NEEDED
import discord
print('Importing ASYNCIO')     #Asyncio library, bundled with Discord.py, allows you to multiprocess, I think, NEEDED HOWEVER.
import asyncio

#Even more technical wizardry, even I don't understand this.

client = discord.Client()

@client.event
async def on_ready():
    print('Signed in as ' + client.user.name + '! ID is ' + client.user.id + '!') #What it says in the command line when it opens

    await client.change_presence(game=discord.Game(name='>>>help for help')) #Use this to set a game status.

@client.event

#End of intro code.

#Asyncio first thing, senses when a message comes in. !test and !sleep are the originals from Discord.py's github

async def on_message(message):
    if message.content.startswith('!test'):  #Checks if the message begins with test. If you want it to be equal, replace it with [ if message.content == '!test': ]
        counter = 0
        tmp = await client.send_message(message.channel, 'Calculating messages...') #Sends a message to a channel, message.channel is where the command was posted.
        async for log in client.logs_from(message.channel, limit=100):               #I have no idea for this, only the guy who made it does, but it counts messages.
            if log.author == message.author:
                counter += 1

        await client.edit_message(tmp, 'You have {} messages.'.format(counter))    #.format() replaces any instance of {} in a string with the thing in (), useful, huh.

    elif message.content.startswith('!sleep'):               #This makes the bot sleep, like time.sleep
        await asyncio.sleep(5) #Sleep for 5 seconds
        await client.send_message(message.channel, 'Done sleeping')  #Send to the channel from which the command was posted 'done sleeping'

    if message.content == ('>>>catme'):          #The catme command from Harpy, useful to have and experiment with.
       print(str(message.author) + ' just requested a cat in ' + str(message.server) + ' : ' + str(message.channel))   #Writes in console that someone wanted a cat.
       #import from catimages.py, complex!
       response = (requests.get("http://random.cat/meow"))
       response = str(response.content)
       print(response)
       response = response.replace("b'","")
       response = response.replace("'","")
       response = response.replace("\\","")
       print(response)
       data = json.loads(response)
       print (data["file"])
       #import end. Complexity END!

       line = data["file"] #Sets a variable with the variable file from the JSON

       await client.send_message(message.channel, 'Meow! ' + data["file"])  #Sends a message with the cat link.
       print('Responded!')

    if message.content.startswith('>>>whereami'):  #If a command starts with >>>whereami, an example, it should be message.content == ('>>>catme')
       print(str(message.author) + ' just requested location in ' + str(message.server) + ' : ' + str(message.channel)) #Posts in console
       await client.send_message(message.channel, 'User: ' + str(message.author)) #Sends to Discord the username of the request
       await client.send_message(message.channel, 'Channel: ' + str(message.channel)) #Sends the channel of the request
       await client.send_message(message.channel, 'Server: ' + str(message.server)) #Sends the server of the request

       print('Responded!') #Posts in console

    if message.content == ('>>>testme'):  #If a command is testme
       print(str(message.author) + ' just requested help in ' + str(message.server) + ' : ' + str(message.channel)) #post to console
       await client.send_message(message.channel, '''This is a line.
This is another line.''')                                                       #This posts in the channel
       print('Responded!') #Post in console.

client.run('token') #runs the bot, replace the 'token' with your bot token, which is on the bot's page under 'token:'


# coding: utf-8
import pygame.display
from settings import Settings
import game_functions as gf
from music import Music
from blocks import Blocks
import time
from pygame.sprite import Group
from scoreboard import Scoreboard
from button import Button

def run_game():
    pygame.init()
    gf.initialize_data()
    sets = Settings()
    music = Music()
    screen = pygame.display.set_mode(
        (sets.screen_width,sets.screen_height))
    pygame.display.set_caption("2048")
    play_button = Button(sets,screen,"Play!")
    failed_and_play_button = Button(sets,screen,"Try again")
    win_and_play_button = Button(sets,screen,"Win! New Game!")
    blocks = Group()
    sb = Scoreboard(sets,screen,blocks)
    while True:
        gf.check_events(sets,screen,sb,play_button,win_and_play_button,
           failed_and_play_button,blocks,music)

        if sets.stats:
            gf.update_blocks(blocks)
        gf.update_screen(sb,sets,screen,blocks,play_button,win_and_play_button,
              failed_and_play_button)
        gf.check_mv(sets,blocks)
run_game()


import re
import locale
from grab.spider import Spider, Task
from datetime import datetime

class MySpider(Spider):

    initial_urls = []

    def prepare(self):
        locale.setlocale(locale.LC_TIME, 'ru_RU.UTF-8')

        self.date_time = []

    def task_initial(self, grab, task):
        pages_nmb = grab.doc.select('//a[@class="block br3 brc8 large tdnone lheight24"]')[-1].text()

        for i in range(int(pages_nmb)):
        # for i in range(2):
            url = '{}?page={}'.format(self.initial_urls[0], i+1)
            yield Task('page', url=url)

    def task_page(self, grab, task):
        ads_list = grab.doc.select('//a[@class="marginright5 link linkWithHash detailsLink"]')

        for elem in ads_list:
            yield Task('ad', url=elem.attr('href'))

    def task_ad(self, grab, task):
        date_time_info = grab.doc.select('//div[@class="offer-titlebox__details"]/em')

        for elem in date_time_info:
            text = elem.text()

            time = re.search(r'\d{2}:\d{2}', text)[0]
            date = re.search(r'\d{1,2} [а-я]+ \d{4}', text)[0]
            date_time = '{} {}'.format(date, time)

            self.date_time.append(datetime.strptime(date_time, '%d %B %Y %H:%M'))


# -*- coding: utf-8 -*-
"""
Created on Mon Aug 29 10:49:47 2016

@author: Naveen
"""

import pandas as pd
import numpy as np
#%matplotlib inline
import seaborn as sns
import matplotlib.pyplot as plt
import os
from sklearn.preprocessing import LabelEncoder
from scipy.sparse import csr_matrix, hstack
from sklearn.linear_model import LogisticRegression
from sklearn.cross_validation import StratifiedKFold
from sklearn.metrics import log_loss

"""LOAD THE FILES"""
from enum import Enum
class Load_Type(Enum):
    readdisk_writeHD5 = 1
    readdisk = 2
    readHD5 = 3

load_type = Load_Type.readdisk
datadir = r'D:\Work\Projects\Programming\Python\Kaggle\TalkingData\Data\Extracted'

if load_type == Load_Type.readdisk or load_type == Load_Type.readdisk_writeHD5:
    gender_age_train_df = pd.read_csv(os.path.join(datadir,'gender_age_train.csv'), index_col = 'device_id')
    gender_age_test_df = pd.read_csv(os.path.join(datadir,'gender_age_test.csv'), index_col = 'device_id')
    label_categories_df = pd.read_csv(os.path.join(datadir, 'label_categories.csv'))
    phone_df = pd.read_csv(os.path.join(datadir, 'phone_brand_device_model.csv'))
    sample_submission_df = pd.read_csv(os.path.join(datadir, 'sample_submission.csv'))
    app_events_df = pd.read_csv(os.path.join(datadir, 'app_events.csv'), usecols=['event_id','app_id','is_active'], dtype={'is_active':bool})
    app_labels_df = pd.read_csv(os.path.join(datadir, 'app_labels.csv'))
    events_df = pd.read_csv(os.path.join(datadir, 'events.csv'), parse_dates=['timestamp'], index_col='event_id')

    if load_type == Load_Type.readdisk_writeHD5:
        store = pd.HDFStore(os.path.join(datadir, 'AllData_TalkingData.h5'))

        store['gender_age_train_df'] = gender_age_train_df
        store['gender_age_test_df'] = gender_age_test_df
        store['label_categories_df'] = label_categories_df
        store['phone_df'] = phone_df
        store['sample_submission_df'] = sample_submission_df
        store['app_events_df'] = app_events_df
        store['app_labels_df'] = app_labels_df
        store['events_df'] = events_df
        store.close()

elif load_type == Load_Type.readHD5:
    store = pd.HDFStore(os.path.join(datadir, 'AllData_TalkingData.h5'),'r')

    gender_age_train_df = store['gender_age_train_df']
    gender_age_test_df = store['gender_age_test_df']
    label_categories_df = store['label_categories_df']
    phone_df = store['phone_df']
    sample_submission_df = store['sample_submission_df']
    app_events_df = store['app_events_df']
    app_labels_df = store['app_labels_df']
    events_df = store['events_df']
    store.close()

print("gender_age_train_df : ")
print(gender_age_train_df.head())
print("gender_age_test_df : ")
print(gender_age_test_df.head())
print("label_categories_df : ")
print(label_categories_df.head())
print("phone_df : ")
print(phone_df.head())
print("sample_submission_df : ")
print(sample_submission_df.head())
print("app_events_df : ")
print(app_events_df.head())
print("app_labels_df : ")
print(app_labels_df.head())
print("events_df : ")
print(events_df.head())

phone_df = phone_df.drop_duplicates('device_id',keep='first').set_index('device_id')

gender_age_train_df['trainrow'] = np.arange(gender_age_train_df.shape[0])
gender_age_test_df['testrow'] = np.arange(gender_age_test_df.shape[0])

"""BRAND MODEL"""
brandencoder = LabelEncoder().fit(phone_df.phone_brand)
phone_df['brand'] = brandencoder.transform(phone_df['phone_brand'])
gender_age_train_df['brand'] = phone_df['brand']
gender_age_test_df['brand'] = phone_df['brand']
Xtr_brand = csr_matrix((np.ones(gender_age_train_df.shape[0]),
                       (gender_age_train_df.trainrow, gender_age_train_df.brand)))
Xte_brand = csr_matrix((np.ones(gender_age_test_df.shape[0]),
                       (gender_age_test_df.testrow, gender_age_test_df.brand)))
print('Brand features: train shape {}, test shape {}'.format(Xtr_brand.shape, Xte_brand.shape))

"""DEVICE MODEL"""
m = phone_df.phone_brand.str.cat(phone_df.device_model)
modelencoder = LabelEncoder().fit(m)
phone_df['model'] = modelencoder.transform(m)
gender_age_train_df['model'] = phone_df['model']
gender_age_test_df['model'] = phone_df['model']
Xtr_model = csr_matrix((np.ones(gender_age_train_df.shape[0]),
                       (gender_age_train_df.trainrow, gender_age_train_df.model)))
Xte_model = csr_matrix((np.ones(gender_age_test_df.shape[0]),
                       (gender_age_test_df.testrow, gender_age_test_df.model)))
print('Model features: train shape {}, test shape {}'.format(Xtr_model.shape, Xte_model.shape))

"""APP FEATURES"""
appencoder = LabelEncoder().fit(app_events_df.app_id)
app_events_df['app'] = appencoder.transform(app_events_df.app_id)
napps = len(appencoder.classes_)
deviceapps = (app_events_df.merge(events_df[['device_id']], how='left',left_on='event_id',right_index=True)
                       .groupby(['device_id','app'])['app'].agg(['size'])
                       .merge(gender_age_train_df[['trainrow']], how='left', left_index=True, right_index=True)
                       .merge(gender_age_test_df[['testrow']], how='left', left_index=True, right_index=True)
                       .reset_index())
deviceapps.head()

d = deviceapps.dropna(subset=['trainrow'])
Xtr_app = csr_matrix((np.ones(d.shape[0]), (d.trainrow, d.app)),
                      shape=(gender_age_train_df.shape[0],napps))
d = deviceapps.dropna(subset=['testrow'])
Xte_app = csr_matrix((np.ones(d.shape[0]), (d.testrow, d.app)),
                      shape=(gender_age_test_df.shape[0],napps))
print('Apps data: train shape {}, test shape {}'.format(Xtr_app.shape, Xte_app.shape))

"""APP LABEL FEATURES"""
app_labels_df = app_labels_df.loc[app_labels_df.app_id.isin(app_events_df.app_id.unique())]
app_labels_df['app'] = appencoder.transform(app_labels_df.app_id)
labelencoder = LabelEncoder().fit(app_labels_df.label_id)
app_labels_df['label'] = labelencoder.transform(app_labels_df.label_id)
nlabels = len(labelencoder.classes_)

devicelabels = (deviceapps[['device_id','app']]
                .merge(app_labels_df[['app','label']])
                .groupby(['device_id','label'])['app'].agg(['size'])
                .merge(gender_age_train_df[['trainrow']], how='left', left_index=True, right_index=True)
                .merge(gender_age_test_df[['testrow']], how='left', left_index=True, right_index=True)
                .reset_index())
devicelabels.head()

d = devicelabels.dropna(subset=['trainrow'])
Xtr_label = csr_matrix((np.ones(d.shape[0]), (d.trainrow, d.label)),
                      shape=(gender_age_train_df.shape[0],nlabels))
d = devicelabels.dropna(subset=['testrow'])
Xte_label = csr_matrix((np.ones(d.shape[0]), (d.testrow, d.label)),
                      shape=(gender_age_test_df.shape[0],nlabels))
print('Labels data: train shape {}, test shape {}'.format(Xtr_label.shape, Xte_label.shape))

"""CONCATENATE ALL FEATURES """
Xtrain = hstack((Xtr_brand, Xtr_model, Xtr_app, Xtr_label), format='csr')
Xtest =  hstack((Xte_brand, Xte_model, Xte_app, Xte_label), format='csr')
print('All features: train shape {}, test shape {}'.format(Xtrain.shape, Xtest.shape))

"""CROSS VALIDATION"""
targetencoder = LabelEncoder().fit(gender_age_train_df.group)
y = targetencoder.transform(gender_age_train_df.group)
nclasses = len(targetencoder.classes_)

def score(clf, random_state = 0):
    kf = StratifiedKFold(y, n_folds=5, shuffle=True, random_state=random_state)
    pred = np.zeros((y.shape[0],nclasses))
    for itrain, itest in kf:
        Xtr, Xte = Xtrain[itrain, :], Xtrain[itest, :]
        ytr, yte = y[itrain], y[itest]
        clf.fit(Xtr, ytr)
        pred[itest,:] = clf.predict_proba(Xte)
        # Downsize to one fold only for kernels
        return log_loss(yte, pred[itest, :])
        print("{:.5f}".format(log_loss(yte, pred[itest,:])), end=' ')
    print('')
    return log_loss(y, pred)

#Cs = np.logspace(-3,0,4)
#Cs = np.arange(0.01,0.025,0.001)
res = []
for C in Cs:
    #res.append(score(LogisticRegression(C = C)))
    res.append(score(LogisticRegression(C=C, multi_class='multinomial',solver='lbfgs')))
plt.semilogx(Cs, res,'-o');

clf = LogisticRegression(C=0.019, multi_class='multinomial',solver='lbfgs')
clf.fit(Xtrain, y)
pred = pd.DataFrame(clf.predict_proba(Xtest), index = gender_age_test_df.index, columns=targetencoder.classes_)
pred.head()
pred.to_csv('logreg_subm.csv',index=True)

# Get rid of duplicate device ids in phone
#==============================================================================
# events = pd.read_csv(os.path.join(datadir,'events.csv'),
#                      parse_dates=['timestamp'], index_col='event_id')
# appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
#                         usecols=['event_id','app_id','is_active'],
#                         dtype={'is_active':bool})
# applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))
#
# gatrain = pd.DataFrame(gender_age_train_df, index='device_id')
#==============================================================================

# Get rid of duplicate device ids in phone
#phone = phone.drop_duplicates('device_id',keep='first').set_index('device_id')
#events = pd.read_csv(os.path.join(datadir,'events.csv'),#
#                     parse_dates=['timestamp'], index_col='event_id')
#appevents = pd.read_csv(os.path.join(datadir,'app_events.csv'),
#                        usecols=['event_id','app_id','is_active'],
#                        dtype={'is_active':bool})
#applabels = pd.read_csv(os.path.join(datadir,'app_labels.csv'))


import csv
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()

from math import sqrt
from scipy.stats import norm

priceE = []
dateE = []
priceA = []
dateA = []

tahun = 2001
with open('AIR.PA bulanan.csv') as csv_file:
    csv_reader = csv.reader(csv_file, delimiter=',')
    line_count = 0
    sementara = 0
    for row in csv_reader:
        if line_count == 0:
            #print(f'Column names are {", ".join(row)}')
            line_count += 1
        else:

            if line_count % 12 == 0:
                sementara = float(sementara / 12)
                priceE.append(sementara)
                dateE.append(tahun)
                tahun += 1
                sementara = 0
            else:
                sementara += float(row[1])
            #print(f'\t{row[0]} works in the {row[1]} department, and was born in {row[2]}.')
            line_count += 1

tahun  = 2001
with open('BA bulanan.csv') as csv_file:
    csv_reader = csv.reader(csv_file, delimiter=',')
    line_count = 0
    sementara = 0
    for row in csv_reader:
        if line_count == 0:
            #print(f'Column names are {", ".join(row)}')
            line_count += 1
        else:

            if line_count % 12 == 0:
                sementara = float(sementara / 12)
                priceA.append(sementara)
                dateA.append(tahun)
                tahun += 1
                sementara = 0
            else:
                sementara += float(row[1])
            #print(f'\t{row[0]} works in the {row[1]} department, and was born in {row[2]}.')
            line_count += 1

def brownian(x0, n, dt, delta, out=None):

    x0 = np.asarray(x0)

    # For each element of x0, generate a sample of n numbers from a
    # normal distribution.
    r = norm.rvs(size=x0.shape + (n,), scale=delta*sqrt(dt))

    # If `out` was not given, create an output array.
    if out is None:
        out = np.empty(r.shape)

    # This computes the Brownian motion by forming the cumulative sum of
    # the random samples.
    np.cumsum(r, axis=-1, out=out)

    # Add the initial condition.
    out += np.expand_dims(x0, axis=-1)

    return out

import numpy
from pylab import plot, show, grid, xlabel, ylabel

# The Wiener process parameter.
deltaE = 10
deltaA = 50
# Total time.
T = 5.0
# Number of steps.
N = 5
# Time step size
dt = T/N
# Number of realizations to generate.
m = 4
# Create an empty array to store the realizations.
x = np.empty((m,N+1))
y = np.empty((m,N+1))
# Initial values of x.
x[:, 0] = priceE[len(priceE)-1]
y[:, 0] = priceA[len(priceA)-1]

brownian(x[:,0], N, dt, deltaE, out=x[:,1:])
brownian(y[:,0], N, dt, deltaA, out=y[:,1:])

tE = numpy.linspace(dateE[len(dateE)-1], dateE[len(dateE)-1]+5, N+1)
tA = numpy.linspace(dateA[len(dateA)-1], dateA[len(dateA)-1]+5, N+1)

# plotting the points
plt.plot(dateE,priceE)
plt.plot(dateA,priceA)
plt.xlabel('Bulan')
plt.ylabel('Harga (USD) and (EUR)')
plt.title('Saham Boeing (Oranye) dan Airbus (Biru) per Bulan'
          '\ndari September 2001 hingga Desember 2019'
          '\nDisertakan prediksi dengan metode Geometric Brownian Motion')
for k in range(m):
    plot(tE, x[k])
    plot(tA, y[k])
xlabel('t', fontsize=16)
ylabel('x', fontsize=16)
grid(True)
show()


from similarity_ratings_tests import *
from subset_experiments import *
import sys
import os
import datetime
# Used to compute all tests in a single run

now = datetime.datetime.now()

RESULTS_PATH = 'results/'

def perform_complete_test_suite(subset_test_filename, zero_shot_filename, spearman_filename, table_filename, subset = False, zero = False, spearman = False, tables = False, projection_modes = 'best', verbosity = 0):
    """
    Computes all test necessary for the ba thesis
    :param subset_test_filename: Path for saving subset tests.
    :param zero_shot_filename: Path for saving zero shot tests.
    :param spearman_filename: Path for spearman tests.
    :param table_filename: Path for saving explicit tables.
    :param subset: Flag to include or exclude subset tests from the test suite.
    :param zero: Flag to include or exclude zero shot tests from the test suite.
    :param spearman: Flag to include or exclude spearman tests from the test suite.
    :param tables: Flag to include or exclude explicit tables from the test suite.
    :param projection_modes: certain keywords for specific list of projection modes to test.
    :param verbosity: determines if errors are printed or not.
    :return: null
    """
    print("Subset = {}, Zero = {}, Spearman = {}, Table = {}, projection_modes = {}, Verbosity = {}".format(subset,zero,spearman,tables,projection_modes,verbosity))
    result_dir = RESULTS_PATH + '{}_{}_{}'.format(now.year,now.month,now.day)

    if not '{}_{}_{}'.format(now.year,now.month,now.day) in os.listdir(RESULTS_PATH):
            os.mkdir(result_dir)

    projection_mode_list = PROJECTION_MODE_LIST
    if projection_modes == 'best':
        projection_mode_list = SMALL_PROJECTION_MODE_LSIT
    elif projection_modes == 'medium':
        projection_mode_list = MEDIUM_PROJECTION_LIST
    elif projection_modes == 'test':
        projection_mode_list = TEST_PROJECTION_MODE_LIST

    check_attribute_set_sizes()

    if subset:
        sys.stdout = open(result_dir + '/' + subset_test_filename, 'w')

        sys.stdout.flush()

        print("\n\n----------------------Subset-Tests----------------------".upper())
        perform_subset_tests(train_subsets=ATTRIBUTE_SETS,
                             test_subsets=ATTRIBUTE_SETS,
                             tables=False, quantitive=True,
                             proj_mode_list=projection_mode_list,
                             train_test_exclusivity=False,
                             verbosity=verbosity)

        sys.stdout.flush()

    if zero:
        sys.stdout = open(result_dir + '/' + zero_shot_filename, 'w')

        print("\n\n----------------------Subset-Tests (Testmenge explizit ohne die Attribute aus der Trainingsmenge)----------------------".upper())
        perform_subset_tests(train_subsets=[attribute_set for attribute_set in ATTRIBUTE_SETS if attribute_set[0] != 'ALL'],
                             test_subsets=[ALL_ATTRIBUTES],
                             tables=False, quantitive=True,
                             proj_mode_list=projection_mode_list,
                             train_test_exclusivity=True,
                             verbosity=verbosity)

        sys.stdout.flush()

    if spearman:
        sys.stdout = open(result_dir + '/' + spearman_filename + '.txt', 'w')
        print("\n\n----------------------spearman tests----------------------".upper())
        for attribute_set in ATTRIBUTE_SETS:
            print("\n---------------------Teste mit Attribut-Menge {}----------------".format(attribute_set[0]))
            similarity_experiment(attribute_set,attribute_set,
                                  ['add','nn_weighted_adjective_noun_identity','mult','mitchell_lapata_reversed_2','mitchell_lapata_2'], rating_cutoff_list=[4,5,6],
                                  verbosity=verbosity)

        sys.stdout.flush()

        sys.stdout = open(result_dir + '/' + spearman_filename + '_no_tables.txt', 'w')
        print("\n\n----------------------spearman tests----------------------".upper())
        for attribute_set in ATTRIBUTE_SETS:
            print("\n---------------------Teste mit Attribut-Menge {}----------------".format(attribute_set[0]))
            similarity_experiment(attribute_set,attribute_set,
                                  ['add','nn_weighted_adjective_noun_identity','mult','mitchell_lapata_reversed_2','mitchell_lapata_2'], rating_cutoff_list=[0,1,2,3,4,5,6],
                                  tables=False,
                                  verbosity=verbosity)

        sys.stdout.flush()

    if tables:
        sys.stdout = open(result_dir + '/' + table_filename, 'w')

        print("\n\n----------------------Tabellen----------------------".upper())
        perform_subset_tests(train_subsets=ATTRIBUTE_SETS,
                             test_subsets=ATTRIBUTE_SETS,
                             tables=True, quantitive=False,
                             proj_mode_list=projection_mode_list,
                             train_test_exclusivity=False,
                             verbosity=verbosity)

arglist = sys.argv[1:]

# print(arglist)

subset, zero, spearman, tables, verbosity, projection_modes = False, False, False, False, 0, 'all'

if arglist[0] == 'True':
    # print(arglist[0])
    subset = True
if arglist[1] == 'True':
    # print(arglist[1])
    zero = True
if arglist[2] == 'True':
    # print(arglist[2])
    spearman = True
if arglist[3] == 'True':
    # print(arglist[3])
    tables = True

if arglist[4]:
    verbosity = int(arglist[4])

if arglist[5]:
    projection_modes = arglist[5]

# perform_complete_test_suite('subset_results.txt','zero_shot_results.txt','spearman_results','tables.txt', subset=subset, zero=zero, spearman=spearman, tables=tables, projection_modes=projection_modes, verbosity = verbosity)

check_attribute_set_sizes()