# -*- coding: utf-8 -*-
"""
Created on Wed Aug 19 14:36:47 2020

@author: allison
"""

def readATS(filename):
    infile = open(filename)
    ats = []
    for line in infile:
        ats.append(line.replace("\n",""))
    infile.close()
    return ats

def readCouplingMatrix(filename):
    infile = open(filename)
    Csca = []
    i = 0
    for line in infile:
        Csca.append([])
        entries = line.split(",")
        entries = entries[0:len(entries)-1]
        for v in entries:
            Csca[i].append(float(v))
        i+=1
    return Csca

def getPairwiseCouplingValues(Csca, ats):
    coupling_values = []
    for i in range(0, len(ats)):
        for j in range(i+1, len(ats)):
            coupling_values.append(Csca[i][j])
    coupling_values.sort()
    return coupling_values

def getCouplingInfo(pos1, pos2):
    ats = readATS("Data/ats.txt")
    if str(pos1) not in ats:
        return (None, 1)
    if str(pos2) not in ats:
        return (None, 2)

    Csca = readCouplingMatrix("Data/SCA_matrix.csv")
    coupling_values = getPairwiseCouplingValues(Csca, ats)
    index1 = ats.index(str(pos1))
    index2 = ats.index(str(pos2))
    coupling = Csca[index1][index2]
    first_index = coupling_values.index(coupling)
    last_index = len(coupling_values) - 1 - coupling_values[::-1].index(coupling)
    if first_index == -1:
        return (coupling, -1)
    percentile = 100.0*(first_index+last_index)/(2*len(coupling_values))
    return (coupling, percentile)

#position1 = 4
#position2 = 11
#(coupling, percentile) = getCouplingInfo(position1,position2)


import pandas as pd
import numpy as np
from scipy.sparse import csr_matrix
from sklearn.neighbors import NearestNeighbors
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity, linear_kernel

def GetMoviesByDescription(movieName):
    movie_list=[]

    csv_path = "cleaned data/movies.csv"
    df = pd.read_csv(csv_path)
    df['description'] = df['description'].fillna('')
    #create the matrix
    tf = TfidfVectorizer(analyzer='word',ngram_range=(1, 2),min_df=0, stop_words='english')
    tfidf_matrix = tf.fit_transform(df['description'])

    #calaculate the  Cosine Similarity Score
    cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix)

    md = df.reset_index()
    titles = df['title']
    indices = pd.Series(df.index, index=df['title'])

    try:
        idx = indices[movieName]
        sim_scores = list(enumerate(cosine_sim[idx]))
        sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
        sim_scores = sim_scores[1:31]
        movie_indices = [i[0] for i in sim_scores]
        recdf = titles.iloc[movie_indices]
        count = 0
        for index, value in recdf.items():
            count = count + 1
            movie_list.append(value)
            if(count == 8):
                break
    except:
        movie_list.append("No Recommendation available.")
    return movie_list

def GetMoviesByUserRating(movieName):
    movies = pd.read_csv("raw data/movies.csv")
    ratings = pd.read_csv("raw data/ratings.csv")
    movievsuser = ratings.pivot(index='movieId',columns='userId',values='rating')
    movievsuser.fillna(0, inplace=True)
    ratingsByMovie = ratings.groupby('movieId')['rating'].agg('count')
    ratingsByUser = ratings.groupby('userId')['rating'].agg('count')
    movievsuser = movievsuser.loc[ratingsByMovie[ratingsByMovie > 40].index,:]
    csr_data = csr_matrix(movievsuser.values)
    movievsuser.reset_index(inplace=True)

    # Using KNN algorithm to predict similarity with cosine distance
    knn = NearestNeighbors(metric='cosine', algorithm='brute', n_neighbors=20, n_jobs=-1)
    knn.fit(csr_data)

    reccomendCount = 8
    listing = []
    movieList = movies[movies['title'].str.contains(movieName)]
    if len(movieList):
        movie_id= movieList.iloc[0]['movieId']
        movie_id = movievsuser[movievsuser['movieId'] == movie_id].index[0]
        distances , indices = knn.kneighbors(csr_data[movie_id],n_neighbors=reccomendCount+1)
        recommendedMovies = sorted(list(zip(indices.squeeze().tolist(),distances.squeeze().tolist())),key=lambda x: x[1])[:0:-1]
        recMoviesList = []
        for val in recommendedMovies:
            movie_id = movievsuser.iloc[val[0]]['movieId']
            idx = movies[movies['movieId'] == movie_id].index
            recMoviesList.append({'Title':movies.iloc[idx]['title'].values[0],'Distance':val[1]})
        df = pd.DataFrame(recMoviesList,index=range(1,reccomendCount+1))
        df['Distance'] = pd.to_numeric(df['Distance'])
        df= df.sort_values('Distance')
        listing = df['Title']

    return listing

def GetMoviesByGenre(movieName):
    file_to_load = "raw data/movies.csv"
    ratings = pd.read_csv("raw data/ratings.csv")

    data = pd.read_csv(file_to_load)
    data['genres'] = data['genres'].str.replace(r'|', ' ')
    genre_data=data[['title','genres']]
    genre_data=genre_data.set_index('title')

    #convert genre column to array
    cv = CountVectorizer()
    X = cv.fit_transform(genre_data["genres"]).toarray()

    similarities = cosine_similarity(X)
    movie_index = data.loc[data['title'].str.contains(movieName)].index[0]

    similarity_values = pd.Series(similarities[movie_index])

    #We converted list into series in order to preserve the actual indexes of dataset even after sorting
    similarity_values.sort_values(ascending=False)
    similar_movie_indexes = list(similarity_values.sort_values(ascending=False).index)

    #Remove the already watched movie from index list
    similar_movie_indexes.remove(movie_index)

    movie_list=[]
    for i in range(8):
        movie_list.append(genre_data.index[similar_movie_indexes[i]])

    return movie_list

def GetPredictionsForMovie(moviename):
    complete_df = pd.read_csv("cleaned data/complete_df_with_predictions.csv")

    mv = complete_df.loc[complete_df['title'].str.contains(moviename),['title']]
    movie = mv.head(1)
    names = movie.to_numpy()
    name = names[0][0]

    #based on all the users in the dataframe and their predictions what is the average rating the movie will get
    movie_rating=round((complete_df.loc[complete_df['title']==name,['predicted rating']].values).mean(),2)

    #from data already available what is the average of the movie
    movie_gavg=round((complete_df.loc[complete_df['title']==name,['MAvg']].values).mean(),2)

    percdiff = round(((movie_rating-movie_gavg)/movie_gavg*100),2)

    summary = {'Predicted Rating': movie_rating, 'Actual Rating': movie_gavg ,"Percentage Difference%":percdiff}
    return summary

def GetPredictions(moviename, userid):
    complete_df = pd.read_csv("cleaned data/complete_df_with_predictions.csv")
    #print(complete_df.head())
    userid=int(userid)
    try:
        mv = complete_df.loc[complete_df['title'].str.contains(moviename),['title']]
        movie = mv.head(1)
        names = movie.to_numpy()
        name = names[0][0]
        #print(name)

        #based on users past ratings what is the prediction for a particular movie
        pred_rating=round(complete_df.loc[(complete_df['user']==userid) & (complete_df['title']==name),['predicted rating']].values[0][0],2)

        #from data already available what is the average of the movie
        user_rating=round(complete_df.loc[(complete_df['user']==userid) & (complete_df['title']==name),['rating']].values[0][0],2)

        percdiff = round(((pred_rating-user_rating)/user_rating*100),2)

        summary = {'Predicted Rating': pred_rating, 'Actual Rating': user_rating ,"Percentage Difference%":percdiff}
        return summary
    except:
        pred_rating=0
        user_rating=0
        percdiff = 0

        #based on all the users in the dataframe and their predictions what is the average rating a user gives
        user_rating=round((complete_df.loc[complete_df['user']==userid,['predicted rating']].values).mean(),2)

        #from data already available what is the average of the movie
        user_uavg=round((complete_df.loc[complete_df['user']==userid,['UAvg']].values).mean(),2)

        percdiff = round(((user_rating-user_uavg)/user_uavg*100),2)

        print ("this user has not rated this movie")
        summary = {'Status': "this user has not rated this movie, showing a prediction of what this user is likely to predict",'Predicted Rating': user_rating, 'Actual Rating': user_uavg ,"Percentage Difference%":percdiff}
    return summary


import argparse
import logging
import random
import sys
import time
from copy import deepcopy

import numpy as np
import torch
from scipy.stats import kendalltau

from datasets.dataloader import get_dataloader
from models.cell_operations import NAS_BENCH_201
from models.supernet import Supernet201
from utils import obtain_accuracy, AverageMeter, set_seed, run_func, time_record

parser = argparse.ArgumentParser("Train 201 Supernet")
# dataset
parser.add_argument("--data_root", type=str, default='./dataset/', help="The path to dataset")
parser.add_argument("--dataset", type=str, default='cifar10', help="Dataset.")
parser.add_argument("--search_space_name", type=str, default='nas-bench-201', help="The search space name.")
parser.add_argument("--num_classes", type=int, default=10, help="Dataset Classes")
# supernet
parser.add_argument("--max_nodes", type=int, default=4, help="The maximum number of nodes.")
parser.add_argument("--channel", type=int, default=16, help="The number of channels.")
parser.add_argument("--num_cells", type=int, default=5, help="The number of cells in one stage.")
# training settings
parser.add_argument("--exp_name", type=str, default='debug_baseline', help='exp_name for saving results')
parser.add_argument("--method", type=str, default='spos', choices=['spos', 'fairnas', 'sumnas'])
parser.add_argument("--lr", type=float, default=0.05, help="Learning rate")
parser.add_argument("--inner_lr", type=float, default=0.05, help="Learning rate")
parser.add_argument("--momentum", type=float, default=0.9, help="Momentum")
parser.add_argument("--wd", type=float, default=2.5e-4, help="Weight decay")
parser.add_argument("--epochs", type=int, default=250, help="Training epochs")
parser.add_argument("--gpu_id", type=int, default=0, help="Training GPU")
parser.add_argument("--train_batch_size", type=int, default=256, help="Train batch size")
parser.add_argument("--valid_batch_size", type=int, default=512, help="Valid batch size")
parser.add_argument("--print_freq", type=int, default=50, help="print frequency when training")
parser.add_argument("--rank_print_freq", type=int, default=100, help="print frequency when ranking")
parser.add_argument("--seed", type=int, default=0, help="manual seed")
parser.add_argument("--debug", default=False, action='store_true', help="for debug")

args = parser.parse_args()
args.device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
args.ckpt_path = 'checkpoints/%s.pt' % args.exp_name
args.pred_path = 'results/%s.npy' % args.exp_name
if args.debug:
    args.epochs = 5

# logging config
log_format = '%(asctime)s %(message)s'
logging.basicConfig(stream=sys.stdout, level=logging.INFO,
                    format=log_format, datefmt='%m/%d %I:%M:%S %p')
logging.info(args)
set_seed(args.seed)

def mix_grad(grad_list, weight_list):
    """
    calc weighted average of gradient
    """
    mixed_grad = []
    for g_list in zip(*grad_list):
        g_list = torch.stack([weight_list[i] * g_list[i] for i in range(len(weight_list))])
        mixed_grad.append(torch.sum(g_list, dim=0))
    return mixed_grad

def apply_grad(model, grad):
    """
    assign gradient to model(nn.Module) instance. return the norm of gradient
    """
    for p, g in zip(model.parameters(), grad):
        if p.grad is None:
            p.grad = g
        else:
            p.grad += g

def train(epoch, train_loader, model, criterion, optimizer, inner_optimizer=None):
    train_loss = AverageMeter()
    train_top1 = AverageMeter()
    train_top5 = AverageMeter()

    model.train()
    path_list = []
    num_candidate_ops = 5
    candidate_ops = list(range(5))
    candidate_edges = 6

    for step, (inputs, targets) in enumerate(train_loader):
        inputs = inputs.to(args.device)
        targets = targets.to(args.device)
        if args.method == 'spos':
            # randomly sample an arch
            sampled_arch = [
                random.choice(candidate_ops) for _ in range(candidate_edges)
            ]
            optimizer.zero_grad()
            logits = model(inputs, sampled_arch)
            loss = criterion(logits, targets)
            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), 5)
            optimizer.step()

        elif args.method == 'fairnas':
            # shuffle the ops to get sub-models with strict fairness
            for _ in range(candidate_edges):
                random.shuffle(candidate_ops)
                path_list.append(deepcopy(candidate_ops))
            # inner loop
            optimizer.zero_grad()
            for _path_id in range(num_candidate_ops):
                sampled_arch = [_operations[_path_id] for _operations in path_list]
                logits = model(inputs, sampled_arch)
                loss = criterion(logits, targets)
                loss.backward()
                # record training metrics
                prec1, prec5 = obtain_accuracy(logits.data, targets.data, topk=(1, 5))
                train_loss.update(loss.item(), inputs.size(0))
                train_top1.update(prec1.item(), inputs.size(0))
                train_top5.update(prec5.item(), inputs.size(0))
            torch.nn.utils.clip_grad_norm_(model.parameters(), 5)
            optimizer.step()

        elif args.method == 'sumnas':
            # record the supernet weights
            weights_before = deepcopy(model.state_dict())
            grad_list = []

            # shuffle the ops to get sub-models fairly
            for _ in range(candidate_edges):
                random.shuffle(candidate_ops)
                path_list.append(deepcopy(candidate_ops))
            # inner loop
            for _path_id in range(num_candidate_ops):
                sampled_arch = [_operations[_path_id] for _operations in path_list]
                # inner optimization
                for _step in range(args.adaption_steps):
                    inner_optimizer.zero_grad()
                    logits = model(inputs, sampled_arch)
                    loss = criterion(logits, targets)
                    loss.backward()
                    torch.nn.utils.clip_grad_norm_(model.parameters(), 5)
                    inner_optimizer.step()
                    # record training metrics
                    prec1, prec5 = obtain_accuracy(logits.data, targets.data, topk=(1, 5))
                    train_loss.update(loss.item(), inputs.size(0))
                    train_top1.update(prec1.item(), inputs.size(0))
                    train_top5.update(prec5.item(), inputs.size(0))
                # record reptile gradient
                outer_grad = []
                weights_after = deepcopy(model.state_dict())
                for p_0, p_T in zip(weights_before.items(), weights_after.items()):
                    outer_grad.append(-(p_T[1] - p_0[1]).detach())
                grad_list.append(outer_grad)
                model.load_state_dict(weights_before)
            # outer loop
            optimizer.zero_grad()
            weight = torch.ones(len(grad_list)) / len(grad_list)
            grad = mix_grad(grad_list, weight)
            apply_grad(model, grad)
            optimizer.step()

        else:
            raise ValueError('Wrong training method for the supernet: %s' % args.method)
        # record
        prec1, prec5 = obtain_accuracy(logits.data, targets.data, topk=(1, 5))
        train_loss.update(loss.item(), inputs.size(0))
        train_top1.update(prec1.item(), inputs.size(0))
        train_top5.update(prec5.item(), inputs.size(0))

        if step % args.print_freq == 0 or step + 1 == len(train_loader):
            logging.info('[Training] Epoch %03d/%03d, step %03d/%03d, loss: %.3f, top1: %.3f, top5: %.3f'
                         % (epoch, args.epochs, step, len(train_loader), train_loss.avg, train_top1.avg, train_top5.avg))
    return train_loss.avg, train_top1.avg, train_top5.avg

def valid(valid_loader, model, criterion):
    val_loss, val_top1, val_top5 = AverageMeter(), AverageMeter(), AverageMeter()
    model.eval()
    with torch.no_grad():
        for step, (val_inputs, val_targets) in enumerate(valid_loader):
            val_inputs = val_inputs.to(args.device)
            val_targets = val_targets.to(args.device)

            # randomly sample an arch
            candidate_ops = range(5)
            candidate_edges = 6
            sampled_arch = [
                random.choice(candidate_ops) for _ in range(candidate_edges)
            ]

            # prediction
            logits = model(val_inputs, sampled_arch)
            loss = criterion(logits, val_targets)
            # record
            prec1, prec5 = obtain_accuracy(
                logits.data, val_targets.data, topk=(1, 5)
            )
            val_loss.update(loss.item(), val_inputs.size(0))
            val_top1.update(prec1.item(), val_inputs.size(0))
            val_top5.update(prec5.item(), val_inputs.size(0))
    return val_loss.avg, val_top1.avg, val_top5.avg

def valid_specific_path(valid_loader, model, sampled_arch, criterion, device):
    val_loss, val_top1, val_top5 = AverageMeter(), AverageMeter(), AverageMeter()
    model.eval()
    with torch.no_grad():
        for step, (val_inputs, val_targets) in enumerate(valid_loader):
            val_inputs = val_inputs.to(device)
            val_targets = val_targets.to(device)

            # prediction
            logits = model(val_inputs, sampled_arch)
            loss = criterion(logits, val_targets)
            # record
            prec1, prec5 = obtain_accuracy(
                logits.data, val_targets.data, topk=(1, 5)
            )
            val_loss.update(loss.item(), val_inputs.size(0))
            val_top1.update(prec1.item(), val_inputs.size(0))
            val_top5.update(prec5.item(), val_inputs.size(0))
    return val_loss.avg, val_top1.avg, val_top5.avg

def rank_supernet(valid_loader, model, criterion):
    logging.info('---------- Start to rank on NAS-Bench-201 ----------')
    nasbench201 = np.load('./dataset/nasbench201/nasbench201_dict.npy', allow_pickle=True).item()
    if args.debug:
        new_dict = {}
        for i in range(5):
            new_dict[str(i)] = deepcopy(nasbench201[str(i)])
        nasbench201 = deepcopy(new_dict)
    nasbench201_len = len(nasbench201)
    tmp_pred = []
    tmp_target = []
    prediction = {}
    for step, item in enumerate(nasbench201):
        model_id = int(item)
        operation = nasbench201[item]['operation']
        target = nasbench201[item]['cifar10_test']
        val_loss, val_top1, val_top5 = valid_specific_path(valid_loader, model, operation, criterion, args.device)
        tmp_pred.append(val_top1)
        tmp_target.append(target)
        prediction[model_id] = {'id': model_id, 'model_gene': operation, 'pred': val_top1, 'target': target}
        if step % args.rank_print_freq == 0 or (step + 1) == nasbench201_len:
            logging.info("model_id: %d  gene: %s  loss: %.3f  top1: %.3f  target: %.3f"
                         % (model_id, str(operation), val_loss, val_top1, target))
            logging.info("Evaluated: %05d\tWaiting: %05d\tCurrent Kendall's Tau: %.5f" %
                         (len(tmp_pred), nasbench201_len-len(tmp_pred), kendalltau(tmp_pred, tmp_target)[0]))

    # save predictions
    print('\n')
    np.save(args.pred_path, prediction)
    logging.info('Finish ranking and save predictions to : %s' % args.pred_path)
    final_ranking = kendalltau(tmp_pred, tmp_target)[0]
    logging.info("Final_pred: %05d\tFinal_target: %05d\tFinal_Kendall's Tau: %.5f" %
                 (len(tmp_pred), len(tmp_target), final_ranking))

    return final_ranking

def main():
    # time record
    train_start = time.time()

    # dataloader
    train_loader, valid_loader = get_dataloader(args, model=None, dataset=args.dataset)

    # supernet
    model = Supernet201(
        C=args.channel, N=args.num_cells, max_nodes=args.max_nodes,
        num_classes=args.num_classes, search_space=NAS_BENCH_201
    ).to(args.device)

    # training settings
    optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.wd)
    if args.method == 'sumnas':
        args.adaption_steps = 2
        optimizer = torch.optim.SGD(model.parameters(), 1.0, weight_decay=4e-5)
        inner_optimizer = torch.optim.SGD(model.parameters(), 0.05, momentum=0.9)
    else:
        inner_optimizer = None
    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, args.epochs)
    criterion = torch.nn.CrossEntropyLoss()
    best_val_top1 = 0.0

    logging.info('---------- Start to train supernet ----------')
    for epoch in range(args.epochs):
        # train supernet
        train_loss, train_top1, cnn_top5 = train(epoch, train_loader, model, criterion, optimizer, inner_optimizer)
        logging.info(
            "[Epoch: %s/%s] train_loss=%.3f, train_top1=%.3f, train_top5=%.3f" %
            (epoch, args.epochs, train_loss, train_top1, cnn_top5)
        )

        # valid supernet
        val_loss, val_top1, val_top5 = valid(valid_loader, model, criterion)
        logging.info(
            "[Validation], val_loss=%.3f, val_top1=%.3f, val_top5=%.3f, best_top1=%.3f" %
            (val_loss, val_top1, val_top5, best_val_top1)
        )
        if best_val_top1 < val_top1:
            best_val_top1 = val_top1

        # save latest checkpoint
        torch.save(model.state_dict(), args.ckpt_path)
        logging.info('Save latest checkpoint to %s' % args.ckpt_path)

        # scheduler step
        scheduler.step()
        print('\n')

    # time record
    supernet_training_elapse = time_record(train_start, prefix='Supernet training')
    print('\n')

    # load best supernet weights
    latest_pretrained_weights = torch.load(args.ckpt_path)
    model.load_state_dict(latest_pretrained_weights)
    model.eval()

    # ranking supernet
    final_ranking = rank_supernet(valid_loader, model, criterion)

    # write results
    with open('./results/ranking.txt', 'a') as f:
        f.write("EXP: %s \t Seed: %s \t Kendall' Tau: %.6f \t Training_Elapse: %s \n"
                % (args.exp_name, args.seed, final_ranking, supernet_training_elapse))

if __name__ == "__main__":
    run_func(args, main)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import gzip
from collections import defaultdict
import math
import scipy.optimize
import numpy
import string
import random
from sklearn import linear_model
import sklearn

# In[2]:

# This will suppress any warnings, comment out if you'd like to preserve them
import warnings
warnings.filterwarnings("ignore")

# In[3]:

# Check formatting of submissions
def assertFloat(x):
    assert type(float(x)) == float

def assertFloatList(items, N):
    assert len(items) == N
    assert [type(float(x)) for x in items] == [float]*N

# In[4]:

answers = {}

# In[5]:

f = open("spoilers.json.gz", 'r')

# In[6]:

dataset = []
for l in f:
    d = eval(l)
    dataset.append(d)

# In[7]:

f.close()

# In[8]:

# A few utility data structures
reviewsPerUser = defaultdict(list)
reviewsPerItem = defaultdict(list)

for d in dataset:
    u,i = d['user_id'],d['book_id']
    reviewsPerUser[u].append(d)
    reviewsPerItem[i].append(d)

# Sort reviews per user by timestamp
for u in reviewsPerUser:
    reviewsPerUser[u].sort(key=lambda x: x['timestamp'])

# Same for reviews per item
for i in reviewsPerItem:
    reviewsPerItem[i].sort(key=lambda x: x['timestamp'])

# In[9]:

# E.g. reviews for this user are sorted from earliest to most recent
[d['timestamp'] for d in reviewsPerUser['b0d7e561ca59e313b728dc30a5b1862e']]

# In[10]:

### 1

# In[11]:

def MSE(y, ypred):
    return sum([(a-b)**2 for (a,b) in zip(y,ypred)]) / len(y)

# In[12]:

# (a)
y = []
y_pred = []
for u in reviewsPerUser:
    cur = []
    reviews = reviewsPerUser[u]
    for i in range(0, len(reviews) - 1):
        cur.append(reviews[i]['rating'])
    if len(cur) == 0:
        continue
    y_pred.append(sum(cur)/len(cur))
    y.append(reviews[-1]['rating'])
answers['Q1a'] = MSE(y, y_pred)
assertFloat(answers['Q1a'])

# In[13]:

# (b)
y = []
y_pred = []
for u in reviewsPerItem:
    cur = []
    reviews = reviewsPerItem[u]
    for i in range(0, len(reviews) - 1):
        cur.append(reviews[i]['rating'])
    if len(cur) == 0:
        continue
    y_pred.append(sum(cur)/len(cur))
    y.append(reviews[-1]['rating'])
answers['Q1b'] = MSE(y, y_pred)
assertFloat(answers['Q1b'])

# In[14]:

### 2
answers['Q2'] = []

for N in [1,2,3]:
    y = []
    y_pred = []
    for u in reviewsPerUser:
        cur = []
        reviews = reviewsPerUser[u]
        for i in range(0, len(reviews) - 1):
            cur.append(reviews[i]['rating'])
        if len(cur) == 0:
            continue

        if len(cur) < N:
            cur_new = cur

        if len(cur) >= N:
            cur_new = cur[-N:]

        y_pred.append(sum(cur_new)/len(cur_new))
        y.append(reviews[-1]['rating'])

    answers['Q2'].append(MSE(y,y_pred))

# In[15]:

assertFloatList(answers['Q2'], 3)

# In[16]:

answers

# In[17]:

### 3a

# In[18]:

def feature3(N, u): # For a user u and a window size of N

    cur = []
    reviews = reviewsPerUser[u]
    for i in range(0, len(reviews) - 1):
        cur.append(reviews[i]['rating'])

    feat = [1]
    for n in range(1, N + 1):
        feat.append(cur[-n])

    return feat

# In[19]:

answers['Q3a'] = [feature3(2,dataset[0]['user_id']), feature3(3,dataset[0]['user_id'])]

# In[20]:

assert len(answers['Q3a']) == 2
assert len(answers['Q3a'][0]) == 3
assert len(answers['Q3a'][1]) == 4

# In[21]:

### 3b
answers['Q3b'] = []
def feat(N, u):
    feat = [1]
    data = reviewsPerUser[u]
    for d in data[-N-1:-1]:
        feat.append(d['rating'])
    return feat

for N in [1,2,3]:
    X = []
    y = []
    for u,data in reviewsPerUser.items():
        if len(data) <= N:
            continue
        else:
            X.append(feat(N,u))
            y.append(data[-1]['rating'])
    model = sklearn.linear_model.LinearRegression(fit_intercept=False)
    model.fit(X, y)
    y_pred = model.predict(X)
    mse = MSE(y, y_pred)
    answers['Q3b'].append(mse)
assertFloatList(answers['Q3b'], 3)
answers

# In[22]:

### 4a
globalAverage = [d['rating'] for d in dataset]
globalAverage = sum(globalAverage) / len(globalAverage)

def featureMeanValue(N, u): # For a user u and a window size of N
    feat = [1]
    data = reviewsPerUser[u]
    if len(data) < N + 1:
        if len(data) < 2:
            for j in range(N):
                feat.append(globalAverage)
        elif len(data) >= 2:
            rate = [review['rating'] for review in data[:-1]]
            avg = sum(rate)/len(rate)
            for i in range(len(data)-1):
                feat.append(data[-i-2]['rating'])
            for i in range(N-len(data)+1):
                feat.append(avg)
    else:
        for i in range(N):
            feat.append(data[-i-2]['rating'])
    return feat

def featureMissingValue(N, u):
    feat = [1]
    data = reviewsPerUser[u]

    if len(data) < N + 1:
        if len(data) < 2:
            for j in range(N):
                feat.append(1)
                feat.append(0)
        elif len(data) >= 2:
            for i in range(len(data)-1):
                feat.append(0)
                feat.append(data[- i - 2]['rating'])
            for i in range(N + 1-len(data)):
                feat.append(1)
                feat.append(0)
    else:
        for i in range(N):
            feat.append(0)
            feat.append(data[-i-2]['rating'])
    return feat

answers['Q4a'] = [featureMeanValue(10, dataset[0]['user_id']), featureMissingValue(10, dataset[0]['user_id'])]

answers

# In[23]:

answers['Q4b'] = []

for featFunc in [featureMeanValue, featureMissingValue]:
    X = []
    y = []
    for user,rating in reviewsPerUser.items():
        if len(rating) < 1:
            continue
        else:
            X.append(featFunc(10,user))
            y.append(rating[-1]['rating'])

    model = linear_model.LinearRegression()
    model.fit(X,y)
    y_pred = model.predict(X)
    mse = MSE(y, y_pred)
    answers['Q4b'].append(mse)

# In[24]:

answers['Q4b']

# In[25]:

### 5
#(a)
def feature5(sentence):
    feat = [1]
    feat.append(len(sentence))
    feat.append(sentence.count('!')) # Quadratic term
    feat.append(sum(i.isupper() for i in sentence))
    return feat

X = []
y = []

for d in dataset:
    for spoiler,sentence in d['review_sentences']:
        X.append(feature5(sentence))
        y.append(spoiler)

# In[26]:

answers['Q5a'] = X[0]

# In[27]:

###5(b)
mod = sklearn.linear_model.LogisticRegression( class_weight='balanced', C=1)
mod.fit(X,y)
predictions = mod.predict(X)

TP = sum([(p and l) for (p,l) in zip(predictions, y)])
FP = sum([(p and not l) for (p,l) in zip(predictions, y)])
TN = sum([(not p and not l) for (p,l) in zip(predictions, y)])
FN = sum([(not p and l) for (p,l) in zip(predictions, y)])
TPR = TP / (TP + FN)
TNR = TN / (TN + FP)
BER = 1 - 1/2 * (TPR + TNR)
answers['Q5b'] = [TP, TN, FP, FN, BER]

# In[28]:

assert len(answers['Q5a']) == 4
assertFloatList(answers['Q5b'], 5)

# In[29]:

### 6
def feature6(review):
    review = review['review_sentences']
    feat = [1]
    for i in range(0, 5):
        feat.append(review[i][0])
    feat.append(len(review[5][1]))
    feat.append(review[5].count('!')) # Quadratic term
    feat.append(sum(i.isupper() for i in review[5][1]))

    return feat

# In[30]:

y = []
X = []

for d in dataset:
    sentences = d['review_sentences']
    if len(sentences) < 6: continue
    X.append(feature6(d))
    y.append(sentences[5][0])

# In[31]:

answers['Q6a'] = feature6(dataset[0])
answers

# In[32]:

answers['Q6a'] = X[0]
answers

# In[33]:

mod = sklearn.linear_model.LogisticRegression(class_weight='balanced', C = 1)
mod.fit(X,y)
predictions = mod.predict(X)

TP = sum([(p and l) for (p,l) in zip(predictions, y)])
FP = sum([(p and not l) for (p,l) in zip(predictions, y)])
TN = sum([(not p and not l) for (p,l) in zip(predictions, y)])
FN = sum([(not p and l) for (p,l) in zip(predictions, y)])
TPR = TP / (TP + FN)
TNR = TN / (TN + FP)
BER = 1 - 1/2 * (TPR + TNR)

answers['Q6b'] = BER

# In[34]:

assert len(answers['Q6a']) == 9
assertFloat(answers['Q6b'])
answers

# In[35]:

### 7

# In[36]:

# 50/25/25% train/valid/test split
Xtrain, Xvalid, Xtest = X[:len(X)//2], X[len(X)//2:(3*len(X))//4], X[(3*len(X))//4:]
ytrain, yvalid, ytest = y[:len(X)//2], y[len(X)//2:(3*len(X))//4], y[(3*len(X))//4:]

# In[37]:

def pipeline(reg, bers, BER_test):
    mod = linear_model.LogisticRegression(class_weight='balanced', C=reg)

    # 50/25/25% train/valid/test split
    Xtrain, Xvalid, Xtest = X[:len(X)//2], X[len(X)//2:(3*len(X))//4], X[(3*len(X))//4:]
    ytrain, yvalid, ytest = y[:len(X)//2], y[len(X)//2:(3*len(X))//4], y[(3*len(X))//4:]

    mod.fit(Xtrain,ytrain)
    ypredValid = mod.predict(Xvalid)
    ypredTest = mod.predict(Xtest)

    # validation

    TP = sum([(a and b) for (a,b) in zip(yvalid, ypredValid)])
    TN = sum([(not a and not b) for (a,b) in zip(yvalid, ypredValid)])
    FP = sum([(not a and b) for (a,b) in zip(yvalid, ypredValid)])
    FN = sum([(a and not b) for (a,b) in zip(yvalid, ypredValid)])

    TPR = TP / (TP + FN)
    TNR = TN / (TN + FP)

    BER = 1 - 0.5*(TPR + TNR)

    print("C = " + str(reg) + "; validation BER = " + str(BER))
    bers = bers.append(BER)

     # test

    TP = sum([(a and b) for (a,b) in zip(ytest, ypredTest)])
    TN = sum([(not a and not b) for (a,b) in zip(ytest, ypredTest)])
    FP = sum([(not a and b) for (a,b) in zip(ytest, ypredTest)])
    FN = sum([(a and not b) for (a,b) in zip(ytest, ypredTest)])

    TPR = TP / (TP + FN)
    TNR = TN / (TN + FP)

    BER = 1 - 0.5*(TPR + TNR)

    BER_test = BER_test.append(BER)

    return mod

# In[38]:

bers = []
BER_test = []
for c in [0.01, 0.1, 1, 10, 100]:
    pipeline(c, bers, BER_test)
bers
BER_test

# In[39]:

bestC = 0.1
ber = 0.21299572460563176
answers['Q7'] = bers + [bestC] + [ber]
assertFloatList(answers['Q7'], 7)
answers

# In[40]:

### 8
def Jaccard(s1, s2):
    numer = len(s1.intersection(s2))
    denom = len(s1.union(s2))
    if denom == 0:
        return 0
    return numer / denom

# In[41]:

# 75/25% train/test split
dataTrain = dataset[:15000]
dataTest = dataset[15000:]

# In[42]:

# A few utilities

itemAverages = defaultdict(list)
ratingMean = []

for d in dataTrain:
    itemAverages[d['book_id']].append(d['rating'])
    ratingMean.append(d['rating'])

for i in itemAverages:
    itemAverages[i] = sum(itemAverages[i]) / len(itemAverages[i])

ratingMean = sum(ratingMean) / len(ratingMean)

# In[43]:

reviewsPerUser = defaultdict(list)
usersPerItem = defaultdict(set)

for d in dataTrain:
    u,i = d['user_id'], d['book_id']
    reviewsPerUser[u].append(d)
    usersPerItem[i].add(u)

# In[44]:

# From my HW2 solution, welcome to reuse
def predictRating(user,item):
    ratings = []
    similarities = []
    for d in reviewsPerUser[user]:
        i2 = d['book_id']
        if i2 == item: continue
        ratings.append(d['rating'] - itemAverages[i2])
        similarities.append(Jaccard(usersPerItem[item],usersPerItem[i2]))
    if (sum(similarities) > 0):
        weightedRatings = [(x*y) for x,y in zip(ratings,similarities)]
        return itemAverages[item] + sum(weightedRatings) / sum(similarities)
    else:
        # User hasn't rated any similar items
        if item in itemAverages:
            return itemAverages[item]
        else:
            return ratingMean

# In[45]:

predictions = [predictRating(d['user_id'], d['book_id']) for d in dataTest]
labels = [d['rating'] for d in dataTest]

# In[46]:

answers["Q8"] = MSE(predictions, labels)
assertFloat(answers["Q8"])

# In[ ]:

# In[56]:

### 9
item = [d['book_id'] for d in dataTrain]
data0, rating0 =  [], []

for d in dataTest:
    num = item.count(d['book_id'])
    if num == 0:
        data0.append([d['user_id'], d['book_id']])
        rating0.append(d['rating'])

pred0 = [predictRating(u, i) for u, i in data0]

mse0 = MSE(pred0, rating0)
mse0

# In[57]:

data1, rating1 = [],[]

for d in dataTest:
    num = item.count(d['book_id'])

    if 1 <= num <= 5:
        data1.append([d['user_id'], d['book_id']])
        rating1.append(d['rating'])

pred1 = [predictRating(u, i) for u, i in data1]

mse1to5= MSE(pred1, rating1)
mse1to5

# In[58]:

data5, rating5 = [], []

for d in dataTest:
    num = item.count(d['book_id'])

    if num > 5:
        data5.append([d['user_id'], d['book_id']])
        rating5.append(d['rating'])

pred5 = [predictRating(u, i) for u, i in data5]

mse5 = MSE(pred5, rating5)
mse5

# In[ ]:

# In[50]:

answers["Q9"] = [mse0, mse1to5, mse5]
assertFloatList(answers["Q9"], 3)

answers

# In[51]:

### 10

# In[52]:

userAverages = defaultdict(list)

for d in dataTrain:
    userAverages[d['user_id']].append(d['rating'])

for i in userAverages:
    userAverages[i] = sum(userAverages[i]) / len(userAverages[i])

def predictRating(user,item):
    ratings = []
    similarities = []
    for d in reviewsPerUser[user]:
        i2 = d['book_id']
        if i2 == item: continue
        ratings.append(d['rating'] - itemAverages[i2])
        similarities.append(Jaccard(usersPerItem[item],usersPerItem[i2]))
    if (sum(similarities) > 0):
        weightedRatings = [(x*y) for x,y in zip(ratings,similarities)]
        return itemAverages[item] + sum(weightedRatings) / sum(similarities)
    else:
        # User hasn't rated any similar items
        if item in itemAverages:
            return itemAverages[item]
        else:
            # return RatingMean
            if user in userAverages:
                return userAverages[user]
            else:
                return ratingMean

item = [d['book_id'] for d in dataTrain]
data10, rating10 = [], []

for d in dataTest:
    num = item.count(d['book_id'])
    if num == 0:
        data10.append([d['user_id'], d['book_id']])
        rating10.append(d['rating'])

pred10 = [predictRating(u, i) for u, i in data10]

mse10 = MSE(pred10, rating10)
mse10

# In[59]:

answers["Q10"] = ("To improve the prediction function for unseen items, we can modify the predictRating function. Since previously the predictRating only use itemAverages for prediction function, we can add the userAverage to specify the condition and make mse smaller, inside of just categorize data into ratingMean. We can see that the mse become smaller for unseen data.", mse10)
assert type(answers["Q10"][0]) == str
assertFloat(answers["Q10"][1])

# In[60]:

answers

# In[55]:

f = open("answers_midterm.txt", 'w')
f.write(str(answers) + '\n')
f.close()


#
# demo.py
#
import argparse
import os
import numpy as np
import time

from modeling.deeplab import *
from dataloaders import custom_transforms as tr
from PIL import Image
from torchvision import transforms
from dataloaders.utils import *
from torchvision.utils import make_grid, save_image
torch.set_printoptions(profile="full")

def main():
    parser = argparse.ArgumentParser(description="PyTorch DeeplabV3Plus Training")
    parser.add_argument('--in-path', type=str, default=r'D:\PT\archive\Testingset10class\dataset\结果\test_96_label',
                        help='image to test')
    parser.add_argument('--out-path', type=str, default=r'D:\PT\archive\Testingset10class\dataset\结果\96', help='mask image to save')
    parser.add_argument('--backbone', type=str, default='mobilenet',
                        choices=['resnet', 'xception', 'drn', 'mobilenet'],
                        help='backbone name (default: resnet)')
    parser.add_argument('--ckpt', type=str, default=r'D:\PT\超分辨率语义分割\模型保存\10_class\dsrl\128/model_best.pth.tar',
                        help='saved model')
    parser.add_argument('--out-stride', type=int, default=16,
                        help='network output stride (default: 8)')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='disables CUDA training')
    parser.add_argument('--gpu-ids', type=str, default='0',
                        help='use which gpu to train, must be a \
                        comma-separated list of integers only (default=0)')
    parser.add_argument('--dataset', type=str, default='rockdataset',
                        choices=['pascal', 'coco', 'cityscapes', 'rockdataset'],
                        help='dataset name (default: pascal)')
    parser.add_argument('--crop-size', type=int, default=96,
                        help='crop image size')
    parser.add_argument('--num_classes', type=int, default=11,
                        help='crop image size')
    parser.add_argument('--sync-bn', type=bool, default=None,
                        help='whether to use sync bn (default: auto)')
    parser.add_argument('--freeze-bn', type=bool, default=False,
                        help='whether to freeze bn parameters (default: False)')

    args = parser.parse_args()
    args.cuda = not args.no_cuda and torch.cuda.is_available()
    if args.cuda:
        try:
            args.gpu_ids = [int(s) for s in args.gpu_ids.split(',')]
        except ValueError:
            raise ValueError('Argument --gpu_ids must be a comma-separated list of integers only')

    if args.sync_bn is None:
        if args.cuda and len(args.gpu_ids) > 1:
            args.sync_bn = True
        else:
            args.sync_bn = False
    composed_transforms = transforms.Compose([
        tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
        tr.ToTensor()])
    for name in os.listdir(args.in_path):
        image = Image.open(args.in_path + "/" + name).convert('RGB')

        # image = Image.open(args.in_path).convert('RGB')
        target = Image.open(args.in_path + "/" + name)
        sample = {'image': image, 'label': target}
        tensor_in = composed_transforms(sample)['label'].unsqueeze(0)
        print(tensor_in.shape)

        grid_image = make_grid(decode_seg_map_sequence(tensor_in.detach().cpu().numpy()),
                               3, normalize=False, range=(0, 255))
        save_image(grid_image, args.out_path + "/" + "{}_label.png".format(name[0:-4]))
        # save_image(grid_image, args.out_path)
        # print("type(grid) is: ", type(grid_image))
        # print("grid_image.shape is: ", grid_image.shape)
    print("image save in in_path.")

if __name__ == "__main__":
    main()

# python demo.py --in-path your_file --out-path your_dst_file


# 집합의 표현
# 합집합 연산과, 두 원소가 같은 집합에 포함되어 있는지 확인
# 0 a b ( a가 포함되어 있는 집합과 b가 포함되어 있는 집합을 합친다는 의미 )
# 1 a b ( a와 b가 같은 집합에 포함되어 있는지 확인)
import sys
input = sys.stdin.readline
sys.setrecursionlimit(10**9)

n, m = map(int, input().split())

# parent 테이블 자기 자신으로 초기화
parent = [i for i in range(n+1)]

# 루트 노드 찾을 때까지 재귀 호출
def find(x):
  if parent[x] != x:
    parent[x] = find(parent[x])
  return parent[x]

# 합집합 테이블
def union(x,y):
  x = find(x)
  y = find(y)

  if x < y:
    parent[y] = x
  else:
    parent[x] = y

for i in range(m):
  num, a, b = map(int, input().split())
  # 0이면 합집합 호출
  if num == 0:
    union(a, b)
  elif num == 1:
    # a와 b가 같은 집합에 포함되어 있을 때
    if find(a) == find(b):
      print('YES')
    else:
      print('NO')


from flask import Flask, render_template, request
import sqlite3

app = Flask(__name__)

@app.route('/',methods = ['POST', 'GET'])
def home():
    if request.method == 'GET':
        return render_template('index.html')

@app.route('/thankyou',methods = ['POST', 'GET'])
def thankyou():
    if request.method == 'GET':
        return render_template('thankyou.html')
    elif request.method == 'POST':
        emailid = request.form.get('eid')
        conn = sqlite3.connect("emailist")
        cur=conn.cursor()
        cur.execute("SELECT * from emails_table")
        print(cur.fetchall())
#cur.execute('INSERT INTO emails_table (email) VALUES (?), ("t@iron.com")')
#cur.execute("INSERT INTO movie VALUES(%s,%s)",(movID,Name))
        cur.execute("INSERT INTO emails_table (email) VALUES (?)", (emailid,))
        conn.commit()
        conn.close()
        return render_template('thankyou.html')

if __name__ == '__main__':
   app.run(debug=True)


n, s = [int(i) for i in input().split()]
nums = [int(i) for i in input().split()]

f = False

i = 0
j = len(nums)-1

while i != j:
    if nums[i] + nums[j] == s:
        f = True
        break
    elif nums[i] + nums[j] > s:
        j -= 1
    else: i += 1

if f:
    print("YES")
else: print("NO")


from . import Funktioner
import datetime
import MyModules.GUIclasses2 as GUI
import numpy as np
import os
from . import FileOps
#fix 14.04.10, simlk. Changed "centering error", which should make the test more forgiving at small distances - at large distances it has no effect.
# Last edit: 2012-01-09 fixed mtl test. Unified, 2012-04-23: fixed possible None return val in TestStretch
#Rejcection criteria reflects a model variance for meas. of a strectch, should correpond to a variance of half the parameter used in the test.

# Test is really - for two meas:
#par=prec=reject_par/2.....
#|diff|<2*sqrt(var_model(d,par)) - always linear in par.
#No more Centering err/ constant 'avoid zero' term!
#Thus the var-models are artificial close to zero. Instead a global min is defined (0,3 mm for now)!!!!!

GLOBAL_MIN_DEV=0.3 #twice precision on mean

def MTL_var_model_linear(dist,parameter):
	dist=dist/1000.0
	return (dist*parameter)**2

def MTL_var_model(dist,parameter):
	dist=dist/1000.0
	DLIM=0.2 #km
	c_err=0 #divided by two below because 'precision' is (defined to be)  half of 'reject par'
	if dist<DLIM:
		FKLIN=(np.sqrt(DLIM)*parameter-c_err*0.5)/DLIM
		return (FKLIN*dist+c_err*0.5)**2
	else:
		return (parameter**2*dist)

def MGL_var_model(dist,parameter):
	dist=dist/1000.0
	c_err=0.0 #divided by two below because 'precision' is (defined to be)  half of 'reject par'
	return (np.sqrt(dist)*parameter+c_err*0.5)**2   #add a centering err....

class FBreject(object):
	def __init__(self,database,program="MGL",parameter=2.0,unit="ne"):
		if program=="MGL":
			self.var_model=MGL_var_model
		else:
			if unit=="ne":
				self.var_model=MTL_var_model
			else:
				self.var_model=MTL_var_model_linear
		self.unit=unit
		self.parameter=parameter
		self.precision=parameter*0.5 #this is the correpsonding 'precision'
		self.initialized=False
		self.found=False
		self.wasok=False
		self.database=database
		self.initialized=True
	def GetData(self):
		data=""
		for key in list(self.database.keys()):
			s=self.database[key]
			data+="%s->%s: dist: %.2f m\n" %(key[0],key[1],s.dist)
			for i in range(len(s.hdiffs)):
				data+="dh: %.4f m tid: %s j-side: %s\n" %(s.hdiffs[i],s.times[i].isoformat().replace("T",","),s.jpages[i])
		return data
	def GetDatabase(self):
		return self.database
	def TestStretch(self,start,end,hdiff): #returns foundstretch,testresult,#found,msg
		self.found=False
		self.wasok=False
		msg=""
		key_back=(end,start)
		key_forward=(start,end)
		nforward=0
		nback=0
		hdiffs_all=np.empty((0,))
		dists=[]
		if key_back in self.database:
			s_back=self.database[key_back]
			nback+=len(s_back.hdiffs)
			if nback>0:
				dists.append(s_back.dist)
				hdiffs_all=np.append(hdiffs_all,np.array(s_back.hdiffs)*-1.0)
		if key_forward in self.database:
			s_forward=self.database[key_forward]
			nforward+=len(s_forward.hdiffs)
			if nforward>0:
				dists.append(s_forward.dist)
				hdiffs_all=np.append(hdiffs_all,np.array(s_forward.hdiffs))
		msg+="%s->%s er tidligere m\u00E5lt %d gang(e), og %d gang(e) i modsat retning.\n" %(start,end,nforward,nback)
		nall=len(hdiffs_all)
		if len(hdiffs_all)>0:
			d=np.mean(dists)
			norm_d=np.sqrt(d/1e3)
			msg+="Afstand: %.2f m\n" %d
			if len(hdiffs_all)>1:
				raw_mean=np.mean(hdiffs_all)
				raw_std=np.std(hdiffs_all,ddof=1)
				raw_prec=raw_std/np.sqrt(len(hdiffs_all))
				raw_max_diff=hdiffs_all.max()-hdiffs_all.min()
				msg+="hdiff_middel: %.4f m, max-diff: %.2f mm (%.2f ne)\n" %(raw_mean,raw_max_diff*1000,raw_max_diff*1e3/norm_d)
				msg+="std_dev: %.2f mm, std_dev(middel): %.2f mm (%.2f ne)\n" %(raw_std*1000,raw_prec*1000,raw_prec*1e3/norm_d)
			msg+="\nEfter inds\u00E6ttelse af ny m\u00E5ling:\n"
			hdiffs_new=np.append(hdiffs_all,[hdiff])
			new_mean=np.mean(hdiffs_new)
			new_std=np.std(hdiffs_new,ddof=1)
			new_prec=new_std/np.sqrt(len(hdiffs_new))
			new_max_diff=hdiffs_new.max()-hdiffs_new.min()
			msg+="hdiff_middel: %.4f m, max-diff: %.2f mm (%.2f ne)\n" %(new_mean,new_max_diff*1000,new_max_diff*1e3/norm_d)
			msg+="std_dev: %.2f mm, std_dev(middel): %.2f mm (%.2f ne)\n" %(new_std*1000,new_prec*1000,new_prec*1e3/norm_d)
			msg+="\nForkastelsesparameter: %.3f %s." %(self.parameter,self.unit)
			max_dev=self.GetMaxDev(d) #in mm!!
			if len(hdiffs_new)==2:
				msg+=" Vil acceptere |diff|<%.2f mm" %(2*max_dev)
			isok=(new_prec*1e3<=max_dev)
			self.found=True
			self.wasok=isok
			if isok:
				msg+="\nDen samlede standardafvigelse p\u00E5 middel er OK.\n"
			else:
				msg+="\nDen samlede standarafvigelse p\u00E5 middel er IKKE OK\n"
				msg+="Foretag flere m\u00E5linger!\n"
			if len(hdiffs_all)>1 and new_prec>raw_prec: #or something more fancy
				msg+="Den nye m\u00E5ling er tilsyneladende en outlier og kan evt. omm\u00E5les!\n"
				isok=False
			return True,isok,len(hdiffs_all),msg
		else:
			msg="%s->%s er ikke m\u00E5lt tidligere" %(start,end)
			self.found=False
			self.wasok=True
			return True,True,0,msg

	def GetMaxDev(self,dist): #max dev in mm!
		return max(np.sqrt(self.var_model(dist,self.precision)),GLOBAL_MIN_DEV*0.5)

	def InsertStretch(self,start,end,hdiff,dist,dato,tid,jside=""):
		if not self.initialized:
			return True #we havent done anyting
		data=self.database
		try:
			start=start.strip()
			end=end.strip()
			key=(start,end)
			m,h=Funktioner.GetTime(tid)
			day,month,year=Funktioner.GetDate(dato)
			date=datetime.datetime(year,month,day,h,m)
			if key in data:
				data[key].AddStretch(hdiff,dist,date,jside)
			else:
				data[key]=Stretch()
				data[key].AddStretch(hdiff,dist,date,jside)
		except Exception as msg:
			print(repr(msg))
			return False
		else:
			return True
	def OutlierAnalysis(self):
		data=self.database.copy()
		msg=""
		noutliers=0
		nbad=0
		keys=list(data.keys())
		for key_forward in keys:
			l_msg="%s->%s:" %key_forward
			key_back=(key_forward[1],key_forward[0])
			if not key_forward in data: #could happen since we delete stuff below
				continue
			s_forward=data[key_forward]
			hdiffs_all=np.array(s_forward.hdiffs)
			nforward=len(s_forward.hdiffs)
			dists=[s_forward.dist]
			nback=0
			if key_back in data:
				s_back=data[key_back]
				nback=len(s_back.hdiffs)
				if nback>0:
					dists.append(s_back.dist)
					hdiffs_all=np.append(hdiffs_all,np.array(s_back.hdiffs)*-1.0)
			d=np.mean(dists)
			l_msg+=" m\u00E5lt %d gange frem og %d gange tilbage." %(nforward,nback)
			report=False
			if len(hdiffs_all)>1:
				std_dev=np.std(hdiffs_all,ddof=1)
				m=np.mean(hdiffs_all)
				#same test as above#
				prec=std_dev/np.sqrt(len(hdiffs_all))
				max_dev=self.GetMaxDev(d) #in mm
				#print max_dev,prec
				is_ok=(prec*1e3<=max_dev)
				if not is_ok:
					nbad+=1
					report=True
					l_msg+="\nForkastelseskriterie IKKE overholdt."
					l_msg+="\nTilladt fejl p\u00E5 middel: %.2f mm, aktuel fejl: %.2f mm" %(max_dev,prec*1e3)
				if len(hdiffs_all)>2:
					dh=np.fabs(hdiffs_all-m)
					outlier_limit=1.5*std_dev
					if len(hdiffs_all)==3:
						outlier_limit=1.1*std_dev
					I=np.where(np.fabs(dh)>outlier_limit)[0]
					if I.size>0:
						report=True
						l_msg+="\nOutliere:"
						for i in I:
							noutliers+=1
							if i>nforward-1:
								i-=nforward
								s=s_back
							else:
								s=s_forward
							l_msg+="\nHdiff: %.4f m, m\u00E5lt %s, journalside: %s" %(s.hdiffs[i],s.times[i].isoformat().replace("T"," "),s.jpages[i])
							hdiffs_new=np.delete(hdiffs_all,i)
							new_prec=np.std(hdiffs_new,ddof=1)/np.sqrt(len(hdiffs_new))
							l_msg+="\nFejl p\u00E5 middel: %.2f mm, fejl p\u00E5 middel uden denne m\u00E5ling: %.2f mm" %(prec*1e3,new_prec*1e3)
				if report:
					msg+="\n"+"*"*60+"\n"+l_msg

			#Finally delete that entry#
			del data[key_forward]
			if nback>0:
				del data[key_back]
		nprob=noutliers+nbad
		if nprob==0:
			return True,"Ingen problemer fundet"
		lmsg="%*s %d\n" %(-42,"#overtr\u00E6delser af forkastelseskriterie:",nbad)
		lmsg+="%*s %d\n" %(-42,"#outliere:",noutliers)
		return False,lmsg+msg

	def IsInitialized(self):
		return self.initialized
	def GetNumber(self):
		return len(self.database)
	def Disconnect(self):
		pass

def GetPlotData(program="MGL",parameter=2.0,unit="ne"):
	if program=="MGL":
		var_model=MGL_var_model
	else:
		if unit=="ne":
			var_model=MTL_var_model
		else:
			var_model=MTL_var_model_linear
	dists=np.arange(0,1500,10)
	precision=0.5*parameter  #since parameter is 'reject-parameter' and we define precison as half of dat - man :-)
	out=2*np.sqrt([var_model(x,precision) for x in dists])
	return np.column_stack((dists,out))

def GetGlobalMinLine(program="MGL"):
	dists=[0,400.0]
	hs=[GLOBAL_MIN_DEV,GLOBAL_MIN_DEV]
	return np.column_stack((dists,hs))

class Stretch(object):
	def __init__(self):
		self.hdiffs=[]
		self.dist=0
		self.times=[]
		self.jpages=[]
	def AddStretch(self,hdiff,dist,date,jpage=""):
		n=float(len(self.hdiffs))+1
		self.dist=self.dist*(n-1)/n+dist/n
		self.hdiffs.append(hdiff)
		self.times.append(date)
		self.jpages.append(jpage)

def MakeRejectData(resfiles):
	data=dict()
	nstrk=0
	nerrors=0
	for file in resfiles:
		heads=FileOps.Hoveder(file)
		for head in heads:
			try:
				key=(head[0],head[1])
				hdiff=float(head[5])
				dist=float(head[4])
				jside=head[6]
				tid=head[3]
				dato=head[2]
				m,h=Funktioner.GetTime(tid)
				day,month,year=Funktioner.GetDate(dato)
				date=datetime.datetime(year,month,day,h,m)
			except Exception as msg:
				print(repr(msg),head)
				nerrors+=1
			else:
				if key in data:
					data[key].AddStretch(hdiff,dist,date,jside)
				else:
					data[key]=Stretch()
					data[key].AddStretch(hdiff,dist,date,jside)
				nstrk+=1
	return data,nerrors


import numpy as np
import matplotlib.pyplot as plt
from cartoplot import cartoplot
import imageio
from netCDF4 import Dataset
import pickle

def get(string):
    """
                    "Lakes":0,
                    "Oceans":1,
                    "Okhotsk":2,
                    "Bering":3,
                    "Hudson":4,
                    "St Lawrence":5,
                    "Baffin":6,
                    "Greenland":7,
                    "Barents":8,
                    "Kara":9,
                    "Laptev":10,
                    "East Siberian":11,
                    "Chukchi":12,
                    "Beaufort":13,
                    "Canadian Archipelago":14,
                    "Central Arctic":15,
                    "Land":20,
                    "Coast":21}
    """
    path_grid = "/home/robbie/Dropbox/Data/grid.nc"

    if string == 'lon':
        grid_data = Dataset(path_grid)
        lon = np.array(grid_data.variables["lon"])
        return(lon)
    elif string == 'lat':
        grid_data = Dataset(path_grid)
        lat = np.array(grid_data.variables["lat"])
        return(lat)
    elif string == 'mask':
        im = imageio.imread('J_Mask.tif')
        mask = np.flipud(np.array(im))
        return(mask)

def EASE():
    """
                    "Lakes":0,
                    "Oceans":1,
                    "Okhotsk":2,
                    "Bering":3,
                    "Hudson":4,
                    "St Lawrence":5,
                    "Baffin":6,
                    "Greenland":7,
                    "Barents":8,
                    "Kara":9,
                    "Laptev":10,
                    "East Siberian":11,
                    "Chukchi":12,
                    "Beaufort":13,
                    "Canadian Archipelago":14,
                    "Central Arctic":15,
                    "Land":20,
                    "Coast":21}
    """
    mask = pickle.load( open( "/home/robbie/Dropbox/Code/mask_348x348.p", "rb" ) )
    return(mask)

def OSISAF():
    """
                    "Lakes":0,
                    "Oceans":1,
                    "Okhotsk":2,
                    "Bering":3,
                    "Hudson":4,
                    "St Lawrence":5,
                    "Baffin":6,
                    "Greenland":7,
                    "Barents":8,
                    "Kara":9,
                    "Laptev":10,
                    "East Siberian":11,
                    "Chukchi":12,
                    "Beaufort":13,
                    "Canadian Archipelago":14,
                    "Central Arctic":15,
                    "Land":20,
                    "Coast":21}
    """
    mask = pickle.load( open( "/home/robbie/custom_modules/mask_1120x760.p", "rb" ) )
    return(mask)

def plot(region_string):

    regions_dict = {"Lakes":0,
                    "Oceans":1,
                    "Okhotsk":2,
                    "Bering":3,
                    "Hudson":4,
                    "St Lawrence":5,
                    "Baffin":6,
                    "Greenland":7,
                    "Barents":8,
                    "Kara":9,
                    "Laptev":10,
                    "East Siberian":11,
                    "Chukchi":12,
                    "Beaufort":13,
                    "Canadian Archipelago":14,
                    "Central Arctic":15,
                    "Land":20,
                    "Coast":21}

    code = regions_dict[region_string]

    fig = plt.figure(figsize=(10, 8))

    cartoplot(get('lon'), get('lat'), get('mask'),color_scale=(code+1,code-1))

    print(code)

    plt.show()


import os
import random
import string

CLIENT_SECRET = os.getenv("CLIENT_SECRET")
CLIENT_ID = os.getenv("CLIENT_ID")
SCOPE = "user-library-read playlist-modify-public playlist-modify-private ugc-image-upload"
REDIRECT_URI = "http://127.0.0.1:8080/callback" if os.getenv("LOCAL_DEV") else "https://spotify-recently-liked.herokuapp.com/callback"
AUTH_URL = "https://accounts.spotify.com/api/token"
STATE = os.getenv("STATE")
API_URL = "https://api.spotify.com/v1"
FLASK_PORT = 8080
PLAYLIST_DELETE_LIMIT = 100
DATABASE_URL = os.getenv("DATABASE_URL").replace("postgres://", "postgresql://", 1)
DEFAULT_PLAYLIST_NAME = "Recently liked"


#Count the Number of Words: Write a program that counts the number of words in a string.
def count_words(string):
    # Remove leading and trailing whitespace
    string = string.strip()
    # Split the string into words
    words = string.split()
    # Return the count of words
    return len(words)

# User interface & Test the function
print("Word Count Program")
print("------------------")
while True:
    input_string = input("Enter a string (or 'q' to quit): ")
    if input_string.lower() == 'q':
        break
    word_count = count_words(input_string)
    print("The number of words in the string is:", word_count)
    print()

print("Thank you for using the Word Count Program. Goodbye!")

#new_solution:
def CountOfWord(text):
    return len([i for i in text.split() if i.isalpha()])
text=input("enter string: ")
print(CountOfWord(text))


def fib(n):
    if n < 3:
        return 1
    else:
        return fib(n - 1) + fib(n - 2)

def fast_fib(n):
    if n < 3:
        return 1
    first = 1
    second = 1
    for i in range(3, n+1):
        sum = first + second
        first = second
        second = sum

    return second


from utils import pickle_load
from matplotlib import cm
import matplotlib.pyplot as plt
import collections

def show_results(res_paths):
    results = {}
    for path in res_paths:
        result = pickle_load(path)
        for k, v in result.items():
            if k not in results.keys():
                results[k] = result[k]
    results = collections.OrderedDict(sorted(results.items()))
    fig, ax = plt.subplots(figsize=(9, 5.5))
    colors = cm.Dark2(np.linspace(0, 1, len(results)))
    count = 0
    for k, res in results.items():
        mean, std = np.nanmean(res, axis=0), np.nanstd(res, axis=0)
        # ax.errorbar(np.arange(mean.shape[0]), mean, yerr=std, color=colors[count], label=k, fmt='-o')
        plt.plot(np.arange(mean.shape[0]) + 1, mean, '-o', color=colors[count], label=k)
        count += 1
        print(np.array_str(mean[8:], precision=3))
        print("Average precision of %s for future prediction: %f" % (k, mean[8:].mean()))

    # Now add the legend with some customizations.
    legend = ax.legend(loc='upper right')

    ax.set_xlabel("time step")
    ax.set_ylabel("average precision")

    plt.axvline(x=8.5, color='r', linestyle='--')
    plt.text(3, 0.1, 'tracking', fontsize=18, color='grey')
    plt.text(11, 0.1, 'prediction', fontsize=18, color='grey')

    plt.show()

def show_best(filename, metric, k=1):

    def line_to_list(line):
        exclude_next_line = lambda x: x[:-1] if x.endswith('\n') else x
        entries = map(exclude_next_line, line.split(','))
        return entries

    items = []

    def print_dict(dic, attrs=None):

        if attrs is None:
            attrs = ['omega', 'noise_var', 'extent', metric, metric + ' mean']
            if 'keep_motion' in dic and dic['keep_motion']:
                attrs += ['window_size', 'initial_motion_factor', 'keep_motion_factor']
            if 'blur_spatially' in dic and dic['blur_spatially']:
                attrs += ['blur_extent', 'blur_var']

        for k, v in dic.items():
            if attrs is not None and k not in attrs:
                continue
            print("{}: {}".format(k, v))

    with open(filename, 'r') as f:
        line = f.readline()
        #print(line)
        attrs = line_to_list(line)

        for i, line in enumerate(f):
            #print(line)
            values = line_to_list(line)
            #print(values)
            dict_ = {k: v for (k, v) in zip(attrs, values)}
            items.append(dict_)
    #print(items[0])

    items = sorted(items, key=lambda item: item[metric + ' mean'])
    if metric == 'f1_score' or metric == 'average_precision':
        items = items[::-1]

    for i in range(k):
        print("------- {}th best ------- ".format(i+1))
        print_dict(items[i])


from __future__ import print_function
from builtins import str
from optparse import OptionParser
import sys

from opendiamond.config import DiamondConfig
from opendiamond.protocol import PORT
from opendiamond.server.server import DiamondServer

# Create option parser
# pylint: disable=invalid-name
parser = OptionParser()
attrs = set()

def add_option(*args, **kwargs):
    opt = parser.add_option(*args, **kwargs)
    attrs.add(opt.dest)

# Configure options
# dest should reflect attr names in DiamondConfig
add_option('-d', dest='daemonize', action='store_true', default=False,
           help='Run as a daemon')
add_option('-e', metavar='SPEC',
           dest='debug_filters', action='append', default=[],
           help='filter name/signature to run under debugger')
add_option('-E', metavar='COMMAND',
           dest='debug_command', action='store', default='valgrind',
           help='debug command to use with -e (default: valgrind)')
add_option('-f', dest='path',
           help='config file')
add_option('-n', dest='oneshot', action='store_true', default=False,
           help='do not fork for a new connection')
add_option('-p', dest='diamondd_port', default=PORT, help='accept new clients on port')

def run():
    opts, args = parser.parse_args()
    if args:
        parser.error('unrecognized command-line arguments')

    # Calculate DiamondConfig arguments
    kwargs = dict([(attr, getattr(opts, attr)) for attr in attrs])
    # If we are debugging, force single-threaded filter execution
    if kwargs['debug_filters']:
        kwargs['threads'] = 1

    # Create config object and server
    try:
        config = DiamondConfig(**kwargs)
        server = DiamondServer(config)
    except Exception as e:  # pylint: disable=broad-except
        print(str(e))
        sys.exit(1)

    # Run the server
    server.run()

if __name__ == '__main__':
    run()


# -*- encoding: UTF-8 -*-
from django.http import Http404
from django.db.models.loading import get_model
from django.contrib.staticfiles.storage import staticfiles_storage
from django.contrib.admin.views.decorators import staff_member_required
from django.core.urlresolvers import reverse
from django.shortcuts import render
from django.forms.widgets import Select
import models

def detail(request, app, cls, slug):
    """ generic view that return direct CMS model rendered content """
    model = get_model(app, cls)
    if model and issubclass(model, models.CMSModel):
        return model.get_response(request, slug)
    raise Http404

@staff_member_required
def imagechooser(request, app, cls):
    model = get_model(app, cls)
    datas = {
        'tinymce_path': staticfiles_storage.url('tiny_mce'),
        'chosen_path': staticfiles_storage.url('chosen'),
        'admin_path': staticfiles_storage.url('admin')
    }
    if model and issubclass(model, models.CMSModel):
        if getattr(model.CMSMeta, 'image_model', None):
            images = [('', '----')]
            for fileitem in model.CMSMeta.image_model.objects.all().order_by('title'):
                if fileitem.file.name.lower().endswith(('.jpg', '.jpeg', '.gif', '.png')):
                    images.append((fileitem.get_absolute_url(), fileitem.title))
            datas['select_files'] = Select(choices=images, attrs={'class': 'chosen-single', 'style': 'width:200px'}).render('file', '')
            #datas['form_upload'] = None
    # gestion upload if any
    # send result back
    return render(request, 'imagechooser.html', datas)

@staff_member_required
def tinymcejs(request, app, cls):
    datas = {
        'tinymce_path': staticfiles_storage.url('tiny_mce'),
        'imagechooser_path': reverse('picocms-imagechooser', args=(app, cls))
    }
    return render(request, 'tiny_mce_src.js', datas, content_type='application/javascript')


import tensorflow as tf
import tensorflow_datasets as tfds

config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=config)

def load_celeba_dataset(args, shuffle_files=False, batch_size=128):
  ds_train, ds_test = tfds.load(name='celeb_a', split=['train', 'test'], data_dir=args.data_dir,
      batch_size=batch_size, download=True, shuffle_files=shuffle_files)
  return ds_train, ds_test


import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn import datasets
from sklearn import linear_model
import matplotlib.pyplot as plt
def sigmoid(z):
    return 1/(1+np.exp(-z))
def costfunction(X, y, w):
    cost = 0
    size = y.shape[0]
    for i in range(size):
        if y[i] == 1:
            cost -= np.log(sigmoid(X[i]*w))
        else:
            cost -= np.log(1 - sigmoid(X[i]*w))
    return cost / size
def gradAscent(traindata,label,iter,alpha,step,lamda=0.001):
    dataMat=np.mat(traindata)
    labelMat=np.mat(label)
    m,n=np.shape(dataMat)
    weights=np.ones((n,1))
    weights=np.mat(weights)
    for k in range(iter):
        temp=costfunction(dataMat,labelMat,weights)
        weights=weights-alpha*((dataMat.transpose())*(sigmoid(dataMat*weights)-labelMat)+lamda*weights)
        if k%200==0:
            print("Loss is: ",temp,weights.transpose())
        if (k/step==0 and k!=0):
            alpha=alpha/5
    return weights
def preprocessing(x_train,x_test):
    sc=StandardScaler()
    sc.fit(x_train)
    x_train_scaled=sc.transform(x_train)
    x_test_scaled=sc.transform(x_test)
    return x_train_scaled,x_test_scaled
def split(ratio):
    Data = datasets.load_iris()
    #Data = datasets.load_wine()  #for Dataset wine
    x = Data.data
    y=Data.target
    x_train, x_test, y_train, y_test = train_test_split(x,y,test_size = ratio, random_state = 0)
    return x_train,x_test,y_train,y_test
def plot(X,Y):
    x_min, x_max = X[:, 0].min() - .2, X[:, 0].max() + .2
    y_min, y_max = X[:, 1].min() - .2, X[:, 1].max() + .2
    h = .02
    logreg =linear_model.LogisticRegression(C=1e5, solver='lbfgs', multi_class='multinomial')
    logreg.fit(X,Y)
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
    Z = logreg.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    plt.pcolormesh(xx, yy, Z, cmap=plt.cm.Paired)
    plt.scatter(X[:, 0], X[:, 1], c=Y, edgecolors='k', cmap=plt.cm.Paired)
    plt.xlabel('Sepal length')
    plt.ylabel('Sepal width')
    plt.show()
if __name__=='__main__':
    x_train,x_test,y_train,y_test=split(0.3)
    x_train_scaled,x_test_scaled=preprocessing(x_train,x_test)
    #logreg=linear_model.LogisticRegression(C=1e4) #for ovr
    logreg=linear_model.LogisticRegression(C=1e4,multi_class='multinomial',solver='lbfgs') #ovm
    logreg.fit(x_train_scaled,y_train)
    print("Accuracy:",logreg.score(x_test_scaled,y_test))
    plot(x_train_scaled[:,:2],y_train)


# given an integer array nums, handle multiple queries of the
# following type: calculate the sum of the elements of nums
# between indices left and right inclusive where left <= right

class prefix_sum:
    def __init__(self,arr):
        self.arr = arr
        prefix = []
        total = 0
        for i in range(len(arr)):
            total += arr[i]
            prefix.append(total)
        self.prefix = prefix

    def range_sum(self,left,right):
        if left-1<0:
            left_val = 0
        else:
            left_val = self.prefix[left-1]
        right_val = self.prefix[right]
        return right_val-left_val


import json
import logging
import os
import random
import time
from datetime import datetime
from uuid import uuid4

import paho.mqtt.client as mqtt

# MQTT broker details
BROKER_ADDRESS = os.getenv("BROKER_HOST")
BROKER_PORT = 1883

# Configuring file handler for logging
log_file = f"{__file__}.log"
# Logging setup
logging.basicConfig(
    filename=log_file,
    filemode="w",
    format="%(asctime)s - %(levelname)s - %(message)s",
    level=logging.INFO,
)
logger = logging.getLogger(__name__)

# Creating unique sensor IDs for each sensor
temp_sensor_id = str(uuid4())
hum_sensor_id = str(uuid4())

# Simulated sensor data generation for temperature
def generate_temperature_data() -> dict:
    """
    Generate random temperature data.

    Returns:
        dict: Generated sensor data.
    """
    temperature = round(20 + (30 * random.random()), 2)
    timestamp = datetime.utcnow().isoformat()  # ISO8601 format

    data = {
        "sensor_id": temp_sensor_id,
        "topic": "temperature",
        "value": temperature,
        "timestamp": timestamp,
    }
    return data

def generate_humidity_data() -> dict:
    """
    Generate random humidity data.

    Returns:
        dict: Generated sensor data.
    """
    humidity = round(40 + (60 * random.random()), 2)
    timestamp = datetime.utcnow().isoformat()

    data = {
        "sensor_id": hum_sensor_id,
        "topic": "humidity",
        "value": humidity,
        "timestamp": timestamp,
    }
    return data

def on_publish(client, userdata, mid):
    """
    MQTT on_publish callback function.

    Args:
        client: The MQTT client instance.
        userdata: User data.
        mid: Message ID.
    """
    logger.info(f"Message Published: {mid}")

def on_connect(client, userdata, flags, rc):
    """
    MQTT on_connect callback function.

    Args:
        client: The MQTT client instance.
        userdata: User data.
        flags: Flags.
        rc: Return code.
    """
    if rc == 0:
        logger.info("Connected to Mosquitto MQTT Broker!")
    else:
        logger.error(f"Failed to connect, return code: {rc}")

# Create MQTT client instance
client = mqtt.Client()
client.on_connect = on_connect
client.on_publish = on_publish

# Connect to broker
client.connect(BROKER_ADDRESS, port=BROKER_PORT)

# Start the MQTT loop
client.loop_start()

try:
    while True:
        sensor_data_temp = generate_temperature_data()
        sensor_data_hum = generate_humidity_data()
        temperature_payload = json.dumps(sensor_data_temp)
        humidity_payload = json.dumps(sensor_data_hum)

        # Publishing the topics
        client.publish("sensors/temperature", temperature_payload)
        client.publish("sensors/humidity", humidity_payload)

        time.sleep(15)  # Publish every 5 seconds
except KeyboardInterrupt:
    logger.info("Publisher stopped.")
    client.loop_stop()
    client.disconnect()


from fastapi import APIRouter
from pydantic import BaseModel
from starlette.requests import Request

from ozz_backend import app_logger
from ozz_backend.persistence_layer import User

router = APIRouter(
    prefix="/user",
    tags=["user"],
    # dependencies=[Depends(get_token_header)],
)

class UserOngoingOut(BaseModel):
    user_id: str
    mission_id: int
    quest_id: int

@router.get('/test')
def test_api():
    app_logger.info('test')
    return {'test'}

@router.get('/user-ongoing', response_model=UserOngoingOut)
def get_ongoing_info(request: Request, user_id: int, mission_id: int):
    app_logger.info(f'[{request.method}] {request.url}: {request.client.host}:{request.client.port}')
    result = User.get_user_ongoing_info(user_id, mission_id)
    ongoing = UserOngoingOut(user_id=result.user_id, mission_id=result.mission_id, quest_id=result.quest_id)
    return ongoing


import argparse
import json
EXAMPLE_USAGE = """
Example Usage via RLlib CLI:
    rllib rollout /tmp/ray/checkpoint_dir/checkpoint-0 --run DQN
    --env CartPole-v0 --steps 1000000 --out rollouts.pkl

Example Usage via executable:
    ./rollout.py /tmp/ray/checkpoint_dir/checkpoint-0 --run DQN
    --env CartPole-v0 --steps 1000000 --out rollouts.pkl
"""

def create_parser(parser_creator = None):
    #parser = argparse.ArgumentParser("Ray training with custom IG environment")

    ## parser for rollouts
    parser_creator = parser_creator or argparse.ArgumentParser
    parser = parser_creator(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="Roll out a reinforcement learning agent "
                    "given a checkpoint.",
        epilog=EXAMPLE_USAGE)

    parser.add_argument(
        "--checkpoint", default='' ,type=str, help="Checkpoint from which to roll out.")
    required_named = parser.add_argument_group("required named arguments")
    required_named.add_argument(
        "--run",
        type=str,
        required=True,
        help="The algorithm or model to train. This may refer to the name "
             "of a built-on algorithm (e.g. RLLib's DQN or PPO), or a "
             "user-defined trainable function or class registered in the "
             "tune registry.")
    required_named.add_argument(
        "--env", type=str, help="The gym environment to use.")
    parser.add_argument(
        "--no-render",
        default=False,
        action="store_const",
        const=True,
        help="Suppress rendering of the environment.")
    parser.add_argument(
        "--monitor",
        default=False,
        action="store_true",
        help="Wrap environment in gym Monitor to record video. NOTE: This "
             "option is deprecated: Use `--video-dir [some dir]` instead.")
    parser.add_argument(
        "--video-dir",
        type=str,
        default=None,
        help="Specifies the directory into which videos of all episode "
             "rollouts will be stored.")
    parser.add_argument(
        "--steps",
        default=20000,
        help="Number of timesteps to roll out (overwritten by --episodes).")
    parser.add_argument(
        "--episodes",
        default=0,
        help="Number of complete episodes to roll out (overrides --steps).")
    parser.add_argument("--out", default=None, help="Output filename.")
    parser.add_argument(
        "--config",
        default="{}",
        type=json.loads,
        help="Algorithm-specific configuration (e.g. env, hyperparams). "
             "Gets merged with loaded configuration from checkpoint file and "
             "`evaluation_config` settings therein.")
    parser.add_argument(
        "--save-info",
        default=False,
        action="store_true",
        help="Save the info field generated by the step() method, "
             "as well as the action, observations, rewards and done fields.")
    parser.add_argument(
        "--use-shelve",
        default=False,
        action="store_true",
        help="Save rollouts into a python shelf file (will save each episode "
             "as it is generated). An output filename must be set using --out.")
    parser.add_argument(
        "--track-progress",
        default=False,
        action="store_true",
        help="Write progress to a temporary file (updated "
             "after each episode). An output filename must be set using --out; "
             "the progress file will live in the same folder.")

    # save and restore file management
    parser.add_argument(
        "--policy-dir", type=str, help="folder name of the policy.", default="")
    parser.add_argument(
        "--experiment", type=str, help="chosen experiment to reload.", default="")
    parser.add_argument(
        "--ncheckpoint", type=str, help="chosen checkpoint to reload.", default="")

    parser.add_argument(
        "--heuristic-policy", type=bool, help="chosen checkpoint to reload.", default=False)

    parser.add_argument(
        "--static-targets", type=bool, help="chosen checkpoint to reload.", default=False)

    parser.add_argument(
        "--video_dir", type=str, help="chosen folder to save video.", default="")

    parser.add_argument(
        "--horizon", type=int, help="limit of timesteps.", default=40)

    ### Old arguments needs a cleanup

    parser.add_argument("--scenario", type=str, default="simple_spread_assigned",
                        choices=['simple', 'simple_speaker_listener',
                                 'simple_crypto', 'simple_push',
                                 'simple_tag', 'simple_spread', 'simple_adversary', 'simple_spread_assigned',
                                 'matlab_simple_spread_assigned','matlab_simple_spread_assigned_hardcoll', 'matlab_simple_spread_assigned_checkpoints'],
                        help="name of the scenario script")
    parser.add_argument("--max-episode-len", type=int, default=100,
                        help="maximum episode length")
    parser.add_argument("--num-episodes", type=int, default=60000,
                        help="number of episodes")
    parser.add_argument("--num-adversaries", type=int, default=0,
                        help="number of adversaries")
    parser.add_argument("--good-policy", type=str, default="maddpg",
                        help="policy for good agents")
    parser.add_argument("--adv-policy", type=str, default="maddpg",
                        help="policy of adversaries")

    # Core training parameters
    parser.add_argument("--lr", type=float, default=1e-3,
                        help="learning rate for Adam optimizer")
    parser.add_argument("--gamma", type=float, default=0.99,
                        help="discount factor")
    # NOTE: 1 iteration = sample_batch_size * num_workers timesteps * num_envs_per_worker
    parser.add_argument("--sample-batch-size", type=int, default=25,
                        help="number of data points sampled /update /worker")
    parser.add_argument("--train-batch-size", type=int, default=1024,
                        help="number of data points /update")
    parser.add_argument("--n-step", type=int, default=1,
                        help="length of multistep value backup")
    parser.add_argument("--num-units", type=int, default=128,
                        help="number of units in the mlp")
    parser.add_argument("--replay-buffer", type=int, default=1000000,
                        help="size of replay buffer in training")
    parser.add_argument("--seed", type=int, default=100,
                        help="initialization seed for the network weights")

    # Checkpoint
    parser.add_argument("--checkpoint-freq", type=int, default = 10, #75,
                        help="save model once every time this many iterations are completed")
    parser.add_argument("--local-dir", type=str, default="./ray_results",
                        help="path to save checkpoints")
    parser.add_argument("--restore", type=str, default=None,
                        help="directory in which training state and model are loaded")
    parser.add_argument("--in-evaluation", type=bool, default=False, help="trigger evaluation procedure")

    # Parallelism
    #parser.add_argument("--num-workers", type=int, default=0)
    #parser.add_argument("--num-envs-per-worker", type=int, default=1)
    #parser.add_argument("--num-gpus", type=int, default=0)

    parser.add_argument("--num-workers", type=int, default=0)  #0
    parser.add_argument("--num-envs-per-worker", type=int, default=1)  #1
    parser.add_argument("--num-gpus", type=int, default=0)  #0
    #parser.add_argument("--num-cpus-per-worker", type=int, default=1)
    parser.add_argument("--num-gpus-per-worker", type=int, default=0)  #0

    # From the ppo
    parser.add_argument("--stop-iters", type=int, default=100)
    parser.add_argument("--stop-timesteps", type=int, default=160000000)
    # parser.add_argument("--stop-reward", type=float, default=7.99)

    # For rollouts
    parser.add_argument("--stop-iters-rollout", type=int, default=1)
    parser.add_argument("--nagents", type=int, default=1)
    parser.add_argument("--ntargets", type=int, default=1)
    parser.add_argument("--nrobots", type=int, default=1)

    # mode of hand-engineered comm. policy (-1 no hand-engineered)
    parser.add_argument("--mode", type=int, default=-1)
    parser.add_argument("--test", type=int, default=0, choices = [0,1], help="whether we want to test the policy or not")
    parser.add_argument("--test-env", type=int, default=0, choices = [0,1], help="whether we want to act in the test environment or not")
    parser.add_argument("--deterministic", type=int, default=1, choices=[0, 1],
                        help="enable exploration or not during execution")

    return parser


from atelier_4_ex1 import gen_list_random_int

import matplotlib.pyplot as plt
import numpy as np
import time ,random

def extract_elements_list(list_in_which_to_choose,int_nbr_of_element_to_extract=10):
     list_in_which_to_choose_length,mix_length  = len(list_in_which_to_choose),0
     mixList = list()
     while mix_length < int_nbr_of_element_to_extract :
        random_ = gen_list_random_int(0,list_in_which_to_choose_length)
        if random_ not in mixList :
            mixList.append(random_)
            mix_length += 1
        else :
            continue

     return [ list_in_which_to_choose[elem] for elem in mixList ]

# Test de votre code

# def extract_elements_list2(list_in_which_to_choose,int_nbr_of_element_to_extract=10):
#      list_in_which_to_choose_length,mix_length  = len(list_in_which_to_choose),0
#      mixList = list()
#      while mix_length < int_nbr_of_element_to_extract :
#         random_ = gen_list_random_int(0,list_in_which_to_choose_length)
#         mixList.append(random_)
#         mix_length += 1

#      return [ list_in_which_to_choose[elem] for elem in mixList ]

# print(extract_elements_list( [ i for i in range(1,11)],4))

def pref_mix(func1,func2,lst,num=100):

   result = ([],[])

   for elem in lst :
    data1 ,data2= [],[]
    nb_elements = int(elem / 2)
    for index in range(num) :
      lst_elem = list(range(elem))
      # first function
      start = time.perf_counter()
      func1(lst_elem,nb_elements)
      end = time.perf_counter() - start
      data1.append(end)

      start = time.perf_counter()
      func2(lst_elem,nb_elements)
      end = time.perf_counter() - start
      data2.append(end)
    result[0].append(sum(data1)/len(data1))
    result[1].append(sum(data2)/len(data2))
   return result

list_test = [500,1000,2500,5000,7500]

result = pref_mix(extract_elements_list,random.sample, list_test ,100)

print(result)

#Ici on décrit les abscisses
#Entre 0 et 5 en 10 points
fig, ax = plt.subplots()
#Dessin des courbes, le premier paramètre
#correspond aux point d'abscisse le
#deuxième correspond aux points d'ordonnées
#le troisième paramètre, optionnel permet de
#choisir éventuellement la couleur et le marqueur

ax.plot(list_test,result[0], 'bo-',label='extract_elements_list')
ax.plot(list_test,result[1], 'r*-',label='random.sample')

ax.set(xlabel='temps', ylabel='nombre d\'elements',
 title='temps d’exécution moyen pour extract_elements_list et random.sample')
ax.legend(loc='upper center', shadow=True, fontsize='x-large')
#fig.savefig("test.png")
plt.show()


from netCDF4 import Dataset
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
import argodb as argo
import research_tools as research

plt.ion()
plt.close('all')

dirtopo = '/datawork/fsi2/mars/DATA/BATHY/ETOPO2'
topofile = 'etopo2.nc'

dirtopo = '/net/alpha/exports/sciences/data/BATHYMETRIE/BATHYMETRIE'
topofile = 'ETOPO2v2c_f4.nc'
dirtile = '/net/libra/local/tmp/1/herry/tiles'

itile = 50

figsize = (9, 7)

reso = 0.5

argodic = research.read_argo_filter(itile)
minlon, maxlon, minlat, maxlat = argodic['LONMIN_NO_M'], argodic['LONMAX_NO_M'], argodic['LATMIN_NO_M'], argodic['LATMAX_NO_M']

lon = np.arange(reso*np.floor(minlon/reso), reso*np.floor(maxlon/reso)+reso, reso)
lat = np.arange(reso*np.floor(minlat/reso), reso*np.floor(maxlat/reso)+reso, reso)

#lon_deg, lat_deg = define_grid(minlon, maxlon, minlat, maxlat, reso_deg)

with Dataset('%s/%s' % (dirtopo, topofile)) as nc:
    z = nc.variables['z'][:,:]

dl0 = 1/30.  # 1/30deg for etopo
lontopo = np.arange(-180, 180+dl0, dl0)
lattopo = np.arange(-90, 90+dl0, dl0)

def get_idx_of_box(lontopo, lattopo, cell):
    minlon, maxlon, minlat, maxlat = cell
    ilon = [i for i, x in enumerate(lontopo) if (x>=minlon) and (x<=maxlon)]
    jlon = [j for j, x in enumerate(lattopo) if (x>=minlat) and (x<=maxlat)]
    return ilon[0], ilon[-1], jlon[0], jlon[-1]

domain = [minlon, maxlon, minlat, maxlat]
i0, i1, j0, j1 = get_idx_of_box(lontopo, lattopo, domain)

def average_topo_on_box(depth, cell):
    """ average high resolution depth array on cell """
    i0, i1, j0, j1 = get_idx_of_box(lontopo, lattopo, cell)
    return np.mean(depth[j0:j1, i0:i1].ravel())

def box(cell, d=0):
    x1, x2, y1, y2 = cell
    plt.plot([x1-d, x1-d, x2+d, x2+d, x1-d],
           [y1-d, y2+d, y2+d, y1-d, y1-d], 'k')

plt.figure(figsize=figsize)
plt.imshow(z[j0:j1, i0:i1],
           origin='lower', extent=[minlon, maxlon, minlat, maxlat])
plt.axis('tight')
plt.colorbar()

reso = 0.5
lon = np.arange(minlon, maxlon, reso)
lat = np.arange(minlat, maxlat, reso)

nlon = len(lon)
nlat = len(lat)
bathy = np.zeros((nlat, nlon))
for j in range(nlat-1):
    for i in range(nlon-1):
        reso2 = reso*0.5
        gridcell = [lon[i]-reso2, lon[i]+reso2, lat[j]-reso2, lat[j]+reso2]
        box(gridcell)
        get_idx_of_box(lontopo, lattopo, gridcell)
        bathy[j, i] = average_topo_on_box(z, gridcell)

msk = bathy < 0

fig, ax = plt.subplots(2,1)

divider = make_axes_locatable(ax[0])
ax_cb = divider.new_horizontal(size="4%", pad=0.2)
im = ax[0].imshow(bathy,
           origin='lower', interpolation='nearest',
           extent=[minlon, maxlon, minlat, maxlat])
ax[0].set_title('tile #%03i' % itile)
fig.add_axes(ax_cb)
fig.colorbar(im, cax=ax_cb)

divider = make_axes_locatable(ax[1])
ax_cb = divider.new_horizontal(size="4%", pad=0.2)
ax[1].imshow(msk,
           origin='lower', interpolation='nearest',
           extent=[minlon, maxlon, minlat, maxlat])


# Author: Sirui Feng
'''
This file splits each review on periods and conjuctions.
'''

import re
import json
from textblob import TextBlob
from textblob.sentiments import NaiveBayesAnalyzer
import csv
from word_stemmer import word_stemmer

public_utilities_path = 'data/public_utilities.json'

def split_period(review):
	'''
	Splits sentences on periods.
	'''
	p = re.compile(r'[^\s\.][^\.\n]+')
	sentences = p.findall(review)
	return sentences

def split_conjunctions(sentence):
	'''
	Splits each sentence on conjuctions.
	'''
	conjuctions = [';', 'for', 'and', 'nor', 'but', 'or', 'yet', 'so']
	clause = re.split('; | and | nor | but | or | yet | so | although | despite | though | however | on the other hand | in contrast ', sentence)
	clause = [x.strip() for x in clause]
	clause = [x for x in clause if len(x) != 0]
	return clause

def gen_sentences():
	'''
	Reads in the sentences and splits on periods and conjuctions.
	'''
	with open(public_utilities_path) as datafile:
		with open('data/full_data.csv', 'w') as outfile:
			writer = csv.DictWriter(outfile, fieldnames = ['review_id', \
				'business_id', 'user_id', 'stars', 'blob_polarity', 'review', \
				'label'])
			writer.writeheader()
			i=0

			for line in datafile:
				i += 1
				print(i)
				row = json.loads(line)
				review = row['text']
				review = review.lower()

				#split only on periods
				sentences = split_period(review)
				for s in sentences:
					blob = TextBlob(s, analyzer = NaiveBayesAnalyzer())
					polarity = blob.polarity
					#s = word_stemmer(s)

					writer.writerow({'review_id':row['review_id'], \
						'business_id': row['business_id'], \
						'user_id':row['user_id'], 'stars':row['stars'], \
						'blob_polarity': polarity, 'review': s})

gen_sentences()


import imaplib
import socket

class IMAP4WithTimeout(imaplib.IMAP4):
    def __init__(self, address, port, timeout):
        self._timeout = timeout
        imaplib.IMAP4.__init__(self, address, port)

    def open(self, host="", port=143, timeout=None):
        # This is overridden to make it consistent across Python versions.
        self.host = host
        self.port = port
        self.sock = self._create_socket(timeout)
        self.file = self.sock.makefile("rb")

    def _create_socket(self, timeout=None):
        return socket.create_connection(
            (self.host, self.port), timeout if timeout is not None else self._timeout
        )


import tkinter, threading
from tkinter import ttk
from interface.onglets.onglets_map import OngletsMap
from interface.onglets.onglets_packets import OngletsPackets
from interface.onglets.onglets_personnage import OngletsPersonnage
from interface.onglets.onglets_sorts import OngletsSorts
import time

class MainInterface(threading.Thread):

    def __init__(self):
        threading.Thread(None,self.launch).start()
        while True:
            time.sleep(1)
            if self.ongletsSorts:
                break

    def set_character(self, character):
        self.character = character
        self.ongletsMap.set_character(character)
        self.ongletsSorts.set_character(character)
        self.ongletsPersonnage.set_character(character)
        threading.Thread(None,self.character_statue).start()

    def character_statue(self):
        en_mouvement = tkinter.Label(self.main, bg="red", text = "En mouvement")
        en_mouvement.place(relx=0.05, rely=0.05, relwidth=0.08, relheight=0.04)

        en_recolte = tkinter.Label(self.main, bg="red", text = "En recolte")
        en_recolte.place(relx=0.05, rely=0.10, relwidth=0.08, relheight=0.04)

        en_combat = tkinter.Label(self.main, bg="red", text = "En combat")
        en_combat.place(relx=0.05, rely=0.15, relwidth=0.08, relheight=0.04)
        while True:
            time.sleep(1)
            if self.character.deplacement.ismouving:
                en_mouvement.configure(bg = "Green")
            else:
                en_mouvement.configure(bg = "Red")
            if self.character.isharvest:
                en_recolte.configure(bg = "Green")
            else:
                en_recolte.configure(bg = "red")
            if self.character.isfighting:
                en_combat.configure(bg = "Green")
            else:
                en_combat.configure(bg = "red")

    def launch(self):
        self.main = tkinter.Tk()
        self.main.title("LeafBot")
        self.main.geometry('1200x900')
        self.create_notebook()
        self.main.mainloop()

    def create_notebook(self):
        self.onglets = tkinter.ttk.Notebook(self.main)
        self.onglets.pack()
        self.onglets.place(relx=0.15, rely=0.05, relwidth=0.83, relheight=0.83)

        self.ongletsPackets = OngletsPackets(self.onglets)
        self.ongletsPersonnage = OngletsPersonnage(self.onglets)
        self.ongletsMap = OngletsMap(self.onglets)
        self.ongletsSorts = OngletsSorts(self.onglets)

    def base_start(self,character):
        self.vita = tkinter.Label(self.main, bg="red", text = character.vie_actuelle +" / " + character.vie_max)
        self.vita.pack()
        self.vita.place(relx=0.20, rely=0.90, relwidth=0.08, relheight=0.08)

        self.energie = tkinter.Label(self.main, bg="yellow", text = character.ennergie_actuelle +" / " + character.ennergie_max)
        self.energie.pack()
        self.energie.place(relx=0.40, rely=0.90, relwidth=0.08, relheight=0.08)

        self.xp = tkinter.Label(self.main,bg="deep sky blue", text = character.xp_actuelle +" / " + character.xp_fin)
        self.xp.pack()
        self.xp.place(relx=0.60, rely=0.90, relwidth=0.1, relheight=0.08)

        self.kamas = tkinter.Label(self.main, bg="orange", text = character.kamas)
        self.kamas.pack()
        self.kamas.place(relx=0.80, rely=0.90, relwidth=0.08, relheight=0.08)

if __name__ == "__main__":
    MainInterface()


# Definir una función inversa() que calcule la inversión de una cadena. Por ejemplo la cadena "estoy probando" debería devolver la cadena "odnaborp yotse".

def inversa(cad1):
    cad2 = ""
    for i in range(1, len(cad1)+1):   #arranco en 1 porque no existe el -0
        cad2 += cad1[-i]
    return cad2

assert(inversa('Hola como estas') == 'satse omoc aloH')
assert(inversa('123456789') == '987654321')


# https://atcoder.jp/contests/past202004-open/tasks/past202004_h
N, M = list(map(int, input().split()))
A = []
for _ in range(N):
    A.append(input())

group = []
for _ in range(11):
    group.append([])

for i in range(N):
    for j in range(M):
        if A[i][j] == 'S':
            n = 0
        elif A[i][j] == 'G':
            n = 10
        else:
            n = int(A[i][j])
        group[n].append([i ,j])

# cost[i][j]:(i,j)にたどり着くまで最小移動回数の総和
cost = []
INF = 10**3
for i in range(N):
    cost.append([INF]*M)

# 初期条件
si, sj = group[0][0]
cost[si][sj] = 0

for n in range(1, 11):
    for i, j in group[n]:
        for i2, j2 in group[n-1]:
            cost[i][j] = min(cost[i][j], cost[i2][j2]+abs(i-i2)+abs(j-j2))

gi, gj = group[10][0]
if cost[gi][gj] ==  INF:
    print(-1)
else:
    print(cost[gi][gj])