from collections import defaultdict
import Id3_dubbed
import copy

def get_valid_examples(attributes):
    examples = defaultdict(list)
    loop = 0
    with open("../Top5000Words/training-set.feat", 'r') as file:
        for line in file:
            # print(line)
            loop = loop + 1
            words = line.split(' ')
            # print(words[0:1])
            comb = words[1:]
            comb = [int(i.split(':', 1)[0]) for i in comb]
            # print(comb)
            # attributes.sort()
            # print(attributes)
            subt = [x for x in comb if x in attributes]
            # print(subt)
            attr = words[0:1] + subt
            # print(words[0:1][0])
            # print(attr)
            for i in range(len(attr)):
                examples[loop].append(attr[i])
            if int(words[0:1][0]) >= 7:
                examples[loop].append('y')
            else:
                examples[loop].append('n')
    for k, v in examples.items():
        examples[k] = examples[k][1:]
    # print(examples)
    return examples

def accuracy(test,node):
    accurate = 0
    sizes = len(test)
    for k,v in test.items():
        temp = copy.copy(node)
        while temp.feature != -1:
            if temp.feature in v:
                temp = temp.right
            else:
                temp = temp.left
        if v[len(v)-1] == 'y':
            if temp.sample == 1:
                accurate = accurate+1
        else:
            if temp.sample == 0:
                accurate = accurate+1
    # print(accurate)
    return accurate/sizes

def accuracy2(test,node, node1, node2):
    accurate = 0
    sizes = len(test)
    for k,v in test.items():
        temp = copy.copy(node)
        temp1 = copy.copy(node1)
        temp2 = copy.copy(node2)
        while temp.feature != -1:
            if temp.feature in v:
                temp = temp.right
            else:
                temp = temp.left
        while temp1.feature != -1:
            if temp1.feature in v:
                temp1 = temp1.right
            else:
                temp1 = temp1.left
        while temp2.feature != -1:
            if temp2.feature in v:
                temp2 = temp2.right
            else:
                temp2 = temp2.left
        count1 = 0
        count2 = 0
        if temp.sample == 1:
            count1 = count1+1
        else:
            count2 = count2+1
        if temp1.sample == 1:
            count1 = count1 + 1
        else:
            count2 = count2 + 1
        if temp2.sample == 1:
            count1 = count1 + 1
        else:
            count2 = count2 + 1

        if count1 >= count2:
            temp.sample = 1
        else:
            temp.sample = 0
        if v[len(v)-1] == 'y':
            if temp.sample == 1:
                accurate = accurate+1
        else:
            if temp.sample == 0:
                accurate = accurate+1
    # print(accurate)
    return accurate/sizes

def accuracy3(test,node, node1, node2, node3, node4):
    accurate = 0
    sizes = len(test)
    for k,v in test.items():
        temp = copy.copy(node)
        temp1 = copy.copy(node1)
        temp2 = copy.copy(node2)
        temp3 = copy.copy(node3)
        temp4 = copy.copy(node4)
        while temp.feature != -1:
            if temp.feature in v:
                temp = temp.right
            else:
                temp = temp.left
        while temp1.feature != -1:
            if temp1.feature in v:
                temp1 = temp1.right
            else:
                temp1 = temp1.left
        while temp2.feature != -1:
            if temp2.feature in v:
                temp2 = temp2.right
            else:
                temp2 = temp2.left
        while temp3.feature != -1:
            if temp3.feature in v:
                temp3 = temp3.right
            else:
                temp3 = temp3.left

        while temp4.feature != -1:
            if temp4.feature in v:
                temp4 = temp4.right
            else:
                temp4 = temp4.left
        count1 = 0
        count2 = 0
        if temp.sample == 1:
            count1 = count1+1
        else:
            count2 = count2+1
        if temp1.sample == 1:
            count1 = count1 + 1
        else:
            count2 = count2 + 1
        if temp2.sample == 1:
            count1 = count1 + 1
        else:
            count2 = count2 + 1
        if temp3.sample == 1:
            count1 = count1 + 1
        else:
            count2 = count2 + 1
        if temp4.sample == 1:
            count1 = count1 + 1
        else:
            count2 = count2 + 1

        if count1 >= count2:
            temp.sample = 1
        else:
            temp.sample = 0
        if v[len(v)-1] == 'y':
            if temp.sample == 1:
                accurate = accurate+1
        else:
            if temp.sample == 0:
                accurate = accurate+1
    # print(accurate)
    return accurate/sizes

def main(test_file):
    loop = 0
    lst = []
    with open("../Top5000Words/selected-features-indices.txt", 'r') as file:
        for line in file:
            lst.append([int(x) for x in line.split()])
        attributes = [x[0] for x in lst]
    attributes2_1 = []
    attributes2_1[0:1250] = attributes[0:1250]
    attributes2_1[1250:2500] = attributes[2500:3750]
    attributes2_2 = []
    attributes2_2[0:1250] = attributes[1250:2500]
    attributes2_2[1250:2500] = attributes[3750:5000]
    attributes2_3 = []
    attributes2_3[0:1250] = attributes[0:1250]
    attributes2_3[1250:2500] = attributes[3750:5000]

    attributes3_1 = []
    attributes3_1[0:500] = attributes[0:500]
    attributes3_1[500:1000] = attributes[2500:3000]
    attributes3_2 = []
    attributes3_2[0:500] = attributes[500:1000]
    attributes3_2[500:1000] = attributes[3000:3500]
    attributes3_3 = []
    attributes3_3[0:500] = attributes[1000:1500]
    attributes2_1[500:1000] = attributes[3500:4000]
    attributes3_4 = []
    attributes3_4[0:500] = attributes[1500:2000]
    attributes3_4[500:1000] = attributes[4000:4500]
    attributes3_5 = []
    attributes3_5[0:500] = attributes[2000:2500]
    attributes3_5[500:1000] = attributes[4500:5000]
    # print(attributes2_1)

    examples = get_valid_examples(attributes)
    examples2_1 = get_valid_examples(attributes2_1)
    examples2_2 = get_valid_examples(attributes2_2)
    examples2_3 = get_valid_examples(attributes2_3)
    examples3_1 = get_valid_examples(attributes3_1)
    examples3_2 = get_valid_examples(attributes3_2)
    examples3_3 = get_valid_examples(attributes3_3)
    examples3_4 = get_valid_examples(attributes3_4)
    examples3_5 = get_valid_examples(attributes3_5)
    testExamples = defaultdict(list)
    loop = 0
    with open(test_file, 'r') as file:
        for line in file:
            loop = loop + 1
            words = line.split(' ')
            # print(words[0:1])
            comb = words[1:]
            comb = [int(i.split(':', 1)[0]) for i in comb]
            subt = [x for x in comb if x in attributes]
            # print(subt)
            attr = words[0:1] + subt
            # print(words[0:1][0])
            # print(attr)
            for i in range(len(attr)):
                testExamples[loop].append(attr[i])
            if int(words[0:1][0]) >= 7:
                testExamples[loop].append('y')
            else:
                testExamples[loop].append('n')
    for k, v in testExamples.items():
        testExamples[k] = testExamples[k][1:]
    # print(testExamples)

    # Run ID3
    print("1 decision tree of 5k attributes")
    node = Id3_dubbed.decisionTree(examples, attributes, 0)
    # temp = copy.copy(node)
    # print("No. of nodes",tree_size(temp))
    print("generated decision tree")
    # print("Accuracy on test set is ", accuracy(testExamples, node)*100)
    # print("Accuracy on training set is ", accuracy(examples, node)*100)
    print("1    ",accuracy(examples, node)*100,"    ", accuracy(testExamples, node)*100)
    print("3 decision trees of 2.5k attributes each")
    node = Id3_dubbed.decisionTree(examples2_1, attributes2_1, 0)
    node1 = Id3_dubbed.decisionTree(examples2_2, attributes2_2, 0)
    node2 = Id3_dubbed.decisionTree(examples2_3, attributes2_3, 0)
    # temp = copy.copy(node)
    # print("No. of nodes", tree_size(temp))
    print("generated decision tree")
    # print("Accuracy on test set is ", accuracy2(testExamples, node, node1, node2)*100)
    # print("Accuracy on training set is ", accuracy2(examples, node, node1, node2)*100)
    print("3    ", accuracy2(examples, node, node1, node2)*100, "    ", accuracy2(testExamples, node, node1, node2)*100)
    print("5 decision trees of 1k attributes each")
    node = Id3_dubbed.decisionTree(examples3_1, attributes3_1, 0)
    node1 = Id3_dubbed.decisionTree(examples3_2, attributes3_2, 0)
    node2 = Id3_dubbed.decisionTree(examples3_3, attributes3_3, 0)
    node3 = Id3_dubbed.decisionTree(examples3_4, attributes3_4, 0)
    node4 = Id3_dubbed.decisionTree(examples3_5, attributes3_5, 0)
    # temp = copy.copy(node)
    # print("No. of nodes", tree_size(temp))
    print("generated decision tree")
    # print("Accuracy on test set is ", accuracy3(testExamples, node, node1, node2, node3, node4)*100)
    # print("Accuracy on training set is ", accuracy3(examples, node, node1, node2, node3, node4)*100)
    print("5    ", accuracy3(examples, node, node1, node2, node3, node4)*100,"  ",accuracy3(testExamples, node, node1, node2, node3, node4)*100)
if __name__ == '__main__':
    import sys
    testing_data = sys.argv[1]
    print("Learning Random Forest using Feature Bagging and calculating accuracy on training and test data")
    print("result will be displayed as")
    print("No. of Trees     Training Accuracy      Testing Accuracy")
    main(testing_data)


from flask import Flask, render_template, request
from keras.preprocessing import image

import testModel as func

app = Flask(__name__)

@app.route('/')
def home():
    return render_template('home.html')

# json example， {"best":"0","classify":"thing","line_score":"51","score":"50"}
@app.route('/up_photo',methods=['post'])
def up_photo():
    file = request.form.get('photo')
    json = func.getScore(file)
    return json

# json example， {"classify":"thing","line_score":"51","score":"50"}
@app.route('/up_one_photo',methods=['post'])
def up_one_photo():
    file = request.form.get('photo')
    json = func.getOneScore(file)
    return json

@app.route('/style_transfer',methods=['post'])
def style_transfer():
    photo = request.form.get('photo')
    style = request.form.get('style')
    json = func.style_transfer(photo,style)
    return json

if __name__ == "__main__":
    app.run(host='0.0.0.0',port='80')


import argparse
from heapq import heappush, heappop, heapify
import math

def valid_stream(stream: list[int], default_stream: list[int]) -> list[int]:
    if stream is None:
        return default_stream
    return stream

def get_args(default_args: dict):
    parser = argparse.ArgumentParser(description="Compute the running median of a sequence of numbers. That is, given a stream of numbers, print out the median of the list so far on each new element.")

    parser.add_argument('--stream', nargs='*', type=int, metavar='list', default=default_args['stream'], help=f'The stream. By default his param is equal to "{default_args["stream"]}"')

    args = parser.parse_args()

    args.stream = valid_stream(args.stream, default_args['stream'])

    return args

def daily(stream):

    upper = []
    lower = []
    for e in stream:
        heappush(lower, -e)
        heappush(upper, -heappop(lower))

        if len(upper) > len(lower):
            heappush(lower, -heappop(upper))

        if len(upper) != len(lower):
            print(-lower[0])
        else:
            print((upper[0] - lower[0])/2)

if __name__ == "__main__":

    default_args = {
        "stream" : [2, 1, 5, 7, 2, 0, 5]
    }

    args = get_args(default_args)
    daily(args.stream)


import asyncio
import os
import random
from discord.ext import commands
import dotenv

dotenv.load_dotenv()

TOKEN = os.environ.get('TOKEN')
bot = commands.Bot('>_ ', self_bot=True)
fish_channel = None

@bot.event
async def on_ready():
    print('ready >:)')

@bot.event
async def on_message(message):
    global fish_channel

    if message.content == 'fish':
        if message.author.id == bot.user.id:
            fish_channel = message.channel.id
            while True:
                await message.channel.send('p!fish')
                await asyncio.sleep(62)
    elif message.author.bot and message.author.name == 'Pancake' \
            and message.author.discriminator == '3691' and message.channel.id == fish_channel:

        print(message.content)

        await asyncio.sleep(1)

        if 'Your fishing rod broke!' in message.content:
            await asyncio.sleep(3)
            await message.channel.send('p!buy fishing rod')
        elif 'purchase' in message.content or 'sell' in message.content:
            if message.reactions:
                await message.add_reaction(message.reactions[0])
        elif 'casted out a line...' in message.content:
            await message.channel.send('p!sell all')
            await asyncio.sleep(random.randint(80, 120))
            await message.channel.send('p!give olive 100')

bot.run(TOKEN)


from collections import defaultdict
from copy import deepcopy
import os
import re
import warnings

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import transforms
from matplotlib.path import Path
from matplotlib.patches import PathPatch
import matplotlib.patheffects
from matplotlib.text import TextPath
import pkg_resources
from plotly.offline import download_plotlyjs, plot
import plotly.graph_objs as go
import seaborn as sns
import tabix

from ..sequences import Genome

_SVG_PATHS = {'T': "M 0,100 l 100, 0 l 0,-25 l -37.5, 0 l 0,-75 l -25, 0 " +
                   "l 0,75 l -37.5,0 l 0,25 z",
              'C': ("M 100,12.5 l 0,25 c 0,0 -25,-15 -50,-12.5 " +
                    "c 0,0 -25,0 -25,25 c 0,0 0,25 25,25 c 0,0 25,2.5 50,-15" +
                    " l 0, 25 C 100,87.5 75,100 50,100 C 50,100 0,100 0,50 " +
                    "C 0,50 0,0 50,0 C 50,0 75,0 100,12.5 z"),
              'G': ("M 100,12.5 l 0,25 c 0,0 -25,-15 -50,-12.5 " +
                    "c 0,0 -25,0 -25,25 c 0,0 0,25 25,25 c 0,0 25,2.5 50,-15" +
                    " l 0, 25 C 100,87.5 75,100 50,100 C 50,100 0,100 0,50 " +
                    "C 0,50 0,0 50,0 C 50,0 75,0 100,12.5 M 100,37.5 " +
                    "l 0,17.5 l -50,0 l 0,-17 l 25,0 l 0,-25 l 25,0 z"),
              'A': ("M 0,0 l 37.5,100 l 25,0 l 37.5,-100 l -25,0 l -9.375,25" +
                    " l -31.25,0 l -9.375,-25 l -25,0 z 0,0 M 43.75, 50 " +
                    "l 12.5,0 l -5.859375,15.625 l -5.859375,-15.625 z"),
              'U': ("M 0,100 l 25,0 l 0,-50 C 25,50 25,25 50,25" +
                    " C 50,25 75,25 75,50 l 0,50 l 25,0 L 100,50 " +
                    "C 100,50 100,0, 50,0 C 50,0 0,0 0,50 l 0,50 z")}

def _svg_parse(path_string):
    commands = {'M': (Path.MOVETO,),
                'L': (Path.LINETO,),
                'Q': (Path.CURVE3,) * 2,
                'C': (Path.CURVE4,) * 3,
                'Z': (Path.CLOSEPOLY,)}
    path_re = re.compile(r'([MLHVCSQTAZ])([^MLHVCSQTAZ]+)', re.IGNORECASE)
    float_re = re.compile(r'(?:[\s,]*)([+-]?\d+(?:\.\d+)?)')
    vertices = []
    codes = []
    last = (0, 0)
    for cmd, values in path_re.findall(path_string):
        points = [float(v) for v in float_re.findall(values)]
        points = np.array(points).reshape((len(points) // 2, 2))
        if cmd.islower():
            points += last
        cmd = cmd.capitalize()
        if len(points) > 0:
            last = points[-1]
        codes.extend(commands[cmd])
        vertices.extend(points.tolist())
    return np.array(vertices), codes

for k in _SVG_PATHS.keys():
    _SVG_PATHS[k] = _svg_parse(_SVG_PATHS[k])

class _TextPathRenderingEffect(matplotlib.patheffects.AbstractPathEffect):
    def __init__(self, bar, x_translation=0., y_translation=0.,
                 x_scale=1., y_scale=1.):
        self._bar = bar
        self._x_translation = x_translation
        self._y_translation = y_translation
        self._x_scale = x_scale
        self._y_scale = y_scale

    def draw_path(self, renderer, gc, tpath, affine, rgbFace=None):
        b_x, b_y, b_w, b_h = self._bar.get_extents().bounds
        t_x, t_y, t_w, t_h = tpath.get_extents().bounds
        translation = [b_x - t_x, b_y - t_y]
        translation[0] += self._x_translation
        translation[1] += self._y_translation
        scale = [b_w / t_w, b_h / t_h]
        scale[0] *= self._x_scale
        scale[1] *= self._y_scale
        affine = affine.identity().scale(*scale).translate(*translation)
        renderer.draw_path(gc, tpath, affine, rgbFace)

def sequence_logo(score_matrix, order="value", width=1.0, ax=None,
                  sequence_type=Genome, font_properties=None,
                  color_scheme=None,
                  **kwargs):
    # Note that everything will break if we do not deepcopy.
    score_matrix = deepcopy(score_matrix)

    score_matrix = score_matrix.transpose()
    if font_properties is not None:
        warnings.warn(
            "Specifying a value for `font_properties` (other than `None`) "
            "will use the `matplotlib`-based character paths, and causes "
            "distortions in the plotted motif. We recommend leaving "
            "`font_properties=None`. See the documentation for details.",
            UserWarning)

    if color_scheme is None:
        color_scheme = sns.color_palette("Set1",
                                         n_colors=len(sequence_type.BASES_ARR))
        color_scheme = color_scheme.as_hex()
    if len(color_scheme) < len(sequence_type.BASES_ARR):
        raise ValueError(
            "Color scheme is shorter than number of bases in sequence.")

    if score_matrix.shape[0] != len(sequence_type.BASES_ARR):
        raise ValueError(
            "Got score with {0} bases for sequence with {1} bases.".format(
                score_matrix.shape[0], len(sequence_type.BASES_ARR)))
    if ax is None:
        _, ax = plt.subplots(figsize=score_matrix.shape)

    # Determine offsets depending on sort order.
    positive_offsets = np.zeros_like(score_matrix)
    negative_offsets = np.zeros_like(score_matrix)
    bases = np.empty(score_matrix.shape, dtype=object)
    bases[:, :] = "?"  # This ensures blanks are visually obvious.

    # Change ordering of things based on input arguments.
    if order == "alpha":
        for i in range(score_matrix.shape[0]):
            bases[i, :] = sequence_type.BASES_ARR[i]

    elif order == "value":
        if np.sum(score_matrix < 0) != 0:
            sorted_scores = np.zeros_like(score_matrix)
            for j in range(score_matrix.shape[1]):
                # Sort the negative values and put them at bottom.
                div = np.sum(score_matrix[:, j] < 0.)
                negative_idx = np.argwhere(score_matrix[:, j] < 0.).flatten()
                negative_sort_idx = np.argsort(score_matrix[negative_idx, j],
                                               axis=None)
                sorted_scores[:div, j] = score_matrix[
                    negative_idx[negative_sort_idx], j]
                bases[:div, j] = sequence_type.BASES_ARR[
                    negative_idx[negative_sort_idx]].flatten()

                # Sort the positive values and stack atop the negatives.
                positive_idx = np.argwhere(score_matrix[:, j] >= 0.).flatten()
                positive_sort_idx = np.argsort(score_matrix[positive_idx, j],
                                               axis=None)
                sorted_scores[div:, j] = score_matrix[
                    positive_idx[positive_sort_idx], j]
                bases[div:, j] = sequence_type.BASES_ARR[
                    positive_idx[positive_sort_idx]].flatten()
            score_matrix = sorted_scores
        else:
            for j in range(score_matrix.shape[1]):
                sort_idx = np.argsort(score_matrix[:, j], axis=None)[::-1]
                bases[:, j] = sequence_type.BASES_ARR[sort_idx]
                score_matrix[:, j] = score_matrix[sort_idx, j]

    # Create offsets for each bar.
    for i in range(score_matrix.shape[0] - 1):
        y_coords = score_matrix[i, :]
        if i > 0:
            negative_offsets[i + 1, :] = negative_offsets[i, :]
            positive_offsets[i + 1, :] = positive_offsets[i, :]
        neg_idx = np.argwhere(y_coords < 0.)
        pos_idx = np.argwhere(y_coords >= 0.)
        negative_offsets[i + 1, neg_idx] += y_coords[neg_idx]
        positive_offsets[i + 1, pos_idx] += y_coords[pos_idx]

    for i in range(score_matrix.shape[0]):
        x_coords = np.arange(score_matrix.shape[1]) + 0.5
        y_coords = score_matrix[i, :]

        # Manage negatives and positives separately.
        offsets = np.zeros(score_matrix.shape[1])
        negative_idx = np.argwhere(y_coords < 0.)
        positive_idx = np.argwhere(y_coords >= 0.)
        offsets[negative_idx] = negative_offsets[i, negative_idx]
        offsets[positive_idx] = positive_offsets[i, positive_idx]
        bars = ax.bar(x_coords, y_coords, color="black", width=width,
                      bottom=offsets)
        for j, bar in enumerate(bars):
            base = bases[i, j]
            bar.set_color(color_scheme[sequence_type.BASE_TO_INDEX[base]])
            bar.set_edgecolor(None)

    # Iterate over the barplot's bars and turn them into letters.
    new_patches = []
    for i, bar in enumerate(ax.patches):
        base_idx = i // score_matrix.shape[1]
        seq_idx = i % score_matrix.shape[1]
        base = bases[base_idx, seq_idx]
        # We construct a text path that tracks the bars in the barplot.
        # Thus, the barplot takes care of scaling and translation,
        #  and we just copy it.
        if font_properties is None:
            text = Path(_SVG_PATHS[base][0], _SVG_PATHS[base][1])
        else:
            text = TextPath((0., 0.), base, fontproperties=font_properties)
        b_x, b_y, b_w, b_h = bar.get_extents().bounds
        t_x, t_y, t_w, t_h = text.get_extents().bounds
        scale = (b_w / t_w, b_h / t_h)
        translation = (b_x - t_x, b_y - t_y)
        text = PathPatch(text, facecolor=bar.get_facecolor(), lw=0.)
        bar.set_facecolor("none")
        text.set_path_effects([_TextPathRenderingEffect(bar)])
        transform = transforms.Affine2D().translate(*translation).scale(*scale)
        text.set_transform(transform)
        new_patches.append(text)

    for patch in new_patches:
        ax.add_patch(patch)
    ax.set_xlim(0, score_matrix.shape[1])
    ax.set_xticks(np.arange(score_matrix.shape[1]) + 0.5)
    ax.set_xticklabels(np.arange(score_matrix.shape[1]))
    return ax

def rescale_score_matrix(score_matrix, base_scaling="identity",
                         position_scaling="identity"):
    # Note that things can break if we do not deepcopy.
    score_matrix = deepcopy(score_matrix)

    score_matrix = score_matrix.transpose()
    rescaled_scores = score_matrix

    # Scale individual bases.
    if base_scaling == "identity" or base_scaling == "probability":
        pass
    elif base_scaling == "max_effect":
        rescaled_scores = score_matrix - np.min(score_matrix, axis=0)
    else:
        raise ValueError(
            "Could not find base scaling \"{0}\".".format(base_scaling))

    # Scale each position
    if position_scaling == "max_effect":
        max_effects = np.max(score_matrix, axis=0) - np.min(score_matrix,
                                                            axis=0)
        rescaled_scores /= rescaled_scores.sum(axis=0)[np.newaxis, :]
        rescaled_scores *= max_effects[np.newaxis, :]
    elif position_scaling == "probability":
        rescaled_scores /= np.sum(score_matrix, axis=0)[np.newaxis, :]
    elif position_scaling != "identity":
        raise ValueError(
            "Could not find position scaling \"{0}\".".format(
                position_scaling))
    return rescaled_scores.transpose()

def heatmap(score_matrix, mask=None, sequence_type=Genome, **kwargs):
    # Note that some things can break if we do not deepcopy.
    score_matrix = deepcopy(score_matrix)

    # This flipping is so that ordering is consistent with ordering
    # in the sequence logo.
    if mask is not None:
        mask = mask.transpose()
        mask = np.flip(mask, axis=0)
    score_matrix = score_matrix.transpose()
    score_matrix = np.flip(score_matrix, axis=0)

    if "yticklabels" in kwargs:
        yticklabels = kwargs.pop("yticklabels")
    else:
        yticklabels = sequence_type.BASES_ARR[::-1]
    if "cbar_kws" in kwargs:
        cbar_kws = kwargs.pop("cbar_kws")
    else:
        cbar_kws = dict(use_gridspec=False, location="bottom", pad=0.2)
    if "cmap" in kwargs:
        cmap = kwargs.pop("cmap")
    else:
        cmap = "Blues"
    ret = sns.heatmap(score_matrix, mask=mask, yticklabels=yticklabels,
                      cbar_kws=cbar_kws, cmap=cmap, **kwargs)
    ret.set_yticklabels(labels=ret.get_yticklabels(), rotation=0)
    return ret

def load_variant_abs_diff_scores(input_path):
    features = []
    labels = []
    diffs = []
    with open(input_path, 'r') as file_handle:
        colnames = file_handle.readline()
        features = colnames.strip().split('\t')[5:]
        for line in file_handle:
            cols = line.strip().split('\t')
            scores = [float(f) for f in cols[5:]]
            label = tuple(cols[:5])
            diffs.append(scores)
            labels.append(label)
    diffs = np.array(diffs)
    return diffs, labels, features

def sort_standard_chrs(chrom):
    chrom = chrom[3:]
    if chrom.isdigit():
        return int(chrom)
    if chrom == 'X':
        return 23
    elif chrom == 'Y':
        return 24
    elif chrom == 'M':
        return 25
    else:  # unknown chr
        return 26

def ordered_variants_and_indices(labels):
    labels_dict = defaultdict(list)
    for i, l in enumerate(labels):
        chrom, pos, name, ref, alt = l
        pos = int(pos)
        info = (i, pos, ref, alt)
        labels_dict[chrom].append(info)
    for chrom, labels_list in labels_dict.items():
        labels_list.sort(key=lambda tup: tup[1:])
    ordered_keys = sorted(labels_dict.keys(),
                          key=sort_standard_chrs)

    ordered_labels = []
    ordered_label_indices = []
    for chrom in ordered_keys:
        for l in labels_dict[chrom]:
            index, pos, ref, alt = l
            ordered_label_indices.append(index)
            ordered_labels.append((chrom, pos, ref, alt))
    return ordered_labels, ordered_label_indices

def _label_tuple_to_text(label, diff, genes=None):
    chrom, pos, ref, alt = label
    if genes is not None:
        if len(genes) == 0:
            genes_str = "none found"
        else:
            genes_str = ', '.join(genes)
        text = ("max diff score: {0}
{1} {2}, {3}/{4}
"
                "closest protein-coding gene(s): {5}").format(
            diff, chrom, pos, ref, alt, genes_str)
    else:
        text = "max diff score: {0}
{1} {2}, {3}/{4}".format(
            diff, chrom, pos, ref, alt)
    return text

def _variant_closest_genes(label, tabix_fh, chrs_gene_intervals):
    chrom = label[0]
    pos = label[1]
    closest_genes = []
    try:
        overlaps = tabix_fh.query(chrom, pos, pos + 1)
        for o in overlaps:
            closest_genes.append(o[-1])
    except tabix.TabixError:
        pass
    if len(closest_genes) != 0:
        return closest_genes
    gene_intervals = chrs_gene_intervals[chrom]
    closest = None
    for (start, end, strand, gene) in gene_intervals:
        if start < pos and closest and closest == pos - end:
            closest_genes.append(gene)
        elif start < pos and (closest is None
                              or closest > pos - end):
            closest = pos - end
            closest_genes = [gene]
        elif start > pos and closest and closest == start - pos:
            closest_genes.append(gene)
        elif start > pos and (closest is None
                              or closest > start - pos):
            closest = start - pos
            closest_genes = [gene]
    return closest_genes

def _load_chrs_gene_intervals(gene_intervals_bed):
    chrs_gene_intervals = defaultdict(list)
    with open(gene_intervals_bed, 'r') as file_handle:
        for line in file_handle:
            cols = line.strip().split('\t')
            chrom = cols[0]
            start = int(cols[1])
            end = int(cols[2])
            strand = cols[3]
            gene = cols[4]
            chrs_gene_intervals[chrom].append((start, end, strand, gene))
    return chrs_gene_intervals

def _variants_closest_protein_coding_gene(labels, version="hg38"):
    gene_intervals_tabix = pkg_resources.resource_filename(
        "skylar",
        ("interpret/data/gencode_v28_{0}/"
         "protein_coding_l12_genes.bed.gz").format(version))
    gene_intervals_bed = pkg_resources.resource_filename(
        "skylar",
        ("interpret/data/gencode_v28_{0}/"
         "protein_coding_l12_genes.bed").format(version))

    tabix_fh = tabix.open(gene_intervals_tabix)
    chrs_gene_intervals = _load_chrs_gene_intervals(gene_intervals_bed)

    labels_gene_information = []
    for l in labels:
        labels_gene_information.append(_variant_closest_genes(
            l, tabix_fh, chrs_gene_intervals))

    return labels_gene_information

def variant_diffs_scatter_plot(data,
                               labels,
                               features,
                               output_path,
                               filter_features=None,
                               labels_sort_fn=ordered_variants_and_indices,
                               nth_percentile=None,
                               hg_reference_version=None,
                               threshold_line=None,
                               auto_open=False):
    labels_ordered, label_indices = labels_sort_fn(labels)
    variant_closest_genes = None
    if hg_reference_version is not None:
        variant_closest_genes = _variants_closest_protein_coding_gene(
            labels_ordered, version=hg_reference_version)
    ordered_data = data[label_indices, :]

    feature_indices = None
    if filter_features is not None:
        feature_indices = filter_features(features)
        ordered_data = data[:, feature_indices]
    variants_max_diff = np.amax(ordered_data, axis=1)

    display_labels = None
    if nth_percentile:
        p = np.percentile(variants_max_diff, nth_percentile)
        keep = np.where(variants_max_diff >= p)[0]
        print("{0} variants with max abs diff score above {1} are in the "
              "{2}th percentile.".format(len(keep), p, nth_percentile))
        variants_max_diff = variants_max_diff[keep]
        display_labels = []
        for i, l in enumerate(labels_ordered):
            if i not in keep:
                continue
            display_labels.append(l)
    else:
        display_labels = labels_ordered

    if variant_closest_genes:
        text_labels = [
            _label_tuple_to_text(l, d, g) for l, d, g in
            zip(display_labels, variants_max_diff, variant_closest_genes)]
    else:
        text_labels = [_label_tuple_to_text(l, d) for l, d in
                       zip(display_labels, variants_max_diff)]

    label_x = [' '.join([l[0], str(l[1])]) for l in display_labels]
    data = [go.Scatter(x=label_x,
                       y=variants_max_diff,
                       mode='markers',
                       marker=dict(
                           color="#39CCCC",
                           line=dict(width=1)),
                       text=text_labels,
                       hoverinfo="text")]

    go_layout = {
        "title": "Max probability difference scores",
        "hovermode": "closest",
        "hoverlabel": {
            "font": {"size": 16}
        },
        "xaxis": {
            "title": "Genome coordinates",
            "showticklabels": True,
            "tickangle": 35,
            "titlefont": {"family": "Arial, sans-serif",
                          "size": 16},
            "nticks": 25,
            "tickmode": "auto",
            "automargin": True
        },
        "yaxis": {
            "title": "Absolute difference",
            "titlefont": {"family": "Arial, sans-serif",
                          "size": 16}
        }
    }
    if isinstance(threshold_line, float):
        layout = go.Layout(
            **go_layout,
            shapes=[
                {"type": "line",
                 "x0": label_x[0],
                 "y0": threshold_line,
                 "x1": label_x[-1],
                 "y1": threshold_line,
                 "line": {
                     "color": "rgba(255, 99, 71, 1)",
                     "width": 2
                 }
                 }
            ])
    else:
        layout = go.Layout(**go_layout)
    fig = go.Figure(data=data, layout=layout)
    path, filename = os.path.split(output_path)
    os.makedirs(path, exist_ok=True)
    plot(fig, filename=output_path, auto_open=auto_open)
    return fig


import Project_Lib as pl
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split

# GPU memory management
gpu = tf.config.experimental.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(gpu[0], True)

# Load and analyze train and test dataset
dir_path_train = r"C:/Users/ASUS/Desktop/AMLS_Assignment/AMLS_PROJECT/UCI_HAR_Dataset/train"
dir_path_test = r"C:/Users/ASUS/Desktop/AMLS_Assignment/AMLS_PROJECT/UCI_HAR_Dataset/test"
data_train, label_train = pl.build_dataset(dir_path_train, 'Inertial Signals', 'train')
data_test, label_test = pl.build_dataset(dir_path_test, 'Inertial Signals', 'test')
pl.distribution_ana(label_train)
pl.distribution_ana(label_test)
print(data_train.shape)
print(label_train.shape)
print(data_test.shape)
print(label_test.shape)

# plot one data
"""
print('plot begin')
time_steps= range(0, 128, 1)
fig, ax = plt.subplots(1,1, figsize=(10,6))
ax.plot(time_steps, data_train[1,:,1], label='body acc x')
ax.set_title('Body acceleration for one time window)', fontsize=22)
ax.set_xlabel(r'Time steps', fontsize=22)
ax.set_ylabel(r'Acceleration', fontsize=22)
ax.tick_params(labelsize=22)
ax.legend(fontsize=24)
plt.show()
print('plot finished')
"""

# manually shuffle prepared train set and test set
"""
index_train = [i for i in range(len(data_train))]
index_test = [i for i in range(len(data_test))]
np.random.shuffle(index_train)
np.random.shuffle(index_test)
data_train = data_train[index_train, :, :]
data_test = data_test[index_test, :, :]
label_train = label_train[index_train]
label_test = label_test[index_test]
"""

# manually create and shuffle train set and test set; note: the result is very bad! Why?
"""
data = np.concatenate((data_train, data_test), axis=0)
label = np.concatenate((label_train, label_test), axis=0)
print("New data shape: " + str(data.shape))
print("New label shape: " + str(label.shape))
index = [i for i in range(len(data))]
np.random.shuffle(index)
data_ = data[index, :, :]
label = label[index]
data_train, data_test, label_train, label_test = train_test_split(data, label, test_size=0.2, random_state=1)
print("Balance after shuffle: ")
pl.distribution_ana(label_train)
pl.distribution_ana(label_test)
print(data_train.shape)
print(label_train.shape)
print(data_test.shape)
print(label_test.shape)
"""

# Train and test all models without standardization
"""
histories_cnn = pl.evaluate_repreat(data_train, label_train, data_test, label_test, 'cnn', 5)
histories_lstm = pl.evaluate_repreat(data_train, label_train, data_test, label_test, 'lstm', 5)
histories_cnn_lstm = pl.evaluate_repreat(data_train, label_train, data_test, label_test, 'cnn_lstm', 5)
histories_convlstm = pl.evaluate_repreat(data_train, label_train, data_test, label_test, 'conv_lstm', 5)
"""
"""
# Train and test all models with standardization
data_train_std, data_test_std = pl.standardization(data_train, data_test)
data_train_std = data_train
data_test_std = data_test
histories_cnn = pl.evaluate_repreat(data_train_std, label_train, data_test_std, label_test, 'cnn', 5)
histories_lstm = pl.evaluate_repreat(data_train_std, label_train, data_test_std, label_test, 'lstm', 5)
histories_cnn_lstm = pl.evaluate_repreat(data_train_std, label_train, data_test_std, label_test, 'cnn_lstm', 5)
histories_convlstm = pl.evaluate_repreat(data_train_std, label_train, data_test_std, label_test, 'conv_lstm', 5)

# Plot learning curve
fig, ax = plt.subplots(figsize=(15, 50), ncols=1, nrows=4)
histories = [histories_cnn[4], histories_lstm[4], histories_cnn_lstm[4], histories_convlstm[4]]
histories_name = ['cnn', 'lstm', 'cnn-lstm', 'convlstm']
#histories = [histories_cnn[4]]
#histories_name = ['cnn']

for n in range(4):
    print('plot begin')
    time_steps = range(1, len(histories[n].history['acc'])+1)
    ax[n].plot(time_steps, histories[n].history['acc'], 'ro', label=str(histories_name[n])+'acc')
    ax[n].plot(time_steps, histories[n].history['val_acc'], 'b', label=str(histories_name[n])+'val_acc')
    ax[n].set_title(str(histories_name[n]), fontsize=22)
    ax[n].set_xlabel(r'Epochs', fontsize=22)
    ax[n].set_ylabel(r'Accuracy', fontsize=22)
    ax[n].tick_params(labelsize=22)
    ax[n].legend(fontsize=24)
    print('plot finished')
plt.show()
"""
fig, ax = plt.subplots(figsize=(10, 6), ncols=1, nrows=1)
print('plot begin')
sample_num = range(1, len(label_train[0:175])+1)
ax.plot(sample_num, label_train[0:175], 'ro', label=r'labels')
ax.set_title(r'training labels', fontsize=22)
ax.set_xlabel(r'sample number', fontsize=22)
ax.set_ylabel(r'label', fontsize=22)
ax.tick_params(labelsize=22)
ax.legend(fontsize=24)
print('plot finished')
plt.show()


from copy import deepcopy
import threading
import json
import logging
from datetime import datetime
import time
import uuid
from psycopg2 import pool as pgpool
import psycopg2
from kafka import KafkaConsumer

logger = logging.getLogger(__name__)

class KafkaConsumerWrapper:
    def __init__(self, connection_args, topic):
        self.topic = topic
        consumer_args = {
            'auto_offset_reset': "earliest",
            'group_id': 'httpmon',
            'client_id': uuid.uuid4(),
            'value_deserializer': lambda v: json.loads(v.decode('utf-8')),
        }
        connection_args.update(consumer_args)
        self.consumer = KafkaConsumer(**connection_args)
        self.consumer.subscribe([self.topic])

    def messages(self):
        raw_msgs = self.consumer.poll(timeout_ms=1000)
        for tp, msgs in raw_msgs.items():
            for msg in msgs:
                logger.info('got msg: %s' % str(msg))
                yield msg.value
        self.consumer.commit()

class PGExporter:
    def __init__(self, connection_args, pool_size=10):
        self.pool = pgpool.ThreadedConnectionPool(1, pool_size, connection_args)

    @staticmethod
    def worker(pool, sql, params):
        conn = None
        while not conn:
            try:
                conn = pool.getconn()
            except pgpool.PoolError:
                logger.warning("No free connection in pool")
                time.sleep(1)

        with conn.cursor() as curs:
            try:
                curs.execute(sql, params)
                conn.commit()
                logger.info("Put to DB: %s" % params)
            except psycopg2.Error as e:
                logger.error(e)
        pool.putconn(conn)

    def submit(self, message):
        sql = """INSERT INTO checks
        (url, timestamp, code, body_check_valid, time)
        VALUES
        (%(url)s, %(timestamp_dt)s, %(code)s, %(body_check_valid)s, %(time)s)
        """
        message = deepcopy(message)
        message['timestamp_dt'] = datetime.fromtimestamp(message['timestamp'])
        thread = threading.Thread(
            target=self.worker, args=(self.pool, sql, message))
        thread.start()

def consumer_loop(consumer, exporter):
    while True:
        for message in consumer.messages():
            exporter.submit(message)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from graphviz import Digraph

class Edge:
    def __init__(self, start, finish, weight=0, min_weight=0, redge=False):
        self.start = start
        self.finish = finish
        self.min_weight = min_weight
        self.weight = weight
        self.redge = redge  # flag for saying, that its incoming edge

    def __repr__(self):
        return '({0})-{1}{2}->({3})'.format(
            self.start,
            '{},'.format(self.min_weight) if self.min_weight != 0 else '',
            self.weight, self.finish)

    def __hash__(self):
        return hash(self.__repr__())

    def __eq__(self, another):
        return self.start == another.start \
               and self.finish == another.finish \
               and self.weight == another.weight \
               and self.min_weight == another.min_weight

class Mark:
    def __init__(self, parent=None, flow=float('inf'), edge=None, sign=1):
        self.parent = parent
        self.flow = flow
        self.edge = edge
        self.sign = sign

    def __repr__(self):
        return '({}, {})'.format(self.parent, self.flow)

class FlowNetwork:
    def __init__(self, file_name):
        self.adj = {}
        self.flow = {}
        self.file_name = file_name

    def add_vertex(self, vertex):
        self.adj[vertex] = []

    def get_vertex(self):
        vertex = self.adj.keys()
        vertex.sort()
        return vertex

    def add_edge(self, u, v, w=0, min_w=0, f=0):
        if u == v:
            raise ValueError('u == v')

        edge = Edge(u, v, w, min_w)
        redge = Edge(u, v, w, min_w, redge=True)

        self.adj[u].append(edge)
        self.adj[v].append(redge)

        self.flow[edge] = f

    def get_edges(self, v):
        """
        Return outcoming edges for v
        :param v: node
        :return: list of Edge object's
        """
        return filter(lambda e: not e.redge, self.adj[v])

    def get_redges(self, v):
        """
        Return incoming edges for v
        :param v: node
        :return: list of Edge object's
        """
        return filter(lambda e: e.redge, self.adj[v])

    def get_marks(self, _from, _to):
        """
        Set marks for nodes
        :param _from: a source
        :param _to: a sink
        :return: list og Mark object's
        """
        marks = dict.fromkeys(self.get_vertex())
        marks[_from] = Mark()
        queue = [_from]

        while queue:
            v = queue.pop(0)
            if v == _to:
                return marks

            for edge in self.get_edges(v):
                if not marks[edge.finish] and self.flow[edge] < edge.weight:
                    flow = min(marks[v].flow, edge.weight - self.flow[edge])
                    marks[edge.finish] = Mark(v, flow, edge)
                    queue.append(edge.finish)

            for redge in self.get_redges(v):
                if not marks[redge.start] and self.flow[redge] > redge.min_weight:
                    flow = min(marks[v].flow, self.flow[redge] - redge.min_weight)
                    marks[redge.start] = Mark(v, flow, redge, sign=-1)
                    queue.append(redge.start)

        return marks

    def max_flow(self, _from, _to, out=None, write=True):
        """
        Find max flow in network
        :param _from: a source
        :param _to: a sink
        :param write: param for disabling file outputs
        :return: max flow, marks from last iteration
        """
        if write:
            out.write('* table with marks by iteration:\n\n')
            self.print_table_head(out)
        marks = self.get_marks(_from, _to)

        while marks[_to]:
            if write:
                self.print_table_row(out, marks)
            i = _to
            min_flow = marks[_to].flow
            while marks[i].parent:
                self.flow[marks[i].edge] += marks[i].sign*min_flow
                i = marks[i].parent
            marks = self.get_marks(_from, _to)

        flow = sum(self.flow[edge] for edge in self.get_edges(_from))
        if write:
            self.print_table_row(out, marks)
            out.write('\n* maximal flow = {}:\n'.format(flow))
            self.print_flow(out)
        return flow, marks

    def print_flow(self, out):
        out.write('```\n')
        keys = self.flow.keys()
        keys.sort()
        for key in keys:
            out.write('\t{0} - [{1}]\n'.format(key, self.flow[key]))
        out.write('```\n\n')

    def print_table_head(self, out):
        out.write(reduce(lambda s, x: '{0} | {1} '.format(s, x), self.get_vertex()) + '\n')
        out.write('--- | '*len(self.get_vertex()) + '\n')

    def print_table_row(self, out, marks):
        for v in self.get_vertex():
            out.write('`{}` | '.format(marks[v]))
        out.write('\n')

    def draw(self, show=False, postfix=''):
        """
        Render graph to png and save to filesystem
        :param show: flag to open image in new window
        :return: None
        """
        g = Digraph('G', filename='{}.gv'.format(self.file_name + postfix), directory='out', format='png')
        for v in self.get_vertex():
            for e in self.get_edges(v):
                flow = ''
                if self.flow[e] != 0:
                    if self.flow[e] != e.weight:
                        flow = '[{0}/{1}]'.format(self.flow[e], e.weight)
                    else:
                        flow = '[{}]'.format(self.flow[e])
                color = 'red' if self.flow[e] != 0 else ''
                g.edge(e.start, e.finish, label=flow, color=color)
        g.render(view=show)


# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
import json
from datetime import date,datetime,timedelta
from flyDB import FlyDB
import time
from fly import Fly
'''
机票价格爬虫
'''
def request(url):
    result = requests.get(url)
    if result.status_code == 200:
        return result.content
    else:
        print("请求失败." + str(result.status_code))
        return False

def process(content,fly):
    result = []
    soup = BeautifulSoup(content,"html.parser")
    rows = soup.find_all(class_="cssRadio row3")
    while len(rows) > 0:
        row = rows.pop()
        if row is not None:
            label = row.contents[1]
            labelDict = label.attrs
            departingtime = labelDict['data-departingtime']
            value = labelDict['value']
            label = row.contents[3]
            labelDict = label.attrs
            price = labelDict['data-amount']
            now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            fly.setValue(departingtime,price,value,now)
            if float(price) < 3000:
                result.append(fly.toTuple())
        else:
            print("请求失败,价格行为空")
        print(json.dumps(fly.toDict()))
    return result

def conf(date,fromdis,todis):
    url = 'https://beta.cebupacificair.com/Flight/InternalSelect?'
    param = 'o1=' + fromdis + "&d1=" + todis + "&o2=&d2=&dd1=" + date + "&p=&ADT=1&CHD=0&INF=0&s=true&mon=true"
    fly = Fly(fromdis,todis,"cebu")
    return url + param,fly

def test():
    day = date(2020,2,1)
    url,fly = conf(day.strftime('%Y-%m-%d'),'MNL','CEB')
    result = request(url)
    rows = process(result,fly)
    print("================================")
    for row in rows:
        print(row)

# HKG,CEB,ILO,
def main():
    day = date(2019,2,1)
    for i in range(3):
        url,fly = conf(day.strftime('%Y-%m-%d'),'HKG','ILO')
        result = request(url)
        if result is False:
            print("获取数据失败")
            return
        rows = (process(result,fly))

        url,fly = conf(day.strftime('%Y-%m-%d'),'HKG','CEB')
        result = request(url)
        if result is False:
            print("获取数据失败")
            return
        rows.extend(process(result,fly))

        url,fly = conf(day.strftime('%Y-%m-%d'),'HKG','MNL')
        result = request(url)
        if result is False:
            print("获取数据失败")
            return
        rows.extend(process(result,fly))

        flyDB = FlyDB("mysql")
        flyDB.insertValue(rows)

        time.sleep(2)
        print("current date is %s ,flydate been Get" % day.strftime('%Y-%m-%d'))
        day = day + timedelta(days=1)

test()


import xml.etree.ElementTree as ET
from threading import Lock

# global configuration managers
overall_config = None
motor_config = None
cam_config = None
auto_config = None

class ConfigurationError(Exception):
    '''Exception class for configuration.'''
    pass

class Configuration:
    def __init__(self, name="settings.xml"):
        try:
            self.tree = ET.parse(name)
        except FileNotFoundError as e:
            raise ConfigurationError(str(e))
        self.root = self.tree.getroot()
        self.tree_lock = Lock()

    @staticmethod
    def ready():
        global overall_config
        global motor_config
        global cam_config
        global auto_config
        if overall_config is None \
           or motor_config is None \
           or cam_config is None \
           or auto_config is None:
            return False
        return True

    @staticmethod
    def settings_file(name="settings.xml"):
        global overall_config
        global motor_config
        global cam_config
        global auto_config
        overall_config = OverallConfiguration(name)
        motor_config = MotorConfiguration(name)
        cam_config = CameraConfiguration(name)
        auto_config = AutonomousConfiguration(name)

    def _getsubelemvalue(self,elem, pred, match):
        for things in elem:
            for subthing in things:
                try:
                    if subthing.attrib[pred] == match:
                        return subthing
                except KeyError:
                    return None

    def _getsubelemvaluebyname(self, elem, subname, pred, match):
        for things in elem:
            for subthing in things:
                if subthing.tag == subname:
                    try:
                        if subthing.attrib[pred] == match:
                            return subthing
                    except KeyError:
                        return None

    def _make_searchstr_list(self, req_list):
        searchstr_list = []
        for elemstr in req_list:
            searchstr = "."
            elem_list = elemstr.split('.')
            for elem in elem_list:
                idx1 = elem.find('{')
                idx2 = elem.find('}')
                tagname = elem[idx1+1:idx2]
                searchstr = searchstr + "/" + tagname.upper()
                idx3 = elem.find('[')
                idx4 = elem.find(']')
                if (idx3 == -1) or (idx4 == -1):
                    continue
                attrtype = elem[idx3+1:idx4].upper()
                attrname = elem[idx4+1:]
                searchstr = searchstr + "[@"+attrtype+"='"+attrname+"']"
            searchstr_list.append(searchstr)
        return searchstr_list

    def provide_settings(self, req_list):
        if not req_list:
            return None
        searchstr_list = self._make_searchstr_list(req_list)
        res_list = []
        with self.tree_lock:
            for searchstr in searchstr_list:
                res = self.root.findall(searchstr)
                res_list.append(res)
        if not res_list[0]:
            return None
        return res_list

    def top_level_element_value(self, name):
        elem_list = self.provide_settings(["{"+name+"}"])
        if not elem_list:
            return None
        return (elem_list[0][0].text.strip())

class MotorConfiguration(Configuration):
    def __init__(self, name="settings.xml"):
        Configuration.__init__(self, name)

    def get_pwm_pin(self, motor_group ,motor_name):
        req = ["{Motors}[name]"+motor_group+".{Motor}[name]"
               +motor_name]
        motor = self.provide_settings(req)
        # pin = self._getsubelemvalue(motor[0],"TYPE","PWM")
        pin = self._getsubelemvaluebyname(motor[0], "PIN", "TYPE", "PWM")
        try:
            return int(pin.text)
        except ValueError as e:
            raise ConfigurationError("Bad value in settings file for pwm pin of motor '"+motor_name+"'. Error: "+str(e))

    def get_digital_pin(self, motor_group, motor_name):
        req = ["{Motors}[name]"+motor_group+".{Motor}[name]"
               +motor_name]
        motor = self.provide_settings(req)
        # pin = self._getsubelemvalue(motor[0], "TYPE", "Digital")
        pin = self._getsubelemvaluebyname(motor[0], "PIN", "TYPE", "Digital")
        try:
            return int(pin.text)
        except ValueError as e:
             raise ConfigurationError("Bad value in settings file for digital pin of motor '"+motor_name+"'. Error: "+str(e))

    def get_pin(self, motor_group, motor_name):
        req = ["{Motors}[name]"+motor_group+".{Motor}[name]"
               +motor_name]
        motor = self.provide_settings(req)
        # pin = self._getsubelemvalue(motor[0], "TYPE", "Other")
        pin = self._getsubelemvaluebyname(motor[0], "PIN", "TYPE", "Other")
        try:
            return int(pin.text)
        except ValueError as e:
            raise ConfigurationError("Bad value in settings file for pin of motor '"+motor_name+"'. Error: "+str(e))

    def get_limit(self, motor_group, motor_name, lim_name):
        req = ["{Motors}[name]"+motor_group+".{Motor}[name]"+motor_name]
        motor = self.provide_settings(req)
        lim_name = str.capitalize(lim_name)
        lim = self._getsubelemvaluebyname(motor[0], "LIMIT" ,"NAME", lim_name)
        try:
            return float(lim.text)
        except ValueError as e:
            raise ConfigurationError("Bad value in settings file for limit '{}'\
 of motor '{}'. Error: {}".format(lim_name,motor_name,str(e)))
        except AttributeError as e:
            raise ConfigurationError("Search for limit '{}' on motor '{}' \
of group '{}' returned no result".format(lim_name,motor_name,motor_group))

class CameraConfiguration(Configuration):
    def __init__(self, name="settings.xml"):
        Configuration.__init__(self,name)

    def _generic_integer_setting(self, name):
        req = ["{Camera}.{"+name+"}"]
        elem_list = self.provide_settings(req)
        if not elem_list:
            return None
        val = int(elem_list[0][0].text.strip())
        return val

    def _generic_string_setting(self, name):
        req = ["{Camera}.{"+name+"}"]
        elem_list = self.provide_settings(req)
        if not elem_list:
            return None
        val = elem_list[0][0].text.strip()
        return val

    def device(self):
        dev = self._generic_string_setting('device')
        if dev == 'rpicamsrc' or dev == 'v4l2src' or dev == 'avfvideosrc' or \
           dev == 'autovideosrc':
            return dev
        raise ConfigurationError("Error in camera settings: invalid device specified, can only be 'rpicamsrc','v4l2src' or 'avfvideosrc'.")

    def capture_framerate(self):
        return self._generic_integer_setting('capture_framerate')

    def capture_frame_width(self):
        return self._generic_integer_setting('capture_frame_width')

    def capture_frame_height(self):
        return self._generic_integer_setting('capture_frame_height')

    def stream_port(self):
        return self._generic_integer_setting('stream_port')

    def stream_framerate(self):
        return self._generic_integer_setting('stream_framerate')
    def stream_frame_width(self):
        return self._generic_integer_setting('stream_frame_width')

    def stream_frame_height(self):
        return self._generic_integer_setting('stream_frame_height')

class AutonomousConfiguration(Configuration):
    def __init__(self, name="settings.xml"):
        Configuration.__init__(self,name)

    def _generic_integer_setting(self,name):
        req = ["{Autonomous}.{"+name+"}"]
        elem_list = self.provide_settings(req)
        if not elem_list:
            return None
        val = int(elem_list[0][0].text.strip())
        return val

    def _generic_string_setting(self,name):
        req = ["{Autonomous}.{"+name+"}"]
        elem_list = self.provide_settings(req)
        if not elem_list:
            return None
        val = elem_list[0][0].text.strip()
        return val

    def cv_engine(self):
        return self._generic_string_setting('cvengine')

    def capture_framerate(self):
        return self._generic_integer_setting('capture_framerate')

    def capture_frame_width(self):
        return self._generic_integer_setting('capture_frame_width')

    def capture_frame_height(self):
        return self._generic_integer_setting('capture_frame_height')

    def stream_port(self):
        return self._generic_integer_setting('stream_port')

    def stream_framerate(self):
        return self._generic_integer_setting('stream_framerate')

    def stream_frame_width(self):
        return self._generic_integer_setting('stream_frame_width')

    def stream_frame_height(self):
        return self._generic_integer_setting('stream_frame_height')

    def ultrasound_trigger_pin(self):
        return self._generic_integer_setting('ultrasound_trigger_pin')

    def ultrasound_echo_pin(self):
        return self._generic_integer_setting('ultrasound_echo_pin')

    def ultrasound_power_pin(self):
        return self._generic_integer_setting('ultrasound_power_pin')

    def ultrasound_ground_pin(self):
        return self._generic_integer_setting('ultrasound_ground_pin')

class OverallConfiguration(Configuration):
    def __init__(self, name="settings.xml"):
        Configuration.__init__(self, name)
        self.connected_ip = '0.0.0.0' # ip address to which remote has connected
        self.running_as_unit = False # if start_rover was called with --as-unit

    def opmodes_directories(self):
        dir_list_str = self.top_level_element_value("OPMODES_DIRECTORIES")
        if dir_list_str is None:
            raise ConfigurationError("No settings found for operational mode directories.")
        dir_list_str = dir_list_str.replace('\n','').replace(' ','')
        dir_list = dir_list_str.split(',')
        if '' in dir_list and len(dir_list) > 1:
            raise ConfigurationError("Error in operational mode settings: probably extra comma in list.")
        return dir_list

    def resources_directories(self):
        dir_list_str = self.top_level_element_value("RESOURCES_DIRECTORIES")
        if dir_list_str is None:
            raise ConfigurationError("No settings found for resources directories.")
        dir_list_str = dir_list_str.replace('\n','').replace(' ','')
        dir_list = dir_list_str.split(',')
        if '' in dir_list and len(dir_list) > 1:
            raise ConfigurationError("Error in resource settings: probably extra comma in list.")
        return dir_list

    def diagnostics_enabled(self):
        req = ["{Diagnostics}.{state}"]
        elem_list = self.provide_settings(req)
        if not elem_list:
            raise ConfigurationError("No settings found for Diagnostics state.")
        en_str = elem_list[0][0].text.strip()
        if en_str == "Enabled":
            return True
        elif en_str == "Disabled":
            return False
        raise ConfigurationError("Error in Diagnostics settings: invalid state, can only be 'Enabled' or 'Disabled'.")

    def diagnostics_port(self):
        req = ["{Diagnostics}.{port}"]
        elem_list = self.provide_settings(req)
        if not elem_list:
            raise ConfigurationError("No settings found for Diagnostics.")
        port_str = elem_list[0][0].text.strip()
        try:
            port = int(port_str)
        except TypeError:
            raise ConfigurationError("Error in Diagnostics settings: port number setting must be an integer.")
        if port < 1000:
            raise ConfigurationError("Error in Diagnostics settings: port number needs to be larger than 1000 to prevent conflict with reserved ports.")
        return port

    def set_unitname(self, unitname):
        self.unitname = unitname

    def main_hostname(self):
        hostname = self.top_level_element_value("MAIN_HOSTNAME")
        if hostname is None:
            raise ConfigurationError("No settings found for main unit hostname.")
        hostname = hostname.replace(' ','')
        return hostname

    def set_ip(self, ip):
        self.connected_ip = ip

    def get_connected_ip(self):
        return self.connected_ip


import numpy as np
from sklearn.cluster import KMeans
from sklearn.decomposition import TruncatedSVD

from preprocessing import MTL_preprocessing

class MTL:
    def __init__(self, data, link = 'linear', intercept = True, n_class = 2, penalty = 'new', standardization = True):
        # task j has n_j samples and an empirical risk f_j (sample average)
        # objective fcn: \sum_j n_j [ f_j(w - v_j) + lambda_j \| v_j \|_2 ]
        # data = [X, y]
        # X: a list of m feature matrices, each of which is (n_j, d)
        # y: a list of m response vectors, each of which is (n_j, 1)
        # link = 'linear' or 'logistic'
        # n_class: number of classes in logistic regression, ignored for linear regression
        # class indices in y range from 0, 1, ..., n_class - 1
        # penalty: 'new' or 'ridge'
            # 'new': lbd_j * ||v|| on the j-th node
            # 'ridge': lbd_j * ||v||^2

        [self.X, self.y, self.X_means, self.X_stds, self.y_mean, self.y_std, self.n_list, self.d_out] = MTL_preprocessing(data, link, intercept, n_class, standardization)
        self.y_raw = data[1] # used in clustered and low-rank ARMUL

        self.intercept, self.n_class, self.standardization = intercept, n_class, standardization

        self.m = len(data[0])
        self.d = data[0][0].shape[1]
        if intercept: # No. of feature + 1 (when intercept = True)
            self.d += 1
        self.N = np.sum(self.n_list)
        self.link = link
        self.penalty = penalty
        self.models = dict()

    def vanilla_train(self, lbd = None, eta_global = 0.01, eta_local = 0.01, T_global = 100, T_local = 1):
        # vanilla ARMUL
        # lbd: a list of m penalty parameters
        if lbd is None:
            lbd = 0.1 * np.sqrt(self.d / self.n_list)
        gamma = np.zeros((self.d, self.d_out)) # global model
        V = np.zeros((self.m, self.d, self.d_out)) # local corrections

        for t in range(T_global):
            g = np.zeros((self.d, self.d_out))
            for idx in range(self.m):
                # update v_idx
                v = pgd_v(self.X[idx], self.y[idx], self.link, lbd[idx], gamma, V[idx], self.penalty, eta_local, T_local)
                V[idx] = v

                # compute the local gradient of gamma
                g += gradient_loss(self.X[idx], self.y[idx], gamma - v, self.link) / self.N

            # gd on gamma
            gamma -= eta_global * g

        # global model
        self.models['vanilla_gamma'] = gamma

        # get local models
        self.models['vanilla'] = gamma - V

    def clustered_train(self, lbd = None, K = 2, eta_B = 0.01, eta_local = 0.01, T_global = 100, T_B = 1, T_local = 1):
        if lbd is None:
            lbd = 0.1 * np.sqrt(self.d / self.n_list)

        ####################
        # warm start by STL
        if self.link == 'linear' or self.n_class == 1:
            base = baselines([self.X, self.y], self.link, False, self.n_class, False)
        else:
            base = baselines([self.X, self.y_raw], self.link, False, self.n_class, False)

        base.STL_train(eta = eta_local, T = T_global)
        stl = base.models['STL'] # STL

        kmeans = KMeans(n_clusters = K, random_state = 0).fit(stl.reshape(self.m, self.d * self.d_out))
        B = np.zeros((K, self.d, self.d_out))
        Z = np.zeros((self.m, K))
        for k in range(K):
            B[k] = kmeans.cluster_centers_[k].reshape(self.d, self.d_out)
        Z[:, kmeans.labels_] = 1
        V = np.zeros((self.m, self.d, self.d_out))
        for j in range(self.m):
            V[j] = B[kmeans.labels_[j]] - stl[j]

        ####################

        for t in range(T_global):
            # update v
            for idx in range(self.m):
                # compute the global model
                gamma = np.zeros((self.d, self.d_out))
                for r in range(self.d_out):
                    tmp1 = B[:, :, r].T
                    tmp2 = Z[idx].reshape(-1, 1)
                    gamma[:, r] = (tmp1 @ tmp2).reshape(-1,)
                # update v_idx
                v = pgd_v(self.X[idx], self.y[idx], self.link, lbd[idx], gamma, V[idx], self.penalty, eta_local, T_local)
                V[idx] = v

            # update B and Z
            B = gd_B(self.X, self.y, self.link, B, Z, V, eta_B, T_B)
            Z = hard_Z(self.X, self.y, self.link, B, V)

        # get local models
        Theta_clustered = np.zeros((self.m, self.d, self.d_out))
        for r in range(self.d_out):
            Br = B[:, :, r]
            Theta_clustered[:, :, r] = Z @ Br - V[:, :, r] # m-by-d

        self.models['clustered'] = Theta_clustered
        self.models['clustered_B'] = B
        self.models['clustered_Z'] = Z

    def lowrank_train(self, lbd = None, K = 2, eta_B = 0.01, eta_Z = 0.01, eta_local = 0.01, T_global = 100, T_B = 1, T_Z = 1, T_local = 1):
        if lbd is None:
            lbd = 0.1 * np.sqrt(self.d / self.n_list)

        ####################
        # warm start by STL
        if self.link == 'linear' or self.n_class == 1:
            base = baselines([self.X, self.y], self.link, False, self.n_class, False)
        else:
            base = baselines([self.X, self.y_raw], self.link, False, self.n_class, False)

        base.STL_train(eta = eta_local, T = T_global)
        stl = base.models['STL'] # STL

        ########################################

        svd = TruncatedSVD(n_components = K, n_iter = 10, random_state = 0)
        svd.fit( stl.reshape(self.m, self.d * self.d_out).T )
        Z = svd.components_.T # m by K
        B = np.zeros((K, self.d, self.d_out))
        tmp = np.zeros((self.m, self.d, self.d_out)) # low-rank model
        for r in range(self.d_out):
            B[:, :, r] = Z.T @ stl[:, :, r]
            tmp[:, :, r] = Z @ B[:, :, r]
        V = tmp - stl

        #####################
        # balancing
        tmp = np.linalg.norm(B.reshape(-1,))
        B, Z = B / np.sqrt(tmp), Z * np.sqrt(tmp)

        for t in range(T_global):
            # Step 1: update v
            for idx in range(self.m):
                # compute the global model
                gamma = np.zeros((self.d, self.d_out))
                for r in range(self.d_out):
                    tmp1 = B[:, :, r].T
                    tmp2 = Z[idx].reshape(-1, 1)
                    gamma[:, r] = (tmp1 @ tmp2).reshape(-1,)
                # update v_idx
                v = pgd_v(self.X[idx], self.y[idx], self.link, lbd[idx], gamma, V[idx], self.penalty, eta_local, T_local)
                V[idx] = v

            # Step 2: update B and Z
            # compute the current objective value and save the current B, Z
            obj = self.get_loss_lowrank(B, Z, V)
            B_cache, Z_cache = B.copy(), Z.copy()

            # update B and Z
            B = gd_B(self.X, self.y, self.link, B, Z, V, eta_B, T_B)
            Z = gd_Z(self.X, self.y, self.link, B, Z, V, eta_Z, T_Z)

            # if the objective increases, use the old B, Z
            tmp = self.get_loss_lowrank(B, Z, V)
            if tmp > obj:
                B, Z = B_cache.copy(), Z_cache.copy()
            obj = tmp

        self.models['lowrank'] = np.zeros((self.m, self.d, self.d_out))
        for r in range(self.d_out):
            self.models['lowrank'][:, :, r] = Z @ B[:, :, r] - V[:, :, r] # m-by-d
        self.models['lowrank_B'] = B
        self.models['lowrank_Z'] = Z

    def predict(self, X_test, model):
        return prediction(X_test, self.models[model], self.link, self.X_means, self.X_stds, self.y_mean, self.y_std)

    def evaluate(self, data_test, model):
        y_pred = self.predict(data_test[0], model)
        return evaluation(y_pred, data_test[1], self.link)

    def get_loss_lowrank(self, B, Z, V):
        model = np.zeros((self.m, self.d, self.d_out))
        for r in range(self.d_out):
            model[:, :, r] = Z @ B[:, :, r] - V[:, :, r] # m-by-d
        ans = 0
        for j in range(self.m):
            ans += self.n_list[j] * loss(self.X[j], self.y[j], model[j], link = self.link)
        return ans

# baselines
# single-task learning, hard parameter sharing, clustered MTL and low-rank MTL
class baselines:
    def __init__(self, data, link = 'linear', intercept = True, n_class = 2, standardization = True):
        [self.X, self.y, self.X_means, self.X_stds, self.y_mean, self.y_std, self.n_list, self.d_out] = MTL_preprocessing(data, link, intercept, n_class, standardization)
        self.m = len(self.X)
        self.d = self.X[0].shape[1]
        self.N = np.sum(self.n_list)
        self.link = link
        self.models = dict()

    def STL_train(self, eta = 0.01, T = 1000): # STL
        Theta_STL = np.zeros((self.m, self.d, self.d_out))
        for t in range(T):
            for j in range(self.m):
                theta = Theta_STL[j]
                g = gradient_loss(self.X[j], self.y[j], theta, self.link) / self.n_list[j]
                Theta_STL[j] = theta - eta * g
        self.models['STL'] = Theta_STL

    def DP_train(self, eta = 0.01, T = 1000): # DP
        gamma = np.zeros((self.d, self.d_out))
        for t in range(T):
            g = np.zeros((self.d, self.d_out))
            for j in range(self.m):
                g += gradient_loss(self.X[j], self.y[j], gamma, self.link) / self.N
            gamma -= eta * g
        Theta_DP = np.zeros((self.m, self.d, self.d_out))
        for j in range(self.m):
            Theta_DP[j] = gamma
        self.models['DP'] = Theta_DP

    def clustered_train(self, K = 2, eta_B = 0.01, T = 100, T_B = 1):

        ####################
        # warm start by STL
        self.STL_train(eta = eta_B, T = T)
        stl = self.models['STL'] # STL

        kmeans = KMeans(n_clusters = K, random_state = 0).fit(stl.reshape(self.m, self.d * self.d_out))
        B = np.zeros((K, self.d, self.d_out))
        Z = np.zeros((self.m, K))
        for k in range(K):
            B[k] = kmeans.cluster_centers_[k].reshape(self.d, self.d_out)
        Z[:, kmeans.labels_] = 1

        #########################

        for t in range(T):
            # update B and Z
            B = gd_B(X = self.X, y = self.y, link = self.link, B = B, Z = Z, eta = eta_B, T = T_B)
            Z = hard_Z(X = self.X, y = self.y, link = self.link, B = B)

        # get local models
        Theta_clustered = np.zeros((self.m, self.d, self.d_out))
        for r in range(self.d_out):
            Br = B[:, :, r]
            Theta_clustered[:, :, r] = Z @ Br # m-by-d

        self.models['clustered'] = Theta_clustered
        self.models['clustered_B'] = B
        self.models['clustered_Z'] = Z

    def lowrank_train(self, K = 2, eta_B = 0.01, eta_Z = 0.01, T = 100, T_B = 1, T_Z = 1):

        ####################
        # warm start by STL
        self.STL_train(eta = eta_B, T = T)
        stl = self.models['STL'] # STL
        svd = TruncatedSVD(n_components = K, n_iter = 10, random_state = 0)
        svd.fit( stl.reshape(self.m, self.d * self.d_out).T )
        Z = svd.components_.T # m by K
        B = np.zeros((K, self.d, self.d_out))
        for r in range(self.d_out):
            B[:, :, r] = Z.T @ stl[:, :, r]
        #####################
        # balancing
        tmp = np.linalg.norm(B.reshape(-1,))
        B, Z = B / np.sqrt(tmp), Z * np.sqrt(tmp)
        self.models['lowrank'] = np.zeros((self.m, self.d, self.d_out))
        for r in range(self.d_out):
            self.models['lowrank'][:, :, r] = Z @ B[:, :, r] # m-by-d
        obj = self.get_loss(model = 'lowrank')

        for t in range(T):
            # save the current model
            model_cache = self.models['lowrank'].copy

            # update B and Z
            B = gd_B(X = self.X, y = self.y, link = self.link, B = B, Z = Z, eta = eta_B, T = T_B)
            Z = gd_Z(X = self.X, y = self.y, link = self.link, B = B, Z = Z, eta = eta_Z, T = T_Z)
            #####################
            self.models['lowrank'] = np.zeros((self.m, self.d, self.d_out))
            for r in range(self.d_out):
                self.models['lowrank'][:, :, r] = Z @ B[:, :, r] # m-by-d
            tmp = self.get_loss(model = 'lowrank')

            if tmp > obj:
                self.models['lowrank'] = model_cache
                break
            obj = tmp
            #####################

        # get local models
        self.models['lowrank'] = np.zeros((self.m, self.d, self.d_out))
        for r in range(self.d_out):
            self.models['lowrank'][:, :, r] = Z @ B[:, :, r] # m-by-d
        self.models['lowrank_B'] = B
        self.models['lowrank_Z'] = Z

    def predict(self, X_test, model):
        return prediction(X_test, self.models[model], self.link, self.X_means, self.X_stds, self.y_mean, self.y_std)

    def evaluate(self, data_test, model):
        y_pred = self.predict(data_test[0], model)
        return evaluation(y_pred, data_test[1], self.link)

    def get_loss(self, model):
        ans = 0
        for j in range(self.m):
            ans += self.n_list[j] * loss(self.X[j], self.y[j], self.models[model][j], link = self.link)
        return ans

####################################################
## Auxiliaries

def gradient_loss(X, y, b, link = 'linear'):
    z = X @ b

    if link == 'logistic':
        d_out = b.shape[1]
        if d_out == 1: # y_i = 0 or 1
            z = 1 / (1 + np.exp(-z))
        else: # one-hoc encoding
            tmp = np.exp(z)
            tmp2 = tmp @ np.ones((d_out, 1))
            z = tmp / tmp2

    return X.T @ (z - y)

def pgd_v(X, y, link, lbd, w, v, penalty = 'new', eta = 0.01, T = 100):
    # find an (approximately) optimal v_j for a given w
    # i.e. approximate minimizer of f_j(w - v_j) + lambda_j \| v_j \|_2
    # optimization algorithm: proximal gd (new) or gd (ridge)

    # X: (n, d), y: (n, 1)
    # lbd: penalty parameter
    # w: global model
    # v: local correction
    # eta: stepsize
    # T: number of iterations

    v_t = v
    for t in range(T):
        g = - gradient_loss(X, y, w - v_t, link) / len(y)
        tmp = v_t - eta * g # gradient descent

        if penalty == 'new':
            r = np.linalg.norm(tmp, ord = 'fro')
            if r > 0:
                v_t = max(0, 1 - lbd * eta / r) * tmp
            else:
                v_t = 0
        else: # 'ridge'
            v_t -= eta * (g + lbd * v_t)

    return v_t

def gd_B(X, y, link, B, Z, V = None, eta = 0.01, T = 100):
    # B: K-d-d_out
    # Z: m-K
    # V: m-d-d_out
    K, d, d_out = B.shape
    m = Z.shape[0]
    N = sum([len(l) for l in y])
    if V is None:
        V = np.zeros((m, d, d_out))
    B_t = B
    for t in range(T):
        g = np.zeros((d, d_out))
        for idx in range(m):
            # compute the global model
            gamma = np.zeros((d, d_out))
            for r in range(d_out):
                gamma[:, r] = ( B_t[:, :, r].T @ Z[idx].reshape(-1, 1) ).reshape(-1,)
            g += gradient_loss(X[idx], y[idx], gamma - V[idx], link) / N

        # GD on B
        for k in range(K):
            B_t[k] = B_t[k] - eta * np.sum(Z[:, k]) * g
    return B_t

def gd_Z(X, y, link, B, Z, V = None, eta = 0.01, T = 100):
    # B: K-d-d_out
    # Z: m-K
    # V: m-d-d_out
    K, d, d_out = B.shape
    m = Z.shape[0]
    n_list = [len(l) for l in y]
    if V is None:
        V = np.zeros((m, d, d_out))
    Z_t = Z
    for t in range(T):
        for idx in range(m):
            # compute the global model
            gamma = np.zeros((d, d_out))
            for r in range(d_out):
                gamma[:, r] = ( B[:, :, r].T @ Z_t[idx].reshape(-1, 1) ).reshape(-1,)
            g_idx = gradient_loss(X[idx], y[idx], gamma - V[idx], link) / n_list[idx]

            # GD on Z
            for k in range(K):
                Z_t[idx, k] = Z_t[idx, k] - eta * np.sum(g_idx * B[k])
    return Z_t

def hard_Z(X, y, link, B, V = None):
    # B: K-d-d_out
    # Z: m-K
    # V: m-d-d_out
    K, d, d_out = B.shape
    m = len(X)
    Z = np.zeros((m, K))
    if V is None:
        V = np.zeros((m, d, d_out))

    for idx in range(m):
        tmp = np.zeros(K)
        for k in range(K):
            tmp[k] = loss(X[idx], y[idx], B[k] - V[idx], link)
        Z[idx, np.argmin(tmp)] = 1
    return Z

def loss(X, y, b, link = 'linear'):
    # X: (n, d), y: (n, d_out)
    n = X.shape[0]
    z = X @ b   # (n, d_out)
    if link == 'linear':
        return np.mean( (z - y) ** 2 )
    if link == 'logistic':
        d_out = b.shape[1]
        if d_out == 1: # y_i = 0 or 1
            return np.mean( np.log(1 + np.exp(z)) - y * z )
        else: # one-hoc encoding
            tmp = np.exp(z)
            tmp2 = tmp @ np.ones((d_out, 1))
            return np.mean( np.log(tmp2) ) - np.sum(y * z) / n

def prediction(X_test, Theta, link = 'linear', X_means = np.zeros((1, 1)), X_stds = np.ones((1, 1)), y_mean = 0, y_std = 1):
    # X_test: m tasks
    # output: m arrays of shape = (n,)

    intercept = True
    if X_test[0].shape[1] == Theta.shape[1]: # no all-one column added
        intercept = False

    y_pred = []
    for (j, X) in enumerate(X_test):
        if intercept: # add an all-one column
            X = np.concatenate((np.ones((X.shape[0], 1)), X), axis = 1)

        # standardization using the means and stds of the training data
        X0 = (X - X_means.T) / X_stds.T
        z = X0 @ Theta[j]

        if link == 'linear':
            y_pred_j = (z * y_std + y_mean)
        if link == 'logistic':
            if Theta.shape[2] == 1:
                y_pred_j = ((z >= 0) + 0)
            else:
                y_pred_j = np.argmax(z, axis = 1)

        y_pred.append(y_pred_j.reshape(-1,))
    return y_pred

def evaluation(y_pred, y_true, link = 'linear'):
    # both y_pred and y_true have m tasks
    m = len(y_pred)
    n_list = np.zeros(m) # local sample sizes
    err = np.zeros(m) # MSE or misclassification error
    tmp = np.zeros(m) # for computing R2 in linear regression

    for (j, y_pred_j) in enumerate(y_pred):
        y_pred_j = y_pred_j.reshape(-1,)
        n_list[j] = len(y_pred_j)
        y_true_j = y_true[j].reshape(-1,)
        if link == 'linear':
            err[j] = np.mean( (y_pred_j - y_true_j) ** 2 )
            tmp[j] = np.mean( ( y_true[j] - np.mean(y_true_j) ) ** 2 )
        if link == 'logistic':
            err[j] = np.mean(y_pred_j != y_true_j)
    N = np.sum(n_list)

    ans = dict()
    ans['errors'] = err
    ans['average error'] = np.sum(err * n_list) / N

    if link == 'linear':
        total_variance = np.sum(tmp * n_list) / N
        ans['R2'] = 1 - ans['average error'] / total_variance

    return ans