from datetime import datetime

import constants

class Date:
    def __init__(self, date: str) -> None:
        """Convert and validate the input date

        Args:
            date (str): The input date in the ISO format
        """
        self._date = date
        self._initial_date = datetime.strptime(
            constants.INITIAL_DATE, constants.DATE_FORMAT
        ).date()
        self.convert_to_datetime()

    def convert_to_datetime(self):
        """Convert the date string to datetime.date"""
        self._date = datetime.strptime(
            self._date, constants.DATE_FORMAT
        ).date()
        self.validate_initial_date()

    def validate_initial_date(self):
        """Validate if provided date is greater than the initial date

        Raises:
            ValueError: If the provided date is smaller that the initial date
        """
        if self._date < self._initial_date:
            raise ValueError(
                f"The date provided: {self._date} is older than the initial date: {self._initial_date}"
            )
        self.validate_current_date()

    def validate_current_date(self):
        """Validate if the provided date is not in the future

        Raises:
            ValueError: If the provided date is in the future
        """
        if self._date >= datetime.now().date():
            raise ValueError(
                f"The date provided {self._date} is in the future"
            )

    @property
    def date(self) -> datetime.date:
        """Return the converted and validated date

        Returns:
            datetime.date: The input date as datetime.date
        """
        return self._date


from enum import Enum

# Entity IDs
WALL_ID = 1
FRUIT_ID = 2

# Entity Scores
FRUIT_SCORE = 1

# Action IDs
NOTHING = 0
LEFT = 1
RIGHT = 2
UP = 3
DOWN = 4

class Result(Enum):
    FAIL = "failed to act"
    NOTHING = "did nothing"
    LEFT = "moved left"
    RIGHT = "moved right"
    UP = "moved up"
    DOWN = "moved down"
    FRUIT = "ate fruit"

class Agent:
    """Agent class that interacts in the environment."""
    def __init__(self, idx=9, x=1, y=1):
        """
        Initialise the agent with fixed parameters.
        """
        self.idx = idx
        self.x, self.y = x, y
        self.score = 0

    def nothing(self, state):
        """
        Do nothing as the action taken.
        Args:
            state:

        Returns:

        """
        return [Result.NOTHING]

    def left(self, state):
        """
        Move left as the action taken.
        Args:
            state: np.array - 2D array representing game state
        Returns:
            0 if failure, 1 if success
        """
        if self.check_wall(state, LEFT):
            return [Result.FAIL]
        results = [Result.LEFT]
        self.x -= 1
        if self.check_fruit(state):
            results.append(Result.FRUIT)
            self.eat_fruit(state)
        return results

    def right(self, state):
        """
        Move right as the action taken.
        Args:
            state: np.array - 2D array representing game state

        Returns:
            0 if failure, 1 if success
        """
        if self.check_wall(state, RIGHT):
            return [Result.FAIL]
        results = [Result.RIGHT]
        self.x += 1
        if self.check_fruit(state):
            results.append(Result.FRUIT)
            self.eat_fruit(state)
        return results

    def up(self, state):
        """
        Move up as the action taken.
        Args:
            state: np.array - 2D array representing game state

        Returns:
            0 if failure, 1 if success
        """
        if self.check_wall(state, UP):
            return [Result.FAIL]
        results = [Result.UP]
        self.y -= 1
        if self.check_fruit(state):
            results.append(Result.FRUIT)
            self.eat_fruit(state)
        return results

    def down(self, state):
        """
        Move down as the action taken.
        Args:
            state: np.array - 2D array representing game state

        Returns:
            0 if failure, 1 if success
        """
        if self.check_wall(state, DOWN):
            return [Result.FAIL]
        results = [Result.DOWN]
        self.y += 1
        if self.check_fruit(state):
            results.append(Result.FRUIT)
            self.eat_fruit(state)
        return results

    def act(self, action, state):
        """
        Take the action
        Args:
            action: int - action taken by the agent
            state: np.array - 2D array representing game state

        Returns:
            string - result having taken the action
        """
        results = [Result.FAIL]
        state[self.y][self.x] = 0  # clear past position
        if action == NOTHING:
            results = self.nothing(state)
        elif action == LEFT:
            results = self.left(state)
        elif action == RIGHT:
            results = self.right(state)
        elif action == UP:
            results = self.up(state)
        elif action == DOWN:
            results = self.down(state)
        self.update_state(state)
        return ", ".join([r.value for r in results])

    def eat_fruit(self, state):
        """
        Eat fruit to obtain score of 1.
        Args:
            state: np.array - 2D array representing game state
        """
        self.score += FRUIT_SCORE

    def check_wall(self, state, action):
        """
        Check if taking an action in a given state would lead to a wall.
        Args:
            state: np.array - 2D array representing game state
            action: int - action taken by the agent

        Returns:
            True if agent would hit a wall else False
        """
        if action == LEFT:
            return state[self.y, self.x - 1] == WALL_ID
        elif action == RIGHT:
            return state[self.y, self.x + 1] == WALL_ID
        elif action == UP:
            return state[self.y - 1, self.x] == WALL_ID
        elif action == DOWN:
            return state[self.y + 1, self.x] == WALL_ID
        raise Exception(f"Action movement invalid. Expected: [1, 2, 3, 4], Given: {action}")

    def check_fruit(self, state):
        """
        Check if there exists a fruit at the agent's current position.
        Args:
            state: np.array - 2D array representing game state

        Returns:
            True if agent is on a fruit else False
        """
        return state[self.y, self.x] == FRUIT_ID

    def update_state(self, state):
        """
        Update the state with agent's identifier
        Args:
            state: np.array - 2D array representing game state
        """
        state[self.y, self.x] = self.idx


#! /usr/bin/env python3

import sys
from pathlib import Path
from math import *

def encode(x, y):
    return (x+7) + 15*(y+7)

"""
    static String n = "\00\01\02";

    static boolean isIn2(Direction dir) {
        return n.indexOf(dir.ordinal()) != -1;
    }
"""
RADII = {'34': 34, '20': 20, '10': 10}
SMALLER_RADII = {'34': 20, '20': 10, '10': 5}

DIRECTIONS = {
    (1, 0): 'EAST',
    (-1, 0): 'WEST',
    (0, 1): 'NORTH',
    (0, -1): 'SOUTH',
    (1, 1): 'NORTHEAST',
    (-1, 1): 'NORTHWEST',
    (1, -1): 'SOUTHEAST',
    (-1, -1): 'SOUTHWEST',
}

def min_chain(vals):
    if len(vals) == 1:
        return vals[0]
    return f"Math.min({vals[0]}, {min_chain(vals[1:])})"

def opposite(dir):
    if dir == 'EAST':
        return 'WEST'
    if dir == 'WEST':
        return 'EAST'
    if dir == 'NORTH':
        return 'SOUTH'
    if dir == 'SOUTH':
        return 'NORTH'
    if dir == 'NORTHEAST':
        return 'SOUTHWEST'
    if dir == 'NORTHWEST':
        return 'SOUTHEAST'
    if dir == 'SOUTHEAST':
        return 'NORTHWEST'
    if dir == 'SOUTHWEST':
        return 'NORTHEAST'
    return 'CENTER'

def direction_to(dx, dy):
    if (abs(dx) >= 2.414 * abs(dy)):
        if (dx > 0):
            return "EAST"
        elif (dx < 0):
            return "WEST"
        else:
            return "CENTER"
    elif (abs(dy) >= 2.414 * abs(dx)):
        if (dy > 0):
            return "NORTH"
        else:
            return "SOUTH"
    else:
        if (dy > 0):
            if (dx > 0):
                return "NORTHEAST"
            else:
                return "NORTHWEST"
        else:
            if (dx > 0):
                return "SOUTHEAST"
            else:
                return "SOUTHWEST"

def dist(x, y):
    return x*x + y*y

def gen_constants(radius):
    out = f""""""
    for x in range(-7, 8):
        for y in range(-7, 8):
            if dist(x, y) <= radius:
                out += f"""
    static MapLocation l{encode(x,y)}; // location representing relative coordinate ({x}, {y})
    static double d{encode(x,y)}; // shortest distance to location from current location
    static double score{encode(x,y)}; // heuristic distance from location to target
"""
    return out

def sign(x):
    if x > 0:
        return 1
    if x < 0:
        return -1
    return 0

def gen_init(radius):
    out = f"""
        l{encode(0,0)} = rc.getLocation();
        d{encode(0,0)} = 0;
"""
    for r2 in range(1, radius+1):
        for x in range(-7, 8):
            for y in range(-7, 8):
                if dist(x, y) == r2:
                    out += f"""
        l{encode(x,y)} = l{encode(x - sign(x), y - sign(y))}.add({DIRECTIONS[(sign(x), sign(y))]}); // ({x}, {y}) from ({x - sign(x)}, {y - sign(y)})
        d{encode(x,y)} = 99999;
"""
    return out

def gen_bfs(radius):
    visited = set([encode(0,0)])
    out = f""""""
    for r2 in range(1, radius+1):
        for x in range(-7, 8):
            for y in range(-7, 8):
                if dist(x, y) == r2:
                    indent = ""
                    out += f"""
        // check ({x}, {y})
        if (rc.canSenseLocation(l{encode(x,y)}) && rc.sensePassability(l{encode(x,y)})) {{"""
                    indent = "    "
                    dxdy = [(dx, dy) for dx in range(-1, 2) for dy in range(-1, 2) if (dx, dy) != (0, 0) and dist(x+dx,y+dy) <= radius]
                    dxdy = sorted(dxdy, key=lambda dd: dist(x+dd[0], y+dd[1]))
                    vals = []
                    for dx, dy in dxdy:
                        if encode(x+dx, y+dy) in visited:
                            vals.append(str([1/16, 5/16, 2/16, 6/16, 3/16, 7/16, 4/16, 8/16][(round(atan2(-dy,-dx)/pi*4)+8)%8]) if (x+dx,y+dy) == (0, 0) else f'd{encode(x+dx,y+dy)}')
            #                 out += f"""
            # {indent}if (d{encode(x,y)} > d{encode(x+dx,y+dy)}) {{ // from ({x+dx}, {y+dy})
            #     {indent}d{encode(x,y)} = {str([5/16, 1/16, 6/16, 2/16, 7/16, 3/16, 8/16, 4/16][(round(atan2(-dy,-dx)/pi*4)+8)%8]) if (x+dx,y+dy) == (0, 0) else f'd{encode(x+dx,y+dy)}'};
            # {indent}}}"""
                    out += f"""
        {indent}d{encode(x,y)} = {min_chain(vals)};"""

                    out += f"""
        {indent}mapInfo = rc.senseMapInfo(l{encode(x,y)});
        {indent}currentDir = mapInfo.getCurrentDirection();
        {indent}d{encode(x,y)} += 10 * (mapInfo.getCooldownMultiplier(team)) +
        {indent}{indent} (currentDir == CENTER ? 0 :
        {indent}{indent} (TARGET_DIR_ADJ.indexOf(currentDir.ordinal()) >= 0 ? -5 : 5));"""
                    out += f"""
        }}"""
                    visited.add(encode(x,y))
                    out += f"""
"""
    return out

def gen_selection(radius, smaller_radius):
    out = f"""
        int target_dx = target.x - l{encode(0,0)}.x;
        int target_dy = target.y - l{encode(0,0)}.y;
        switch (target_dx) {{"""
    for tdx in range(-7, 8):
        if tdx**2 <= radius:
            out += f"""
                case {tdx}:
                    switch (target_dy) {{"""
            for tdy in range(-7, 8):
                if dist(tdx, tdy) <= radius:
                    out += f"""
                        case {tdy}:
                            return direction(d{encode(tdx, tdy)}); // destination is at relative location ({tdx}, {tdy})"""
                            # if (dir{encode(tdx, tdy)} != null)
                            #     return dir{encode(tdx, tdy)}; // destination is at relative location ({tdx}, {tdy})
                            # break;"""
            out += f"""
                    }}
                    break;"""
    out += f"""
        }}

        ans = Double.POSITIVE_INFINITY;
        bestScore = 0;
        currDist = Math.sqrt(l{encode(0,0)}.distanceSquaredTo(target));
        """
    for x in range(-7, 8):
        for y in range(-7, 8):
            if smaller_radius < dist(x, y) <= radius: # on the edge of the radius radius
                out += f"""
        score{encode(x,y)} = (currDist - Math.sqrt(l{encode(x,y)}.distanceSquaredTo(target))) / d{encode(x,y)};
        if (score{encode(x,y)} > bestScore) {{
            bestScore = score{encode(x,y)};
            ans = d{encode(x,y)};
        }}
"""
    return out

def gen_print(radius):
    out = f"""
        // System.out.println("LOCAL DISTANCES:");"""
    for y in range(7, -8, -1):
        if y**2 <= radius:
            out += f"""
        // System.out.println("""
            for x in range(-7, 8):
                if dist(x, y) <= radius:
                    out += f""""\\t" + d{encode(x,y)} + """
                else:
                    out += f""""\\t" + """
            out = out[:-3] + """);"""
    return out

def gen_full(bot, rad):
    radius = RADII[rad]
    smaller_radius = SMALLER_RADII[rad]
    out_file = Path('./src/') / bot / f'bfs/BFSCooldown{rad}.java'
    with open(out_file, 'w') as f:
        f.write(f"""package {bot}.bfs;

import battlecode.common.*;

public class BFSCooldown{rad} {{

    public static RobotController rc;
{gen_constants(radius)}

    public static void init(RobotController r) {{
        rc = r;
        team = rc.getTeam();
    }}

    private static final Direction[] DIRECTIONS = new Direction[] {{null, Direction.EAST, Direction.NORTH, Direction.WEST, Direction.SOUTH, Direction.NORTHEAST, Direction.NORTHWEST, Direction.SOUTHWEST, Direction.SOUTHEAST}};

    public final static Direction NORTH = Direction.NORTH;
    public final static Direction NORTHEAST = Direction.NORTHEAST;
    public final static Direction EAST = Direction.EAST;
    public final static Direction SOUTHEAST = Direction.SOUTHEAST;
    public final static Direction SOUTH = Direction.SOUTH;
    public final static Direction SOUTHWEST = Direction.SOUTHWEST;
    public final static Direction WEST = Direction.WEST;
    public final static Direction NORTHWEST = Direction.NORTHWEST;
    public final static Direction CENTER = Direction.CENTER;

    public static MapInfo mapInfo;
    public static Direction currentDir;
    public static Team team;
    public static double ans;
    public static double bestScore;
    public static double currDist;

    public static String NORTH_ADJ = "\\07\\00\\01";
    public static String NORTHEAST_ADJ = "\\00\\01\\02";
    public static String EAST_ADJ = "\\01\\02\\03";
    public static String SOUTHEAST_ADJ = "\\02\\03\\04";
    public static String SOUTH_ADJ = "\\03\\04\\05";
    public static String SOUTHWEST_ADJ = "\\04\\05\\06";
    public static String WEST_ADJ = "\\05\\06\\07";
    public static String NORTHWEST_ADJ = "\\06\\07\\00";
    public static String TARGET_DIR_ADJ = "";

    public static Direction direction(double dist) {{
        if (dist==Double.POSITIVE_INFINITY) {{
            return null;
        }}
        return DIRECTIONS[(int)(dist * 16 % 16)];
    }}

    public static Direction bestDir(MapLocation target) throws GameActionException {{
        switch (rc.getLocation().directionTo(target)) {{
            case NORTH: TARGET_DIR_ADJ = NORTH_ADJ; break;
            case NORTHEAST: TARGET_DIR_ADJ = NORTHEAST_ADJ; break;
            case EAST: TARGET_DIR_ADJ = EAST_ADJ; break;
            case SOUTHEAST: TARGET_DIR_ADJ = SOUTHEAST_ADJ; break;
            case SOUTH: TARGET_DIR_ADJ = SOUTH_ADJ; break;
            case SOUTHWEST: TARGET_DIR_ADJ = SOUTHWEST_ADJ; break;
            case WEST: TARGET_DIR_ADJ = WEST_ADJ; break;
            case NORTHWEST: TARGET_DIR_ADJ = NORTHWEST_ADJ; break;
            default: TARGET_DIR_ADJ = ""; break;
        }}
{gen_init(radius)}
{gen_bfs(radius)}
{gen_print(radius)}
{gen_selection(radius, smaller_radius)}

        return direction(ans);
    }}
}}
""")

if __name__ == '__main__':
    if len(sys.argv) == 2:
        for rad in ["34", "20", "10"]:
            gen_full(sys.argv[1], rad)
    else:
        gen_full(sys.argv[1], sys.argv[2])


import os
import anthem
import requests
API = 'https://loripsum.net/api/'

def loremsum(nbr, size):
    return requests.get('https://loripsum.net/api/%s/%s' % (nbr, size)).text

@anthem.log
def setup(ctx):
    env = ctx.env
    backend = env.ref('shopinvader.backend_1')
    backend.write({
        'auth_api_key_name': 'api_key_shopinvader_demo'
        })
    countries = env['res.country'].search([])
    backend.write({
        'allowed_country_ids': [(6, 0, countries.ids)],
        'category_binding_level': 3,
        })
    products = env['product.product'].search([
        ('image_ids', '!=', False),
        ])
    wizard = env['shopinvader.variant.binding.wizard'].create({
        'backend_id': backend.id,
        'product_ids': [(6, 0, products.ids)],
        })
    wizard.bind_products()
    for binding in env['shopinvader.product'].search([]):
        binding.write({
            'description': loremsum('3', 'medium'),
            'short_description': loremsum('1', 'short'),
            })


import matplotlib.pyplot as plt
import numpy as np
import math

x = [int(i) for i in input().split()]
y = [int(i) for i in input().split()]

N = len(x)
Sx = 0
Sy = 0
Sxy = 0
Sx2 = 0
Sy2 = 0

for i in range(0, N):
    Sx += x[i]
    Sy += y[i]
    Sxy += x[i] * y[i]
    Sx2 += x[i] ** 2
    Sy2 += y[i] ** 2

print(Sx)
print(Sy)
print(Sxy)
print(Sx2)
print(Sy2)

r = (N * Sxy - Sx * Sy) / (math.sqrt(N * Sx2 - Sx ** 2) * math.sqrt(N * Sy2 - Sy ** 2))
print("The Correlation coefficient is %0.3f" % r)

byx = (N * Sxy - Sx * Sy) / (N * Sx2 - Sx ** 2)
print(byx)

xmean = Sx / N
ymean = Sy / N
print("The Regression line Y on X is ::: y = %0.3f + %0.3f (x-%0.3f)" % (ymean, byx, xmean))
xmean = Sx / N
ymean = Sy / N
print("The Regression line Y on X is ::: y = %0.3f + %0.3f (x-%0.3f)" % (ymean, byx, xmean))

plt.scatter(x, y)

def Reg(x):
    return ymean + byx * (x - xmean)

x = np.linspace(0, 80, 51)
y1 = Reg(x)

plt.plot(x, y1, 'r')
plt.xlabel('x-data')
plt.ylabel('y-data')
plt.legend(['Regression Line', 'Data points'])
plt.show()


from tree_utils import *
import itertools
import copy
import numpy as np
import networkx as nx

import spacy
spacy_nlp = spacy.load("en_core_web_sm")

from evaluate_metric import eval_tree_task3

class Action:
    retrieve = 'retrieve'
    reason = 'reason'
    end = 'end'

    @classmethod
    def linearize_action(cls, action):
        action_str = ""

        if action['type'] == Action.retrieve:
            action_str += f"{action['type']}: {action['query_id']}"

        elif action['type'] == Action.reason:
            action_str += f"{action['type']}: "
            action_str += " & ".join(sorted(action['step']['pre_id']))
            if 'con_sent' in action['step']:
                action_str += f" -> {action['step']['con_sent']}"

        elif action['type'] == Action.end:
            action_str += f"{action['type']}"
            if 'is_proved' in action:
                action_str += f": {action['is_proved']}"

        else:
            raise NotImplementedError

        return action_str

    @classmethod
    def parse_action(cls, action_str):

        if ':' in action_str:
            action_type, paras_str = action_str.split(':', maxsplit=1)
            action_type = action_type.strip()
        else:
            action_type = action_str.strip()
            paras_str = None

        if action_type == Action.retrieve:
            action = {
                'type': action_type,
                'query_id': paras_str.strip(),
            }

        elif action_type == Action.reason:
            if '->' not in paras_str:
                pre_id = [p.strip() for p in paras_str.split('&')]
                action = {
                    'type': action_type,
                    'step': {
                        'pre_id': pre_id,
                    },
                    'use_module': True,
                }
            else:
                pre_id_str, con_sent = paras_str.split('->', maxsplit=1)
                pre_id = [p.strip() for p in pre_id_str.split('&')]
                action = {
                    'type': action_type,
                    'step': {
                        'pre_id': pre_id,
                        'con_sent': con_sent.strip(),
                    },
                    'use_module': False,
                }

        elif action_type == Action.end:
            if paras_str is None:
                action = {
                    'type': action_type,
                }
            else:
                action = {
                    'type': action_type,
                    'is_proved': paras_str.strip(),
                }
        else:
            action = None

        return action

class State:
    def __init__(self, data_item = None):

        self.data_item = data_item

        self.H = data_item.get('H','')
        self.Q = data_item.get('Q','')
        self.A = data_item.get('A','')
        self.S = data_item.get('S', [])
        self.P = data_item.get('P', [])
        self.used_S = data_item.get('used_S', [])
        self.sent2id = data_item.get('sent2id', {})
        self.history_actions = data_item.get('history_actions', [])
        self.choices_str = data_item.get('choices_str', [])

        if not self.H: self.H = data_item['hypothesis']
        if not self.Q: self.Q = data_item['question']
        if not self.A: self.A = data_item['answer']
        if not self.S: self.S = list(data_item.get('meta',{}).get('triples',{}).values())
        if not self.sent2id: self.sent2id = {v:k for k,v in data_item.get('meta',{}).get('triples',{}).items()}
        if not self.choices_str: self.choices_str = " ".join([f"({choice['label']}) {choice['text']}" for choice in data_item.get('choices', [])])

        self.sent2id[self.H] = 'hypothesis'

    def expand_P(self, step):
        # move premises from S to used_S
        assert all([p in self.S for p in step['pre_sent']])
        for p in set(step['pre_sent']):
            self.S.remove(p)
            self.used_S.append(p)

        # add conclusion to S at index 0
        self.S = [step['con_sent']] + self.S

        # add step to P
        self.P.append(step)

        # add conclusion to sent2id
        # some intermediate conclusions could be found in the courpus!!!
        if step['con_sent'] != self.H:
            self.sent2id[step['con_sent']] = self.next_id('int')

    def update_S(self, new_S):
        new_S = [sent for sent in new_S if sent not in self.used_S] # if sent has been used, do not add

        remain_S = []
        for sent in set(self.S):
            if not self.sent2id[sent].startswith('sent'):
                remain_S.append(sent)
            else:
                # print(f"del: {self.sent2id[sent]} {sent}")
                del self.sent2id[sent]

        # self.S = list(set(remain_S + new_S)) # 'set' will disrupt the order
        self.S = []
        for s in remain_S + new_S:
            if s not in self.S:
                self.S.append(s)

        for sent in self.used_S + self.S:
            if sent not in self.sent2id:
                # print(self.next_id('sent'), self.sent2id.values())
                self.sent2id[sent] = self.next_id('sent')

        assert all([sent in self.sent2id for sent in self.S+self.used_S])

    def visualize_gt_graph(self):
        try:
            visualize_nx_graph(get_gt_nx_graph(self.data_item))
        except Exception as e:
            print(e)

    def get_pred_nx_graph(self):
        return State.P_to_nx_graph(self.P, self.sent2id)

    def get_pred_node_list(self, root_node = None):
        if root_node is None:
            root_node = self.H
        pred_tree = nx_graph_to_node_list(self.get_pred_nx_graph(), root_node = root_node)
        if len(pred_tree) == 0:
            pred_tree = [
                {'id':'hypothesis', 'sent':self.H, 'pre':[]},
            ]
        return pred_tree

    def visualize_pred_graph(self):
        try:
            visualize_nx_graph(self.get_pred_nx_graph())
        except Exception as e:
            print(e)

    def get_pred_highest_conclusions(self):
        ### take too much time
        # pred_nx_graph = self.get_pred_nx_graph()
        # return list(set([path[-1] for path in nx.all_topological_sorts(pred_nx_graph) if path]))

        # find nodes which has in_edge but does not have out_edge
        highest_conclusions = []
        nx_graph = self.get_pred_nx_graph()
        for node in nx_graph:
            if nx_graph.in_degree(node) > 0 and nx_graph.out_degree(node) == 0:
                highest_conclusions.append(str(node))
        return highest_conclusions

    def get_pred_height(self):
        nx_graph = self.get_pred_nx_graph()

        # if nx.is_directed_acyclic_graph(nx_graph):
        #     return nx.dag_longest_path_length(nx_graph)
        # else:
        #     print("predicted graph is not DAG")
        #     return 1e2

        try:
            return nx.dag_longest_path_length(nx_graph)
        except Exception as e:
            print(e)
            return 1e2

    def __str__(self):
        s = ""
        s += f"H: {self.H}\n"

        s += f"S:\n"
        for sent in self.S:
            s += f"\t{self.sent2id[sent]}:  {sent}\n"

        s += f"used S:\n"
        for sent in self.used_S:
            s += f"\t{self.sent2id[sent]}:  {sent}\n"

        s += f"P:\n"
        for step in self.P:
            s += f"{' & '.join([self.sent2id[p] for p in step['pre_sent']])} -> {self.sent2id[step['con_sent']]}; "

        # s += f"\nQ: {self.Q}\n"
        # s += f"A: {self.A}\n"
        return s

    def next_id(self, ident='int'):
        assert ident in ['sent', 'int']
        for i in itertools.count(1):
            if f"{ident}{i}" not in self.sent2id.values():
                return f"{ident}{i}"

    def reassign_sent2id(self):
        """
        reassign the identifier of sent and int
        sent2id contain sentences in S / used_S / H
        """
        sent_count = 1
        int_count = 1

        for sent in self.used_S:
            if self.sent2id[sent].startswith('sent'):
                self.sent2id[sent] = f"sent{sent_count}"
                sent_count += 1
            elif self.sent2id[sent].startswith('int'):
                self.sent2id[sent] = f"int{int_count}"
                int_count += 1

        for sent in self.S:
            if self.sent2id[sent].startswith('sent'):
                self.sent2id[sent] = f"sent{sent_count}"
                sent_count += 1
            elif self.sent2id[sent].startswith('int'):
                self.sent2id[sent] = f"int{int_count}"
                int_count += 1

    @classmethod
    def P_to_nx_graph(cls, P, sent2id):
        """
        convert the partial proof to nx-graph
        """

        G = nx.DiGraph()
        for step in P:
            # add node
            for sent in step['pre_sent']+[step['con_sent']]:
                G.add_node(sent, id=sent2id[sent], sent=sent)
            # add edge
            for sent in step['pre_sent']:
                G.add_edge(sent, step['con_sent'])
        return G

    def linearize_proof(self):
        proof_str = ""
        for step in self.P:
            proof_str += " & ".join(sorted([self.sent2id[sent] for sent in step['pre_sent']]))
            proof_str += f" -> {self.sent2id[step['con_sent']]}; "
        proof_str = proof_str[:-2]
        return proof_str

    def linearize_context(self, sent_list = None):
        s = ""
        # for sent in sent_list:
        #     s += f"{self.sent2id[sent]}: {add_fullstop(sent)} "

        sentX_sents = []
        not_sentX_sents = []
        for sent in sent_list:
            if self.sent2id[sent].startswith('sent'):
                sentX_sents.append(sent)
            else:
                not_sentX_sents.append(sent)

        for sent in not_sentX_sents + sentX_sents:
            s += f"{self.sent2id[sent]}: {add_fullstop(sent)} "

        return s

    def linearize_state(self, form = 'default'):

        if form == 'default' or form == 'HPS':
            state_str = ""
            state_str += f"$hypothesis$ {add_fullstop(self.H)} "
            state_str += f"$proof$ {self.linearize_proof()} "
            state_str += f"$context$ {self.linearize_context(self.S)} "
        elif form == 'QAHPS':
            state_str = ""
            state_str += f"$question$ {self.Q} $answer$ {add_fullstop(self.A.lower())} "
            state_str += f"$hypothesis$ {add_fullstop(self.H)} "
            state_str += f"$proof$ {self.linearize_proof()} "
            state_str += f"$context$ {self.linearize_context(self.S)} "
        elif form == 'QAHPN':
            node_sents = []
            for step in self.P[::-1]: #  from root to leaves
                for sent in [step['con_sent']] + step['pre_sent']:
                    if sent not in node_sents:
                        node_sents.append(sent)

            state_str = ""
            state_str += f"$question$ {self.Q} $answer$ {add_fullstop(self.A.lower())} "
            state_str += f"$hypothesis$ {add_fullstop(self.H)} "
            state_str += f"$proof$ {self.linearize_proof()} "
            state_str += f"$node$ {self.linearize_context(node_sents)} "
        elif form == 'QACHPN':
            node_sents = []
            for step in self.P[::-1]: #  from root to leaves
                for sent in [step['con_sent']] + step['pre_sent']:
                    if sent not in node_sents:
                        node_sents.append(sent)

            state_str = ""
            state_str += f"$question$ {self.Q} $answer$ {add_fullstop(self.A.lower())} "
            state_str += f"$choices$ {self.choices_str} "
            state_str += f"$hypothesis$ {add_fullstop(self.H)} "
            state_str += f"$proof$ {self.linearize_proof()} "
            state_str += f"$node$ {self.linearize_context(node_sents)} "
        else:
            # state_str += f"$used$ {self.linearize_context(self.used_S)}"
            raise NotImplementedError

        return state_str

    def to_dict(self):
        return copy.deepcopy({
            'H': self.H,
            'Q': self.Q,
            'A': self.A,
            'S': self.S,
            'used_S': self.used_S,
            'P': self.P,
            'sent2id': self.sent2id,
            'history_actions': self.history_actions,
            'choices_str': self.choices_str,
        })

# class MyEnv(gym.Env): # ref: https://www.gymlibrary.ml/content/environment_creation/
class EntailmentTreeEnv:
    def __init__(self,  retriever = None,
                        entailment_module = None,
                        verifier = None,
                        env_args = {}
                        ):
        """

        """
        self.retriever = retriever
        self.entailment_module = entailment_module
        self.verifier = verifier

        self.env_args = env_args
        self.action_budget = env_args.get('action_budget', {})

        self.retrieve_top_n = env_args.get('retrieve_top_n', 25)
        self.max_height = env_args.get('max_height', 10)
        self.check_premise_overlap = bool(env_args.get('check_premise_overlap', 0))
        self.step_score_thre = env_args.get('step_score_thre', 0.0)
        self.module_num_return = env_args.get('module_num_return', 5)

        self.state = None
        self.history_actions = []
        self.history_trace = []
        self.done = False

        # ! remember to check the self.copy_env function while changing arguments

    def reset(self, data_item = None):
        """
        The reset method will be called to initiate a new episode.
        """
        # super().reset(seed=seed)
        self.state = State(data_item)
        self.history_actions = []
        self.history_trace = []
        self.done = False

    def step(self, action):
        """
        The step method accepts an action, computes the state of the environment
        after applying that action and returns the 4-tuple (observation, reward, done, info)
            observation (object): state
            reward (float)
            done (boolen)
            info: other helpful infomation

        action (dict): {'type':'retrieve', 'query':'the sun', ...}
        """

        if self.done:
            return self.state, 0.0, self.done, []

        if self.check_action_executable(action) == False:
            # print(f"action is not executable: {action}")
            self.done = True
            return self.state, 0.0, self.done, []

        state_before_action = self.state.to_dict()

        # ----- execute action -----
        action_failed = False
        if action['type'] == Action.retrieve:
            """
            retrieve with action['query'] and update the state S with retrieval result + query
            If a query is consecutively queried k times, we scroll down the retrieval result k times
            """

            if 'query' in action:
                query = action['query']
            else:
                id2sent = {sent_id:sent for sent, sent_id in self.state.sent2id.items()}
                query = id2sent.get(action['query_id'], "")
                action['query'] = query

            if not query:
                action_failed = True
            else:
                repeat_time = 0
                for ac in self.history_actions[::-1]:
                    if ac['type'] == Action.retrieve:
                        if ac['query'] == query:
                            repeat_time += 1
                            # print(repeat_time, query)
                        else:
                            pass
                    else:
                        break

                if repeat_time == 0:
                    retrieval_result = self.retriever(query, n = self.retrieve_top_n)
                else:
                    retrieval_result = self.retriever(query, n = self.retrieve_top_n*(repeat_time+1))
                    retrieval_result = retrieval_result[self.retrieve_top_n*repeat_time: self.retrieve_top_n*(repeat_time+1)]

                retrieval_result = [r['text'] for r in retrieval_result]
                if query != self.state.H:
                    retrieval_result = [query] + retrieval_result
                self.state.update_S(retrieval_result)

        elif action['type'] == Action.reason:
            if 'pre_sent' in action['step']:
                pre_sent = action['step']['pre_sent']
            else:
                id2sent = {sent_id:sent for sent, sent_id in self.state.sent2id.items()}
                pre_sent = [id2sent.get(p, "") for p in action['step']['pre_id']]
                action['step']['pre_sent'] = pre_sent

            if action.get('use_module', False) == True:
                # use entailment module in the env to reason a step
                action['step'] = self.reason_entailment_module_steps([action['step']])[0]

                # filter steps
                if action['step']['step_scorer_score'] < self.step_score_thre:
                    # print(f"Filter step with score: {self.step_score_thre}")
                    # print(action['step'])
                    action_failed = True

            assert 'con_sent' in action['step']

            if action['step']['con_sent'] in self.state.S:
                if action['step']['con_sent'] != self.state.H:
                    del self.state.sent2id[action['step']['con_sent']]

            self.state.expand_P(action['step'])

        elif action['type'] == Action.end:
            # print("done: action end")
            self.done = True

        else:
            raise NotImplementedError

        # ----- check action_failed -----
        if action_failed:
            # print(f"done: action failed! {action}")
            self.done = True
            # return self.state, 0.0, self.done, []

        # ----- add history actions -----
        self.history_trace.append((copy.deepcopy(state_before_action), copy.deepcopy(action)))
        self.history_actions.append(copy.deepcopy(action))

        # print(State(state_before_action).linearize_state())
        # print(action)
        # print("*"*20)

        self.state.history_actions = self.history_actions

        # ----- check action budget -----
        for action_type, action_budge in self.action_budget.items():
            if sum([ac['type']==action_type for ac in self.history_actions]) >= action_budge:
                # print(f"done: {action_type} reach budget {action_budge}")
                self.done = True

        # ----- check proved -----
        # Sometimes H can be retrieved directly
        if any([same_sent(sent, self.state.H) for sent in self.state.S]):
            # print("done: H in S")
            self.done = True
        elif any([same_sent(step['con_sent'], self.state.H) for step in self.state.P]):
            # print("done: H in P")
            self.done = True

        # ----- check predicted tree height -----
        if self.state.get_pred_height() >= self.max_height:
            # print(f"done: Reach maximal height {self.state.get_pred_height()} >= {self.max_height}")
            self.done = True

        # ----- check S and P for task 1 -----
        if len(self.state.P) > 0 and len(self.state.S) < 2:
            # print(f"done: length of S {len(self.state.S)}")
            self.done = True

        observation = self.state
        reward = 0.0
        done = self.done
        info = []

        return observation, reward, done, info

    def render(self):
        pass

    def check_action_executable(self, action_str):
        if action_str is None:
            return False

        # try to parse the str
        if type(action_str) == str:
            try:
                action = Action.parse_action(action_str)
            except Exception as e:
                # print(f"check_action_executable failed: {str(e)}. action_str: {action_str}")
                action = None

            if action is None:
                return False
        else:
            action = action_str
            if 'type' not in action:
                return False

        # check by action type
        if action['type'] == Action.retrieve:
            if 'query' in action:
                query = action['query']
            elif 'query_id' in action:
                id2sent = {sent_id:sent for sent, sent_id in self.state.sent2id.items()}
                query = id2sent.get(action['query_id'], "")
                action['query'] = query
            else:
                return False

            if not query:
                return False

        elif action['type'] == Action.reason:
            if 'step' not in action:
                return False

            # check 'pre_id' / 'pre_sent'
            if 'pre_sent' in action['step']:
                pre_sent = action['step']['pre_sent']
            elif 'pre_id' in action['step']:
                id2sent = {sent_id:sent for sent, sent_id in self.state.sent2id.items()}
                pre_sent = [id2sent.get(p, "") for p in action['step']['pre_id']]
                action['step']['pre_sent'] = pre_sent
            else:
                return False

            if any([sent not in self.state.S for sent in pre_sent]):
                # premise not in state.S
                return False
            if len(set(action['step']['pre_sent'])) != len(action['step']['pre_sent']):
                # premises: sent1 & sent1
                return False

            # filter pre_sent by rules
            if self.check_premise_overlap:
                # Rule: if the pre_sent have no overlap or have too much overlap, we reject the step
                if len(action['step']['pre_sent']) < 2:
                    return False
                pre_iou = max([sent_IoU(ps[0], ps[1], spacy_nlp) for ps in itertools.combinations(action['step']['pre_sent'], 2)])
                if pre_iou == 0.0 or pre_iou >= 0.7:
                    # print("*** filter step ***", action['step']['pre_sent'])
                    return False

            # check 'con_sent'
            if 'con_sent' not in action['step']:
                # use module to get con_sent
                pass
            else:
                con_sent = action['step']['con_sent']
                if any([same_sent(con_sent, s) for s in pre_sent]):
                    # conclusion repeats one of the premises
                    return False
                if any([same_sent(con_sent, s) for s in self.state.used_S]):
                    # print('con_sent in used_S')
                    return False

                S_int = [sent for sent in self.state.S if self.state.sent2id[sent].startswith('int')]
                if any([same_sent(con_sent, s) for s in S_int]):
                    # some intermediate conclusions could be found as facts in corpus
                    # we only return false when the con_sent has been a int
                    # print('con_sent has been int')
                    return False

        elif action['type'] == Action.end:
            pass

        else:
            raise NotImplementedError

        return True

    def get_score_compare_gold(self, bleurt_scorer):

        def is_step_pre_in_S(step, S):
            return all([p in S for p in step['pre_sent']])

        state = self.state
        score = {}

        # ----- gold info -----
        gold_node_list = get_gt_node_list(state.data_item)
        id2sent = {node['id']:node['sent'] for node in gold_node_list}
        gold_sents = [node['sent'] for node in gold_node_list if node['id'].startswith('sent')]
        gold_steps = [{'pre':sorted(node['pre']), 'pre_sent': sorted([id2sent[p] for p in node['pre']]),
                    'con':node['id'], 'con_sent': id2sent[node['id']]}
                    for node in gold_node_list
                    if len(node['pre'])]

        gold_next_steps = []
        for step in gold_steps:
            if is_step_pre_in_S(step, state.S):
                gold_next_steps.append(step)

        # ----- S -----
        # used_sent = sum([step['pre_sent'] for step in state.P],[])
        # unused_gold_sents = set(gold_sents) - set(used_sent)
        # if len(unused_gold_sents) == 0: # bug here? for int1 & int2
        #     recall = -1.0
        # else:
        #     recall = sum([sent in state.S for sent in unused_gold_sents])/(len(unused_gold_sents)+1e-10)
        # score['S_recall'] = recall

        recall = sum([sent in (state.S + state.used_S) for sent in gold_sents])/(len(gold_sents)+1e-20)
        score['S_recall'] = recall

        score['S_contain_step'] = len(gold_next_steps)

        # ----- P (tree) -----
        pred_node_list = state.get_pred_node_list()
        eval_result = eval_tree_task3(pred_node_list, gold_node_list, bleurt_scorer)
        score['tree_score'] = eval_result

        return score

    def reason_entailment_module_steps(self, steps):

        # module generate parameters
        prefixes = [
            'deductive substitution:',
            'deductive conjunction:',
            'deductive if-then:',
        ]
        mode = 'beam_search'
        generate_args = {
            'num_beams' : self.module_num_return,
            'num_return_sequences': self.module_num_return,
        }

        # module inference
        input_sents = []
        for step in steps:
            input_sents.append(self.entailment_module.make_input_sent(step['pre_sent'],
                                        H=self.state.H, Q=self.state.Q, A=self.state.A))

        module_preds, module_generate_scores, input_infos = self.entailment_module(input_sents=input_sents, prefixes=prefixes,
                                                                         mode = mode, generate_args=generate_args)

        # filter by rule
        S_int = [sent for sent in self.state.S if self.state.sent2id[sent].startswith('int')]
        for idx, input_sent in enumerate(input_sents):
            for pred_idx in range(len(module_preds[idx])):
                # see self.check_action_executable
                forbidden_cons = steps[idx]['pre_sent'] + self.state.used_S + S_int
                if any([same_sent(module_preds[idx][pred_idx], s) for s in forbidden_cons]):
                    module_generate_scores[idx][pred_idx] = -1

        # verify and select the best conclusion sentence
        for idx in range(len(steps)):
            pre_sent = steps[idx]['pre_sent']

            preds_ = module_preds[idx]
            generate_scores_ = module_generate_scores[idx]
            module_inputs_ = input_infos[idx]

            verifier_inputs = [
                    {'pre_sent': pre_sent, 'con_sent': con_sent}
                    for con_sent in preds_
            ]
            step_scorer_scores_ = self.verifier.verify_steps(verifier_inputs)

            # merge_scores_ = 0.5*np.array(generate_scores_) + 0.5*np.array(verifier_scores_)
            merge_scores_ = np.array(generate_scores_)*np.array(step_scorer_scores_)

            max_i = np.argmax(merge_scores_)

            steps[idx]['con_sent'] = preds_[max_i]
            steps[idx]['generate_score'] = generate_scores_[max_i]
            steps[idx]['step_scorer_score'] = step_scorer_scores_[max_i]
            steps[idx]['merge_score'] = merge_scores_[max_i]
            steps[idx]['module_input'] = module_inputs_[max_i]

        return steps

    def copy_env(self):

        new_env = EntailmentTreeEnv(retriever = self.retriever,
                                    entailment_module = self.entailment_module,
                                    verifier = self.verifier,
                                    env_args = self.env_args,
                                    )

        new_env.state = copy.deepcopy(self.state)
        new_env.history_actions = copy.deepcopy(self.history_actions)
        new_env.history_trace = copy.deepcopy(self.history_trace)
        new_env.done = self.done

        return new_env


import argparse
from collections import defaultdict
from sklearn.datasets import load_svmlight_file
from math import log

import torch
import torch.nn as nn
import torch.optim
import torch.utils.data as data_utils
import random
import numpy as np
import matplotlib.pyplot as plt

DATASETS=("abalone" ,"bodyfat", "cpusmall", "housing", "mg", "mpg", "space_ga")
DATASET_FEATURES = {
    "abalone": 8,
    "bodyfat": 14,
    "cpusmall": 12,
    "housing": 13,
    "mg": 6,
    "mpg": 7,
    "space_ga": 6,
}

def parse_args():
    parser = argparse.ArgumentParser(
        description="Compare (regularized) linear regresssion "\
        "training of mult models vs. single model")

    parser.add_argument("-s", "--seed", type=int, default=None,
                        help='random seed')
    parser.add_argument("-w", "--num_workers", type=int, default=0,
                        help="number of data workers")
    parser.add_argument("--cuda", action="store_true",
                        help="whether to use cuda or not")

    parser.add_argument("--dataset", type=str, choices=DATASETS, default="space_ga",
                        help="name of dataset (expected to be in 'datasets' directory")
    parser.add_argument("--feature_dim", type=int, default=None,
                        help="features dimension (default: infer from dataset)")
    parser.add_argument("--output_dim", type=int, default=1,
                        help="output dimension (default: 1)")

    parser.add_argument("-M", "--num_models", dest="M", type=int, default=10,
                        help="number of multiple models")
    parser.add_argument("-T", "--num_iters", dest="T", type=int, default=100,
                        help="number of iterations")
    parser.add_argument("--agg_shots", type=int, default=1,
                        help="number of aggregating steps (not implemented yet)")
    parser.add_argument("--val_freq", type=int, default=10,
                        help="frequency of validation per gradient computation")

    parser.add_argument("-B", "--batch_size", type=int, default=1,
                        help="batch size")
    parser.add_argument("-lr", "--learning_rate", dest="lr", type=float, default=0.05,
                        help="base learning rate")
    parser.add_argument("--models_lrs", dest="lrs", type=float, nargs="+", default=None,
                        help="learning rates per model (default: same as base lr if)")

    parser.add_argument("--reg_pow", type=int, choices=(2, 3, 4), default=None,
                        help="power of regularization term (default: no regularization)")
    parser.add_argument("--reg_coeff", type=float, default=1.,
                        help="regularization coefficient")

    parser.add_argument("--save_fig", type=str, default=None,
                        help="save validation plot under this name (default: don't save)")
    parser.add_argument("--log_scale", action="store_true",
                        help="use log scale for validation plot")

    # Parse command line args
    args = parser.parse_args()

    # Handle args specifications
    if args.feature_dim is None:
        args.feature_dim = DATASET_FEATURES[args.dataset]
    if args.lrs is None:
        args.lrs = [args.lr] * args.M
    if args.cuda and torch.cuda.is_available():
        args.device = "cuda:0"  # TODO: assign all available/chosen devices?
    else:
        args.device = "cpu"

    return args

### Dataset ###
class MyDataset(data_utils.Dataset):
    def __init__(self, dataset):
        super().__init__()
        self.dataset = f"datasets/{dataset}"
        with open(self.dataset, "rb") as f:
            self.X, self.y = load_svmlight_file(f)

    def __len__(self):
        return self.X.shape[0]

    def __getitem__(self, idx):
        X_tensor = torch.Tensor(self.X[idx].todense()).squeeze(0)
        y_tensor = torch.Tensor([self.y[idx]]).squeeze(0)
        return X_tensor, y_tensor

class DataSampler:
    def __init__(self, dataloader):
        self.dataloader = dataloader
        self.reset_sampler()

    def reset_sampler(self):
        self.sampler = iter(self.dataloader)

    def sample(self):
        try:
            x, y = next(self.sampler)
        except StopIteration:
            self.reset_sampler()
            x, y = next(self.sampler)
        return x, y

### Optimizer ###
class LocalSGD(torch.optim.SGD):
    def aggregate(self, param_groups=None):
        if param_groups is None:
            param_groups = self.param_groups
        # This function simply averages parameters across all groups/models
        num_params = len(param_groups[0]["params"])
        agg_params = [None] * num_params
        # Find average per parameter
        for param_idx in range(num_params):
            param_list = [param_groups[model_idx]["params"][param_idx].data
                          for model_idx in range(len(param_groups))]
            agg_params[param_idx] = torch.mean(torch.stack(param_list, dim=0), dim=0)

        return agg_params

    def sync(self, agg_params, param_groups=None):
        if param_groups is None:
            param_groups = self.param_groups
        # Synchronize
        num_params = len(param_groups[0]["params"])
        for model_idx in range(len(param_groups)):
            for param_idx in range(num_params):
                param_groups[model_idx]["params"][param_idx] = agg_params[param_idx]

    def aggregate_and_sync(self):
        agg_params = self.aggregate()
        self.sync(agg_params)

### Report Utils ###
class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self, name, fmt=':f'):
        self.name = name
        self.fmt = fmt
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

    def __str__(self):
        fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})'
        return fmtstr.format(**self.__dict__)

class ProgressMeter(object):
    def __init__(self, num_batches, meters, prefix=""):
        self.batch_fmtstr = self._get_batch_fmtstr(num_batches)
        self.meters = meters
        self.prefix = prefix

    def display(self, batch):
        entries = [self.prefix + self.batch_fmtstr.format(batch)]
        entries += [str(meter) for meter in self.meters]
        print('\t'.join(entries))

    def _get_batch_fmtstr(self, num_batches):
        num_digits = len(str(num_batches // 1))
        fmt = '{:' + str(num_digits) + 'd}'
        return '[' + fmt + '/' + fmt.format(num_batches) + ']'

def plot_lines(losses_dict, title="", log_scale=False, show_plot=True, filename=None):
    plt.figure(figsize=(10,5))
    if log_scale:
        title += " (log scale)"
    plt.title(title)
    for label, losses in losses_dict.items():
        if log_scale:
            losses = list(map(log, losses))
        plt.plot(losses, label=label)
    plt.xlabel("t")
    plt.legend()
    if filename is not None:
        plt.savefig(filename)
    if show_plot:
        plt.show()
    plt.close()

### Data Utils ###
def manual_seed(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

def create_seed_worker(seed):
    if seed is None:
        return None
    def seed_worker(worker_id):
        worker_seed = seed % 2**32
        np.random.seed(worker_seed)
        random.seed(worker_seed)
    return seed_worker

def create_generator(seed):
    if seed is None:
        return None
    g = torch.Generator()
    g.manual_seed(seed)
    return g

def create_trainloader(dataset, args):
    return data_utils.DataLoader(dataset, batch_size=args.batch_size,
                                 num_workers=args.num_workers, shuffle=True, pin_memory=True,
                                 worker_init_fn=create_seed_worker(args.seed),
                                 generator=create_generator(args.seed))
def create_testloader(dataset, args):
    return data_utils.DataLoader(dataset, batch_size=len(dataset),
                                 num_workers=args.num_workers, shuffle=False, pin_memory=True,
                                 worker_init_fn=create_seed_worker(args.seed),
                                 generator=create_generator(args.seed))

### Aggregation schedule ###
def n_shot_schedule(N, T):
    return {i for i in range(T-1, 0, -T // N)}

### Model ###
def create_model(args):
    return nn.Linear(args.feature_dim, args.output_dim)

def init_model_(model):
    def init_weights(m):
        if isinstance(m, nn.Linear):
            # Use default initialization
            pass
    model.apply(init_weights)

def init_like_(m1, m2):
    with torch.no_grad():
        for p1, p2 in zip(m1.parameters(), m2.parameters()):
            p1.copy_(p2)

### Regularization ###
def create_regularizer(p):
    assert p in (2, 3, 4)
    c = (0., 1., 1./2, 1./6, 1./24)
    def regularizer(model):
        params = nn.utils.parameters_to_vector(model.parameters())
        return torch.norm(params).pow(p) * c[p]
    return regularizer

### Training ###
def validate(testloader, model, loss_fn, args):
    with torch.no_grad():
        running_loss = 0.0
        running_acc = 0
        for x, y in testloader:
            x = x.to(device=args.device)
            y = y.to(device=args.device)
            y_pred = model(x).view_as(y)
            loss = loss_fn(y_pred, y)
            acc = torch.sum(y_pred.round() == y).float() / y.size(0)
            running_loss += loss.item()
            running_acc += acc.item()
        mean_loss = running_loss / len(testloader)
        mean_acc = running_acc / len(testloader)
    return mean_loss, mean_acc

def train_models(T, trainloader, testloader, models, optimizer, loss_fn, regularizer, args):
    data_sampler = DataSampler(trainloader)
    stats = defaultdict(list)
    train_losses = AverageMeter("train_loss", ":.4f")
    val_losses = AverageMeter("val_loss", ":.4f")
    progress = ProgressMeter(T, [train_losses, val_losses])
    # Create dummy model for validating aggregated model
    agg_model = create_model(args).to(device=args.device)

    for t in range(T):
        # Validate
        if (t * len(models)) % args.val_freq == 0:
            with torch.no_grad():
                if len(models) > 1:
                    agg_params = optimizer.aggregate()
                    for p1, p2 in zip(agg_model.parameters(), agg_params):
                        p1.copy_(p2)
                    val_loss, _ = validate(testloader, agg_model, loss_fn, args)
                else:
                    val_loss, _ = validate(testloader, models[0], loss_fn, args)
            stats["val_loss"] += [val_loss]
            val_losses.update(val_loss)

        for model in models:
            x, y = data_sampler.sample()
            x = x.to(device=args.device)
            y = y.to(device=args.device)
            y_pred = model(x).view_as(y)
            loss = loss_fn(y_pred, y)
            if regularizer is not None:
                loss = loss + args.reg_coeff * regularizer(model)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            stats["train_loss"] += [loss.item()]
            train_losses.update(loss.item())

        progress.display(t)

    return stats

def main(args):

    # Set seed if given
    if args.seed is not None:
        manual_seed(args.seed)

    # Initialize dataset
    trainset = MyDataset(args.dataset)
    testset = trainset  # test on full dataset

    # Define a group of models (optimized in parallel) and a singled out model (optimized alone)
    one_model = create_model(args).to(device=args.device)
    one_optimizer = torch.optim.SGD(one_model.parameters(), lr=args.lr)
    mult_models = [create_model(args).to(device=args.device) for i in range(args.M)]
    param_groups = [{"params": mult_models[i].parameters(), "lr": args.lrs[i]} for i in range(args.M)]
    mult_optimizer = LocalSGD(param_groups, lr=args.lr)

    # Init models
    init_model_(one_model)
    for model in mult_models:
        init_like_(model, one_model)

    # Loss function
    loss_fn = nn.MSELoss().to(device=args.device)
    regularizer = None
    if args.reg_pow is not None:
        regularizer = create_regularizer(args.reg_pow)

    print("Training single model...")
    stats_one = train_models(args.T * args.M,
                             create_trainloader(trainset, args),
                             create_testloader(testset, args),
                             [one_model], one_optimizer, loss_fn, regularizer, args)
    print("Training multiple models...")
    stats_mult = train_models(args.T,
                              create_trainloader(trainset, args),
                              create_testloader(testset, args),
                              mult_models, mult_optimizer, loss_fn, regularizer, args)

    stats = {
        "mult_train_loss": stats_mult["train_loss"],
        "mult_val_loss": stats_mult["val_loss"],
        "one_train_loss": stats_one["train_loss"],
        "one_val_loss": stats_one["val_loss"],
    }

    train_losses = {k: v for k, v in stats.items() if "train_loss" in k}
    val_losses = {k: v for k, v in stats.items() if "val_loss" in k}
    plot_lines(train_losses, "Training Loss",
               log_scale=args.log_scale, show_plot=args.save_fig is None)
    plot_lines(val_losses, "Validation Loss",
               log_scale=args.log_scale, show_plot=args.save_fig is None, filename=args.save_fig)

if __name__ == "__main__":
    args = parse_args()
    main(args)


import numpy as np
import pandas as pd

from pandas import DataFrame, Series

from ewa import EWA
from simulations import simulate_record, alt_simulate_record
from likelihood import set_evaluation, set_evaluation_with_graphs, likelihood
from tests import transformation, likelihood_ratio_test, salmon_set_test
import datetime
from matrices import matrix_set
from metrics import brier_multi, log_score_multi, wide_brier_calc

dt_string = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d_%H:%M')

matrixSet = [[np.array(item[0].T, order='F'), item[1]] for item in matrix_set]

matrixSetTransposed = [[item[0].T, item[1]] for item in matrix_set]

for item in matrixSet:
    item[0] = DataFrame(np.array(item[0]))
    item[1] = DataFrame(np.asarray(item[1]))
# matrixSet
print(matrixSet[0][0])

population_set = {
    'EWA_FP': [1, 1, 1, 0, 1],
    'EWA_CB': [0, 1, 0, 1, 1],
    'EWA_RL': [0, 0, 0.75,1,1],
    'EWA_mdl':[0.5,0.5,0.5,0.5,1],
#     'test': [0.97, 0.6,  0.94,  1, 0.8, ],
            }
params_name = ['ro','delta','phi','N_in','lambd']
set_nubmbs = 50
couple_ides = 40
ages = 40

#Run evaluations over simulated data
for model in population_set:
    for idx, game_matrices in enumerate(matrixSet):
        print("MODEL: {model_name}".format(model_name=model))
        print("Matrix_set index : " + str(idx))
        simulation_df = simulate_record(ages, set_nubmbs,  dict(zip(params_name, population_set[model])),couple_ides, game_matrices, model_name=model)
        simulation =  simulation_df[simulation_df.columns[[0,1,2,3,4,]]]
        ids = sorted(simulation_df.id.unique())
        rounds = sorted(simulation_df['round'].unique())
        set_numbs = sorted(simulation_df['set_numb'].unique())
        simulation = simulation.to_numpy(dtype=None, copy=False)
        game_matrices=[item.values for item in game_matrices]
        set_evaluation_with_graphs(set_numbs, simulation, ids, rounds, game_matrices, dict(zip(params_name,population_set[model])), model, True, True)
        break

is_tested_set  = transformation(population_set)
hypothesis_set = {
#     'rho_1':{0:1},
#     'rho_0':{0:0},
    'delta_1':{1:1},
    'delta_0':{1:0},
     'phi_1':{1:1},
     'phi_0':{1:0}
    }

#Run hypothesis tests for restricted params
res = []
for model in population_set:
    for idx, game_matrices in enumerate(matrixSet):
        print("MODEL: {model_name}".format(model_name=model))
        print("Matrix: " + str(len(game_matrices[0]))+"x"+ str(len(game_matrices[0])))
        simulation_df = simulate_record(ages, set_nubmbs,  dict(zip(params_name, population_set[model])),
                                        couple_ides, game_matrices, model_name=model)
        simulation =  simulation_df[simulation_df.columns[[0,1,2,3,4,]]]
        ids = sorted(simulation_df.id.unique())
        rounds = sorted(simulation_df['round'].unique())
        set_numbs = sorted(simulation_df['set_numb'].unique())
        simulation = simulation.to_numpy(dtype=None, copy=False)
        game_matrices=[item.values for item in game_matrices]
        params_for_likelihood = [simulation, ids, rounds]
        y = salmon_set_test(start_params=[np.random.rand(1,1)[0][0] for i in range(5)], params_for_likelihood=params_for_likelihood,
                        params_restrictions= is_tested_set[model], test_results =  np.array([]), set_numbs = set_numbs,
                        game_matrices=tuple([game_matrices[0], game_matrices[1]]),
                        _bounds=((0, 1),(0, 1),(0, None),(0, 1),(0, None)) )
        percent_of_set_H0_not_false = np.mean((y>=0.95).astype(int))
        hyp_set = []
        for hyp in hypothesis_set.keys():
            z = salmon_set_test(start_params=[np.random.rand(1,1)[0][0] for i in range(5)], params_for_likelihood=params_for_likelihood,
                        params_restrictions= hypothesis_set[hyp], test_results =  np.array([]), set_numbs = set_numbs,
                        game_matrices=tuple([game_matrices[0], game_matrices[1]]),
                        _bounds=((0, 1),(0, 1),(0, None),(0, 1),(0, None)) )
            percent_of_z_set_H0_not_false = np.mean((z>=0.95).astype(int))
            mean_p_val = round(np.mean(z),2)
            hyp_set.append([hyp,'percent this one param hold', percent_of_z_set_H0_not_false,'mean_p_val', mean_p_val])

        for hypo in hyp_set:
            dct = {'model_name':model, 'matrix':(str(len(game_matrices[0]))+"x"+ str(len(game_matrices[0])))}
            dct['hyp'] = hypo[0]
            dct[hypo[1]] = hypo[2]
            dct[hypo[3]] = hypo[4]
            res.append(dct)
        print('percent, 4 params hold', percent_of_set_H0_not_false, 'mean p value', round(np.mean(y),2), hyp_set)
#             break
tests = pd.DataFrame(res)

tests.to_csv("tests_{dt}.csv".format(dt=dt_string))

fun = [0]
res = []
for model in population_set:
    for idx, game_matrices in enumerate(matrixSet):
        print("MODEL: {model_name}".format(model_name=model))
        print("Matrix_set index : " + str(idx))
        simulation_df = simulate_record(ages, set_nubmbs,  dict(zip(params_name, population_set[model])),
                                        couple_ides, game_matrices, model_name=model)
        ids = sorted(simulation_df.id.unique())
        rounds = sorted(simulation_df['round'].unique())
        set_numbers = sorted(simulation_df['set_numb'].unique())
        simulation = simulation_df.to_numpy(dtype=None, copy=False)
#             simulation =  simulation_df[simulation_df.columns[[0,1,2,3,4,]]]

        for alt_model in population_set:

            alt_simulation_df = alt_simulate_record(ages, set_numbers,  simulation,
                                                dict(zip(params_name, population_set[alt_model])),
                                                couple_ides, game_matrices, model_name=alt_model)
            pl1_move_wide_data = simulation_df['pl1_move_wide']

            y_true_pl1 = np.asarray([np.asarray(x)  for x in  pl1_move_wide_data], dtype=np.float32)

            probs_pl1_data = alt_simulation_df['probs_pl1']
            y_prob_pl1 = np.asarray([np.asarray(x)   for x in  probs_pl1_data], dtype=np.float32)
#                 [::-1]
#                 print(probs_pl1_data.head, pl1_move_wide_data.head)
            brier_pl1 = brier_multi(y_true_pl1, y_prob_pl1)
            container = []
            wide_brier = wide_brier_calc(simulation_df, alt_simulation_df, container, model,alt_model, set_numbers)
#                 print(np.round(brier_pl1,3)-np.round(np.mean(wide_brier["brier_score"]),3) )
            for one_set in set_numbers:
                dct = dict((('brier', (wide_brier["brier_score"][one_set])),( "sd_brier", np.std(wide_brier["brier_score"])),
                  ("log_score", np.mean(wide_brier["log_score"])), ("sd_log_score", np.std(wide_brier["log_score"])),
                  ('player','pl_1'),('true_model', model), ('alt_model',  alt_model), ('set', wide_brier["set_numb"][one_set])))
                dct['model_name'] = model
                dct['matrix'] = str(len(game_matrices[0]))+"x"+ str(len(game_matrices[0]))
                res.append(dct)
#                 print('brier:', brier_pl1, "sd_brier", np.std(wide_brier["brier_score"]),
#                       "log_score", np.mean(wide_brier["log_score"]), "sd_log_score",
#                       'pl_1','true_model', model, 'alt_model',  alt_model)

#             break
briers = pd.DataFrame(res)

briers.to_csv('measure_of_accuracy_{dt}.csv'.format(dt=dt_string))


import sys
import os
import tkinter
import pygame

from CaveGenerator import *
from FileManager import *

class Program(object):

    def __init__(self, root, d ,mn, mx):
        #Sets a tkinter instance
        self.root = root
        self.root.title("Cellular Automata v0.5")
        #set the weights of the grid
        self.root.grid_rowconfigure(0, weight=1)
        self.root.grid_columnconfigure(0, weight=1)
        self.root.grid_columnconfigure(1, weight=1)
        #Gets the screen resolution
        self.width = 600
        self.height = 500
        #Gets the values for the cave
        self.density = d
        self.mn = mn
        self.mx = mx
        #Create frame for the pygame window to go in
        self.pyframe = tkinter.Frame(self.root, width=self.width - 100, height=self.height)
        #Create frame for the cave settings
        self.setframe = tkinter.Frame(self.root, width=100, height=self.height) #added
        #Creates a top menu bar
        self.menubar = tkinter.Menu(self.root)
        #Creates a dropdown box from the menu
        self.filemenu = tkinter.Menu(self.menubar, tearoff=0)
        #Creates an Open and Save button in the dropdown
        self.filemenu.add_command(label="Open", command=self.LoadCave)
        self.filemenu.add_command(label="Save", command=self.SaveCave)
        self.filemenu.add_separator()
        #Label the dropdown as File
        self.menubar.add_cascade(label="File", menu=self.filemenu)
        self.root.config(menu=self.menubar)
        #Create the button label
        self.buttonlabel = tkinter.Label(self.setframe, text="Cave Density")
        #Create the button to generate the cave
        self.button = tkinter.Button(self.setframe, text="Generate", command=self.Generate, height = 2, width = 8)
        #Create the slider
        self.slider = tkinter.Scale(self.setframe, from_=0, to=25, orient=tkinter.HORIZONTAL)

        #Put the objects in the right location
        self.pyframe.grid(row=0, column=0)
        self.setframe.grid(row=0, column=1, rowspan=2)
        self.buttonlabel.grid(row=2, column=1)
        self.slider.grid(row=3, column=1, sticky="ew")
        self.button.grid(row=4, column=1, sticky="ew")

        #Update to display
        self.root.update()
        ###Tkinter Done###

        #The three lines below embed a pygame window to a tkinter window
        os.environ['SDL_WINDOWID'] = str(self.pyframe.winfo_id())
        if sys.platform == "win32":
            os.environ['SDL_VIDEODRIVER'] = 'windib'

        #Initialize Pygame
        pygame.display.init()
        #Set the screen size
        self.screen = pygame.display.set_mode((self.width, self.height))
        ###Pygame Done###

    def Generate(self):
        #Create a new cave
        self.cave = Cave(self.height,self.width,(float(self.slider.get()) * 0.1),self.mn,self.mx)
        #Generate the cave
        self.cave.Generate()
        #Get the cave data
        self.pixels = self.cave.Render()
        #Run the render loop
        self.loop()

    def loop(self):
        #Fill the screen black
        self.screen.fill((0,0,0))
        #Create a pixel
        square=pygame.Surface((1, 1)) #changed
        #loop through the positions
        for x in range(0, self.width, 1):
            for y in range(0, self.height, 1):
                #Check if a pixel needs to be displayed at position
                if self.pixels[y][x] == 1:
                    #Fill the colour white
                    square.fill((255,255,255))
                    draw_me=pygame.Rect((x+1), (y+1), 1, 1)
                    self.screen.blit(square,draw_me)
        #Update the screen to render
        pygame.display.flip()
        #self.pyframe.after(5, self.loop)

    def SaveCave(self):
        #Open a save file box and let the user save it in a location of choice
        savename = tkinter.filedialog.asksaveasfile(mode='w', defaultextension=".cave")
        #Check if cancel button has been pressed. If so return
        if savename is None:
            return
        #Runs the default classes to save
        filem = CFile(savename, self.cave)
        filem.WriteData()
        filem.SaveCreateFile()

    def LoadCave(self):
        #Open a load file box and let the user save it in a location of choice
        loadname = tkinter.filedialog.askopenfile(mode='r', defaultextension=".cave")
        #Check if cancel button has been pressed. If so return
        if loadname is None:
            return
        #Runs the default classes to load
        parser = ParseCFile(loadname)
        parser.ReadFile()
        self.cave = parser.Parse()
        #Gets the data for the pixel display
        self.pixels = self.cave.Render()

if __name__ == "__main__":
    #Creates the tkinter instance
    root = tkinter.Tk()
    #Creates a Program instance
    program = Program(root, 0.8, 1, 2) #0.8 is not needed anymore because self.density is not used.
    #Loop tkinter
    root.mainloop()


from lib.ConfigManager import ConfigManager
from lib.FileManager import FileManager
from lib.FixedSizeRotatingFileHandler import FixedSizeRotatingFileHandler
from lib.TableScraper import TableScraper

from flask import Flask, render_template, request, redirect, url_for, send_from_directory, jsonify
import time
import re
import logging
from werkzeug.exceptions import BadRequestKeyError
from threading import Lock

app = Flask(__name__)

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
# 250 MB
MAX_LOG_SIZE_FLASK = 250 * 1024 * 1024
MAX_LOG_SIZE = 250 * 1024 * 1024

app_logger = logging.getLogger("app_log")
app_logger.setLevel(logging.DEBUG)
fh = FixedSizeRotatingFileHandler("app_log.log", maxBytes=MAX_LOG_SIZE)  # 5KB
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
app_logger.addHandler(fh)

# werkzeug_logger = logging.getLogger("werkzeug")
# werkzeug_logger.setLevel(logging.DEBUG)
# werkzeug_logger_fh = FixedSizeRotatingFileHandler("flask_log.log", maxBytes=MAX_LOG_SIZE_FLASK)  # 5KB
# werkzeug_logger_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# werkzeug_logger_fh.setFormatter(werkzeug_logger_formatter )
# werkzeug_logger.addHandler(werkzeug_logger_fh)
# werkzeug_logger.propagate = False

def update_current_status(new_status):
    config = config_manager.load()
    if new_status in config['status']:
        config['current_status'] = new_status
        for property_name, property_value in config['status'][new_status].items():
            config['css_vars'][property_name] = property_value
            print(property_name,config['css_vars'][property_name],property_value,"\n",config)
        config_manager.save(config)
        update_css_file_from_config()
        print(f"New Config: {config}")
        return True
    else:
        return False

def update_css_file_from_config():
    config = config_manager.load()
    css_vars = config['css_vars']

    # Read the file using FileManager
    css_content = file_manager.read_file()

    for var, value in css_vars.items():
        pattern = f'--{var}:\s*[^;]*;'
        if re.search(pattern, css_content):
            css_content = re.sub(pattern, f'--{var}: {value};', css_content)
        else:
            css_content = css_content.replace(':root {', f':root {{\n  --{var}: {value};')

    # Write the updated content using FileManager
    file_manager.write_file(css_content)

def load_css_vars():
    css_content = file_manager.read_file()

    # Extract the content of the :root selector
    root_content_match = re.search(r':root\s*{\s*([^}]+)\s*}', css_content)
    if not root_content_match:
        return {}

    root_content = root_content_match.group(1)

    # Extract all --variable: value; pairs from the :root content
    variables = re.findall(r'--(.*?):\s?(.*?);', root_content)

    return {var_name: value for var_name, value in variables}

def update_times_in_config():
    """Update the opening times in the config."""
    config = config_manager.load()
    times = config['times']

    for day in times.keys():
        morning_time = request.form.get(f'{day}_morning')
        afternoon_time = request.form.get(f'{day}_afternoon')
        times[day] = [morning_time, afternoon_time]

    config_manager.save_if_changed(config)

def load_css_vars_from_config():
    return config_manager.load()['css_vars']
def load_status_word_from_config():
    return config_manager.load()['current_status']
def load_times_from_config():
    return config_manager.load()['times']

def update_config_from_post_request():
    config = config_manager.load()

    # Update CSS variables
    css_vars_from_file = load_css_vars()
    new_css_vars = {key: request.form.get(key, default='') for key in css_vars_from_file.keys()}
    config['css_vars'] = {**config.get('css_vars', {}), **new_css_vars}

    # Update times
    times = config['times']
    for day in times.keys():
        morning_time = request.form.get(f'{day}_morning')
        afternoon_time = request.form.get(f'{day}_afternoon')
        times[day] = [morning_time, afternoon_time]
    config['times'] = times

    # Update statuses
    for status_key in config['status'].keys():
        for property_name in config['status'][status_key].keys():
            input_name = f"{status_key}_{property_name}"
            config['status'][status_key][property_name] = request.form.get(input_name, default='')
    #update Scrapper Config
    config['scrapper_config']['url']=request.form.get('url')
    config['enable_scraping']=request.form.get('enable_scraping') == 'on'
    config['scrapper_config']['frame_id']= request.form.get('frame_id')
    config['scrapper_config']['refresh_time'] = request.form.get('refresh_time')
    config['scrapper_config']['valid_date']= request.form.get('valid_date')

    config_manager.save(config)

def update_config_from_status_change(new_status):
    config = config_manager.load()
    print(new_status)
    print(new_status in config['status'])
    if new_status in config['status']:
        config['current_status'] = new_status
        for property_name, property_value in config['status'][new_status].items():
            config['css_vars'][property_name] = property_value
        config_manager.save(config)
        update_css_file_from_config()
        return True
    else:
        return False

@app.errorhandler(BadRequestKeyError)
def handle_bad_request(e):
    return f'bad request!:{e}', 400
@app.route('/update-css-vars', methods=['POST'])
def update_css_vars():
    received_data = request.json
    print(received_data)
    received_css_vars = received_data.get('cssVars', {})
    received_status_word = received_data.get('currentStatusWord', "")
    received_times = received_data.get('times', {})

    config_data = config_manager.load()
    config_css_vars = config_data['css_vars']
    config_status_word = config_data['current_status']
    config_times = config_data['times']

    # Determine if any CSS variables changed
    css_vars_changed = any(received_css_vars[key] != config_css_vars.get(key) for key in received_css_vars)

    # Check if the status word has changed
    status_word_changed = received_status_word != config_status_word

    # Check if times have changed
    times_changed = received_times != config_times

    # Determine if any changes occurred
    if css_vars_changed or status_word_changed or times_changed:
        global prev_css_vars, prev_status_word, prev_times
        prev_css_vars = received_css_vars
        prev_status_word = received_status_word
        prev_times = received_times
        update_css_file_from_config()
        print("refresh")
        return {"refresh": True}

    return {"refresh": False}

@app.route('/', methods=['GET'])
def configure_times_get():
    config = config_manager.load()
    times = config['times']
    current_status_word=config['current_status']
    valid_date=config['valid_date']
    return render_template(r'index1_new.html', times=times, current_status_word=current_status_word,valid_date=valid_date)

@app.route('/', methods=['POST'])
def configure_times_post():

    config = config_manager.load()
    times = config['times']
    for day in times.keys():
        morning_time = request.form.get(f'{day}_morning')
        afternoon_time = request.form.get(f'{day}_afternoon')
        times[day] = [morning_time, afternoon_time]

    config['times'] = times
    config_manager.save_if_changed(config)
    return redirect(url_for('configure_times_get'))

@app.route('/settings', methods=['GET'])
def settings_get():
    config = config_manager.load()
    times = config['times']
    enable_scraping=config['enable_scraping']
    css_vars = load_css_vars()
    scrapper_config=config['scrapper_config']
    return render_template('settings.html', css_vars=css_vars, times=times, statuses=config['status'],enable_scraping=enable_scraping,scrapper_config=scrapper_config)

@app.route('/settings', methods=['POST'])
def settings_post():

    update_config_from_post_request()
    update_css_file_from_config()  # Update the CSS file based on the new config
    return redirect(url_for('settings_get'))

@app.route('/update-status', methods=['POST'])
def update_status():
    received_data = request.json
    new_status = received_data['current_status']
    if update_config_from_status_change(new_status):
        return {"message": "Status updated successfully"}
    else:
        return {"message": "Invalid status"}, 400

@app.route('/css/<filename>')
def serve_css(filename):
    return send_from_directory("static", filename)

if __name__ == "__main__":
    json_file_path = "config.json"

    config_manager = ConfigManager(json_file_path)
    file_manager = FileManager("static/Home.css")

    app.run(debug=True, port=4000)


from rest_framework.serializers import ModelSerializer
from base.models import MangaTitle, MangaAuthor, Genre

class MangaAuthorSerializer(ModelSerializer):
    class Meta:
        model = MangaAuthor
        fields = '__all__'

class GenreSerializer(ModelSerializer):
    class Meta:
        model = Genre
        fields = '__all__'

class MangaTitleSerializer(ModelSerializer):
    authors = MangaAuthorSerializer(many=True, read_only=True)
    genres = GenreSerializer(many=True, read_only=True)

    class Meta:
        model = MangaTitle
        fields = '__all__'


from pptx import Presentation

prs = Presentation("自動化建立PPT簡報.pptx")
# 新增標題和內容頁投影片
title_content_slide_layout = prs.slide_layouts[1]
slide = prs.slides.add_slide(title_content_slide_layout)
title = slide.shapes.title
title.text = "自動化建立文字內容和清單"
body_shape = slide.placeholders[1]
tf = body_shape.text_frame
tf.text = "Python辦公室自動化套件"
p = tf.add_paragraph()
p.text = "Word自動化"
p.level = 1
p = tf.add_paragraph()
p.text = "python-docx套件"
p.level = 2
p = tf.add_paragraph()
p.text = "Excel自動化"
p.level = 1
p = tf.add_paragraph()
p.text = "openpyxl套件"
p.level = 2
prs.save("自動化建立PPT簡報2.pptx")


import matplotlib.pyplot as plt
# %matplotlib inline
plt.style.use('ggplot')

x = ['Nuclear', 'Hydro', 'Gas', 'Oil', 'Coal', 'Biofuel']
energy = [5, 6, 15, 22, 24, 8]

x_pos = [i for i, _ in enumerate(x)]

plt.bar(x_pos, energy, color='green')
plt.xlabel("Energy Source")
plt.ylabel("Energy Output (GJ)")
plt.title("Energy output from various fuel sources")

plt.xticks(x_pos, x)

plt.show()


import itertools
from itertools import combinations

def find_closure(attributes, fds, X):
    closure = set(X)
    old_closure = set()
    while closure != old_closure:
        old_closure = set(closure)
        for fd in fds:
            if set(fd['left']).issubset(closure):
                closure |= set(fd['right'])
    return closure

def find_candidate_keys(attributes, fds):
    candidate_keys = []
    for i in range(1, len(attributes)+1):
        for subset in itertools.combinations(attributes, i):
            flag = 0
            for j in range(0, len(candidate_keys)):
                if (candidate_keys[j].issubset(set(subset))):
                    flag = 1
                    break
            if flag == 1:
                continue
            closure = find_closure(attributes, fds, subset)
            if closure == set(attributes):
                candidate_keys.append(set(subset))
    return candidate_keys

def find_prime_attributes(candidate_keys):
    prime_attributes = set()
    for key in candidate_keys:
        prime_attributes.update(set(key))
    return prime_attributes

def is_in_2NF(prime_attributes, non_prime_attributes, candidate_keys, fds):
    problem_dependencies = []
    for index, fd in enumerate(fds):
        if set(fd['right']).issubset(non_prime_attributes):
            flag = 0
            for s in candidate_keys:
                if len(set(fd['left'])) != len(s) and set(fd['left']).issubset(s):
                    flag = 1
                    break  # Stop checking if a match is found
            if (flag):
                problem_dependencies.append(index)

    if len(problem_dependencies) == 0:
        return [True, problem_dependencies]
    else:
        return [False, problem_dependencies]

def is_in_3NF(prime_attributes, non_prime_attributes, candidate_keys, fds):

    temp_violated_list = []
    # Check if every functional dependency is in 3NF
    for index, fd in enumerate(fds):
        left = fd['left']
        right = fd['right']
        # Check if every attribute in Y is a prime attribute of R
        # print("This",right,prime_attributes)
        if set(right).issubset(prime_attributes):
            continue
        # Check if X is a superkey for R
        # print(left,candidate_keys)
        flag = 0
        for i in range(0, len(candidate_keys)):
            if candidate_keys[i].issubset(set(left)):
                flag = 1
        if flag == 1:
            continue
        # print("yay")
        temp_violated_list.append(index)
    if len(temp_violated_list) == 0:
        return [True, temp_violated_list]
    else:
        return [False, temp_violated_list]

def is_in_BCNF(prime_attributes, non_prime_attributes, candidate_keys, fds):
    temp_violated_list = []
        # Check if every functional dependency is in 3NF
    for index, fd in enumerate(fds):
        left = fd['left']
        right = fd['right']

        flag = 0
        for i in range(0, len(candidate_keys)):
            if candidate_keys[i].issubset(set(left)):
                flag = 1
        if flag == 1:
            continue
        # print("yay")
        temp_violated_list.append(index)

    if len(temp_violated_list) == 0:
        return [True, temp_violated_list]
    else:
        return [False, temp_violated_list]

def normalize_to_2nf(relation_1nf):

    relation_2nf = relation_1nf.copy()

    j = 0
    while j<(len(relation_2nf)):
        is2nf,pd_index = is_in_2NF(relation_2nf[j]["prime_attributes"], relation_2nf[j]["non_prime_attributes"], relation_2nf[j]["candidate_keys"], relation_2nf[j]["fds"])
        if(is2nf):
            j=j+1
            continue
        else:
            vindex = pd_index[0]
            # Identify the A, B, and C sets
            k = j
            A = set(relation_2nf[j]["fds"][vindex]['left'])
            B = set(relation_2nf[j]["fds"][vindex]['right'])
            Bclosure = find_closure(relation_2nf[k]["attributes"], relation_2nf[k]["fds"], B)
            Aclosure = Bclosure.union(A)
            print("closure", Aclosure)
            # R2 = A.union(B)
            # # R1 = find_closure(normalized_relations[k]["attributes"],normalized_relations[k]["fds"], A)

            # # Create the new relations R1 and R2
            # C = normalized_relations[k]["attributes"]-A-B
            C = set(relation_2nf[k]["attributes"])-Aclosure
            C = C.union(A)
            # R1 = A.union(C)
            # R2 = set(normalized_relations[k]["attributes"].copy())-find_closure(normalized_relations[k]["attributes"],normalized_relations[k]["fds"], B)

            Lfds = relation_2nf[k]["fds"].copy()
            i = 0
            while i < len(Lfds):
                if set(Lfds[i]["left"]).issubset(Aclosure) and set(Lfds[i]["right"]).issubset(Aclosure):
                    i = i
                else:
                    del Lfds[i]
                    i = i-1
                i = i+1

            # Add the new relations to the list of normalized relations
            Rfds = relation_2nf[k]["fds"].copy()
            if fds[vindex] in Rfds:
                Rfds.remove(relation_2nf[j]["fds"][vindex])
            i = 0
            while i < len(Rfds):
                if set(Rfds[i]["left"]).issubset(C) and set(Rfds[i]["right"]).issubset(C):
                    i = i
                # elif Rfds[i] not in Lfds:
                #     C=C.union(Rfds[i]["left"])
                #     C=C.union(Rfds[i]["right"])
                else:
                    del Rfds[i]
                    i = i-1
                i = i+1

            l_cand =  find_candidate_keys(Aclosure, Lfds)
            r_cand =  find_candidate_keys(C, Rfds)
            l_prime = find_prime_attributes(l_cand)
            r_prime = find_prime_attributes(r_cand)
            relation_2nf.append({"attributes": Aclosure, "fds": Lfds,"candidate_keys":l_cand,"prime_attributes":l_prime,"non_prime_attributes":(Aclosure-l_prime)})
            relation_2nf.append({"attributes": C, "fds": Rfds,"candidate_keys":r_cand,"prime_attributes":r_prime,"non_prime_attributes":(C-r_prime)})
            del relation_2nf[k]

    # print(normalized_relations,"\n")
    return relation_2nf

def normalize_to_3nf(relation_2nf):

    relation_3nf = relation_2nf.copy()

    j = 0
    while j<(len(relation_3nf)):
        is3nf,pd_index = is_in_3NF(relation_3nf[j]["prime_attributes"], relation_3nf[j]["non_prime_attributes"], relation_3nf[j]["candidate_keys"], relation_3nf[j]["fds"])
        if(is3nf):
            j=j+1
            continue
        else:
            vindex = pd_index[0]
            # Identify the A, B, and C sets
            k = j
            A = set(relation_3nf[j]["fds"][vindex]['left'])
            B = set(relation_3nf[j]["fds"][vindex]['right'])
            Bclosure = find_closure(relation_3nf[k]["attributes"], relation_3nf[k]["fds"], B)
            Aclosure = Bclosure.union(A)
            print("closure", Aclosure)
            # R2 = A.union(B)
            # # R1 = find_closure(normalized_relations[k]["attributes"],normalized_relations[k]["fds"], A)

            # # Create the new relations R1 and R2
            # C = normalized_relations[k]["attributes"]-A-B
            C = set(relation_3nf[k]["attributes"])-Aclosure
            C = C.union(A)
            # R1 = A.union(C)
            # R2 = set(normalized_relations[k]["attributes"].copy())-find_closure(normalized_relations[k]["attributes"],normalized_relations[k]["fds"], B)

            Lfds = relation_3nf[k]["fds"].copy()
            i = 0
            while i < len(Lfds):
                if set(Lfds[i]["left"]).issubset(Aclosure) and set(Lfds[i]["right"]).issubset(Aclosure):
                    i = i
                else:
                    del Lfds[i]
                    i = i-1
                i = i+1

            # Add the new relations to the list of normalized relations
            Rfds = relation_3nf[k]["fds"].copy()
            if fds[vindex] in Rfds:
                Rfds.remove(relation_3nf[j]["fds"][vindex])
            i = 0
            while i < len(Rfds):
                if set(Rfds[i]["left"]).issubset(C) and set(Rfds[i]["right"]).issubset(C):
                    i = i
                # elif Rfds[i] not in Lfds:
                #     C=C.union(Rfds[i]["left"])
                #     C=C.union(Rfds[i]["right"])
                else:
                    del Rfds[i]
                    i = i-1
                i = i+1

            l_cand =  find_candidate_keys(Aclosure, Lfds)
            r_cand =  find_candidate_keys(C, Rfds)
            l_prime = find_prime_attributes(l_cand)
            r_prime = find_prime_attributes(r_cand)
            relation_3nf.append({"attributes": Aclosure, "fds": Lfds,"candidate_keys":l_cand,"prime_attributes":l_prime,"non_prime_attributes":(Aclosure-l_prime)})
            relation_3nf.append({"attributes": C, "fds": Rfds,"candidate_keys":r_cand,"prime_attributes":r_prime,"non_prime_attributes":(C-r_prime)})
            del relation_3nf[k]

    # print(normalized_relations,"\n")
    return relation_3nf

def normalize_to_bcnf(relation_3nf):
    relation_bcnf = relation_3nf.copy()
    j = 0
    p = 0
    while j<(len(relation_bcnf)):
        # print("random ",relation_bcnf[j],"random ")
        isbcnf,pd_index = is_in_BCNF(relation_bcnf[j]["prime_attributes"], relation_bcnf[j]["non_prime_attributes"], relation_bcnf[j]["candidate_keys"], relation_bcnf[j]["fds"])
        # print(isbcnf,pd_index)
        if(isbcnf):
            j=j+1
            continue
        else:
            vindex = pd_index[0]
            # Identify the A, B, and C sets
            k = j
            A = set(relation_bcnf[j]["fds"][vindex]['left'])
            B = set(relation_bcnf[j]["fds"][vindex]['right'])
            Aclosure = B.union(A)

            C = set(relation_bcnf[j]["attributes"])-Aclosure
            C = C.union(A)

            Lfds = relation_bcnf[k]["fds"].copy()
            i = 0
            while i < len(Lfds):
                if set(Lfds[i]["left"]).issubset(Aclosure) and set(Lfds[i]["right"]).issubset(Aclosure):
                    i = i
                else:
                    del Lfds[i]
                    i = i-1
                i = i+1

            # Add the new relations to the list of normalized relations
            Rfds = relation_bcnf[k]["fds"].copy()
            if fds[vindex] in Rfds:
                Rfds.remove(relation_bcnf[j]["fds"][vindex])
            i = 0
            while i < len(Rfds):
                if set(Rfds[i]["left"]).issubset(C) and set(Rfds[i]["right"]).issubset(C):
                    i = i
                else:
                    del Rfds[i]
                    i = i-1
                i = i+1

            l_cand =  find_candidate_keys(Aclosure, Lfds)
            r_cand =  find_candidate_keys(C, Rfds)
            l_prime = find_prime_attributes(l_cand)
            r_prime = find_prime_attributes(r_cand)
            relation_bcnf.append({"attributes": Aclosure, "fds": Lfds,"candidate_keys":l_cand,"prime_attributes":l_prime,"non_prime_attributes":(Aclosure-l_prime)})
            relation_bcnf.append({"attributes": C, "fds": Rfds,"candidate_keys":r_cand,"prime_attributes":r_prime,"non_prime_attributes":(C-r_prime)})
            del relation_bcnf[k]

    # print(normalized_relations,"\n")
    return relation_bcnf

def printedges(fds):
    for i in range(0, len(fds)):
        input = ''.join(str(num) for num in fds[i]["left"])
        output = ''.join(str(num) for num in fds[i]["right"])
        print(input+"->"+output, end="  ")
    print("")

# def verify_FDs(original_relation, original_fds, decomposed_relations):
#     # Check if the FDs are preserved in each 3NF relation
#     for relation in decomposed_relations:
#         for X, A in original_fds.items():
#             if set(X) <= set(relation["left"]):
#                 closure = compute_closure(X, relation, original_fds)
#                 if set(A) != closure.intersection(set(A)):
#                     return False
#     return True

def LJ_tester(p_relation,c_relation):
    plist = list(p_relation[0]["attributes"])
    n = len(plist)
    m = len(c_relation)
    lj_mat = []
    for i in range(0,m):
        temp = []
        for j in range(0,n):
            temp.extend([0])
        lj_mat.append(temp)

    print(lj_mat)
    for i in range(0,m):
        for k in set(c_relation[i]["attributes"]):
            lj_mat[i][plist.index(k)] = 1
    ans = False
    # print(lj_mat)
    for i in range(0,m):
        j=0
        while j<len(p_relation[0]["fds"]):
            left = p_relation[0]["fds"][j]["left"]
            all_1 = all(lj_mat[i][plist.index(val)] == 1 for val in left)
            # print(all_1)
            if(all_1):
                right = p_relation[0]["fds"][j]["right"]
                all_1l = all(lj_mat[i][plist.index(val)] == 1 for val in right)
                if(all_1l):
                    j = j+1
                    continue
                for val in right:
                    lj_mat[i][plist.index(val)]=1
                j=0
            else:
                j=j+1
        all_cut = all(val==1 for val in lj_mat[i])
        if(all_cut):
            ans = True
    # print(lj_mat)
    return [ans,lj_mat]

# A->C, B->DE, D->C
# Example usage:
# attributes = ['A', 'B', 'C', 'D', 'E']
# fds = [
#     {'left': ['A'], 'right': ['C']},
#     {'left': ['B'], 'right': ['D', 'E']},
#     {'left': ['D'], 'right': ['C']}
# ]
# attributes = ['A', 'B', 'C']
# fds = [
#     {'left': ['A'], 'right': ['B']},
#     {'left': ['B'], 'right': ['C']}
# ]
attributes = ['A', 'B', 'C','D','E','F']
fds = [
    {'left': ['A','B'], 'right': ['C']},
    {'left': ['C'], 'right': ['D']},
    {'left': ['C'], 'right': ['E']},
    {'left': ['E'], 'right': ['F']},
    {'left': ['F'], 'right': ['A']}
]
# attributes = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H']
# fds = [
#     {'left': ['A'], 'right': ['D', 'B']},
#     {'left': ['B'], 'right': ['C']},
#     {'left': ['E'], 'right': ['F', 'G']},
#     {'left': ['A', 'E'], 'right': ['H']}
# ]
output = ''.join(str(num) for num in attributes)
print("R(", output, ") ", end="")
printedges(fds)
candidate_keys = find_candidate_keys(attributes, fds)
print(candidate_keys)

prime_attributes = find_prime_attributes(candidate_keys)
print("Prime Attributes", prime_attributes)

non_prime_attributes = set(attributes) - prime_attributes
# print("Non Prime Attributes",non_prime_attributes)

relation_1nf = [{"attributes":attributes,"fds":fds,"candidate_keys":candidate_keys,"prime_attributes":prime_attributes,"non_prime_attributes":non_prime_attributes}]

relation_2nf = normalize_to_2nf(relation_1nf)
print("\nconverted to 2NF\n\n Relations :=> ",relation_2nf,"\n")
for i in range(0, len(relation_2nf)):
    output = ''.join(relation_2nf[i]["attributes"])
    print("R"+str(i+1)+"("+output+")", end=" ")
    printedges(relation_2nf[i]["fds"])

relation_3nf = normalize_to_3nf(relation_2nf)
print("\nconverted to 3NF\n\n Relations :=> ",relation_3nf,"\n")
for i in range(0, len(relation_3nf)):
    output = ''.join(relation_3nf[i]["attributes"])
    print("R"+str(i+1)+"("+output+")", end=" ")
    printedges(relation_3nf[i]["fds"])

# violated_index_bcnf = []
relation_bcnf= normalize_to_bcnf(relation_3nf)
print("\nconverted to BCNF\n\n Relations :=> ",relation_3nf,"\n")
for i in range(0, len(relation_bcnf)):
    output = ''.join(relation_bcnf[i]["attributes"])
    print("R"+str(i+1)+"("+output+")", end=" ")
    printedges(relation_bcnf[i]["fds"])

# print(LJ_tester(relation_1nf,relation_bcnf))


# coding: UTF-8

import numpy as np
import torch
import torchvision

class TestDataset(torch.utils.data.Dataset):

    def __init__(self, data, transform=None):
        self.transform = transform
        self.data = data
        self.data_index = np.random.choice((0, 1), size=len(self.data), p=(.8, .2))

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        pre_data = self.data[idx]
        idx = self.data_index[idx]
        if self.transform:
            for transform in self.transform:
                pre_data = transform(x)
        else:
            pre_data = torchvision.transforms.ToTensor()(pre_data)
        pre_data = (pre_data, pre_data.view(-1))

        return pre_data, idx

if __name__ == '__main__':

    data = [np.random.random((64, 64, 31)) for _ in range(10)]
    transform = torchvision.transforms.ToTensor()
    dataset = TestDataset(data)
    x = dataset[0]
    dataloader = torch.utils.data.DataLoader(dataset, batch_size=5, shuffle=True)
    for i, (x, y) in enumerate(dataloader):
        print(i, x[0].shape, x[1].shape, y)