import json
import sys
import xml.etree.ElementTree as ET
from ast import literal_eval as make_tuple
from ctypes import windll
from datetime import datetime
from getpass import getuser
from glob import glob
from logging import getLogger
from os import mkdir, remove, scandir
from os.path import (abspath, basename, dirname, exists, getmtime, isdir, join,
                     splitext)
from random import choice
from shutil import copy2, get_terminal_size, move
from threading import Event, Thread
from time import sleep

import pyperclip
from bin.Town_clipper import *
from bin.Town_cooker import *
from bin.Town_shortcuts import shortcut
from keyboard import add_hotkey

# Logging options
root = getLogger("Town.waiter")
stream = getLogger("TownStream.waiter")

# Title
TITLE = r"""
 _____                   ____  _          _ _
|_   _|____      ___ __ / ___|| |__   ___| | |
  | |/ _ \ \ /\ / / '_ \\___ \| '_ \ / _ \ | |
  | | (_) \ V  V /| | | |___) | | | |  __/ | |
  |_|\___/ \_/\_/ |_| |_|____/|_| |_|\___|_|_|
  """

# All available colors
ALLCOLORS = {
    0: "red",
    1: "orange",
    2: "yellow",
    3: "yellow_green",
    4: "light_green",
    5: "grass_green",
    6: "green",
    7: "green_blue",
    8: "blue",
    9: "deep_blue",
    10: "purple",
    11: "mauve",
    12: "beige",
    13: "brown_gray",
    14: "white",
}

# To handle .exe treatment
BUNDLE_DIR = getattr(sys, "_MEIPASS", abspath(dirname(__file__)))

def exePath(mypath):
    if exists(mypath):
        return mypath
    elif exists(abspath(join(BUNDLE_DIR, mypath))):
        return abspath(join(BUNDLE_DIR, mypath))
    else:
        return mypath

# The configuration file
TOWNSHELL_PATH = "townshell.cfg"

# Directory of backup
BACKUPTOWN = "backup\\" + datetime.now().strftime("%Y%m%d%H%M%S")

# Directory for temporary storage
TEMP = "temp"

# Timestamp format
TIMESTAMP_FORMAT = "%Y/%m/%d %H:%M:%S"

# Boolean to know whether Keyboard shortcuts are active or not
global active_shortcuts
active_shortcuts = False

# Dictionary for undo and redo
global version
version = {}

# Dictionary of stored clips
global clipHistory
clipHistory = {"previous": [], "current": "", "future": []}

# Directory where all Townscaper saved files are
scapedir = ""

# Print the colors a pretty way
def print_colors():

    for digit, color in ALLCOLORS.items():
        print("{} ==> {}".format(digit, color))

# Fetch the log level from 'townshell.cfg'
def get_loglevel():

    # Checks that townshell.cfg exists
    if exists(TOWNSHELL_PATH) is False:
        myTownshell = copyc(exePath(r"tmp_townshell.cfg"), TOWNSHELL_PATH)
        if myTownshell is None:
            return "INFO"

    loglevel = read_cfg("loglevel")
    if loglevel not in ("INFO", "WARNING", "DEBUG", "ERROR"):
        print("Invalid log level found in 'townshell.cfg'. INFO will be used")

        return "INFO"
    else:
        return loglevel

# Copy a corvox dictionnary so that it's fully independent from the original one
def dictCopy(corvox):
    return {key: dictvalues.copy() for key, dictvalues in corvox.items()}

# Customized copy2 to handle logging constraints
# rename: If True the copy will be renamed in destination if any file with the same name is found
# erase: If True and the file exists in the destination it's erased by the new file
# Note that rename has higher priority than erase so rename=True and erase=True is useless
def copyc(src, dst, rename=False, erase=False):

    # Conversion for comfort
    if isdir(dst):
        dst = join(dst, basename(src))

    # Case the file exists already in destination and rename = True
    if exists(dst) and rename:
        root.warning("%s already exists in %s and will be renamed", src, dst)

        # The renaming is done by adding the current timestamp to basename of the file
        suffix = datetime.now().strftime("%Y%m%d%H%M%S")
        base_src, ext = splitext(basename(src))
        dst = join(dirname(dst), base_src + suffix + ext)

    # Case the file exists already in destination and erase = True
    elif exists(dst) and erase:
        root.warning("%s already exists in %s and will be updated", src, dst)

        # The previous file is stored in TEMP
        try:
            temp = move(dst, TEMP)
        except:
            root.exception("Backup of %s failed, %s will not be copied", dst, src)

            return

    # Case the file exists already in destination and rename = erase = False
    elif exists(dst):
        root.error("%s already exists in %s, no file copied", src, dst)

        return

    # Copy
    try:
        copy2(src, dst)
        root.debug("Successful copy from %s to %s", src, dst)

    except PermissionError:
        root.exception("Copy of %s in %s not allowed", src, dst)

        if erase:
            move(temp, dst)

    except:
        root.exception("Error during copy of %s in %s", src, dst)

        if erase:
            move(temp, dst)

    # When everything is fine
    else:
        # Erase = True
        if erase:
            remove(temp)
            root.debug("Sucessful cleaning of %s", temp)

        # Return
        return dst

    # Erase = True
    if erase:
        move(temp, dst)
        root.debug("Successful rollback on %s", temp)

    # Return failure
    return

# Read TOWNSHELL_PATH and send back the dictionary associated or the value associated with the key given as input
# key: key in townshell.cfg if exists the value is returned
def read_cfg(key=None):

    # Checks that the configuration file exists
    if exists(TOWNSHELL_PATH) is False:
        root.error("{} does not exist.".format(TOWNSHELL_PATH))
        return

    # Reads the configuration file and store the value in a dictionary
    try:
        cfg = json.load(open(TOWNSHELL_PATH))
        root.debug("Output read :\n{}".format(cfg))
    except:
        root.exception("Invalid format for {}".format(TOWNSHELL_PATH))
        return

    # Checks the key, if 'all' the whole dictionary is provided. If None or does not exists, the dictionary is returned
    if key == "all":
        return cfg
    elif key is None or key not in cfg:
        return
    else:
        return cfg[key]

# Update townshell.cfg.
# key: name of the field to update in config file
# value: new value for the spotted key
def update_cfg(key, value):

    # Read of the configuration file to fetch the dictionary cfg
    cfg = read_cfg("all")
    if cfg is None:
        return

    # Checks the format of the value depending on the key
    # Path of the .scape save files, the value must be an existing path
    if key == "scapedir":

        # Path with spaces, when drag and drop might have this look
        if value.startswith('"'):
            value = value[1:-1]

        if exists(value) is False:
            toprint = "'{}' does not exist. Directory of scape saved files not updated".format(
                value
            )
            root.warning(toprint)

            return

        else:
            cfg[key] = value
            global scapedir
            scapedir = value
            root.debug("'{}' is a valid path".format(value))

    # Update of the config file
    try:
        json.dump(cfg, open(TOWNSHELL_PATH, "w"), indent=1)
    except:
        toprint("Error on saving the updated {}".format(TOWNSHELL_PATH))
        root.error(toprint)

    # Final return
    root.info("%s successfully updated", TOWNSHELL_PATH)
    root.debug("Written :\n{}".format(cfg))
    return True

# Identify the directory where all maps are stored by Townscaper
# Return the path if it does exist or None
def find_scapedir():

    # Fetching the current user
    current_user = getuser()

    # Identifying the potential Townscaper save directory
    scapedir = join(
        r"C:\Users", current_user, r"AppData\LocalLow\Oskar Stalberg\Townscaper\Saves"
    )

    # Checking that the directory exists and return
    if exists(scapedir):
        root.info("Townscaper directory is : %s", scapedir)
        return scapedir
    else:
        root.warning("Townscaper directory not found")
        return ""

# Find the saved file that contains the characters given as input
# char: chain of characters
# extended: If True, the amount of corners and voxels are returned too,
def list_scapefiles(char="", extended=False):

    # Case it's obvious
    if exists(char) and extended is False:
        root.debug("%s returned directly", char)
        return [char]

    elif exists(char):
        char = basename(char)

    # Gathering the files
    if char.endswith(".scape"):
        candidates = glob(join(scapedir, "*" + char + "*"))
    else:
        candidates = glob(join(scapedir, "*" + char + "*.scape"))

    # Case nothing found
    if candidates == []:
        root.warning("No Townscaper saved files were found")
        return

    # Other cases

    # Extended extraction
    if extended:
        scapefiles = {}
        for path in candidates:
            # Parsing XML
            tree = ET.parse(path)
            loot = tree.getroot()
            # Amount of corners : coordinates with at list one case filled
            corners = loot.find("corners")
            amount_corners = len(corners.findall("C"))
            # Amount of voxels
            voxels = loot.find("voxels")
            amount_voxels = len(voxels.findall("V"))

            # Adding the data to dictionnary
            scapefiles[basename(path)] = {
                "path": path,
                "date": getmtime(path),
                "corners": amount_corners,
                "voxels": amount_voxels,
            }
        # Return for extended version
        root.debug("{} files returned with extended True".format(len(scapefiles)))
        return scapefiles

    else:
        # Return
        root.debug("{} files returned with extended False".format(len(candidates)))
        return candidates

# Print the townscaper saved files highlighting key information
# max_amount: Number of files to be displayed at most, if None, there are no limits
# args: cmd dictionary when print_scapefiles is used from Cmd
def print_scapefiles(max_amount=5, args=None):

    # Fill the end the input string with spaces according to a given length,
    # if the string is bigger, it's truncated and ended with '...'
    # Convert other type to str
    def fill(string, length):

        # Convert
        if not isinstance(string, str):
            string = str(string)

        # Fill
        if len(string) > length:
            return string[: length - 3] + "..."

        elif len(string) < length:
            return string + " " * (length - len(string))

        else:
            return string

    ####
    # if full print : all columns are printed. Otherwise only file name and modification date are published
    full_print = True

    # Dealing with args
    # No args means default settings are used otherwise the options are taken in charge
    if args == {} or args is None:
        pass
    elif "input1" in args and args["input1"] in ("-a", "-all"):
        max_amount = None

    # Gather all saved files in a dictionary
    scapefiles = list_scapefiles(extended=True)

    # Case list_scapefiles failed
    if scapefiles is None:
        return

    # Saving the length of list_scapefiles, useful for print outs
    len_scapefiles = len(scapefiles)

    # Fetch the max amount of characters for every column
    max_len_name = max([len(name) for name in scapefiles])
    max_len_corners = max(
        [len(str(data["corners"])) for data in scapefiles.values()] + [len("corners")]
    )
    max_len_voxels = max(
        [len(str(data["voxels"])) for data in scapefiles.values()] + [len("voxels")]
    )

    # Size of the column for dates
    len_date_column = len(datetime.now().strftime(TIMESTAMP_FORMAT))

    # Between each column there are 4 spaces
    SPACES = "  |  "
    len_SPACES = len(SPACES)

    # Length of the full print
    full_len = (
        max_len_name
        + len_SPACES
        + len_date_column
        + len_SPACES
        + max_len_corners
        + len_SPACES
        + max_len_voxels
    )
    # Fetch the terminal size
    max_columns = get_terminal_size(fallback=(80, 24))[0]
    # Compare the length of the full print to the terminal size
    if full_len < max_columns:
        len_name_column = max_len_name
        len_corners_column = max_len_corners
        len_voxels_column = max_len_voxels

    else:
        # If the lenght of the files is too much for the terminal, the size of the field for File name is reduced
        # Half of the terminal size is booked for the File name
        len_name_column = round(max_columns / 2)

        # Check that other field together do not take too much place. If it's too much full print is set to false
        part_len = (
            len_SPACES
            + len_date_column
            + len_SPACES
            + max_len_corners
            + len_SPACES
            + max_len_voxels
        )
        if part_len > max_columns / 2:
            full_print = False
            full_len = len_name_column + len_SPACES + len_date_column

        # Full print stay True
        else:
            len_corners_column = max_len_corners
            len_voxels_column = max_len_voxels
            full_len = (
                len_name_column
                + len_SPACES
                + len_date_column
                + len_SPACES
                + len_corners_column
                + len_SPACES
                + len_voxels_column
            )

    # Handling of max_amount and Sort the files, default key is modification date
    if max_amount is None:
        scapefiles = sorted(
            scapefiles.items(), key=lambda x: x[1]["date"], reverse=True
        )
    else:
        scapefiles = sorted(
            scapefiles.items(), key=lambda x: x[1]["date"], reverse=True
        )[:max_amount]

    # Print of the header
    if full_print:
        print(
            "{0}{1}{2}{1}{3}{1}{4}".format(
                fill("File Name", len_name_column),
                SPACES,
                fill("Last update", len_date_column),
                fill("Corners", len_corners_column),
                fill("Voxels", len_voxels_column),
            )
        )

    else:
        print(
            "{0}{1}{2}".format(
                fill("File Name", len_name_column),
                SPACES,
                fill("Last update", len_date_column),
            )
        )

    # Line break
    print("-" * full_len)

    # Iteration on the dictionary items
    for i, (name, file_data) in enumerate(scapefiles):

        # Format of the date
        date_last_update = datetime.fromtimestamp(file_data["date"])
        date_last_update = date_last_update.strftime(TIMESTAMP_FORMAT)

        if full_print is False:
            print(
                "{}{}{}".format(fill(name, len_name_column), SPACES, date_last_update)
            )
            continue

        # Corners and voxels
        amount_corners = file_data["corners"]
        amount_voxels = file_data["voxels"]

        # Print-out for full_print = True
        print(
            "{0}{1}{2}{1}{3}{1}{4}".format(
                fill(name, len_name_column),
                SPACES,
                date_last_update,
                fill(amount_corners, len_corners_column),
                fill(amount_voxels, len_voxels_column),
            )
        )

    # Line break
    print("-" * full_len)

    # Sum up of the file displayed
    if max_amount is None:
        nb_files_displayed = len_scapefiles
    else:
        nb_files_displayed = max_amount
    print(
        "{} saved file(s) // {} file(s) displayed".format(
            len_scapefiles, nb_files_displayed
        )
    )

    # Line break
    print("-" * full_len)

    # Return
    return True

# Initialize Townshell
def init_townshell():

    global scapedir
    global mypid

    # Print Title (deprecated)

    # Fetch the pid of TownShell
    mypid = windll.user32.GetForegroundWindow()

    # Checks for logs
    if exists("log") is False:
        mkdir("log")

    # Archiving of the previous log
    with open("log\\town.log") as logfile:
        with open("log\\town_old.log", "a") as old_logfile:
            old_logfile.write(logfile.read())

    with open("log\\town.log", "w") as logfile:
        logfile.write("")

    # Start of new logging
    root.info("TownShell started")

    # Start clipboard watch
    clipboard_thread = ClipboardThread()
    clipboard_thread.start()

    # Start keyboard shortcuts
    start_shortcuts()

    # Print keyboard shortcuts (deprecated)
    # print_shortcuts()

    # Checks townshell.cfg to find the directory where Townscaper saved files are
    scapedir = read_cfg("scapedir")

    # Checks that scapedir (the directory where Townscaper saved files are stored is filled) is OK
    if isinstance(scapedir, str) and exists(scapedir):
        pass

    else:

        # Case where the value in townshell.cfg does not exist
        if isinstance(scapedir, str) and not exists(scapedir):
            root.warning("scapedir path in townshell.cfg does not exist")

        # Search for scapedir
        # Search through find_scapedir
        scapedir = find_scapedir()

        # Case where the path was not found the user is asked to provide it
        if scapedir == "":
            root.warning("Townscaper saved files directory was not found")
            # scapedir = input(
            #     """Townscaper saved files directory was not found.
            # Drag and drop the directory to the command line or copy-paste the path of the directory
            # Path : """
            # )

            # if scapedir.startswith('"'):
            #     scapedir = scapedir[1:-1]

            # # Case where the provided path is not correct
            # if exists(scapedir) is False:
            #     root.error(
            #         """%s does not exist.
            #     'listfiles' will not work. Please use command 'loadpath' to provide Townscaper saved files directory""",
            #         scapedir,
            #     )
            #     return False

        # Case the path is correct
        update_cfg("scapedir", scapedir)

    # Look for a job to be read and executed (unsafe not checked for a while)
    job_path = read_cfg("job")
    if isinstance(job_path, str) and exists(job_path):
        with open(job_path) as job_file:
            return job_file.read().splitlines()

# Do all operations to level the town as required
# - Checks the args provided
# - Find the file spotted
# - Load the file using function load
# - Level it as required using function level
# - Save it using function save
def level_town(args):

    root.debug(f"Input settings : {args}")

    # Checking the provided args
    # args is a dictionary so we'll analyze each arg one by one
    iter_args = iter(args.items())

    # Then the other args are analyzed
    # Expected input
    char = None  # chain of character of the file to be modified
    height = None
    coord = None
    max_height = 255
    min_height = -1
    plain = False
    color = None
    color_filter = None
    for key, arg in iter_args:

        # File to modify (char)
        if key == "input1":
            char = arg

        # Height
        if key in ("height", "h") and isinstance(arg, int):
            height = arg

        elif key in ("height", "h") and (
            arg.isnumeric() or (arg.startswith("-") and arg[1:].isnumeric())
        ):
            height = int(arg)

        # Coord
        elif key == "coord":
            coord = make_tuple(arg)

        # Max_height
        elif key in ("max_height", "maxh") and isinstance(arg, int):
            max_height = arg

        elif key in ("max_height", "maxh") and arg.isnumeric():
            if int(arg) > 255:
                root.warning(
                    "Invalid max height : %s ; Maximum height cannot exceed 255", arg
                )
            else:
                max_height = int(arg)

        # Min_height
        elif key in ("min_height", "minh") and isinstance(arg, int):
            min_height = arg

        elif key in ("min_height", "minh") and arg.isnumeric():
            if int(arg) > max_height:

                root.warning(
                    "Invalid min height : %s ; Minimum height cannot be superior to maximum height",
                    arg,
                )
            else:
                min_height = int(arg)

        # Plain
        elif key in ("plain", "p") and isinstance(arg, bool):
            plain = arg
        elif key in ("plain", "p"):
            if (arg.isnumeric() and int(arg) > 0) or (arg == "True"):
                plain = True

        # Color
        elif key in ("color", "c"):
            if arg.isnumeric():
                color = int(arg)

        # color_filter
        elif key in ("color_filter", "cf"):
            if arg.isnumeric():
                color_filter = int(arg)

    # Checking that height has been filled
    if height is None:

        root.warning("No valid height was found")
        return

    # PREVIOUSLY: Checking that 'char' has been provided last_modified is used otherwise
    # 2020/11/15 : lastClip is used instead of last_modified if 'char' was not provided
    if char is None and lastClip is None:

        root.info("No valid clip from Townscaper")
        return
    elif char is None:
        char = lastClip

    # Checks if 'char' is a clip
    elif isClip(char) is False:
        return

    storeClip(char)

    # Corvox from clip
    corvox = clipToCorvox(char)
    if corvox is None:
        return

    # Leveling of appropriate data
    root.debug("Input right before leveling : {}".format([height, coord, max_height, min_height, plain, color, color_filter]))
    corvox = level(
        corvox, height, coord, max_height, min_height, plain, color, color_filter
    )

    # Copy to clipboard
    new_clip = corvoxToClip(corvox)

    # Store the new clip
    storeClip(new_clip)

    # Success
    return True

# Do all operations to level the town as required
# - Checks the args provided
# - Paint it as required using function paint
def paint_town(args):

    root.debug(f"Input settings : {args}")

    # Checking the provided args
    # args is a dictionary so we'll analyze each arg one by one
    iter_args = iter(args.items())

    # Then the other args are analyzed
    # Expected input
    char = None  # chain of character of the file to be modified
    color = None
    color_filter = None
    height = None
    column = None
    coord = ()
    details = None

    for key, arg in iter_args:

        # File to modify (char)
        if key == "input1":
            char = arg

        # Color
        if key in ("color", "c"):

            # color is already a tuple
            if isinstance(arg, tuple):
                color = tuple(c for c in arg if c in ALLCOLORS)
            # color is between 0 and 14
            elif arg.isnumeric() and 0 <= int(arg) <= 14:
                color = int(arg)
            # Invalid color digit
            elif arg.isnumeric():
                root.warning(
                    "Color %s is not a valid one ; Color needs to be between 0 and 14 included",
                    arg,
                )
                return
            # Random color
            elif arg.lower() in ("r", "random"):
                color = "random"

            # Color is a tuple like (color1, color2, color3)
            elif "," in arg:
                color = make_tuple(arg)

        # Old color
        elif key in ("color_filter", "cf"):

            if arg == ():
                pass
            # old color is already a tuple
            elif isinstance(arg, tuple):
                color_filter = tuple(c for c in arg if c in ALLCOLORS)
            # old color needs to be between 0 and 14
            elif arg.isnumeric() and 0 <= int(arg) <= 14:
                color_filter = int(arg)

            # Invalid color digit
            elif arg.isnumeric():

                root.warning("Old color %s is not a valid one", arg)

        # Height
        elif key in ("height", "h"):
            if arg is None:
                pass
            elif isinstance(arg, (int, tuple)):
                height = arg
            elif arg.isnumeric():
                height = int(arg)
            elif "," in arg:
                height = make_tuple(arg)

        ###
        # Column, coord and Details are not implemented yet
        ###

    if char is None and lastClip == "":

        root.info("No valid clip from Townscaper")
        return
    elif char is None:
        char = lastClip

    # Checks if 'char' is a clip
    elif isClip(char) is False:
        return

    # The clip saved for undo/redo
    storeClip(char)

    # Corvox from clip
    corvox = clipToCorvox(char)
    if corvox is None:
        return

    # Painting of appropriate data
    root.debug(f"Settings : color :{color}, cf: {color_filter}, height: {height}")
    corvox = paint(corvox, color, color_filter, height, column, coord, details)

    # Copy to clipboard
    new_clip = corvoxToClip(corvox)

    # Store the new clip
    storeClip(new_clip)

    # Success
    return True

# Do all operations to merge the args provided
def merge_town(args):

    root.debug(f"Input settings : {args}")

    toMerge = []
    for i, arg in enumerate(args.values()):

        # Checks that it's an actual clip
        if isClip(arg) is False:
            root.debug(f"Invalid clip : {arg}")
            root.warning("Invalid clip, no merge done")
            return

        # Converts the clip and add it to toMerge
        else:
            toMerge.append(clipToCorvox(arg))

            # The first clip is stored for undo/redo
            if i == 0:
                storeClip(arg)

    # Merging of dict
    corvox = merge(*toMerge)

    # Copy to clipboard
    new_clip = corvoxToClip(corvox)

    # Store the new clip
    storeClip(new_clip)

    # Success
    return True

# Copy a structure to another height a given amount of time
# Shortly, it's the combination of level and merge
def replicate_town(args):

    root.debug(f"Input settings : {args}")

    # Checking the provided args
    # args is a dictionary so we'll analyze each arg one by one
    iter_args = iter(args.items())

    # Then the other args are analyzed
    # Expected input
    char = None  # chain of character of the file to be modified
    copyAmount = 1
    height = None
    max_height = 255
    min_height = -1
    coord = None
    plain = False
    color = None
    color_filter = None
    for key, arg in iter_args:

        # File to modify (char)
        if key == "input1":
            char = arg

        # Amount of copy
        if key in ("copy", "c") and isinstance(arg, int):
            copyAmount = arg
        elif (
            key in ("copy", "c")
            and isinstance(arg, str)
            and (arg.isnumeric() or (arg.startswith("-") and arg[1:].isnumeric()))
        ):
            copyAmount = int(arg)

        # Height
        elif key in ("height", "h") and isinstance(arg, int):
            height = arg

        elif key in ("height", "h") and (
            arg.isnumeric() or (arg.startswith("-") and arg[1:].isnumeric())
        ):
            height = int(arg)

        # Coord
        elif key == "coord":
            coord = make_tuple(arg)

        # Plain
        elif key in ("plain", "p") and isinstance(arg, bool):
            plain = arg
        elif key in ("plain", "p"):
            if (arg.isnumeric() and int(arg) > 0) or (arg == "True"):
                plain = True

        # Color
        elif key in ("color", "c"):
            if arg.lower() in ("r", "random"):
                color = "random"

            elif arg.isnumeric():
                color = int(arg)

        # color_filter
        elif key in ("color_filter", "cf"):
            if arg.isnumeric():
                color_filter = int(arg)

    # Checking that height has been filled
    if height is None:

        root.warning("No valid height was found")
        return

    # PREVIOUSLY: Checking that 'char' has been provided last_modified is used otherwise
    # 2020/11/15 : lastClip is used instead of last_modified if 'char' was not provided
    if char is None and lastClip is None:

        root.info("No valid clip from Townscaper")
        return
    elif char is None:
        char = lastClip

    # Checks if 'char' is a clip
    elif isClip(char) is False:
        return

    storeClip(char)

    # Corvox from clip
    initCorvox = clipToCorvox(char)

    if initCorvox is None:
        return

    toLevel = dictCopy(initCorvox)
    toMerge = [initCorvox]
    thisHeight = 0
    for i in range(copyAmount):

        # The height used for this copy
        thisHeight += height
        if thisHeight > 255:
            break

        # color choice if random has been selected
        thisColor = choice(ALLCOLORS) if color == "random" else color

        # Leveling of appropriate data
        corvox = level(
            toLevel,
            height,
            coord,
            max_height,
            min_height,
            plain,
            thisColor,
            color_filter,
        )

        # Adding the dictionnary to toMerge
        toMerge.append(dictCopy(corvox))
        # root.debug(toMerge)

    # Merge of all dictionnary : if plain is True, the last becomes the first
    if plain:
        toMerge.reverse()

    finalCorvox = merge(*toMerge)

    # Copy to clipboard
    new_clip = corvoxToClip(finalCorvox)

    # Store the new clip
    storeClip(new_clip)

    # Success
    return True

# Do all operations for using dig function
def dig_town(args):

    root.debug(f"Input settings : {args}")

    # Checking the provided args
    # args is a dictionary so we'll analyze each arg one by one
    iter_args = iter(args.items())

    # Then the other args are analyzed
    # Expected input
    char = None  # chain of character of the file to be modified
    color_filter = None
    height = None
    percent = 1

    for key, arg in iter_args:

        # File to modify (char)
        if key == "input1":
            char = arg

        # Percent
        if key in ("percent", "p"):

            if isinstance(arg, (int, float)) and arg <= 100:
                percent = arg / 100
            else:
                root.warning(f"Invalid percent value {percent}")
                return

        # Old color
        elif key in ("color_filter", "cf"):

            if arg == ():
                pass
            # old color is already a tuple
            elif isinstance(arg, tuple):
                color_filter = tuple(c for c in arg if c in ALLCOLORS)
            # old color needs to be between 0 and 14
            elif arg.isnumeric() and 0 <= int(arg) <= 14:
                color_filter = int(arg)

            # Invalid color digit
            elif arg.isnumeric():

                root.warning("Old color %s is not a valid one", arg)

        # Height
        elif key in ("height", "h"):
            if arg is None:
                pass
            elif isinstance(arg, (int, tuple)):
                height = arg
            elif arg.isnumeric():
                height = int(arg)
            elif "," in arg:
                height = make_tuple(arg)

    if char is None and lastClip == "":

        root.info("No valid clip from Townscaper")
        return
    elif char is None:
        char = lastClip

    # Checks if 'char' is a clip
    elif isClip(char) is False:
        return

    # The clip saved for undo/redo
    storeClip(char)

    # Corvox from clip
    corvox = clipToCorvox(char)
    if corvox is None:
        return

    # Digging of appropriate data
    corvox = dig(corvox, color_filter, percent, height)

    # Copy to clipboard
    new_clip = corvoxToClip(corvox)

    # Store the new clip
    storeClip(new_clip)

    # Success
    return True

# Store the clip
def storeClip(clip):
    global clipHistory

    # Duplicate from the current clip are bypassed
    if clip == clipHistory["current"]:
        return
    else:
        clipHistory["previous"].append(clipHistory["current"])
        clipHistory["current"] = clip
        clipHistory["future"] = []

# Fetch the lastclip
def undoClip():

    global clipHistory

    if clipHistory["previous"] != []:

        clipHistory["future"].append(clipHistory["current"])
        clipHistory["current"] = clipHistory["previous"].pop()

        # To clipboard
        pyperclip.copy(clipHistory["current"])

        # Success
        return True

    # No corresponding files
    else:
        root.info(f"No older clip")

# Fetch the last future clip (Yes I don't know how to say it)
def redoClip():

    global clipHistory

    if clipHistory["future"] != []:

        clipHistory["previous"].append(clipHistory["current"])
        clipHistory["current"] = clipHistory["future"].pop()

        # To clipboard
        pyperclip.copy(clipHistory["current"])

        # Success
        return True

    # No corresponding files
    else:
        root.info(f"No newer clip")

# Print the current keyboard shortcuts
def print_shortcuts():

    output = ""

    # Print whether keyboard shortcuts are active or not
    if active_shortcuts:
        output += "Keyboard shortcuts : ON\n"
    else:
        output += "Keyboard shortcuts : OFF\n"

    # Fetch shortcuts
    shortcuts = read_cfg("shortcuts")
    if shortcuts is None:
        return
    shortcuts = iter(shortcuts.items())

    # Shorcut to open TownShell
    output += "Press '²' to open TownShell from Townscaper\n"

    # First shortcuts
    info_printed = False
    for key, value in shortcuts:

        if key.startswith("custom") and info_printed is False:
            output += "\nTo get precise amount of clicks, enter quickly a few digits, then use the custom shortcuts below :\n"
            info_printed = True

        if key == "pause":
            output += f"\nIntervals between clicks (s) : {value}\n"

        else:
            output += f"'{key}' ==> '{value}'\n"

    # How to modify shortcuts
    output += "Shortcuts and intervals can be modified in 'townshell.cfg'\n"

    # Output is directly printed by default but can be returned
    return output

# Class of Thread to have shortcuts handled in another thread
class ShortcutThread(Thread):
    def __init__(self, event):  # event = objet Event
        Thread.__init__(self)
        self.event = event
        self.daemon = True

    def run(self):
        global mypid

        # Hotkeys
        myShortcuts = read_cfg("shortcuts")
        add_hotkey(myShortcuts["undo_command"], undoClip, args=[], suppress=True)
        add_hotkey(myShortcuts["redo_command"], redoClip, args=[], suppress=True)

        # Fetch the keyboard shortcuts settings from townshell.cfg
        shortcut(self.event, myShortcuts, mypid)

# Start a thread activating the keyboard shortcuts
def start_shortcuts():

    # Event that determine whether keyboard shortcuts should be active or not
    global stopnow
    global active_shortcuts

    # Checking that another thread is not executed already
    if active_shortcuts:
        root.warning("Keyboard shortcuts are already activated")
        return

    # Initialize the event that control whether shortcuts are active or not
    stopnow = Event()
    stopnow.clear()

    # Main program
    new_thread = ShortcutThread(stopnow)
    new_thread.start()
    active_shortcuts = True

    root.info("Keyboard shortcuts are now active")

    # Success
    return True

# Stop the thread handling the shortcuts
def stop_shortcuts():

    global stopnow
    global active_shortcuts

    # Checking that another thread is not executed already
    if active_shortcuts is False:
        root.warning("No shortcuts activated")
        return

    # The event is activated and shortcuts is stopped
    stopnow.set()
    active_shortcuts = False

    root.info("Keyboard shortcuts have been deactivated")

    # Success
    return True

# Return True if the provided string is a clip
# Only bits and dense are checked assuming that it's enough to check
def isClip(clip):

    try:
        bits = clipToBits(clip)
        if bits and bitsToDense(bits):
            return True
        else:
            return False

    except:
        root.exception("Error in isclip but let's keep working")
        return False

# Thread that checks whether TownShell is the active application and checks the content of the clipboard
# NOTE : The implementation could change the "Groups" features are implemented (TODO)
class ClipboardThread(Thread):
    def __init__(self):
        Thread.__init__(self)
        self.daemon = True

    def run(self):

        global mypid
        global lastClip
        lastClip = None
        prev_clip = ""

        while True:
            # Fetch the foreground application
            cur_foreground = windll.user32.GetForegroundWindow()

            if mypid == cur_foreground:

                # The clipboard is checked
                clip = pyperclip.paste()

                # Check that it's a valid clip from Townscaper
                if clip and prev_clip != clip and isClip(clip):

                    lastClip = clip

            # Wait between loops
            prev_clip = clip
            sleep(0.5)

# Transform a dictionary corvox to string and copy it to clipboard
def corvoxToClip(corvox):

    dense = sparseToDense(corvoxToSparse(corvox))
    bits = denseToBits(dense) if dense else None
    clip = bitsToClip(bits) if bits else None

    if clip:
        pyperclip.copy(clip)
        root.info("Corvox copied on clipboard")

        return clip
    else:

        root.error(
            "Copy for 'Load from Clipboard' failed. Set DEBUG on 'loglevel' in 'townshell.cfg' and try again for more information"
        )
        return

# Transform a clipboard content of Townscaper into a corvox dictionary
def clipToCorvox(clip):

    bits = clipToBits(clip)
    dense = bitsToDense(bits) if bits else None
    sparse = denseToSparse(dense) if dense else None
    corvox = sparseToCorvox(sparse) if sparse else None

    if corvox:
        return corvox
    else:

        root.error(
            "Invalid clipboard content. Set DEBUG on 'loglevel' in 'townshell.cfg' and try again for more information"
        )
        return


import xml.sax

class StudentHandler(xml.sax.ContentHandler):
    def __init__(self):
        self.CurrentData = ''
        self.ids = ''
        self.name = ''
        self.date = ''

    # Call when an element starts
    def startElement(self,tag,attributes):
        self.CurrentData = tag
        if tag == 'student':
            print('*****Student*****')
            ids = attributes['id']
            print('ID:',ids)

    # Call when an element ends
    def endElement(self,tag):
        if self.CurrentData == 'id':
            print('ID:', self.ids)
        elif self.CurrentData == 'name':
            print('Name:', self.name)
        elif self.CurrentData == 'date':
            print('Date:', self.date)
        self.CurrentData = ''

    # Call when character is read
    def characters(self,content):
        if self.CurrentData == 'id':
            self.id = content
        elif self.CurrentData == 'name':
            self.name == content
        elif self.CurrentData == 'date':
            self.date == content

if __name__ == "__main__":
    # Create an XMLReader
    parser = xml.sax.make_parser()
    # turn off namespaces
    parser.setFeature(xml.sax.handler.feature_namespaces,0)

    # overide the default ContexHandler
    Handler = StudentHandler()
    parser.setContentHandler(Handler)

    parser.parse('files_xml/student.xml')


# Copyright (c) 2016-2022 Association of Universities for Research in Astronomy, Inc. (AURA)
# For license information see LICENSE or https://opensource.org/licenses/BSD-3-Clause

from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from scheduler.graphql_mid import graphql_server

app = FastAPI()

origins = [
    "https://schedule-subscriber-staging.herokuapp.com",
    "https://gpp-schedule-ui-staging.herokuapp.com",
    "http://scheduler.lucuma.xyz:5173" # dev
]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

app.add_route('/graphql', graphql_server)
app.add_websocket_route('/graphql', graphql_server)


#!/usr/bin/env python
# encoding: utf-8
import subprocess

for experiment in ['linear', 'quadrat', 'maple', 'maple_2', 'branched']:
    for iter in [10**x for x in range(3,5)]:
        iter = str(iter)
        command = ['python', 'python_submit.py', '--nr_iterations', iter, '--nr_jobs', '2', '--outfile', experiment+'.m.'+iter+'.'+'.out',
        '--experiment', experiment]
        subprocess.call(command)


import sys
import time

from .topology import MergeTreeFloat, vectorFloat, vectorString

from . import TopologicalObject

class MergeTree(TopologicalObject):
    """ A wrapper class for the C++ merge tree data structure
    """

    def __init__(
        self,
        graph="beta skeleton",
        gradient="steepest",
        max_neighbors=-1,
        beta=1.0,
        normalization=None,
        connect=False,
        aggregator=None,
        debug=False,
    ):
        """ Initialization method
            @ In, graph, an optional string specifying the type of
            neighborhood graph to use. Default is 'beta skeleton,' but
            other valid types are: 'delaunay,' 'relaxed beta skeleton,'
            'none', or 'approximate knn'
            @ In, gradient, an optional string specifying the type of
            gradient estimator to use. Currently the only available
            option is 'steepest'
            @ In, max_neighbors, an optional integer value specifying
            the maximum number of k-nearest neighbors used to begin a
            neighborhood search. In the case of graph='[relaxed] beta
            skeleton', we will begin with the specified approximate knn
            graph and prune edges that do not satisfy the empty region
            criteria.
            @ In, beta, an optional floating point value between 0 and
            2. This value is only used when graph='[relaxed] beta
            skeleton' and specifies the radius for the empty region
            graph computation (1=Gabriel graph, 2=Relative neighbor
            graph)
            @ In, normalization, an optional string specifying whether
            the inputs/output should be scaled before computing.
            Currently, two modes are supported 'zscore' and 'feature'.
            'zscore' will ensure the data has a mean of zero and a
            standard deviation of 1 by subtracting the mean and dividing
            by the variance. 'feature' scales the data into the unit
            hypercube.
            @ In, connect, an optional boolean flag for whether the
            algorithm should enforce the data to be a single connected
            component.
            @ In, aggregator, an optional string that specifies what
            type of aggregation to do when duplicates are found in the
            domain space. Default value is None meaning the code will
            error if duplicates are identified.
            @ In, debug, an optional boolean flag for whether debugging
            output should be enabled.
        """
        super(MergeTree, self).__init__(
            graph=graph,
            gradient=gradient,
            max_neighbors=max_neighbors,
            beta=beta,
            normalization=normalization,
            connect=connect,
            aggregator=aggregator,
            debug=debug,
        )

    def _internal_build(self):
        """ This function assumes the self.__tree object has been setup,
            though it doesn't care how. It will then setup all python
            side data structures to be used for querying this object.
        """
        self.nodes = self.__tree.Nodes()
        self.edges = self.__tree.Edges()
        self.augmentedEdges = {}
        for key, val in self.__tree.AugmentedEdges().items():
            self.augmentedEdges[key] = list(val)
        self.root = self.__tree.Root()

        seen = set()
        self.branches = set()

        # Find all of the branching nodes in the tree, degree > 1
        # That is, they appear in more than one edge
        for e1, e2 in self.edges:
            if e1 not in seen:
                seen.add(e1)
            else:
                self.branches.add(e1)

            if e2 not in seen:
                seen.add(e2)
            else:
                self.branches.add(e2)

        # The nodes that are not branches are leaves
        self.leaves = set(self.nodes.keys()) - self.branches
        self.leaves.remove(self.root)

    def build(self, X, Y, w=None, edges=None):
        """ Assigns data to this object and builds the Merge Tree
            @ In, X, an m-by-n array of values specifying m
            n-dimensional samples
            @ In, Y, a m vector of values specifying the output
            responses corresponding to the m samples specified by X
            @ In, w, an optional m vector of values specifying the
            weights associated to each of the m samples used. Default of
            None means all points will be equally weighted
            @ In, edges, an optional list of custom edges to use as a
            starting point for pruning, or in place of a computed graph.
        """
        super(MergeTree, self).build(X, Y, w, edges)

        if self.debug:
            sys.stdout.write("Merge Tree Computation: ")
            start = time.clock()

        self.__tree = MergeTreeFloat(
            vectorFloat(self.Xnorm.flatten()),
            vectorFloat(self.Y),
            str(self.gradient),
            self.graph_rep.full_graph(),
            self.debug,
        )

        self._internal_build()

        if self.debug:
            end = time.clock()
            sys.stdout.write("%f s\n" % (end - start))

    def build_for_contour_tree(self, contour_tree, negate=False):
        """ A helper function that will reduce duplication of data by
            reusing the parent contour tree's parameters and data
        """
        if self.debug:
            tree_type = "Join"
            if negate:
                tree_type = "Split"
            sys.stdout.write("{} Tree Computation: ".format(tree_type))
            start = time.clock()

        Y = contour_tree.Y
        if negate:
            Y = -Y

        self.__tree = MergeTreeFloat(
            vectorFloat(contour_tree.Xnorm.flatten()),
            vectorFloat(Y),
            str(contour_tree.gradient),
            contour_tree.graph_rep.full_graph(),
            self.debug,
        )
        self._internal_build()
        if self.debug:
            end = time.clock()
            sys.stdout.write("%f s\n" % (end - start))


import argparse
from glob import glob
import os
import os.path as osp
import numpy as np
import pandas as pd
import torch
from torch_geometric.data import InMemoryDataset, Data
from tqdm import tqdm

from psbody.mesh import Mesh
from utils import get_vert_connectivity
from transform import Normalize

class ComaDataset(InMemoryDataset):
    def __init__(self, root_dir, dtype='train', split='sliced', split_term='sliced', nVal = 100, transform=None, pre_transform=None):
        self.root_dir = root_dir
        self.split = split
        self.split_term = split_term
        self.nVal = nVal
        self.transform = transform
        self.pre_tranform = pre_transform
        # Downloaded data is present in following format root_dir/*/*/*.py
        if not osp.exists(osp.join(root_dir, 'processed', self.split_term)):
            os.makedirs(osp.join(root_dir, 'processed', self.split_term))
        self.data_file = self.gather_paths(self.split)
        super(ComaDataset, self).__init__(root_dir, transform, pre_transform)
        if dtype == 'train':
            data_path = self.processed_paths[0]
        elif dtype == 'val':
            data_path = self.processed_paths[1]
        elif dtype == 'test':
            data_path = self.processed_paths[2]
        else:
            raise Exception("train, val and test are supported data types")

        norm_path = self.processed_paths[3]
        norm_dict = torch.load(norm_path)
        self.mean, self.std = norm_dict['mean'], norm_dict['std']
        self.data, self.slices = torch.load(data_path)
        if self.transform:
            self.data = [self.transform(td) for td in self.data]

    @property
    def raw_file_names(self):
        all_data_files = []
        for key in self.data_file.keys() :
            all_data_files += self.data_file[key]
        return all_data_files

    @property
    def processed_file_names(self):
        processed_files = ['training.pt', 'val.pt', 'test.pt', 'norm.pt']
        processed_files = [self.split_term+'/'+pf for pf in processed_files]
        return processed_files

    def gather_paths(self, split):
        datapaths = dict()
        if split == 'gnrt' :
            datapaths['all'] = []
            obj_list = glob(osp.join(self.root_dir, '*.obj'))
            print('data length: ', len(obj_list))
            datapaths['all'] = datapaths['all'] + obj_list
        elif split == 'gnrtdx' :
            datapaths['ad'] = []
            datapaths['cn'] = []
            ad_ids = pd.read_csv('/Users/jlee/Desktop/JONG/tum/thesis/data/adni2/AD_PTID_IMGID.csv')
            ad_ids = ad_ids['mesh_filename']
            ad_ids = [i.split('_')[0] for i in ad_ids]
            cn_ids = pd.read_csv('/Users/jlee/Desktop/JONG/tum/thesis/data/adni2/CN_PTID_IMGID.csv')
            cn_ids = cn_ids['mesh_filename']
            cn_ids = [i.split('_')[0] for i in cn_ids]

            for i in ad_ids :
                datapaths['ad'].append(self.root_dir+'/'+i+'-L_Hipp_first.obj')
            for n, i in enumerate(cn_ids) :
                if self.split_term == 'gnrtdxb' and n >= len(ad_ids) :
                    break
                datapaths['cn'].append(self.root_dir+'/'+i+'-L_Hipp_first.obj')
            print('AD ', len(datapaths['ad']), ', CN ', len(datapaths['cn']))
        elif split == 'lgtd' :
            print('lgtd dataset file paths...')
            datapaths['lgtd'] = []
            bl_m24_ids = pd.read_csv('/Users/jlee/Desktop/JONG/tum/thesis/data/adni2/bl_m24_imgid.csv')
            bl_ids = bl_m24_ids['bl']; m24_ids = bl_m24_ids['m24'];
            print('length of dataset is %d'%(len(bl_m24_ids)))
            for i in range(len(bl_m24_ids)):
                blm24 = bl_m24_ids.loc[i]
                bl_path = self.root_dir+'/'+str(blm24['bl'])+'-L_Hipp_first.obj'
                m24_path = self.root_dir+'/'+str(blm24['m24'])+'-L_Hipp_first.obj'
                datapaths['lgtd'].append([bl_path, m24_path])
        elif split == 'lgtddxi' :
            print('lgtdc dataset file paths...')
            ad_count, cn_count = 0, 0;
            datapaths['lgtddx'] = []
            bl_m24_ids = pd.read_csv('/Users/jlee/Desktop/JONG/tum/thesis/data/adni2/bl_m24_imgid.csv')
            adni_info = pd.read_csv('/Users/jlee/Desktop/JONG/tum/thesis/data/adni2/adni_info.csv')
            bl_ids = bl_m24_ids['bl']; m24_ids = bl_m24_ids['m24'];

            for i in range(len(bl_m24_ids)):
                blm24 = bl_m24_ids.loc[i]
                # for lgtddxo, change 'bl' to 'm24' in the below code line.
                bl_DX = adni_info[adni_info['ImageUID'] == blm24['bl']].iloc[0]['DX']
                #print(bl_DX)
                if bl_DX in ['CN', 'Dementia'] :
                    bl_path = self.root_dir+'/'+str(blm24['bl'])+'-L_Hipp_first.obj'
                    m24_path = self.root_dir+'/'+str(blm24['m24'])+'-L_Hipp_first.obj'
                    if bl_DX == 'CN' :
                        bl_class = [0, 1];  cn_count += 1
                    elif bl_DX == 'Dementia' :
                        bl_class = [1, 0];  ad_count += 1
                    datapaths['lgtddx'].append([bl_path, m24_path, bl_class])
            print('length of dataset is %d'%(ad_count+cn_count))
            print(ad_count, cn_count)
        elif split == 'lgtddxo' :
            print('lgtdc dataset file paths...')
            ad_count, cn_count = 0, 0;
            datapaths['lgtddx'] = []
            bl_m24_ids = pd.read_csv('/Users/jlee/Desktop/JONG/tum/thesis/data/adni2/bl_m24_imgid.csv')
            adni_info = pd.read_csv('/Users/jlee/Desktop/JONG/tum/thesis/data/adni2/adni_info.csv')
            bl_ids = bl_m24_ids['bl']; m24_ids = bl_m24_ids['m24'];

            for i in range(len(bl_m24_ids)):
                blm24 = bl_m24_ids.loc[i]
                m24_DX = adni_info[adni_info['ImageUID'] == blm24['m24']].iloc[0]['DX']
                #print(bl_DX)
                if m24_DX in ['CN', 'Dementia'] :
                    bl_path = self.root_dir+'/'+str(blm24['bl'])+'-L_Hipp_first.obj'
                    m24_path = self.root_dir+'/'+str(blm24['m24'])+'-L_Hipp_first.obj'
                    if m24_DX == 'CN' :
                        m24_class = [0, 1];  cn_count += 1
                    elif m24_DX == 'Dementia' :
                        m24_class = [1, 0];  ad_count += 1
                    datapaths['lgtddx'].append([bl_path, m24_path, m24_class])
            print('length of dataset is %d'%(ad_count+cn_count))
            print(ad_count, cn_count)
        elif split == 'lgtdvc' :
            print('lgtdvc dataset file paths...')
            ad_count, cn_count = 0, 0;
            bl_count, m12_count, m24_count = 0, 0, 0;
            datapaths['lgtdvc'] = []
            imgids = pd.read_csv('/Users/jlee/Desktop/JONG/tum/thesis/data/adni2/blm12m24_imgids.csv')
            adni_info = pd.read_csv('/Users/jlee/Desktop/JONG/tum/thesis/data/adni2/adni_info.csv')
            for i in range(len(imgids)):
                imgid = imgids.iloc[i]['ImageUID']
                #print(imgid, type(imgid))
                if i % 3 == 0 : # BL
                    bl_count += 1
                    # period = [1, 0, 0]
                    bl_path = self.root_dir+'/'+str(imgid)+'-L_Hipp_first.obj'
                    bl_DX = adni_info[adni_info['ImageUID'] == imgid].iloc[0]['DX']
                    if bl_DX == 'CN' :
                        bl_class = [0, 1];  cn_count += 1
                    elif bl_DX == 'Dementia' :
                        bl_class = [1, 0];  ad_count += 1
                    # datapaths['lgtdp'].append([bl_path, bl_path, bl_class, period])
                elif i % 3 == 1 : # M12
                    m12_count += 1
                    period = [1, 0]
                    m12_path = self.root_dir+'/'+str(imgid)+'-L_Hipp_first.obj'
                    #print([bl_path, m12_path, bl_class, period])
                    datapaths['lgtdvc'].append([bl_path, m12_path, bl_class, period])
                elif i % 3 == 2 : # M24
                    m24_count += 1
                    period = [0, 1]
                    m24_path = self.root_dir+'/'+str(imgid)+'-L_Hipp_first.obj'
                    #print([bl_path, m24_path, bl_class, period])
                    datapaths['lgtdvc'].append([bl_path, m24_path, bl_class, period])
            #print('length of dataset is %d'%((ad_count+cn_count)*3))
            print(bl_count + m12_count + m24_count)
            print(bl_count, m12_count, m24_count)
            #print(ad_count, cn_count)

        return datapaths

    def process(self):
        train_data, val_data, test_data = [], [], []
        train_vertices = []
        for key in self.data_file :
            for idx, data_file in tqdm(enumerate(self.data_file[key])):
                if key == 'lgtd' :
                    mesh = Mesh(filename=data_file[0])
                    mesh_verts = torch.Tensor(mesh.v)
                    adjacency = get_vert_connectivity(mesh.v, mesh.f).tocoo()
                    edge_index = torch.Tensor(np.vstack((adjacency.row, adjacency.col)))
                    mesh_m24 = Mesh(filename=data_file[1])
                    data = Data(x=mesh_verts, y=torch.Tensor(mesh_m24.v), edge_index=edge_index)
                elif key == 'lgtddx' :
                    mesh = Mesh(filename=data_file[0])
                    mesh_verts = torch.Tensor(mesh.v)
                    adjacency = get_vert_connectivity(mesh.v, mesh.f).tocoo()
                    edge_index = torch.Tensor(np.vstack((adjacency.row, adjacency.col)))
                    mesh_m24 = Mesh(filename=data_file[1])
                    data = Data(x=mesh_verts, y=torch.Tensor(mesh_m24.v), edge_index=edge_index,
                        label=torch.Tensor(data_file[2]))
                elif key == 'lgtdvc' :
                    #print(data_file)
                    mesh = Mesh(filename=data_file[0])
                    mesh_verts = torch.Tensor(mesh.v)
                    adjacency = get_vert_connectivity(mesh.v, mesh.f).tocoo()
                    edge_index = torch.Tensor(np.vstack((adjacency.row, adjacency.col)))
                    mesh_fu = Mesh(filename=data_file[1])
                    data = Data(x=mesh_verts, y=torch.Tensor(mesh_fu.v), edge_index=edge_index,
                        label=torch.Tensor(data_file[2]), period=torch.Tensor(data_file[3]))
                else :
                    mesh = Mesh(filename=data_file)
                    mesh_verts = torch.Tensor(mesh.v)
                    adjacency = get_vert_connectivity(mesh.v, mesh.f).tocoo()
                    edge_index = torch.Tensor(np.vstack((adjacency.row, adjacency.col)))
                    if key == 'ad' :
                        data = Data(x=mesh_verts, y=mesh_verts, label=torch.Tensor([1,0]), edge_index=edge_index)
                    elif key == 'cn' :
                        data = Data(x=mesh_verts, y=mesh_verts, label=torch.Tensor([0,1]), edge_index=edge_index)
                    elif key == 'all' :
                        data = Data(x=mesh_verts, y=mesh_verts, edge_index=edge_index)

                if idx % 100 < 10:
                    test_data.append(data)
                    #print(data.period)
                elif idx % 100 < 20:
                    val_data.append(data)
                else:
                    train_data.append(data)
                    train_vertices.append(mesh.v)
        print(len(train_data), len(val_data), len(test_data))

        mean_train = torch.Tensor(np.mean(train_vertices, axis=0))
        std_train = torch.Tensor(np.std(train_vertices, axis=0))
        norm_dict = {'mean': mean_train, 'std': std_train}
        if self.pre_transform is not None:
            if hasattr(self.pre_transform, 'mean') and hasattr(self.pre_transform, 'std'):
                if self.pre_tranform.mean is None:
                    self.pre_tranform.mean = mean_train
                if self.pre_transform.std is None:
                    self.pre_tranform.std = std_train
            train_data = [self.pre_transform(td) for td in train_data]
            val_data = [self.pre_transform(td) for td in val_data]
            test_data = [self.pre_transform(td) for td in test_data]

        torch.save(self.collate(train_data), self.processed_paths[0])
        torch.save(self.collate(val_data), self.processed_paths[1])
        torch.save(self.collate(test_data), self.processed_paths[2])
        torch.save(norm_dict, self.processed_paths[3])

def prepare_gnrt_dataset(path):
    ComaDataset(path, split='gnrt', split_term='gnrt', pre_transform=Normalize())

def prepare_gnrtdx_dataset(path):
    ComaDataset(path, split='gnrtdx', split_term='gnrtdx', pre_transform=Normalize())

def prepare_lgtd_dataset(path):
    ComaDataset(path, split='lgtd', split_term='lgtd', pre_transform=Normalize())

def prepare_lgtddxi_dataset(path):
    ComaDataset(path, split='lgtddxi', split_term='lgtddxi', pre_transform=Normalize())

def prepare_lgtddxo_dataset(path):
    ComaDataset(path, split='lgtddxo', split_term='lgtddxo', pre_transform=Normalize())

def prepare_lgtdvc_dataset(path):
    ComaDataset(path, split='lgtdvc', split_term='lgtdvc', pre_transform=Normalize())

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='ADNI2 Data preparation for Convolutional Mesh Autoencoders')
    parser.add_argument('-s', '--split', default='gnrt', help='split can be gnrt, clsf, lgtd, lgtdvc, or lgtddx')
    parser.add_argument('-d', '--data_dir', help='path where the downloaded data is stored')

    args = parser.parse_args()
    split = args.split
    data_dir = args.data_dir
    if split == 'gnrt':
        prepare_gnrt_dataset(data_dir)
    elif split == 'gnrtdx':
        prepare_gnrtdx_dataset(data_dir)
    elif split == 'lgtd':
        prepare_lgtd_dataset(data_dir)
    elif split == 'lgtddxi':
        prepare_lgtddxi_dataset(data_dir)
    elif split == 'lgtddxo':
        prepare_lgtddxo_dataset(data_dir)
    elif split == 'lgtdvc':
        prepare_lgtdvc_dataset(data_dir)
    else:
        raise Exception("Only gnrt, clsf, and lgtd split are supported")


## Converter output impedance network
#
#  [Zout] -->          _-_  _-_  _-_  _-_
#     __________x_____/   \/   \/   \/   \_____x_______________x___ [Vout]
#    |  [Vfb]   |             Lout+ESRl        |               |
#   _-_         |                              |               |
#  / . \      __|__                          __|__             \
# | /|\ |     ----- Cout                     ----- Cload       /
# |  |  |       |                              |               \ Rload
#  \_ _/      /ESRco/                        /ESRcl/           /
#    :          |                              |               \
#    |          |                              |               |
#    |__________x______________________________x_______________x
#                                                              |
#                                                            -----
#                                                             ---
#                                                              -

import numpy as np

##
## Engineering notation shorthand
##
p = 1e-12
n = 1e-9
u = 1e-6
m = 1e-3
k = 1e3

class outputZ:

    def __init__(self, Cout, Cload, Lout, ESRco, ESRcl, ESRl, Rload, fstart,\
                       fend, npoints):
        # Plotting range defaults
        self.fstart = 10.0
        self.fend = 120.0*k
        self.npoints = 1000
        self.f =  np.logspace(np.log10(self.fstart), np.log10(self.fend), self.npoints)
        self.s = 2j*np.pi*self.f

        # Error amp default component values
        self.Cout = 3.0*330.0*u;
        self.Cload = 330.0*u
        self.Lout = 470.0*n
        self.Rco = 14.0*m/3.0
        self.Rcl = 14.0*m
        self.Rl = 4.0*m
        self.Rload = 4.2

        self.Zout = []

        self.updateParams(Cout, Cload, Lout, ESRco, ESRcl, ESRl, Rload, fstart,\
                          fend, npoints)

    def updateParams(self, Cout, Cload, Lout, ESRco, ESRcl, ESRl, Rload, fstart,\
                       fend, npoints):
        # Plotting ranges
        self.fstart = fstart
        self.fend = fend
        self.npoints = npoints
        self.f =  np.logspace(np.log10(fstart), np.log10(fend), npoints)
        self.s = 2j*np.pi*self.f

        # Error amp component values
        self.Cout = Cout;
        self.Cload = Cload
        self.Lout = Lout
        self.Rco = ESRco
        self.Rcl = ESRcl
        self.Rl = ESRl
        self.Rload = Rload

        self.compute_Z()

        ##
        ## System transfer functions
        ##
    def compute_Z(self) :
        s = self.s

        zCout = 1.0/(s*self.Cout) + self.Rco
        zCload = 1.0/(s*self.Cload) + self.Rcl
        zLout = s*self.Lout + self.Rl

        zLoad = zLout + zCload*self.Rload/(zCload + self.Rload)
        self.Zout = zLoad*zCout/(zLoad + zCout)


# tags: simulation, analysis

import re
from math import sqrt

from compytetive.util import benchmark

def read_input(filename):
    with open(filename, "r", encoding="utf-8") as f:
        data = f.read().strip()
        xs = re.search(r"x=(-?\d+)\.\.(-?\d+)", data)
        ys = re.search(r"y=(-?\d+)\.\.(-?\d+)", data)
    return (int(xs.group(1)), int(xs.group(2)), int(ys.group(1)), int(ys.group(2)))

def simulate(data):
    min_x, max_x, min_y, max_y = data
    P = []

    # the art is in tightening the bounds
    # for x:
    #  any velocity in n*(n+1)/2 < min_x will never hit
    #  any velocity greater than max_x will shoot past
    # for y:
    #  any velocity lower than min_y will shoot below area
    #  when shooting up, the drone comes back down with opposite velocity
    #  any velocity where the returning velocity is greater than -min_y will overshoot the area
    for x in range(round(sqrt(min_x * 2)), max_x + 1):
        # NOTE: min_y is negative in the input, so y is growing
        for y in range(min_y, -min_y):
            px, py, vx, vy = 0, 0, x, y
            path = [(0, 0)]
            hit = False
            while px <= max_x and py >= min_y:
                px, py = px + vx, py + vy
                path.append((px, py))
                if min_x <= px <= max_x and min_y <= py <= max_y:
                    hit = True
                    break

                if vx > 0:
                    vx -= 1
                elif vx < 0:
                    vx += 1
                vy -= 1

            if hit:
                P.append(path)
    return P

def part1(data):
    P = simulate(data)
    highest = 0
    for p in P:
        highest = max(highest, max(y for x, y in p))
    return highest

def part2(data):
    P = simulate(data)
    return len(P)

def main():
    test_input = read_input("test.in")
    real_input = read_input("input.in")

    assert part1(test_input) == 45
    assert part1(real_input) == 30628
    print(benchmark(part1)(real_input))

    assert part2(test_input) == 112
    assert part2(real_input) == 4433
    print(benchmark(part2)(real_input))

if __name__ == "__main__":
    main()


from model.group import Group

class GroupHelper:

    def __init__(self, app):
        self.app = app

    group_cache = None

    def get_group_list(self):
        if self.group_cache is None:
            self.group_cache = []
            self.open_group_editor()
            tree = self.group_editor.window(auto_id="uxAddressTreeView")
            root = tree.tree_root()
            for node in root.children():
                name = node.text()
                self.group_cache.append(Group(name=name))
            self.close_group_editor()
        return list(self.group_cache)

    def add_new_group(self, group):
        self.open_group_editor()
        self.group_editor.window(auto_id="uxNewAddressButton").click()
        input = self.group_editor.window(class_name="Edit")
        input.set_text(group.name)
        input.type_keys("\n")
        self.close_group_editor()
        self.group_cache = None

    def del_group_by_index(self, index):
        self.open_group_editor()
        self.select_group_by_index(index)
        self.open_group_deletion()
        # select delete the selected group, subgroups and contacts
        self.group_deletion.window(auto_id="uxDeleteAllRadioButton").click()
        # submit deletion
        self.group_deletion.window(auto_id="uxOKAddressButton").click()
        self.close_group_editor()
        self.group_cache = None

    def open_group_editor(self):
        self.app.main_window.window(auto_id="groupButton").click()
        self.group_editor = self.app.application.window(title="Group editor")
        self.group_editor.wait("visible")

    def close_group_editor(self):
        self.group_editor.close()

    def open_group_deletion(self):
        self.group_editor.window(auto_id="uxDeleteAddressButton").click()
        self.group_deletion = self.app.application.window(title="Delete group")

    def select_group_by_index(self, index):
        tree = self.group_editor.window(auto_id="uxAddressTreeView")
        root = tree.tree_root()
        root.children()[index].select()

    def check_group_count(self, group):
        # forbidden to delete the last group
        if len(self.get_group_list()) < 2:
            self.add_new_group(group)


from urllib.request import urlopen
from xml.etree.ElementTree import parse

def read_tube_info(path):
	if not path.endswith('.xml'):
		raise TypeError('must be a xml file')
	doc = parse(path)
	shape = {}
	idx = 0
	root = doc.getroot()
	outD = float(root.findtext("D1"))
	thick = float(root.findtext("WALL_THICK"))
	bendR = float(root.findtext("BendingDie"))
	id = root.findtext("MEMBER_NAME")

	mater = root.findtext("MATERIAL")

	shape = {}
	for item in root.find("YBC").findall("Bend"):
		if item is None:
			raise ValueError('No Bend Label')
		seg = {}
		seg["Y"] = float(item.findtext("Y"))
		seg["B"] = float(item.findtext("B"))
		seg["C"] = float(item.findtext("C"))
		idx += 1
		shape[idx] = seg
	tube = {}
	tube["ID"] = id
	tube["D"] = outD
	tube["Thick"] = thick
	tube["Material"] = mater
	tube["Bend_R"] = bendR
	tube["Shape"] = shape
	return tube


"""A bot to list all members of a server."""
import csv
import time

from discord.ext import commands
from discord.ext.commands import Bot

# constants
PREFIX = "~"
TOKEN = "PUT YOUR BOTs TOKEN HERE"

bot = Bot(command_prefix=PREFIX)

@bot.event
async def on_ready():
    print('Logged in as')
    print(bot.user.name)
    print(bot.user.id)
    print('------')

@bot.event
async def on_command_error(error, ctx):
    if isinstance(error, commands.CommandNotFound):
        return
    else:
        print(error)

@bot.command(pass_context=True)
async def stat(ctx):
    """Returns a CSV file of all users on the server."""
    await bot.request_offline_members(ctx.message.server)
    before = time.time()
    nicknames = [m.display_name for m in ctx.message.server.members]
    with open('temp.csv', mode='w', encoding='utf-8', newline='') as f:
        writer = csv.writer(f, dialect='excel')
        for v in nicknames:
            writer.writerow([v])
    after = time.time()
    await bot.send_file(ctx.message.author, 'temp.csv', filename='stats.csv',
                        content="Here you go! Check your PM's. Generated in {:.4}ms.".format((after - before)*1000))

if __name__ == '__main__':
    bot.run(TOKEN)


# ps3_functions.py
# CPSC 453 -- Problem Set 3
#

# Sasha Safonova

# This script contains pytorch shells for a Logistic regression model, a feed forward network, and an autoencoder.
#
from torch.nn.functional import softmax
from torch import optim, nn
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import numpy as np
import torch
import pandas as pd

class LogisticRegression(nn.Module): # initialize a pytorch neural network module
    def __init__(self): # initialize the model
        super(LogisticRegression, self).__init__() # call for the parent class to initialize
        # you can define variables here that apply to the entire model (e.g. weights, biases, layers...)
        # this model only has two parameters: the weight, and the bias.
        # here's how you can initialize the weight:
        # W = nn.Parameter(torch.zeros(shape)) # this creates a model parameter out of a torch tensor of the specified shape
        # ... torch.zeros is much like numpy.zeros, except optimized for backpropogation.
        # We make it a model parameter and so it will be updated by gradient descent.
        self.Nfeatures = 784
        self.Noutcomes = 10
        weight_shape = [self.Nfeatures, self.Noutcomes]
        self.Nsamples = 128
        bias_shape = [self.Nsamples, self.Noutcomes]

        self.W = nn.Parameter(torch.zeros(weight_shape))
        # create a bias variable here
        self.bias = nn.Parameter(torch.zeros(bias_shape))

    def forward(self, x):
        """
        this is the function that will be executed when we call the logistic regression on data.
        INPUT:
            x, an MNIST image represented as a tensor of shape 784
        OUTPUT:
            forward_result, a tensor of shape 10
        """
        forward_result = torch.matmul(x, self.W)
        forward_result += self.bias
        forward_result = torch.reshape(forward_result, [self.Nsamples, 10])

        return forward_result

class FeedForwardNet(nn.Module):
    """ Simple feed forward network with one hidden layer."""
    # Here, you should place an exact copy of the code from the LogisticRegression class, with two modifications:
    # 1. Add another weight and bias vector to represent the hidden layer
    # 2. In the forward function, add some type of nonlinearity to the output of the first layer, then pass it onto the hidden layer.
    def __init__(self): # initialize the model
        super(FeedForwardNet, self).__init__() # call for the parent class to initialize
        self.Nfeatures = 784
        self.Noutcomes = 10
        self.Nsamples = 128
        weight1_shape = [self.Nfeatures, self.Nsamples]
        weight2_shape = [self.Nsamples, self.Noutcomes]

        bias1_shape = [self.Nsamples, self.Nsamples]
        bias2_shape = [self.Nsamples, self.Noutcomes]

        random_tensor_W1 = torch.rand(weight1_shape)*2-1
        random_tensor_b1 = torch.rand(bias1_shape)*2-1
        random_tensor_W2 = torch.rand(weight2_shape)*2-1
        random_tensor_b2 = torch.rand(bias2_shape)*2-1

        self.W = nn.Parameter(random_tensor_W1)
        self.bias1 = nn.Parameter(random_tensor_b1)
        self.W2 = nn.Parameter(random_tensor_W2)
        self.bias2 = nn.Parameter(random_tensor_b2)

    def forward(self, x):
        """
        this is the function that will be executed when we call the logistic regression on data.
        INPUT:
            x, an MNIST image represented as a tensor of shape 784
        OUTPUT:
            forward_result, a tensor of shape 10
        """

        forward_result = torch.matmul(x, self.W)
        forward_result += self.bias1
        #forward_result = torch.sigmoid(forward_result)
        activation_fn = nn.ELU()
        forward_result = activation_fn(forward_result)
        forward_result = torch.matmul(forward_result, self.W2)
        forward_result += self.bias2
        #forward_result = torch.sigmoid(forward_result)

        forward_result = torch.reshape(forward_result, [self.Nsamples, 10])#, self.Nsamples])#, 10])

        return forward_result

class FeedForwardNet2(nn.Module):
    """ Simple feed forward network with two hidden layers."""
    # Here, you should place an exact copy of the code from the LogisticRegression class, with two modifications:
    # 1. Add another weight and bias vector to represent the hidden layer
    # 2. In the forward function, add some type of nonlinearity to the output of the first layer, then pass it onto the hidden layer.
    def __init__(self): # initialize the model
        super(FeedForwardNet2, self).__init__() # call for the parent class to initialize
        self.Nfeatures = 784
        self.Noutcomes = 10
        self.Nsamples = 128
        weight1_shape = [self.Nfeatures, self.Nsamples]
        weight2_shape = [self.Nsamples, self.Nsamples]
        weight3_shape = [self.Nsamples, self.Noutcomes]

        bias1_shape = [self.Nsamples, self.Nsamples]
        bias2_shape = [self.Nsamples, self.Nsamples]
        bias3_shape = [self.Nsamples, self.Noutcomes]

        random_tensor_W1 = torch.rand(weight1_shape)*2-1
        random_tensor_b1 = torch.rand(bias1_shape)*2-1
        random_tensor_W2 = torch.rand(weight2_shape)*2-1
        random_tensor_b2 = torch.rand(bias2_shape)*2-1
        random_tensor_W3 = torch.rand(weight3_shape)*2-1
        random_tensor_b3 = torch.rand(bias3_shape)*2-1

        self.W = nn.Parameter(random_tensor_W1)
        self.bias1 = nn.Parameter(random_tensor_b1)
        self.W2 = nn.Parameter(random_tensor_W2)
        self.bias2 = nn.Parameter(random_tensor_b2)
        self.W3 = nn.Parameter(random_tensor_W3)
        self.bias3 = nn.Parameter(random_tensor_b3)

    def forward(self, x):
        """
        this is the function that will be executed when we call the logistic regression on data.
        INPUT:
            x, an MNIST image represented as a tensor of shape 784
        OUTPUT:
            forward_result, a tensor of shape 10
        """

        forward_result = torch.matmul(x, self.W)
        forward_result += self.bias1
        # forward_result = torch.sigmoid(forward_result)
        activation_fn = nn.ELU()
        forward_result = activation_fn(forward_result)
        forward_result = torch.matmul(forward_result, self.W2)
        forward_result += self.bias2
        forward_result = activation_fn(forward_result)
        forward_result = torch.matmul(forward_result, self.W3)
        forward_result += self.bias3
        forward_result = torch.reshape(forward_result, [self.Nsamples, 10])#, self.Nsamples])#, 10])

        return forward_result

class FeedForwardNet3(nn.Module):
    """ Simple feed forward network with two hidden layers."""
    # Here, you should place an exact copy of the code from the LogisticRegression class, with two modifications:
    # 1. Add another weight and bias vector to represent the hidden layer
    # 2. In the forward function, add some type of nonlinearity to the output of the first layer, then pass it onto the hidden layer.
    def __init__(self): # initialize the model
        super(FeedForwardNet3, self).__init__() # call for the parent class to initialize
        self.Nfeatures = 784
        self.Noutcomes = 10
        self.Nsamples = 128
        weight1_shape = [self.Nfeatures, self.Nsamples]
        weight2_shape = [self.Nsamples, self.Nsamples]
        weight3_shape = [self.Nsamples, self.Nsamples]
        weight4_shape = [self.Nsamples, self.Noutcomes]

        self.W1 = nn.Linear(*weight1_shape)
        self.W2 = nn.Linear(*weight2_shape)
        self.W3 = nn.Linear(*weight3_shape)
        self.W4 = nn.Linear(*weight4_shape)

    def forward(self, x):
        """
        this is the function that will be executed when we call the logistic regression on data.
        INPUT:
            x, an MNIST image represented as a tensor of shape 784
        OUTPUT:
            forward_result, a tensor of shape 10
        """
        activation_fn = nn.ELU()

        forward_result = self.W1(x)
        forward_result = activation_fn(forward_result)

        forward_result = self.W2(forward_result)
        forward_result = activation_fn(forward_result)

        forward_result = self.W3(forward_result)
        forward_result = activation_fn(forward_result)

        forward_result = self.W4(forward_result)

        forward_result = torch.reshape(forward_result, [self.Nsamples, 10])

        return forward_result

class FeedForwardNet4(nn.Module):
    """ Simple feed forward network with two hidden layers."""
    # Here, you should place an exact copy of the code from the LogisticRegression class, with two modifications:
    # 1. Add another weight and bias vector to represent the hidden layer
    # 2. In the forward function, add some type of nonlinearity to the output of the first layer, then pass it onto the hidden layer.
    def __init__(self): # initialize the model
        super(FeedForwardNet4, self).__init__() # call for the parent class to initialize
        self.Nfeatures = 784
        self.Noutcomes = 10
        self.Nsamples = 128
        weight1_shape = [self.Nfeatures, self.Nsamples]
        weight2_shape = [self.Nsamples, self.Nsamples*2]
        weight3_shape = [self.Nsamples*2, self.Nsamples*3]
        weight4_shape = [self.Nsamples*3, self.Nsamples*2]
        weight5_shape = [self.Nsamples*2, self.Noutcomes]

        self.W1 = nn.Linear(*weight1_shape)
        self.W2 = nn.Linear(*weight2_shape)
        self.W3 = nn.Linear(*weight3_shape)
        self.W4 = nn.Linear(*weight4_shape)
        self.W5 = nn.Linear(*weight5_shape)

    def forward(self, x):
        """
        this is the function that will be executed when we call the logistic regression on data.
        INPUT:
            x, an MNIST image represented as a tensor of shape 784
        OUTPUT:
            forward_result, a tensor of shape 10
        """
        activation_fn = nn.ELU()

        forward_result = self.W1(x)
        forward_result = activation_fn(forward_result)

        forward_result = self.W2(forward_result)
        forward_result = activation_fn(forward_result)

        forward_result = self.W3(forward_result)
        forward_result = activation_fn(forward_result)

        forward_result = self.W4(forward_result)
        forward_result = activation_fn(forward_result)

        forward_result = self.W5(forward_result)

        forward_result = torch.reshape(forward_result, [self.Nsamples, 10])

        return forward_result

class Autoencoder(nn.Module):
    def __init__(self):
        super(Autoencoder, self).__init__()
        self.lin1 = nn.Linear(784, 1000)
        # define additional layers here
        # 784-1000-500-250-2-250-500-1000-784
        self.lin2 = nn.Linear(1000, 500)
        self.lin3 = nn.Linear(500, 250)
        self.lin4 = nn.Linear(250, 2)
        self.lin5 = nn.Linear(2, 250)
        self.lin6 = nn.Linear(250, 500)
        self.lin7 = nn.Linear(500, 1000)
        self.lin8 = nn.Linear(1000, 784)

    def encode(self, x):
        x = self.lin1(x)
        # ... additional layers, plus possible nonlinearities.
        for layer in [self.lin2,
                      self.lin3,
                      self.lin4,
                      ]:
            x = torch.tanh(x)
            x = layer(x)
        return x

    def decode(self, z):
        # ditto, but in reverse
        z = self.lin5(z)
        z = torch.tanh(z)
        z = self.lin6(z)
        z = torch.tanh(z)
        z = self.lin7(z)
        z = torch.sigmoid(z)
        z = self.lin8(z)
        return z

    def forward(self, x):
        z = self.encode(x)
        return self.decode(z)

class retinal_Bipolar_Autoencoder(nn.Module):
    def __init__(self):
        super(Autoencoder, self).__init__()
        self.lin1 = nn.Linear(784, 1000)
        # define additional layers here
        # 784-1000-500-250-2-250-500-1000-784
        self.lin2 = nn.Linear(1000, 500)
        self.lin3 = nn.Linear(500, 250)
        self.lin4 = nn.Linear(250, 2)
        self.lin5 = nn.Linear(2, 250)
        self.lin6 = nn.Linear(250, 500)
        self.lin7 = nn.Linear(500, 1000)
        self.lin8 = nn.Linear(1000, 784)

    def encode(self, x):
        x = self.lin1(x)
        # ... additional layers, plus possible nonlinearities.
        for layer in [self.lin2,
                      self.lin3,
                      self.lin4,
                      ]:
            x = torch.tanh(x)
            x = layer(x)
        return x

    def decode(self, z):
        # ditto, but in reverse
        z = self.lin5(z)
        z = torch.tanh(z)
        z = self.lin6(z)
        z = torch.tanh(z)
        z = self.lin7(z)
        # z = torch.sigmoid(z)
        z = self.lin8(z)
        return z

    def forward(self, x):
        z = self.encode(x)
        return self.decode(z)

def train(model,loss_fn, optimizer, train_loader, test_loader):
    """
    This is a standard training loop, which leaves some parts to be filled in.
    INPUT:
    :param model: an untrained pytorch model
    :param loss_fn: e.g. Cross Entropy loss of Mean Squared Error.
    :param optimizer: the model optimizer, initialized with a learning rate.
    :param training_set: The training data, in a dataloader for easy iteration.
    :param test_loader: The testing data, in a dataloader for easy iteration.
    """
    num_epochs = 201
    print("number of epochs", num_epochs)
    for epoch in range(num_epochs):
        print("epoch", epoch)
        # loop through each data point in the training set
        for data, targets in train_loader:

            # run the model on the data
            Nsample = 128
            try:
                model_input = torch.reshape(data, [Nsample, 784]) #  Turn the 28 by 28 image tensors into a 784 dimensional tensor.
            except RuntimeError:
                break

            out = model(model_input)

            # Calculate the loss
            """if torch.isnan(sum(sum(out))) or torch.isinf(sum(sum(out))):
                print("sum(sum(model_input))", sum(sum(model_input)))
                print("model_input", model_input)
                print("sum(sum(out))", sum(sum(out)))
                print('invalid input detected at iteration ', epoch)
            """

            loss = loss_fn(out, targets)

            # Find the gradients of our loss via backpropogation
            loss.backward()

            # Adjust accordingly with the optimizer
            optimizer.step()
            optimizer.zero_grad()
        print("loss", loss.item())
        # Give status reports every 100 epochs
        if epoch % 5==0:
            evaluate_train = np.round(evaluate(model,train_loader, Nsample=Nsample), 4)
            # evaluate_test = np.round(evaluate(model,test_loader, Nsample=Nsample), 4)

            evaluate_test, correct_digit, mislabeled_digit = evaluate(model, test_loader, Nsample=Nsample, return_confusion=True)
            evaluate_test = np.round(evaluate_test, 4)
            dataseries = pd.Series({"epoch":int(epoch),
                                    "Train accuracy": evaluate_train,
                                    "Test accuracy": evaluate_test,
                                    "Loss" : np.round(loss.item(), 4)}
                                   )
            if epoch==0:
                df_accuracy = pd.DataFrame()

            df_accuracy = df_accuracy.append(dataseries, ignore_index=True)
            print(f" EPOCH {epoch}. Progress: {epoch/num_epochs*100}%. ")
            print(f" Train accuracy: {evaluate_train}. Test accuracy: {evaluate_test}")
            df_accuracy.to_csv("dataframe_accuracy.csv")

            confusionmatrix = {"Correct digit": correct_digit,
                                         "Mislabeled digit": mislabeled_digit}

            if epoch == 0:
                df_confusion = pd.DataFrame()

            df_confusion = df_confusion.append(confusionmatrix, ignore_index=True)
            df_confusion.to_csv("dataframe_confusion.csv")

def evaluate(model, evaluation_set, Nsample=128, return_confusion=False):
    """
    Evaluates the given model on the given dataset.
    Returns the percentage of correct classifications out of total classifications.
    """
    correct = []
    correct_digit = []
    mislabeled_digit = []
    with torch.no_grad(): # this disables backpropogation, which makes the model run much more quickly.
        for data, targets in evaluation_set:
            try:
                model_input = torch.reshape(data, [Nsample, 784])
            except RuntimeError:
                break
            result = model(model_input)
            softie = nn.Softmax(dim=1)
            result = softie(result)
            for ii, result_line in enumerate(result):
                most_prob = np.argmax(result_line)
                if most_prob==targets[ii]:
                    correct.append(1)
                else:
                    correct.append(0)
                    correct_digit.append(targets[ii].item())
                    mislabeled_digit.append(most_prob.item())
    accuracy = np.sum(correct)/np.float(len(correct))
    if return_confusion:
        return accuracy, correct_digit, mislabeled_digit
    return accuracy

# ----- Functions for Part 5 -----
def mmd(X,Y, kernel_fn):
    """
    Implementation of Maximum Mean Discrepancy.
    :param X: An n x 1 numpy vector containing the samples from distribution 1.
    :param Y: An n x 1 numpy vector containing the samples from distribution 2.
    :param kernel_fn: supply the kernel function to use.
    :return: the maximum mean discrepancy:
    MMD(X,Y) = Expected value of k(X,X) + Expected value of k(Y,Y) - Expected value of k(X,Y)
    where k is a kernel function
    """
    kxx = kernel_fn(X, X)
    kyy = kernel_fn(Y, Y)
    kxy = kernel_fn(X, Y)
    mmd = np.mean(kxx) + np.mean(kyy) + np.mean(kxy)
    return mmd

def kernel(A, B):
    """
    A gaussian kernel on two arrays.
    :param A: An n x d numpy matrix containing the samples from distribution 1
    :param B: An n x d numpy matrix containing the samples from distribution 2.
    :return K:  An n x n numpy matrix k, in which k_{i,j} = e^{-||A_i - B_j||^2/(2*sigma^2)}
    """
    sigma = np.std(A.flatten())
    K = np.exp(-(abs(A-B)**2)/(0.5*sigma**2))
    return K

def main():
    batch_size = 128
    mnist_train = datasets.MNIST(root='data', train=True, download=True,
                                 transform=transforms.ToTensor())
    mnist_test = datasets.MNIST(root='data', train=False, download=True, transform=transforms.ToTensor())
    train_loader = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size)
    test_loader = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size)

    #======2.2 Logistic Regression======
    '''# initialize the model (adapt this to each model)
    model = LogisticRegression()
    # initialize the optimizer, and set the learning rate
    # SGD === Stochastic Gradient Descent
    SGD = torch.optim.SGD(model.parameters(), lr=0.5)
    SGD.zero_grad()
    loss_fn = nn.CrossEntropyLoss()
    train(model, loss_fn, SGD, train_loader, test_loader)'''

    #======2.3 Feed-Forward Neural Network======
    '''
    model = FeedForwardNet()

    SGD = torch.optim.SGD(model.parameters(), lr=0.005)
    SGD.zero_grad()
    loss_fn = nn.CrossEntropyLoss()

    train(model, loss_fn, SGD, train_loader, test_loader)'''
    #======2.3 varying width======
    '''model = FeedForwardNet2()

    SGD = torch.optim.SGD(model.parameters(), lr=0.005)
    SGD.zero_grad()
    loss_fn = nn.CrossEntropyLoss()

    train(model, loss_fn, SGD, train_loader, test_loader)'''
    '''model = FeedForwardNet3()

    SGD = torch.optim.SGD(model.parameters(), lr=0.001)
    SGD.zero_grad()
    loss_fn = nn.CrossEntropyLoss()

    train(model, loss_fn, SGD, train_loader, test_loader)'''
    model = FeedForwardNet4()

    SGD = torch.optim.SGD(model.parameters(), lr=0.001)
    SGD.zero_grad()
    loss_fn = nn.CrossEntropyLoss()

    train(model, loss_fn, SGD, train_loader, test_loader)

    # ======3.1 Autoencoder======
    '''model = Autoencoder()

    #
    Adam = torch.optim.Adam(model.parameters(), lr=1e-3)
    loss_fn = nn.MSELoss()

    train(model, loss_fn, Adam, train_loader, test_loader)'''

if __name__=="__main__":
    main()


import requests
from requests.adapters import HTTPAdapter
from scrapy import Selector
import csv
import os

# --------------------define variables-------------------
OUTPUT_FILE = 'karnataka_2017.csv'
PDF_FOLDER = 'karnataka_2017/'

class KarnatakaScraper:
    def __init__(self,
                 base_url='http://ceokarnataka.kar.nic.in'
                 ):
        # define session object
        self.session = requests.Session()
        self.session.mount('https://', HTTPAdapter(max_retries=4))

        # set proxy
        # self.session.proxies.update({'http': 'http://127.0.0.1:40328'})

        # define urls
        self.base_url = base_url

        self.form_data = {
            '__EVENTVALIDATION': '',
            '__VIEWSTATE': '',
            '__VIEWSTATEENCRYPTED': ''
        }

    def GetDistrictList(self):
        # set url
        url = 'http://ceokarnataka.kar.nic.in/FinalRoll-2017/Dist_List.aspx'

        # get request
        ret = self.session.get(url)

        if ret.status_code == 200:
            # get district list
            trs = Selector(text=ret.text).xpath('//table[@id="ctl00_ContentPlaceHolder1_GridView1"]/tr').extract()

            district_list = []
            for idx in range(1, len(trs)):
                tr = trs[idx]
                district = {
                    'no': Selector(text=tr).xpath('//td[1]/font/text()').extract()[0],
                    'name': Selector(text=tr).xpath('//td[2]/font/a/text()').extract()[0]
                }
                district_list.append(district)

            return district_list
        else:
            print('fail to get district list')

    def GetACList(self, district_no):
        # set url
        url = 'http://ceokarnataka.kar.nic.in/FinalRoll-2017/AC_List.aspx'

        # set params
        params = {
            'DistNo': district_no
        }

        # get request
        ret = self.session.get(url, params=params)

        if ret.status_code == 200:
            # get district list
            trs = Selector(text=ret.text).xpath('//table[@id="ctl00_ContentPlaceHolder1_GridView1"]/tr').extract()

            ac_list = []
            for idx in range(1, len(trs)):
                tr = trs[idx]
                ac = {
                    'no': Selector(text=tr).xpath('//td[1]/font/text()').extract()[0],
                    'name': Selector(text=tr).xpath('//td[2]/font/a/text()').extract()[0]
                }
                ac_list.append(ac)

            return ac_list
        else:
            print('fail to get ac list')

    def GetPSList(self, ac_no):
        # set url
        url = 'http://ceokarnataka.kar.nic.in/FinalRoll-2017/Part_List.aspx'

        # set params
        params = {
            'ACNO': ac_no
        }

        # get request
        ret = self.session.get(url, params=params)

        if ret.status_code == 200:
            # get district list
            trs = Selector(text=ret.text).xpath('//table[@id="ctl00_ContentPlaceHolder1_GridView1"]/tr').extract()

            ps_list = []
            for idx in range(1, len(trs)):
                tr = trs[idx]
                ps = {
                    'no': Selector(text=tr).xpath('//td[2]/font/text()').extract()[0],
                    'name': Selector(text=tr).xpath('//td[3]/font/a/text()').extract()[0]
                }
                ps_list.append(ps)

            return ps_list
        else:
            print('fail to get ps list')

    def WriteHeader(self):
        # set headers
        header_info = [
            'district_no',
            'district_name',
            'ac_no',
            'ac_name',
            'part_no',
            'polling_station_name',
            'filename'
        ]

        # write header into output csv file
        writer = csv.writer(open(OUTPUT_FILE, 'w'), delimiter=',', lineterminator='\n')
        writer.writerow(header_info)

    def WriteData(self, data):
        # write data into output csv file
        writer = csv.writer(open(OUTPUT_FILE, 'a', encoding='utf-8'), delimiter=',', lineterminator='\n')
        writer.writerow(data)

    def DownloadPdfFile(self, download_url, filename):
        print('downloading %s' % (filename))
        if os.path.isfile(filename) == True:
            print('this file already downloaded.')
            return

        # set url
        url = download_url

        # get request
        ret = self.session.get(url, stream=True)

        if ret.status_code == 200:
            with open(filename, 'wb') as f:
                f.write(ret.content)
            print('success to download %s' % (filename))
        else:
            print('fail to get pdf file: %s' % (download_url))

    def Start(self,
              start_district='',
              start_ac='',
              start_ps=''):
        # write header into output csv file
        if start_district == '' and start_ac == '' and start_ps == '': self.WriteHeader()

        # get district list
        print('getting district list...')
        district_list = self.GetDistrictList()
        print(district_list)

        district_flag = False
        if start_district == '': district_flag = True

        ac_flag = False
        if start_ac == '': ac_flag = True

        ps_flag = False
        if start_ps == '': ps_flag = True

        for district in district_list:
            if start_district == district['no']: district_flag = True
            if district_flag == False: continue

            # get ac list
            print('getting ac list for %s...' % (district['name']))
            ac_list = self.GetACList(district['no'])
            print(ac_list)

            for ac in ac_list:
                if start_ac == ac['no']: ac_flag = True
                if ac_flag == False: continue

                # get ps list
                print('getting ac list for %s:%s...' % (district['name'], ac['name']))
                ps_list = self.GetPSList(ac['no'])
                print(ps_list)

                for ps in ps_list:
                    if start_ps == ps['no']: ps_flag = True
                    if ps_flag == False: continue

                    # process draft roll
                    file_name = 'AC%s%s.pdf' % (
                        str('{0:03d}'.format(int(ac['no']))),
                        str('{0:04d}'.format(int(ps['no']))),
                    )
                    draft_roll_pdf_url = 'http://ceokarnataka.kar.nic.in/FinalRoll-2017/Kannada/WOIMG/AC%s/%s' % (
                        str('{0:03d}'.format(int(ac['no']))),
                        file_name
                    )
                    self.DownloadPdfFile(draft_roll_pdf_url, PDF_FOLDER + file_name)

                    # write data into output csv file
                    data = []
                    data.append(district['no'])
                    data.append(district['name'])
                    data.append(ac['no'])
                    data.append(ac['name'])
                    data.append(ps['no'])
                    data.append(ps['name'])
                    data.append(file_name)
                    self.WriteData(data)

            #         break
            #     break
            # break

def main():
    # create scraper object
    scraper = KarnatakaScraper()

    # start to scrape
    scraper.Start(
        start_district='',
        start_ac='',
        start_ps=''
    )

if __name__ == '__main__':
    main()


import unittest

import pyebnf.util as util

class UtilTestCase(unittest.TestCase):
  def setUp(self):
    self.source = """Harry James Potter
Albus Dumbledore
Tom Marvolo Riddle
Hermione Granger
"""

  def test_get_newline_position(self):
    expected = [18, 35, 54, 71]
    self.assertEqual(expected, util.get_newline_positions(self.source))

  def test_get_line_and_char(self):
    nlp = util.get_newline_positions(self.source)
    self.assertEqual(util.get_line_and_char(nlp, self.source.find("Dumbledore")), (1, 6))

  @unittest.skip
  def test_point_to_source(self):
    nlp = util.get_newline_positions(self.source)
    pos = util.get_line_and_char(nlp, self.source.find("Dumbledore"))
    print()
    print(util.point_to_source(self.source, pos))