from pyjarowinkler.distance import get_jaro_distance
import editdistance
import fuzzy
from tqdm import tqdm

def pheonetic_distance(name1, name2):

    ''' this returns edit distance for phonetic similarity for two words'''
    soundness1 = fuzzy.nysiis(name1)
    soundness2 = fuzzy.nysiis(name2)
    nysiis_score = editdistance.eval(soundness1, soundness2)

    return nysiis_score, soundness1, soundness2

def sort_words(words):
    words = words.split(" ")
    words.sort()
    newSentence = " ".join(words)
    return newSentence

def isSubSequence(str1,str2):
    m = len(str1)
    n = len(str2)

    j = 0    # Index of str1
    i = 0    # Index of str2

    while j<m and i<n:
        if str1[j] == str2[i]:
            j = j+1
        i = i + 1

    # If all characters of str1 matched, then j is equal to m
    return j==m

def abbrv_match(name1, name2):

    n1 = name1.split()
    n2 = name2.split()

    abr_n1 = ""
    abr_n2 = ""

    for word in n1:
        abr_n1 += word[0]

    for word in n2:
        abr_n2 += word[0]

    # print(abr_n1, abr_n2)
    if(isSubSequence(abr_n1, abr_n2) or isSubSequence(abr_n2, abr_n1)):
        return True

    return False

def nameMatch(name1,name2):

    lgth = min(len(name1), len(name2))

    # print("nameMatch", name1, name2)
    if abbrv_match(name1,name2):
        noCommon1 =  " ".join([w for w in name1.split()  if w not in name2.split() ])
        noCommon2 =  " ".join([w for w in name2.split()  if w not in name1.split() ])

        jaro_score = get_jaro_distance(name1, name2)
#         print("Jaro : " , jaro_score)
        editDist_score = editdistance.eval(name1, name2)
#         print("Edit : ", editDist_score)
        sound_score, sond1, sond2 = pheonetic_distance(noCommon1, noCommon2)
#         print("soundness : ", sound_score)
        s_lgth = min(len(sond1), len(sond2))

        if (lgth <= 8  and jaro_score >= 0.94) or (lgth > 8 and  lgth <= 12 and jaro_score >= 0.9) or (lgth > 12 and jaro_score >= 0.87):
#             print("Case 1" )
            return True

        elif (lgth <= 3 and editDist_score < 1) or (lgth > 3 and lgth <= 8 and editDist_score < 2) or (lgth > 8 and editDist_score <= 3):
#             print("Case 2")
            return True

        elif (s_lgth ==0) or (s_lgth <= 3 and sound_score < 1) or (s_lgth > 3 and s_lgth <= 8 and sound_score < 2) or (s_lgth > 8 and sound_score <= 3):
#             print("Case 3")
            return True

    return False


import glob
import json
import os
import pickle

import numpy as np
import torch
import torch.nn.functional as F
from nasbench import api

from nasbench_analysis.search_spaces.search_space_1 import SearchSpace1
from nasbench_analysis.search_spaces.search_space_2 import SearchSpace2
from nasbench_analysis.search_spaces.search_space_3 import SearchSpace3
from nasbench_analysis.utils import get_top_k, INPUT, OUTPUT, CONV1X1, NasbenchWrapper, natural_keys
from optimizers.darts.genotypes import PRIMITIVES

nasbench = NasbenchWrapper(
    dataset_file='../../nasbench_only108.tfrecord')

def softmax(weights, axis=-1):
    return F.softmax(torch.Tensor(weights), axis).data.cpu().numpy()

def get_directory_list(path):
    """Find directory containing config.json files"""
    directory_list = []
    # return nothing if path is a file
    if os.path.isfile(path):
        return []
    # add dir to directorylist if it contains .json files
    if len([f for f in os.listdir(path) if f == 'config.json']) > 0:
        directory_list.append(path)
    for d in os.listdir(path):
        new_path = os.path.join(path, d)
        if os.path.isdir(new_path):
            directory_list += get_directory_list(new_path)
    return directory_list

def eval_one_shot_model(config, model):
    model_list = pickle.load(open(model, 'rb'))

    alphas_mixed_op = model_list[0]
    chosen_node_ops = softmax(alphas_mixed_op, axis=-1).argmax(-1)

    node_list = [PRIMITIVES[i] for i in chosen_node_ops]
    alphas_output = model_list[1]
    alphas_inputs = model_list[2:]

    if config['search_space'] == '1':
        search_space = SearchSpace1()
        num_inputs = list(search_space.num_parents_per_node.values())[3:-1]
        parents_node_3, parents_node_4 = \
            [get_top_k(softmax(alpha, axis=1), num_input) for num_input, alpha in zip(num_inputs, alphas_inputs)]
        output_parents = get_top_k(softmax(alphas_output), num_inputs[-1])
        parents = {
            '0': [],
            '1': [0],
            '2': [0, 1],
            '3': parents_node_3,
            '4': parents_node_4,
            '5': output_parents
        }
        node_list = [INPUT, *node_list, CONV1X1, OUTPUT]

    elif config['search_space'] == '2':
        search_space = SearchSpace2()
        num_inputs = list(search_space.num_parents_per_node.values())[2:]
        parents_node_2, parents_node_3, parents_node_4 = \
            [get_top_k(softmax(alpha, axis=1), num_input) for num_input, alpha in zip(num_inputs[:-1], alphas_inputs)]
        output_parents = get_top_k(softmax(alphas_output), num_inputs[-1])
        parents = {
            '0': [],
            '1': [0],
            '2': parents_node_2,
            '3': parents_node_3,
            '4': parents_node_4,
            '5': output_parents
        }
        node_list = [INPUT, *node_list, CONV1X1, OUTPUT]

    elif config['search_space'] == '3':
        search_space = SearchSpace3()
        num_inputs = list(search_space.num_parents_per_node.values())[2:]
        parents_node_2, parents_node_3, parents_node_4, parents_node_5 = \
            [get_top_k(softmax(alpha, axis=1), num_input) for num_input, alpha in zip(num_inputs[:-1], alphas_inputs)]
        output_parents = get_top_k(softmax(alphas_output), num_inputs[-1])
        parents = {
            '0': [],
            '1': [0],
            '2': parents_node_2,
            '3': parents_node_3,
            '4': parents_node_4,
            '5': parents_node_5,
            '6': output_parents
        }
        node_list = [INPUT, *node_list, OUTPUT]

    else:
        raise ValueError('Unknown search space')

    adjacency_matrix = search_space.create_nasbench_adjacency_matrix(parents)
    # Convert the adjacency matrix in format for nasbench
    adjacency_list = adjacency_matrix.astype(np.int).tolist()
    model_spec = api.ModelSpec(matrix=adjacency_list, ops=node_list)
    # Query nasbench
    data = nasbench.query(model_spec)
    valid_error, test_error, runtime, params = [], [], [], []
    for item in data:
        test_error.append(1 - item['test_accuracy'])
        valid_error.append(1 - item['validation_accuracy'])
        runtime.append(item['training_time'])
        params.append(item['trainable_parameters'])
    return test_error, valid_error, runtime, params

def eval_directory(path):
    """Evaluates all one-shot architecture methods in the directory."""
    # Read in config
    with open(os.path.join(path, 'config.json')) as fp:
        config = json.load(fp)
    # Accumulate all one-shot models
    one_shot_architectures = glob.glob(os.path.join(path, 'one_shot_architecture_*.obj'))
    # Sort them by date
    one_shot_architectures.sort(key=natural_keys)
    # Eval all of them
    test_errors = []
    valid_errors = []
    for model in one_shot_architectures:
        test, valid, _, _ = eval_one_shot_model(config=config, model=model)
        test_errors.append(test)
        valid_errors.append(valid)

    with open(os.path.join(path, 'one_shot_validation_errors.obj'), 'wb') as fp:
        pickle.dump(valid_errors, fp)

    with open(os.path.join(path, 'one_shot_test_errors.obj'), 'wb') as fp:
        pickle.dump(test_errors, fp)


import tempfile
import os
import threading
from typing import Union

from flask import (Flask, g, render_template, request, jsonify, session)
from flask_socketio import SocketIO, emit
from flask_session import Session

from flask_bootstrap import Bootstrap5
from werkzeug.utils import secure_filename
from flask_sslify import SSLify

import jinja2

import logging
from logging.handlers import TimedRotatingFileHandler

import firecrest as f7t

from config import DevConfig

def create_app():
    app = Flask(__name__)
    app.config.from_object(DevConfig)

    Session(app)

    # to have continue, break statements in jinja
    app.jinja_env.add_extension('jinja2.ext.loopcontrols')

    return app

app = create_app()
async_mode = "threading"
socketio = SocketIO(app, async_mode=async_mode)

sslify = SSLify(app) # SSL support
bootstrap = Bootstrap5(app) # bootstrap support

try:

    FIRECREST_URL = app.config["FIRECREST_URL"]
    DEBUG = app.config["DEBUG"]
    OIDC_CLIENT_ID = app.config["OIDC_CLIENT_ID"]
    OIDC_CLIENT_SECRET = app.config["OIDC_CLIENT_SECRET"]
    OIDC_AUTH_REALM = app.config["OIDC_AUTH_REALM"]
    OIDC_AUTH_BASE_URL = app.config["OIDC_AUTH_BASE_URL"]

    OIDC_AUTH_WEB_ISSUER_URL = f"{OIDC_AUTH_BASE_URL}/auth/realms/{OIDC_AUTH_REALM}"
    OIDC_AUTH_TOKEN_URL = f"{OIDC_AUTH_WEB_ISSUER_URL}/protocol/openid-connect/token"

    SYSTEM_NAME = app.config["SYSTEM_NAME"]
    SYSTEM_PARTITIONS = app.config["SYSTEM_PARTITIONS"]
    SYSTEM_CONSTRAINTS = app.config["SYSTEM_CONSTRAINTS"]
    SYSTEM_RESERVATION = app.config["SYSTEM_RESERVATION"]
    USER_GROUP = app.config["USER_GROUP"]

    PROBLEM_SUBDIR = app.config["PROBLEM_SUBDIR"]
    PROBLEM_INI_FILE = app.config["PROBLEM_INI_FILE"]
    PROBLEM_MSH_FILE = app.config["PROBLEM_MSH_FILE"]
    SBATCH_TEMPLATE = app.config["SBATCH_TEMPLATE"]
    POST_TEMPLATE = app.config["POST_TEMPLATE"]

except KeyError as ke:
    app.logger.error(f"Error in configuration file: {ke}")

# global variables
JOB_LIST = {}
JOB_DIR = None
POST_JOB_ID = None
SYSTEM_BASE_DIR = None

authN = f7t.ClientCredentialsAuth(OIDC_CLIENT_ID, OIDC_CLIENT_SECRET, OIDC_AUTH_TOKEN_URL)

f7t_client = f7t.Firecrest(
    firecrest_url=FIRECREST_URL, authorization=authN
)

def is_system_avail_f7t(system_name: str) -> bool:
    '''
        - Name: `is_system_avail_f7t`
        - Description:
           This function should return `True` if the `system_name` is available, otherwise `False`
        - Params:
          - `system_name`: str
        - Returns:
          - `bool`
    '''

    return False

def list_files_with_f7t(system_name: str, target_path: str) -> list:
    '''
        - Name: `list_files_dir`
        - Description:
           This function should return a list of files on `target_path` on the `system_name`.
            In case of error, it should return None
        - Params:
          - `system_name`: str
          - `target_path`: str
        - Returns:
          - `list` | `None`
    '''

    return None

def get_username_with_f7t(system_name):
    '''
        - Name: `get_username_with_f7t`
        - Description:
        This function should return the username of the current OIDC credential owner
        Returns None if the username is not found
        - Params:
          - `system_name`: str
        - Returns:
          - str|None
    '''

    return None

def get_base_fs_with_f7t(system_name: str) -> str:
    '''
        - Name: `get_avail_fs_with_f7t`
        - Description This function should return a python dictionary of
        `{"system_name":"system_base_dir" }` being `system_base_dir` the `mounted` filesystem for the `system_name`
        - Params:
          - `system_name: str`
        - Returns:
          - `None`
    '''

    return None

def submit_job_with_f7t(system_name: str, job_script: str) -> dict:

    '''
        - Name: `submit_job_with_f7t`
        - Description: submit a job in a given `system_name` for a given sbatch path in `job_script`
          - If the job was submitted correctly should return a dictionary : `{"job": job_information, "error": 0}`
          - If the job couldn't be submitted should return a dictionary : `{"job": "error description", "error": 1}`
        - Params:
          - `system_name`: str
          - `job_script` : str

        - Returns:
          - `dict`

    '''

    return {"job": "Error message", "error": 1}

def mkdir_with_f7t(system_name: str, target_path: str) -> bool:

    '''
    - Name: mkdir_with_f7t
    - Description: creates a directory using FirecREST. Returns `True` if the directory was created, or `False` otherwise
      If the parent directory doesn't exist, it has to be created.
    - Params:
      - `system_name`: str
      - `target_path`
    - Returns:
      - `bool`

    '''

    return False

def list_jobs_with_f7t(system_name: str, job_ids: list[int]) -> dict:
    '''
    - Name: list_jobs_with_f7t
    - Description: list queued and passed (completed, cancelled, pending, etc)
    in a specific `system_name` and filtered by a specific list of job IDs (`job_ids`)

    If the job_ids list is empty return {"jobs":[], "error": 0},
    otherwise a list of `firecrest.types.JobQueue`

    - Parameters:
      - `system_name`: str
      - `job_ids`: list[int]

    - Returns
      - dict: {"jobs": firecrest.types.JobQueue | [], "error": int}
    '''

    return []

def mkdir(jobName):

    username = get_username_with_f7t(SYSTEM_NAME)
    if username == None:
        return {"error": 1, "msg":f"Error creating directory: couldn't get username"}

    targetPath = f"{SYSTEM_BASE_DIR}/{username}/{PROBLEM_SUBDIR}/{jobName}"
    if DEBUG:
        app.logger.debug(f"targetPath: {targetPath}")

    if mkdir_with_f7t(SYSTEM_NAME,targetPath):
        return {"error": 0, "msg": f"Directory {targetPath} created"}

    return {"error": 1, "msg":f"Error creating directory {targetPath}"}

@app.route("/list_files", methods=["GET"])
def list_files():

    targetPath = request.args.get('path',None)

    if targetPath == None:
        return jsonify(rows=[]), 400

    files = list_files_with_f7t(SYSTEM_NAME, targetPath)

    if files == None:
        return jsonify(rows=[]), 400

    return jsonify(rows=files), 200

@app.route("/list_jobs", methods=["GET"])
def list_jobs():

    global JOB_LIST

    if DEBUG:
        app.logger.debug(f"Job list to query: {list(JOB_LIST.keys())}")

    f7t_jobs = list_jobs_with_f7t(SYSTEM_NAME, list(JOB_LIST.keys()))

    if f7t_jobs["error"] == 0:
        return {"rows": f7t_jobs["jobs"]}

    return jsonify(response="Error listing jobs"), 400

@app.route("/results", methods=["GET"])
def results():

    global POST_JOB_ID

    try:

        resultImage = f"{session['jobDir']}/imag.gif"
        targetPath=f"/app/src/static/{POST_JOB_ID}.gif"
        imgPath = f"/static/{POST_JOB_ID}.gif"

        if DEBUG:
            app.logger.debug(f"source_path: {resultImage}")
            app.logger.debug(f"target_path: {targetPath}")

        dwn = f7t_client.simple_download(machine=SYSTEM_NAME,
                                         source_path=resultImage,
                                         target_path=targetPath)

        return jsonify(data=imgPath), 200

    except f7t.FirecrestException as fe:
        app.logger.error(f"Download error {fe}")
        return jsonify(data=f"Download error {fe}"), 400

    except Exception as e:
        if DEBUG:
            app.logger.error(f"Download error: {e}")

        return jsonify(data=f"Download error: {e}"), 400

def write_sbatch(jobTemplate, jobName="f7t_test", ntasks=1, account=None, partition=None,
                constraint=None,reservation=None,jobDir=None, problem_ini_file=None, problem_msh_file=None,step=1,lastJobId=None):

    try:
        # sbatch templates directory
        basePath = os.path.abspath(os.path.dirname(__file__))
        templatesPath = f"{basePath}/sbatch_templates/"

        if DEBUG:
            app.logger.debug(f"Sbatch Template: {templatesPath} / {jobTemplate}")

        # create environment
        jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templatesPath))

        jinja_template = jinja_env.get_template(jobTemplate)

    except jinja2.exceptions.TemplateNotFound as tnf:
        if DEBUG:
            app.logger.error(f"Template {jobTemplate} not found")
            app.logger.error(f"Error: {tnf}")
        return {"error": 1, "msg":"Couldn't create sbatch file"}
    except Exception as e:
        if DEBUG:
            app.logger.error(f"Template {jobTemplate} not found")
            app.logger.error(f"Error: {e}")
        return {"error": 1, "msg":"Couldn't create sbatch file"}

    try:
        # creating temp sbatch file in client to upload in SYSTEM_NAME
        td = tempfile.mkdtemp(prefix="job")

        sbatch_file_path = f"{td}/sbatch-job.sh"

        if DEBUG:
            app.logger.info(f"Creating file: {sbatch_file_path}")

        with open(sbatch_file_path, "w") as sf:
            # replace templates variables with values
            sf.write(jinja_template.render(jobName=jobName, partition=partition, account=account,
                                     constraint=constraint, reservation=reservation, lastJobId=lastJobId, ntasks=ntasks, step=step,
                                     jobDir=jobDir, problem_ini_file=problem_ini_file, problem_msh_file=problem_msh_file))

        if DEBUG:
            app.logger.info(f"Created file: {sbatch_file_path}")

    except IOError as ioe:
        if DEBUG:
            app.logger.debug(ioe.message)
        return {"error": 1, "msg":"Couldn't create sbatch file"}

    except Exception as e:
        if DEBUG:
            app.logger.error(e)
            app.logger.error(type(e))
        return {"error": 1, "msg":"Couldn't create sbatch file"}

    return {"error": 0, "path":sbatch_file_path}

def background_submit_postproc(jobTemplate,jobName,ntasks,partition,constraint,reservation,targetPath,
                           problem_ini_file, problem_msh_file):

    global JOB_LIST
    global POST_JOB_ID

    try:

        # write sbatch file using the jobTemplate

        res = write_sbatch(jobTemplate=jobTemplate, jobName=f"{jobName}_post",ntasks=ntasks,account=USER_GROUP, partition=partition,
                constraint=constraint, reservation=reservation, jobDir=targetPath, problem_ini_file=problem_ini_file, problem_msh_file=problem_msh_file)
    except Exception as e:
        app.logger.error(e)

        socketio.emit("error", {"data":f"Error writing sbatch file for {jobName}_post ({e}"})
        # if the first step fails, then return the error to the frontend
        return {"data": f"Error writing sbatch file for {jobName}_post ({e}", "status": 400}

    # if the sbatch writing worked, then continue submitting jobs
    job_script = res["path"]
    job_submitted = submit_job_with_f7t(system_name=SYSTEM_NAME, job_script=job_script)

    if job_submitted["error"] == 1:

        socketio.emit("error", {"data":f"Error submitting job {jobName}_post ({job_submitted['job']}"})
        return {"data": f"Error submitting job {jobName}_post ({job_submitted['job']}", "status": 400}

    job = job_submitted["job"]

    socketio.emit("success", {"data":f"Job {jobName}_post submitted correctly (jobid: {job['jobid']})"})

    POST_JOB_ID = job["jobid"]
    JOB_LIST[job["jobid"]] = job_submitted["error"]

    if DEBUG:
        app.logger.debug(f"Job submission data: {job}")

    return {"data": "Postprocess job submitted correctly", "status": 200}

def background_submit_task(steps,jobTemplate,jobName,ntasks,partition,constraint,reservation,targetPath,
                           problem_ini_file, problem_msh_file, isPostProcess):

    global JOB_LIST
    global POST_JOB_ID

    lastJobId = 0

    for step in range(1,steps+1):

        if step == 1:
            # if this is the first step, then enchained job is not needed

            try:

                # write sbatch file using the jobTemplate

                res = write_sbatch(jobTemplate=jobTemplate, jobName=f"{jobName}_{step}",ntasks=ntasks,account=USER_GROUP, partition=partition,
                        constraint=constraint, reservation=reservation, jobDir=targetPath, problem_ini_file=problem_ini_file, problem_msh_file=problem_msh_file, step=step)
            except Exception as e:
                app.logger.error(e)

                socketio.emit("error", {"data":f"Error writing sbatch file for {jobName}_{step} ({e}"})
                # if the first step fails, then return the error to the frontend
                return {"data": f"Error writing sbatch file for {jobName}_{step} ({e}", "status": 400}

        else:
            # step is not the first one
            try:
                res = write_sbatch(jobTemplate=jobTemplate,jobName=f"{jobName}_{step}", ntasks=1, account=USER_GROUP, partition=partition,
                            constraint=constraint, reservation=reservation, jobDir=targetPath, problem_ini_file=problem_ini_file, problem_msh_file=problem_msh_file, step=step,lastJobId=lastJobId )
            except Exception as e:
                app.logger.error(e)

                socketio.emit("error", {"data":f"Error writing sbatch file for {jobName}_{step} ({e}"})
                # if writing the next sbatch fails, then the whole process can't continue
                return {"data":f"Error writing sbatch file for {jobName}_{step} ({e}", "status": 400}

        if res["error"] == 1:

            socketio.emit("error", {"data":f"Error writing sbatch file for {jobName}_{step} ({res['msg']}"})
            return {"data":f"Error writing sbatch file for {jobName}_{step} ({res['msg']}", "status": 400}

        # if the sbatch writing worked, then continue submitting jobs
        job_script = res["path"]

        try:

            # so token is refreshed
            job_submitted = submit_job_with_f7t(system_name=SYSTEM_NAME, job_script=job_script)

            if job_submitted["error"] == 1:

                socketio.emit("error", {"data":f"Error submitting job {jobName}_{step} ({job_submitted['job']}"})
                return {"data": f"Error submitting job {jobName}_{step} ({job_submitted['job']}", "status": 400}

            job = job_submitted["job"]

            socketio.emit("success", {"data":f"Job {jobName}_{step} submitted correctly (jobid: {job['jobid']})"})

            if DEBUG:
                app.logger.debug(f"Job submission data: {job}")

            lastJobId = job["jobid"]
            data = job
            status = 200
            if step == 1 and not isPostProcess:
                # for the first job:
                #     session information and directory to the targetPath
                JOB_LIST = {}
            elif isPostProcess:
                POST_JOB_ID = lastJobId

            if DEBUG:
                app.logger.debug(f"Job list: {JOB_LIST}")

            # JOB_LIST global variable updated: JOB_LIST = {"<jobid>": "<error_boolean>""}

            JOB_LIST[job["jobid"]] = job_submitted["error"]

        except f7t.FirecrestException as fe:

            if DEBUG:
                app.logger.debug(f"Job submission error: {fe}")

            data = f"Job submission error: {fe}"
            status = 400

            return {"data": data, "status": status}

        except Exception as e:
            if DEBUG:
                app.logger.error(f"Job submission error: {e}")

            data = f"Job submission error: {e}"
            status = 400

            return {"data": data, "status": status}

    return {"data": "All jobs submitted correctly", "status": 200}

@app.route("/submit_job", methods=["POST"])
def submit_job():
    global SYSTEM_RESERVATION
    global PROBLEM_MSH_FILE
    global PROBLEM_INI_FILE

    reservation = None
    if SYSTEM_RESERVATION != '':
        reservation = SYSTEM_RESERVATION
    if DEBUG:
        app.logger.debug(f"Reservation to use: {reservation}")

    problem_ini_file = None
    if PROBLEM_INI_FILE != '':
        problem_ini_file = PROBLEM_INI_FILE
    if DEBUG:
        app.logger.debug(f"Problem Ini File to use: {problem_ini_file}")

    problem_msh_file = None
    if PROBLEM_MSH_FILE != '':
        problem_msh_file = PROBLEM_MSH_FILE
    if DEBUG:
        app.logger.debug(f"Problem Ini File to use: {problem_msh_file}")

    try:
        ntasks = request.form["numberOfNodes"]
    except Exception as e:
        if DEBUG:
            app.logger.warning(e)
        ntasks = 1

    try:
        jobName = request.form["jobName"]
    except Exception as e:
        if DEBUG:
            app.logger.warning(e)
        jobName = "f7t_test"

    try:
        partition = request.form["partition"]
    except Exception as e:
        if DEBUG:
            app.logger.error(e)
        partition = None

    try:
        constraint = request.form["constraint"]
    except Exception as e:
        if DEBUG:
            app.logger.error(e)
        constraint = None

    try:
        steps = int(request.form["steps"])
    except Exception as e:
        if DEBUG:
            app.logger.error(e)
        steps = None

    try:
        isPostProcess = bool(request.form["isPostProcess"])
    except Exception as e:
        if DEBUG:
            app.logger.debug("Not a postprocess job")
        isPostProcess = False

    # created directory for the test in SYSTEM_BASE_DIR/username/PROBLEM_SUBDIR/jobName
    # ie: $SCRATCH/PyFr/examples/cylinder_inc/job_123456
    # if isPostProcess, then the directory /post is attached to JobName
    jobTemplate = SBATCH_TEMPLATE

    username = get_username_with_f7t(SYSTEM_NAME)
    if username == None:
        return jsonify(data=f"Error at getting username"), 400

    targetPath = f"{SYSTEM_BASE_DIR}/{username}/{PROBLEM_SUBDIR}/{jobName}"
    if DEBUG:
        app.logger.debug(f"targetPath: {targetPath}")

    # if it's a postprocess job, add "/post" to the name
    # and change template
    if isPostProcess:
        # jobName=f"{jobName}_post"
        jobTemplate = POST_TEMPLATE

    else:
        # if it is not a postprocess, a new directory should be created
        res_mkdir = mkdir(jobName=jobName)
        if res_mkdir["error"] != 0:
            return jsonify(data=res_mkdir["msg"]), 400

    if not isPostProcess:

        for pf_path in [problem_ini_file, problem_msh_file]:

            if pf_path == None:
                continue

            basePath = os.path.abspath(os.path.dirname(__file__))
            sourcePath = f"{basePath}/problem_files/{pf_path}"

            try:
                if DEBUG:
                    app.logger.debug(f"Uploading {sourcePath} to {targetPath}")

                upload_file = f7t_client.simple_upload(SYSTEM_NAME,source_path=sourcePath,target_path=targetPath)

            except f7t.FirecrestException as fe:
                app.logger.error(f"Error Uploading {sourcePath} to {targetPath}: {fe}")

                return jsonify(data="Error uploading initial data"), 400
    else:
        # if is postprocess, upload the post_proc.py file
        basePath = os.path.abspath(os.path.dirname(__file__))
        sourcePath = f"{basePath}/sbatch_templates/post_proc.py"

        try:

            app.logger.info(f"Checking if file {targetPath}/post_proc.py exists")

            files = f7t_client.list_files(SYSTEM_NAME,targetPath)

            found = False
            for f in files:
                if f["name"] == "post_proc.py":
                    app.logger.info("File found, skipping uploading")
                    found = True
                    break

            if not found:
                if DEBUG:
                    app.logger.debug("File not found, uploading")
                    app.logger.debug(f"Uploading {sourcePath} to {targetPath}")

                    upload_file = f7t_client.simple_upload(SYSTEM_NAME,source_path=sourcePath,target_path=targetPath)

        except f7t.FirecrestException as fe:
            app.logger.error(f"Error Uploading {sourcePath} to {targetPath}: {fe}")

            return jsonify(data="Error copying initial data"), 400

    if isPostProcess:
        bgtask = threading.Thread(target=background_submit_postproc,
                              args=(jobTemplate,jobName,ntasks,partition,constraint,reservation,targetPath,
                                    problem_ini_file, problem_msh_file))
    else:
        bgtask = threading.Thread(target=background_submit_task,
                              args=(steps,jobTemplate,jobName,ntasks,partition,constraint,reservation,targetPath,
                                    problem_ini_file, problem_msh_file, isPostProcess))

    bgtask.start()

    session.clear()

    session["jobDir"] = targetPath
    session["jobName"] = jobName
    session["partition"] = partition
    session["constraint"] = constraint
    session["steps"] = steps
    session["numberOfNodes"] = ntasks
    session["activePost"] = isPostProcess

    if DEBUG:
        app.logger.debug("Information about job submission")
        app.logger.debug(f"jobDir: {session['jobDir']}")
        app.logger.debug(f"jobName: {session['jobName']}")

    return jsonify(data="Batch started"), 200

@app.before_request
def before_request():
    g.user = get_username_with_f7t(SYSTEM_NAME)
    app.logger.debug(g.user)

@app.route("/",methods=["GET","POST"])
def live():
    '''Function to live dashboard'''

    global SYSTEM_BASE_DIR

    system_status = "undefined"
    if is_system_avail_f7t(SYSTEM_NAME):
        system_status = "avail"
        SYSTEM_BASE_DIR = get_base_fs_with_f7t(SYSTEM_NAME)
    else:
        system_status="notavail"

    status = 200
    jobPath = f"{SYSTEM_BASE_DIR}/{g.user}/{PROBLEM_SUBDIR}/"

    data = {"partitions": SYSTEM_PARTITIONS, "constraints": SYSTEM_CONSTRAINTS,
            "system": SYSTEM_NAME, "job_dir":jobPath, "system_status": system_status}

    if request.method == "POST":
        msg = None
        error = None

        try:
            msg = request.form["msg"]
            error = request.form["error"]
        except Exception as e:
            app.logger.warning(e)

        return render_template("live.html",  data=data, status=status, msg=msg, error=error)

    return render_template('live.html', data=data, status=status, error=0)

if __name__ == '__main__':
    logHandler = TimedRotatingFileHandler('/var/log/client.log', when='D', interval=1)

    logFormatter = logging.Formatter('%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
                                     '%Y-%m-%d:%H:%M:%S')
    logHandler.setFormatter(logFormatter)
    logHandler.setLevel(logging.DEBUG)

    # get app log (Flask+werkzeug+python)
    logger = logging.getLogger()

    logger.addHandler(logHandler)

    USE_SSL = app.config['USE_SSL']

    CLIENT_PORT = app.config["CLIENT_PORT"]

    if not USE_SSL:
        socketio.run(app,host='0.0.0.0',port=CLIENT_PORT,allow_unsafe_werkzeug=True)
    else:
        SSL_PEM = app.config['SSL_PEM']
        SSL_KEY = app.config['SSL_KEY']
        socketio.run(app,host='0.0.0.0', ssl_context=(SSL_PEM, SSL_KEY), port=CLIENT_PORT)


from aiogram import types

urls = {
    'atv': 'https://atvarmor.ru/cat/atv',
    'bike': 'https://atvarmor.ru/cat/moto',
    'snowmobile': 'https://atvarmor.ru/cat/snowmobiles',
    'preowned': 'https://atvarmor.ru/cat/preowned'
}

atv_button = types.InlineKeyboardButton('Посмотреть на сайте ➡', url=urls['atv'])
bike_button = types.InlineKeyboardButton('Посмотреть на сайте ➡', url=urls['bike'])
snowmobile_button = types.InlineKeyboardButton('Посмотреть на сайте ➡', url=urls['snowmobile'])
preowned_button = types.InlineKeyboardButton('Посмотреть на сайте ➡', url=urls['preowned'])

see_atv_keyboard = types.InlineKeyboardMarkup()
see_atv_keyboard.add(atv_button)

see_bike_keyboard = types.InlineKeyboardMarkup()
see_bike_keyboard.add(bike_button)

see_snowmobile_keyboard = types.InlineKeyboardMarkup()
see_snowmobile_keyboard.add(snowmobile_button)

see_preowned_keyboard = types.InlineKeyboardMarkup()
see_preowned_keyboard.add(preowned_button)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 24 17:11:08 2019

@author: noahbrauer
"""

import matplotlib.pyplot as plt
import numpy as np
from scipy import stats
from netCDF4 import Dataset, num2date, MFDataset
from datetime import datetime
from matplotlib.dates import YearLocator, MonthLocator, DateFormatter
from matplotlib.ticker import MultipleLocator
import glob

import os
import conda

conda_file_dir = conda.__file__
conda_dir = conda_file_dir.split('lib')[0]
proj_lib = os.path.join(os.path.join(conda_dir, 'share'), 'proj')
os.environ["PROJ_LIB"] = proj_lib

from mpl_toolkits.basemap import Basemap

q_file = 'shum.mon.mean.nc'
u_file = 'uwnd.mon.mean.nc'
v_file = 'vwnd.mon.mean.nc'

ncq = Dataset(q_file, 'r')
ncu = Dataset(u_file, 'r')
ncv = Dataset(v_file, 'r')

narr = {}
lat = ncq.variables['lat'][:]
lon = ncq.variables['lon'][:]

time = ncq.variables['time'][:]
timeUnits = ncq.variables['time'].units
tmpDates = num2date(time,timeUnits,calendar='gregorian')
narr['date'] = np.asarray([datetime(d.year,d.month,d.day) for d in tmpDates])
narr['day'] = np.asarray([d.day for d in narr['date']])
narr['month'] = np.asarray([d.month for d in narr['date']])
narr['year'] = np.asarray([d.year for d in narr['date']])

time_index_climo = np.where((narr['month']==9))[0]

narr2 = {}

time2 = ncu.variables['time'][:]
timeUnits2 = ncu.variables['time'].units
tmpDates2 = num2date(time2,timeUnits2,calendar='gregorian')
narr2['date'] = np.asarray([datetime(d.year,d.month,d.day) for d in tmpDates2])
narr2['day'] = np.asarray([d.day for d in narr2['date']])
narr2['month'] = np.asarray([d.month for d in narr2['date']])
narr2['year'] = np.asarray([d.year for d in narr2['date']])

time_index_climo_wind = np.where((narr2['month']==8))[0]

#Read in data

q = ncq.variables['shum'][time_index_climo, :,:,:]
uwnd = ncu.variables['uwnd'][time_index_climo_wind, :,:,:]
vwnd = ncv.variables['vwnd'][time_index_climo_wind, :,:,:]

pres = ncq.variables['level'][:]
g = 9.8

#Compute mean  with time (29,277,349) or height,lat,lon

q_mean = np.nanmean(q, axis = 0)

#Compute IVT

ivt_u = np.trapz(q_mean*uwnd,pres, axis = 1)*(-1/g)
ivt_v = np.trapz(q_mean*vwnd,pres, axis = 1)*(-1/g)

#Now compute mean IVT over time

ivt_u_mean = np.nanmean(ivt_u, axis = 0)
ivt_v_mean = np.nanmean(ivt_v, axis = 0)

def magnitude(u,v):
    mag = np.sqrt((u**2) +(v**2))
    return mag

ivt_mag_mean = magnitude(ivt_u_mean,ivt_v_mean)

#Compute standard deviations

ivt_u_std = np.nanstd(ivt_u, axis = 0)
ivt_v_std = np.nanstd(ivt_u, axis = 0)

ivt_mag_std = magnitude(ivt_u_std, ivt_v_std)

ivt_mag_nan = np.ones((277,349))*np.nan
ivt_u_nan = np.ones((277,349))*np.nan
ivt_v_nan = np.ones((277,349))*np.nan

for i in range(ivt_mag_nan.shape[0]):
    for j in range(ivt_mag_nan.shape[1]):

        if lon[i,j]>0:
            ivt_mag_nan[i,j] = np.nan
            ivt_u_nan[i,j] = np.nan
            ivt_v_nan[i,j] = np.nan

        else:
            ivt_mag_nan[i,j] = ivt_mag_mean[i,j]
            ivt_u_nan[i,j] = ivt_u_mean[i,j]
            ivt_v_nan[i,j] = ivt_v_mean[i,j]

#%%
#Now Plot

cmin = 0.; cmax = 4.; cint = 0.25; clevs = np.round(np.arange(cmin,cmax,cint),2)
nlevs = len(clevs) - 1; cmap = plt.get_cmap(name='bwr',lut=nlevs)

plt.figure(figsize=(10,6))

#xlim = np.array([-110,-75]); ylim = np.array([15,40])
xlim = np.array([-130,-75]); ylim = np.array([15,50])

m = Basemap(projection='cyl',lon_0=np.mean(xlim),lat_0=np.mean(ylim),llcrnrlat=ylim[0],urcrnrlat=ylim[1],llcrnrlon=xlim[0],urcrnrlon=xlim[1],resolution='i')
m.drawcoastlines(); m.drawstates(), m.drawcountries()
cs = m.contourf(lon,lat,ivt_mag_nan,clevs,cmap='Greens',extend='both')
widths = np.linspace(0, 2, lat.size)
#m.quiver(lon[::6,::6],lat[::6,::6],ivt_u[8,::6,::6],ivt_v[8,::6,::6], scale = 100)
m.quiver(lon[::4,::4],lat[::4,::4],ivt_u_nan[::4,::4],ivt_v_nan[::4,::4], scale = 150)

#m.drawcounties()

cbar = m.colorbar(cs,size='2%')
cbar.ax.set_ylabel(r'kg $m^{-1}$ $s^{-1}$',name='Calibri',size=18)
cticks = []
for i in clevs:
    cticks.append(int(i)) if i.is_integer() else cticks.append(i)

    cbar.set_ticks(clevs[::4])
    cbar.set_ticklabels(cticks[::4])
for i in cbar.ax.yaxis.get_ticklabels():
    i.set_family('Calibri')
    i.set_size(14)

plt.title('August Mean Integrated Vapor Transport (1979-2018 Climatology)', size = 20)

#x2star,y2star = m(-128,21)
#plt.text(x2star,y2star, 'Plot by Noah Brauer', weight = 'bold', color = 'b')

plt.show(block=False)


from __future__ import annotations

from dataclasses import dataclass
from enum import Enum
from math import sin, cos, radians

INPUT = "input"

class Command(Enum):
    N = "N"
    E = "E"
    S = "S"
    W = "W"
    L = "L"
    R = "R"
    F = "F"

class Direction(Enum):
    N = 0
    E = 90
    S = 180
    W = 270

@dataclass
class Point:
    x: int
    y: int

    def __add__(self, other: Point) -> Point:
        return Point(self.x + other.x, self.y + other.y)

    def __iadd__(self, other: Point) -> Point:
        self.x += other.x
        self.y += other.y
        return self

    def __mul__(self, scalar: int) -> Point:
        return Point(self.x * scalar, self.y * scalar)

    def manhatten_distance(self) -> int:
        return abs(self.x) + abs(self.y)

    def rotate_about_origin(self, degrees: int) -> None:
        rad = radians(degrees)
        new_x = self.x * cos(rad) + self.y * sin(rad)
        new_y = - self.x * sin(rad) + self.y * cos(rad)
        self.x = int(round(new_x))
        self.y = int(round(new_y))

    def move(self, direction: Direction, distance: int) -> None:
        self += VECTORS[direction] * distance

MAX_COMPASS = 360

COMMAND_TO_DIR = {
    Command.N: Direction.N,
    Command.E: Direction.E,
    Command.S: Direction.S,
    Command.W: Direction.W
}

ROTATION_DIRECTION = {
    Command.R: 1,
    Command.L: -1
}

VECTORS = {
    Direction.N: Point(0, 1),
    Direction.E: Point(1, 0),
    Direction.S: Point(0, -1),
    Direction.W: Point(-1, 0)
}

class Part1Ship:
    def __init__(self) -> None:
        self.position = Point(0, 0)
        self.direction = Direction.E

    def follow_command(self, command: Command, arg: int) -> None:
        if command in ROTATION_DIRECTION:
            self.direction = Direction((self.direction.value + arg * ROTATION_DIRECTION[command]) % MAX_COMPASS)
        else:
            self.position.move(COMMAND_TO_DIR.get(command, self.direction), arg)

class Part2Ship:
    def __init__(self) -> None:
        self.position = Point(0, 0)
        self.waypoint = Point(10, 1)

    def follow_command(self, command: Command, arg: int) -> None:
        if command in COMMAND_TO_DIR:
            self.waypoint.move(COMMAND_TO_DIR[command], arg)
        elif command == Command.F:
            self.position += self.waypoint * arg
        else:
            self.waypoint.rotate_about_origin(arg * ROTATION_DIRECTION[command])

def main() -> None:
    part_1_ship = Part1Ship()
    part_2_ship = Part2Ship()
    with open(INPUT, "r") as fin:
        for line in fin:
            command = Command(line[0])
            arg = int(line[1:])
            part_1_ship.follow_command(command, arg)
            part_2_ship.follow_command(command, arg)
    print(part_1_ship.position.manhatten_distance())
    print(part_2_ship.position.manhatten_distance())

if __name__ == "__main__":
    main()


import argparse
import time
from pyppeteer import launch
from pyppeteer import errors as pyppeteer_errors
from bs4 import BeautifulSoup
from tqdm import tqdm
import asyncio
from collections import defaultdict
import collections
import json
import logging
import os

# Parse the command-line arguments
parser = argparse.ArgumentParser()
parser.add_argument('url',help='The base url that will be populated',
                        default= None, type=str)

parser.add_argument('--path',
                        dest='path', help='The path of the file that will be generated with the populated urls',
                        default= None, required=False,type=str)

args = parser.parse_args()
baseUrl = args.url
path = args.path

COLOR_CONSTANTS =   [1,2,3,19,20,4,21,6,7,8,9,10,11,12,13,14,16,17,686230]
BEDEN_CONSTANTS = ["beden|xxs","beden|xs","beden|s","beden|m","beden|l","beden|xl","beden|xxl","beden|2xl","beden|3xl","beden|4xl","beden|5xl","beden|6xl","beden|7xl"]
FIT_CONSTANTS = ["179|16064","179|2861","179|2860","179|2042","179|2004","179|2043","179|22116","179|2001","179|2003","179|7199","179|2028","179|279663","179|666291"]
BRAND_CONSTANTS = ["101476","43","486","592","41","38","37","147","859","708","262","224","102767","324","109738","148833","150370","152406","104686","145554","136","639","44","33","101990","160","124","333","768","230","54","104189","125","108793","156","138","46"]

BRAND_ARGNAME = "wb"
COLOR_ARGNAME= "wcl"
BEDEN_ARGNAME= "vr"
FIT_ARGNAME= "attr"
PRICE_ARGNAME = "prc"

PRODUCT_THRESHOLD = 5000
PRODUCT_RISKY_THRESHOLD = 6000
ASSEMBLY_THRESHOLD = 5000

def assertArgs():
    global baseUrl
    global path

    baseUrl = args.url
    path = args.path

    if baseUrl == None:
       logAndPrint("Please specify the base url to populate.")
       exit()

    if path != None:
        if ".txt" not in path:
            logAndPrint("The path should be to a .txt file")

    if path == None:
        onvironDirname = os.environ.get('DIRNAME')
        if onvironDirname != None:
            path = os.path.join("/Dataset",onvironDirname.strip(),"urls.txt")

    if "wc=" not in baseUrl:
        logAndPrint("Please give the url in the parameter format. (https://www.trendyol.com/sr?wc=75&wg=2)")
        exit()
    if BRAND_ARGNAME in baseUrl or COLOR_ARGNAME in baseUrl or BEDEN_ARGNAME in baseUrl or FIT_ARGNAME in baseUrl or PRICE_ARGNAME in baseUrl:
        logAndPrint("Please give the url in the simplest form without any filtering")
        exit()

def tree(): return defaultdict()

def createDir(path):
  if not os.path.exists(path):
    os.makedirs(path)

def createNecessaryDirs():
    createDir(os.path.join(os.path.dirname(__file__), "logs"))
    if path != None:
        createDir(path[:path.rfind("/")])

def logAndPrint(message):
  logging.info(message)
  print(message,end='\n',flush=True)

def constructUrl(url,params):
    urlString = url
    for param in params:
        urlString += "&" + param

    return urlString

def generatePriceTree(argtree):
    PRICE_ARGNAME = "prc"
    priceList = [str(i) + "-" + str(i+5) for i in range(0, 1000, 5)]
    priceList.append("1000-10000")

    for price in priceList:
        argtree[PRICE_ARGNAME +"="+  price] = None

    return argtree

def writeTreeToFile(tree, fileName):
    treeJson = json.dumps(tree)
    with open(fileName, 'w') as f:
        f.write(treeJson)

def writeListToFile(list, fileName):
    with open(fileName, 'w') as f:
        for item in list:
            f.write(item.strip() + '\n')

async def getPage(pageUrl , firstRecursion = True):
    try:
        await page.goto(pageUrl)
        await page.waitForFunction("""() => document.querySelectorAll('.search-landings-container, .prdct-cntnr-wrppr , .errorpagewrapper, .no-rslt-icon').length""", timeout=10000)

    except pyppeteer_errors.TimeoutError:
        if(firstRecursion):
            #logAndPrint("!!! Page get timed out , trying again with "+pageUrl)
            return await getPage(pageUrl, False)
        else:
            #logAndPrint("!!! Page get timed out , giving up with "+pageUrl)
            return False
    except pyppeteer_errors.NetworkError:
        #logAndPrint("!!! Page get network error , trying again with "+pageUrl)
        if(firstRecursion):
            return await getPage(pageUrl, False)
        else:
            #logAndPrint("!!! Page get network error , giving up with "+pageUrl)
            return False

async def getProductCount(url):
    try:
        if(await getPage(url) == False):
            return False
        source = await page.content()
        soup = BeautifulSoup(source, 'html.parser')

        time.sleep(1)

        description = soup.find("div", {"class": "dscrptn"})

        product_count_string = description.contents[-1].split(" ")[-3].strip()

        if(product_count_string == "100.000+"):
            return 100000

        return int(product_count_string)

    except AttributeError:
        ### Pages that show no result at all
        return 0
    except ValueError:
        ## Pages that have no search result but show other recomendations
        return 0

async def populateUrlsAboveThreashold(urlThree, argname, populatees, threashold, legacyArgs = [], recursion = False):
    iteratonCount = 1

    for key in urlThree:

        if not recursion:
            logAndPrint("{0}/{1}".format(iteratonCount, len(urlThree)))
        iteratonCount += 1

        if(type(urlThree[key]) == collections.defaultdict):
            urlThree[key] = await populateUrlsAboveThreashold(urlThree[key], argname, populatees, threashold, legacyArgs + [key],True)
            continue

        if(type(urlThree[key]) == int):
            productCount = urlThree[key]
        else:
            url = constructUrl(baseUrl,  legacyArgs + [key])
            productCount = await getProductCount(url)

        if(productCount == False or productCount == 0):
            urlThree[key] = 0
            continue

        if(productCount < threashold):
            urlThree[key] = productCount
            continue

        branchTree = tree()
        for populatee in populatees:
            branchTree[argname + "=" + str(populatee)] = None

        urlThree[key] = branchTree

    return urlThree

def multiParamMerger(rawparams):
    def assamble_string(params, char):
        param_string = ""
        for param in params:
            param_string += param + char
        return param_string[:-1]

    if(len(rawparams) == 0):
        return []

    if(COLOR_ARGNAME in rawparams[0]):
        param_vals = [param.split("=")[1] for param in rawparams]
        return [COLOR_ARGNAME + "="+ assamble_string(param_vals,",")]

    if(BRAND_ARGNAME in rawparams[0]):
        param_vals = [param.split("=")[1] for param in rawparams]
        return [BRAND_ARGNAME + "="+ assamble_string(param_vals,",")]

    if(BEDEN_ARGNAME in rawparams[0]):
        param_vals = [param.split("|")[1] for param in rawparams]
        return [BEDEN_ARGNAME + "=beden|"+ assamble_string(param_vals,"_")]

    if(FIT_ARGNAME in rawparams[0]):
        param_vals = [param.split("|")[1] for param in rawparams]
        return [FIT_ARGNAME + "=179|"+ assamble_string(param_vals,"_")]

    if(PRICE_ARGNAME in rawparams[0]):

        pricePairs = [[int(prc) for prc in param.split("=")[1].split("-")] for param in rawparams]

        minPrice = min([pair[0] for pair in pricePairs])
        maxPrice = max([pair[1] for pair in pricePairs])

        return [PRICE_ARGNAME + "=" + str(minPrice) + "-" + str(maxPrice)]

    raise Exception("Unknown multi param type")

def AssembleUrlList(urlTree, legacyArgs = []):
    urlList = []

    buffer = []
    buffer_counter = 0

    for key in urlTree:

        if(type(urlTree[key]) == collections.defaultdict):
            urlList += AssembleUrlList(urlTree[key], legacyArgs + [key])

        if(type(urlTree[key]) == int):
            if(urlTree[key] == 0):
                continue

            if(buffer_counter + urlTree[key] > ASSEMBLY_THRESHOLD):
                urlList.append(constructUrl(baseUrl, legacyArgs + multiParamMerger(buffer)))
                buffer = [key]
                buffer_counter = urlTree[key]
                continue
            buffer.append(key)

            buffer_counter += urlTree[key]

    urlList.append(constructUrl(baseUrl, legacyArgs + multiParamMerger(buffer)))
    return urlList

async def main():
    global browser
    global page

    url_tree = tree()

    browser = await launch(headless=True, args=['--start-maximized','--no-sandbox'], defaultViewport=None, autoClose=False)

    page = await browser.newPage()
    priceTree = generatePriceTree(url_tree)

    logAndPrint("Populating price tree with color information")
    populatedTree = await populateUrlsAboveThreashold(priceTree, COLOR_ARGNAME, COLOR_CONSTANTS, PRODUCT_THRESHOLD)

    logAndPrint("Populating price tree with beden information")
    populatedTree = await populateUrlsAboveThreashold(populatedTree, BEDEN_ARGNAME, BEDEN_CONSTANTS, PRODUCT_THRESHOLD)

    logAndPrint("Populating price tree with fit information")
    populatedTree = await populateUrlsAboveThreashold(populatedTree, FIT_ARGNAME, FIT_CONSTANTS, PRODUCT_THRESHOLD)

    logAndPrint("Populating price tree with brand information")
    populatedTree = await populateUrlsAboveThreashold(populatedTree, BRAND_ARGNAME, BRAND_CONSTANTS, PRODUCT_RISKY_THRESHOLD)

    #This serves to only record the product count of end nodes
    logAndPrint("Populating price tree with product count information")
    populatedTree = await populateUrlsAboveThreashold(populatedTree, "", "", float('inf'))

    populatedUrlList = AssembleUrlList(populatedTree)

    if(path != None):
        urlFilename = path

        if "/" in urlFilename:
            treeFilename = os.path.join(urlFilename[:urlFilename.rfind("/")], "populated_tree_"+str(time.time())+".txt")
        else:
            treeFilename = "populated_tree_"+str(time.time())+".txt"
    else:
        urlFilename = "populated_urls_"+str(time.time())+".txt"
        treeFilename = "populated_tree_"+str(time.time())+".txt"

    writeListToFile(populatedUrlList, urlFilename)

    # For debugging purposes
    writeTreeToFile(populatedTree, treeFilename)

    logAndPrint("Generated {0} with {1} urls".format(urlFilename, len(populatedUrlList)))

    await browser.close()

if __name__ == "__main__":
    start_time = time.time()
    assertArgs()
    createNecessaryDirs()
    logging.basicConfig(filename= os.path.join(os.path.dirname(__file__), "logs",'populate_trendyol_url_'+str(time.time())+'.log'), level=logging.INFO,  format='%(asctime)s %(message)s')
    asyncio.run(main())
    logAndPrint("--- %s seconds ---" % (time.time() - start_time))
    exit()


from copy import deepcopy

import pytest
from torch import nn

from pytorch_widedeep.models import (
    Wide,
    TabMlp,
    TabNet,
    Vision,
    BasicRNN,
    WideDeep,
)

embed_input = [(u, i, j) for u, i, j in zip(["a", "b", "c"][:4], [4] * 3, [8] * 3)]
column_idx = {k: v for v, k in enumerate(["a", "b", "c"])}
wide = Wide(10, 1)
tabmlp = TabMlp(
    mlp_hidden_dims=[16, 8], column_idx=column_idx, cat_embed_input=embed_input
)
tabnet = TabNet(column_idx=column_idx, cat_embed_input=embed_input)
deeptext = BasicRNN(vocab_size=100, embed_dim=8)
deepimage = Vision()

###############################################################################
# test warning on head_layers_dim and deephead
###############################################################################

def test_deephead_and_head_layers_dim():
    deephead = nn.Sequential(nn.Linear(32, 16), nn.Linear(16, 8))
    with pytest.raises(ValueError):
        model = WideDeep(  # noqa: F841
            wide=wide,
            deeptabular=tabmlp,
            head_hidden_dims=[16, 8],
            deephead=deephead,
        )

###############################################################################
# test deephead is None and head_layers_dim is not None
###############################################################################

def test_no_deephead_and_head_layers_dim():
    out = []
    model = WideDeep(
        wide=wide, deeptabular=tabmlp, head_hidden_dims=[8, 4]
    )  # noqa: F841
    for n, p in model.named_parameters():
        if n == "deephead.head_layer_0.0.weight":
            out.append(p.size(0) == 8 and p.size(1) == 8)
        if n == "deephead.head_layer_1.0.weight":
            out.append(p.size(0) == 4 and p.size(1) == 8)
    assert all(out)

###############################################################################
# test tabnet warning
###############################################################################

def test_tabnet_warning():
    with pytest.warns(UserWarning):
        model = WideDeep(wide=wide, deeptabular=tabnet)  # noqa: F841


# You have to import push_pop script

from push_pop import *
# %run push_pop.ipynb

import numpy as np
import networkx as nx
import matplotlib.pyplot as plt
import random
import math
from numpy import genfromtxt

# DEFINE FUNCTIONS

def E_walk(walker): # Calculate the walker hamiltonian
    abs_walker = abs(walker)
    return -1*sum(walker*abs_walker)

def Metropolis(G,H,walker,new_walker):

    E_diff = E_walk(new_walker) - E_walk(walker) # Calculate energy diff for Metropolis

    if E_diff <= 0.0000000001: # Tolerance to negate numeric error
        walker = new_walker
        G = H

    else:
        Temp = 0.5
        if random.random() < math.exp(-E_diff/Temp):
            walker = new_walker
            G = H
        else:
            pass

    return G, walker

def K(G,node): # Find curvature at particular node
    return 6 - G.degree(node)

def draw(G):
    plt.figure(figsize=(3,3))
    nx.draw(G, pos=nx.spring_layout(G), node_color='lightgray', edge_color='black', with_labels=True)
    return

def rename_nodes(G): # rename all node in the graph starting from number zero
    num = 0
    G = nx.convert_node_labels_to_integers(H, first_label = num, ordering='default')
    return G

def get_walker(G, walker, node, new_node):

    split = walker[node]//2

    walker[node] -= split # Split the walkers between the nodes
    walker[new_node] += split

    for i in range(0, max( G.nodes() )+1 ): # Adding walker using the curvature at each node
        walker[i] += K(G,i)

    if len(G) > 2:
        listofnodes = [k for k in G] # Get a list of all nodes in the graph G
        for i in range(12):
            walker[random.choice(listofnodes)] -= 1 # Add twelve -1 walkers to random nodes in graph G

    new_walker = np.zeros(len(walker)).astype(int)

    for i in G.nodes(): # loop through all node to move the walkers

        for j in range(0, int(abs(walker[i])) ): # All walker at node i gets to "walk"

            nbr = [n for n in G[i]] # Get list of neighbor for node i in G
            k = random.choice(nbr)  # walker will move to neighbor k

            if walker[i] > 0:  # move the walker depending on whether it's a +1 walker or -1
                new_walker[k] += 1
            if walker[i] < 0:
                new_walker[k] -= 1
            if walker[i] == 0:
                continue

    return new_walker # Make sure to pass this back, i.e. walker = get_walker(G,walker)

def reconfigure_walker(G, walker):
    walker2 = np.zeros(len(walker)).astype(int)

    j = 0
    for i in G.nodes(): # loop through all node, one should be missing since it "popped".
        walker2[j] = walker[i] # read explanation below
        j += 1

    return walker2

# index i will loop through 0,1,2,.... except for the missing node
# index j will is just 0,1,2,3,....
# let's say node 2 is gone cause it "popped", walker2[0] = walker[0], walker2[1] = walker[1],
# then walker2[2] = walker[3], walker2[3] = walker[4], and so on.

# After running this function, we can then run the "rename_nodes()" function to relabel the nodes 0,1,2,3....

# MAIN CODE START HERE

G = nx.octahedral_graph()

s = 2**8

walker = np.zeros(s).astype(int)

for i in range(0,max(G.nodes())+1): # Adding walker using the curvature at each node
    walker[i] += K(G,i)

while len(G) < s:

    H = G.copy()
    new_walker = walker.copy()

    node, new_node = push(H)

    new_walker = get_walker(H, new_walker, node, new_node)

    G, walker = Metropolis(G, H, walker, new_walker)

    if G == H:
        print("Current number of nodes: ", len(G))

# Equilibration / Thermalizing

for i in range(0,8*s):

    H = G.copy()
    pop(H)
    walker = reconfigure_walker(H, walker)
    H = rename_nodes(H)
    push(H)

    new_walker = walker.copy()
    new_walker = get_walker(H, new_walker, node, new_node)

    G, walker = Metropolis(G, H, walker, new_walker)

a = get_adj(G)
np.savetxt("adjacency.csv", a.astype(int), fmt='%i', delimiter=",")
#np.savetxt("walker_array.csv", walker.astype(int), fmt='%i', delimiter=",")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Mar  9 14:30:10 2023

@author: floriancurvaia
"""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import DBSCAN
from sklearn.mixture import GaussianMixture
import scipy.stats as scst

import plotly.express as px

im="B07_px+1257_py-0474"
fn_dist="/Users/floriancurvaia/Desktop/Uni/ETH/Labos/Pelkmans/work/fcurvaia/distances/"+im+"_w_dist_sph_simp.csv" #B08_px+1076_py-0189_w_dist_sph_simp.csv"
feat_filt=pd.read_csv(fn_dist, sep=",", index_col=False)

#plt.scatter(feat_filt.PhysicalSize, feat_filt.dist_out, c=feat_filt.KMeans_3)

scaler = StandardScaler()
#init_centers=np.array([[13620, 62.5], [13010, 8.9], [3100, 31]])
#init_centers=np.array([[12541.60805147059, 62.01264938445141], [14267.907663043474, 10.911887499867083], [3158.7920548301367, 29.126187516429678]])
#scaled_features = scaler.fit_transform(np.array([scst.zscore(np.log(feat_filt.NucSize)), scst.zscore(feat_filt.PhysicalSize)]).T) #feat_filt.PhysicalSize.to_numpy().reshape(-1,1)
scaled_features = scaler.fit_transform(np.array([feat_filt.NucSize, feat_filt.PhysicalSize]).T)

#scaled_features = scaler.fit_transform(np.array([feat_filt.PhysicalSize,feat_filt.dist_out, feat_filt.aspect_ratio_max]).T)
c=np.mean(scaled_features[np.where((feat_filt.NucSize.to_numpy()>1500)& (feat_filt.PhysicalSize.to_numpy()<20000))], axis=0) #(feat_filt.NucSize.to_numpy()>1500)&

#print(np.mean(scaled_features[np.where(feat_filt.PhysicalSize.to_numpy()>13000)], axis=0))

a=np.mean(scaled_features[np.where((feat_filt.NucSize.to_numpy()>1000) & (feat_filt.PhysicalSize.to_numpy()>20000))], axis=0)

b=np.mean(scaled_features[np.where((feat_filt.NucSize.to_numpy()>3000) & (feat_filt.PhysicalSize.to_numpy()<15000))], axis=0)
init_means=np.array([a, b, c])

kmeans_3= KMeans(n_clusters=3, random_state=42, init=np.array([a, b, c]))
#kmeans_2= KMeans(n_clusters=2, random_state=42)
#kmeans_2.fit(scaled_features)
kmeans_3.fit(scaled_features)
#feat_filt["KMeans_2_c"]=kmeans_2.labels_
feat_filt["KMeans_3_c"]=kmeans_3.labels_

dbscan=DBSCAN(eps=0.5)
dbscan.fit(scaled_features)
feat_filt["DBSCAN_05"]=dbscan.labels_

gmm = GaussianMixture(n_components = 3, covariance_type="spherical",  means_init=np.array([a, b,  c])) #, weights_init=[ 0.998, 0.002,]
gmm.fit(scaled_features)
labels = gmm.predict(scaled_features)
feat_filt["GMM"]=labels

path_out_im="/Users/floriancurvaia/Desktop/Uni/ETH/Labos/Pelkmans/work/fcurvaia/Images/Clusters/"

#print(np.mean(feat_filt.loc[(feat_filt.PhysicalSize>10000) & (feat_filt.dist_out>33)]).PhysicalSize)

#print(np.mean(feat_filt.loc[(feat_filt.PhysicalSize>10000) & (feat_filt.dist_out>33)]).dist_out)
#print(np.mean(feat_filt.loc[(feat_filt.PhysicalSize>10000) & (feat_filt.dist_out<33)]).PhysicalSize)
#print(np.mean(feat_filt.loc[(feat_filt.PhysicalSize>10000) & (feat_filt.dist_out<33)]).dist_out)

plt.figure(1)
plt.clf()
plt.scatter(feat_filt.NucSize, feat_filt.PhysicalSize, c=feat_filt.KMeans_3_c) #, s=(feat_filt.KMeans_3_c+1)*100, c=feat_filt.betaCatenin_cyto)

plt.figure(2)
plt.clf()
plt.scatter(feat_filt.NucSize, feat_filt.PhysicalSize, c=feat_filt.DBSCAN_05) #, s=(feat_filt.DBSCAN_05+1)*100, c=feat_filt.betaCatenin_cyto)

plt.figure(3)
plt.clf()
plt.scatter(feat_filt.NucSize, feat_filt.PhysicalSize, c=feat_filt.GMM) #, s=(feat_filt.DBSCAN_05+1)*100, c=feat_filt.betaCatenin_cyto)
plt.savefig(path_out_im+"Size_Dist_scatter_GMM_"+im)

plt.figure(4)
plt.clf()
plt.scatter(scaled_features[:,0], scaled_features[:,1], c=feat_filt.KMeans_3_c) #, s=(feat_filt.KMeans_3_c+1)*100, c=feat_filt.betaCatenin_cyto)

plt.figure(5)
plt.clf()
plt.scatter(feat_filt.NucSize, feat_filt.aspect_ratio_max, c=feat_filt.DBSCAN_05) #, s=(feat_filt.DBSCAN_05+1)*100, c=feat_filt.betaCatenin_cyto)

plt.figure(6)
plt.clf()
plt.scatter(feat_filt.NucSize, feat_filt.aspect_ratio_max, c=feat_filt.GMM)

fig7 = px.scatter_3d(feat_filt, x='x_corr', y='y_corr', z='z_corr', color="GMM", opacity=1, color_continuous_scale="turbo")
fig7.update_layout(margin=dict(l=0, r=0, b=0, t=0))
#fig8.write_html(path_out_im+"dist_out_simp_"+im+"_"+str(n_bins)+".html")
fig7.write_html(path_out_im+"GMM_"+im+".html")

fig8 = px.scatter_3d(feat_filt, x='x_corr', y='y_corr', z='z_corr', color="KMeans_3_c", opacity=1, color_continuous_scale="turbo")
fig8.update_layout(margin=dict(l=0, r=0, b=0, t=0))
#fig8.write_html(path_out_im+"dist_out_simp_"+im+"_"+str(n_bins)+".html")
fig8.write_html(path_out_im+"KMeans_"+im+".html")

fig9 = px.scatter_3d(feat_filt, x='x_corr', y='y_corr', z='z_corr', color="DBSCAN_05", opacity=1, color_continuous_scale="turbo")
fig9.update_layout(margin=dict(l=0, r=0, b=0, t=0))
#fig8.write_html(path_out_im+"dist_out_simp_"+im+"_"+str(n_bins)+".html")
fig9.write_html(path_out_im+"DBSCAN_"+im+".html")


"""Preprocessing data objects (:mod:`fluidimage.data_objects.preproc`)
======================================================================

.. autoclass:: ArraySerie
   :members:
   :private-members:

.. autoclass:: PreprocResults
   :members:
   :private-members:

"""

import math
import os

import h5py

from fluiddyn.util.serieofarrays import SerieOfArraysFromFiles
from fluidimage.util import imsave, imsave_h5

from .piv import ArrayCouple, LightPIVResults

def get_Ni_Nj(serie):
    """Returns number of images in the first and second indices
    of the series.

    """
    if not isinstance(serie, SerieOfArraysFromFiles):
        raise ValueError(
            "serie must be an instance of class SerieOfArraysFromFiles"
        )

    nb_indices = serie.nb_indices
    slices = serie.get_index_slices()
    Ni = slices[0][1] - slices[0][0]
    if nb_indices == 1:
        Nj = 1
    elif nb_indices == 2:
        if len(slices[1]) == 1:
            Nj = 1
        else:
            Nj = slices[1][1] - slices[1][0]
    else:
        raise NotImplementedError(
            "Cannot evaluate series with more than 2 indices"
        )

    return Ni, Nj

def get_ind_middle(serie):
    Ni, Nj = get_Ni_Nj(serie)
    ind_middle_start = int(math.floor(Ni / 2.0)) * Nj
    ind_middle_end = int(math.ceil(Ni / 2.0)) * Nj
    if Ni % 2 == 0:
        ind_middle_start -= 1

    return ind_middle_start, ind_middle_end

def get_name_preproc(serie, name_files, ind_serie, nb_series, out_format):
    ind_middle_start, ind_middle_end = get_ind_middle(serie)

    if ind_serie == 0 and nb_series == 1:
        s = -1
    elif ind_serie == 0:
        s = ind_middle_start
    elif ind_serie == nb_series - 1:
        s = -1
    else:
        s = ind_middle_start

    if out_format == "img":
        return name_files[s]

    else:
        fname, ext = os.path.splitext(name_files[s])
        fname += "." + out_format
        return fname

class ArraySerie(ArrayCouple):
    def __init__(
        self,
        names=None,
        arrays=None,
        serie=None,
        ind_serie=0,
        nb_series=1,
        str_path=None,
        hdf5_parent=None,
    ):

        if str_path is not None:
            self._load(path=str_path)
            return

        if hdf5_parent is not None:
            self._load(hdf5_object=hdf5_parent["serie"])
            return

        if serie is not None:
            names = serie.get_name_arrays()
            paths = serie.get_path_arrays()
            self.paths = tuple(os.path.abspath(p) for p in paths)

            if arrays is None:
                arrays = serie.get_arrays()

        self.ind_serie = ind_serie
        self.nb_series = nb_series
        self.names = tuple(names)
        self.arrays = tuple(arrays)
        self.serie = serie

    def _clear_data(self):
        self.arrays = tuple()

    def save(self, path=None, hdf5_parent=None):
        if path is not None:
            raise NotImplementedError

        if not isinstance(hdf5_parent, (h5py.File,)):
            raise NotImplementedError

        hdf5_parent.create_group("serie")
        group = hdf5_parent["serie"]
        group.attrs["names"] = self.names
        group.attrs["paths"] = self.paths

class PreprocResults(LightPIVResults):
    def __init__(self, params=None, str_path=None, hdf5_object=None):

        self._keys_to_be_saved = ["data"]
        if hdf5_object is not None:
            if params is not None:
                self.params = params

            self._load_from_hdf5_object(hdf5_object)
            return

        if str_path is not None:
            self._load(str_path)
            return

        self.params = params
        self.data = {}

    def _clear_data(self):
        self.data = {}

    def save(self, path=None):
        out_format = self.params.saving.format
        for k, v in self.data.items():
            path_file = os.path.join(path, k)
            if out_format == "img":
                imsave(path_file, v, as_int=True)
            elif out_format == "h5":
                attrs = {
                    "class_name": "PreprocResults",
                    "module_name": self.__module__,
                }
                imsave_h5(path_file, v, self.params, attrs, as_int=True)
            else:
                # Try to save in formats supported by PIL.Image
                imsave(path_file, v, format=out_format, as_int=True)

        self._clear_data()
        return self


from rest_framework import serializers

from spacare.quyen.models import Quyen

class QuyenSerializer(serializers.ModelSerializer):
    class Meta:
        model = Quyen
        fields = "__all__"
        extra_kwargs = {
            "id": {"read_only": True},
            "created_at": {"read_only": True},
            "updated_at": {"read_only": True},
        }


import mmcv
import torch
from mmcv.parallel import collate, scatter
from mmcv.runner import load_checkpoint

from mmcompression.datasets.pipelines import Compose
from mmcompression.models import build_compressor

def init_compressor(config, checkpoint=None, device="cuda:0"):
    """Initialize a compressor from config file.

    Args:
        config (str or :obj:`mmcv.Config`): Config file path or the config
            object.
        checkpoint (str, optional): Checkpoint path. If left as None, the model
            will not load any weights.
        device (str, optional) CPU/CUDA device option. Default 'cuda:0'.
            Use 'cpu' for loading model on CPU.
    Returns:
        nn.Module: The constructed compressor.
    """
    if isinstance(config, str):
        config = mmcv.Config.fromfile(config)
    elif not isinstance(config, mmcv.Config):
        raise TypeError(
            f"config must be a filename or Config object,but got {type(config)}"
        )
    if checkpoint is not None:
        config.model.pretrained = None
    config.model.train_cfg = None
    model = build_compressor(config.model, test_cfg=config.get("test_cfg"))
    if checkpoint is not None:
        checkpoint = load_checkpoint(model, checkpoint, map_location="cpu")
    model.cfg = config  # save the config in the model for convenience
    model.to(device)
    model.eval()
    return model

class LoadImage:
    """A simple pipeline to load image."""

    def __call__(self, results):
        """Call function to load images into results.

        Args:
            results (dict): A result dict contains the file name
                of the image to be read.

        Returns:
            dict: ``results`` will be returned containing loaded image.
        """

        if isinstance(results["img"], str):
            results["filename"] = results["img"]
            results["ori_filename"] = results["img"]
        else:
            results["filename"] = None
            results["ori_filename"] = None
        img = mmcv.imread(results["img"])
        results["img"] = img
        results["img_shape"] = img.shape
        results["ori_shape"] = img.shape
        return results

def inference_compressor(model, img, return_image=False):
    """Inference image(s) with the compressor.

    Args:
        model (nn.Module): The loaded compressor.
        img (str/ndarray or list[str/ndarray]): Either image files or loaded
            images.
        return_image (bool): Whether return the image after inference.
    """
    cfg = model.cfg
    device = next(model.parameters()).device  # model device
    # build the data pipeline
    test_pipeline = [LoadImage()] + cfg.data.test.pipeline[1:]
    # test_pipeline = cfg.data.test.pipeline
    test_pipeline = Compose(test_pipeline)
    # prepare data
    data = dict(img=img)
    data = test_pipeline(data)
    data = collate([data], samples_per_gpu=1)
    if next(model.parameters()).is_cuda:
        # scatter to specified GPU
        data = scatter(data, [device])[0]
    else:
        data["img_metas"] = [i[0] for i in data["img_metas"].data]
    # forward the model
    with torch.no_grad():
        result = model(return_loss=False, return_image=return_image, **data)
    return result


import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

from plotly.graph_objs import Scatter, Data
from plotly.offline import plot

from scipy.optimize import curve_fit

neurons_count = 2048

def weight_variable(shape):
    # Since we're using ReLU neurons, it is also good practice to initialize them
    # with a slightly positive initial bias to avoid "dead neurons".
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)

def bias_variable(shape):
    # just constant
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)

def conv2d(x, W):
    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

def avg_pool_2x2(x):
    return tf.nn.avg_pool(x, ksize=[1, 2, 2, 1],
                          strides=[1, 2, 2, 1], padding='SAME')

# Import data
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

# Create the model
x = tf.placeholder(tf.float32, [None, 784])
y_ = tf.placeholder(tf.float32, [None, 10])
# 28x28 pixels, 1 color, -1 means calculate automatically this dimension such that
# in total we have 784 elements
x_image = tf.reshape(x, [-1, 28, 28, 1])

# Define architecture of CNN

# 1-st convolutional layer
# 5x5 patch, 1 color (input channel), 32 features (output channel)
W_conv1 = weight_variable([5, 5, 1, 32])
# 32 features requires 32 biases
b_conv1 = bias_variable([32])

h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)  # 28x28x32
h_pool1 = avg_pool_2x2(h_conv1)  # 14x14x32

# Fully-connected layer
# 7x7x64=3136 neurons -> 1024 neurons (fully-connected, usual matrix multiplication)
W_fc1 = weight_variable([14 * 14 * 32, neurons_count])
b_fc1 = bias_variable([neurons_count])

h_pool1_flat = tf.reshape(h_pool1, [-1, 14 * 14 * 32])  # 1x3136
h_fc1 = tf.nn.relu(tf.matmul(h_pool1_flat, W_fc1) + b_fc1)  # 1x1024

# Readout layer
# 1024 learned features and 10 output digits
W_fc2 = weight_variable([neurons_count, 10])
b_fc2 = bias_variable([10])

# TODO: experiment with W_fc1
y_conv = tf.matmul(h_fc1, W_fc2) + b_fc2  # 1x10

# Define loss and optimizer
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(y_conv, y_))
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)

# Start session
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()

correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
sess.run(tf.global_variables_initializer())
train_accuracy = []
for i in range(1000):
    batch = mnist.train.next_batch(50)
    train_accuracy.append(accuracy.eval(feed_dict={x: batch[0], y_: batch[1]}))
    print("step %d, training accuracy %g" % (i, train_accuracy[-1]))
    train_step.run(feed_dict={x: batch[0], y_: batch[1]})

def fit_f(x, a, b, c, d):
    return a*x**0.5 + b*x**0.25 + c*x**0.125 + d*x**0.0625

fit_args, _ = curve_fit(fit_f, list(range(len(train_accuracy))), train_accuracy)
train_accuracy_smoothed = [fit_f(x, *fit_args) for x in range(len(train_accuracy))]
plot(Data([
    Scatter(y=train_accuracy, name='Train accuracy'),
    Scatter(y=train_accuracy_smoothed, line=dict(shape='spline'), name='Train accuracy (smoothed)'),
]), image='svg')

print("test accuracy %g"%accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels}))


import cv2
import matplotlib.pyplot as plt
import numpy as np
from utils.ocg_utils import get_line_pixels

def visualize_spa_info(
    spa,
    draw_plane=False,
    corners=None,
    start_uv=None,
    end_uv=None,
    break_line=None
):

    H, W = spa.ocg.get_shape()
    image = np.zeros((H, W, 3), dtype=np.uint8)
    if draw_plane:
        plane_map = spa.get_plane_density()
        plane_map[plane_map > 0] = 1
        vs, us = plane_map.nonzero()
        image[vs, us, :] = np.array([127, 127, 127])
    else:
        all_nodes = spa.get_valid_pixels()
        image[all_nodes[:, 1], all_nodes[:, 0], :] = np.array([127, 127, 127])

    area = spa.get_interior_area()
    image[area] = np.array([0, 127, 0])

    if start_uv is not None:
        # Pink
        cv2.circle(image, (start_uv[0], start_uv[1]), 1, (254, 140, 173), -1)
    if end_uv is not None:
        # Green
        cv2.circle(image, (end_uv[0], end_uv[1]), 1, (0, 255, 0), -1)
    if break_line is not None:
        line_u, line_v = get_line_pixels(break_line[0], break_line[1])
        image[line_v, line_u, :] = np.array([255, 0, 0])

    return image

def visualize_room_result(spa, corners_uv, start_uv=None, end_uv=None, draw_plane=False):
    ocg = spa.ocg
    H, W = ocg.get_shape()
    image = np.zeros((H, W, 3), dtype=np.uint8)

    # Draw valid pixels
    if draw_plane:
        plane_map = spa.get_plane_density()
        plane_map[plane_map > 0] = 1
        vs, us = plane_map.nonzero()
        image[vs, us, :] = np.array([127, 127, 127])
    else:
        all_nodes = spa.get_valid_pixels()
        image[all_nodes[:, 1], all_nodes[:, 0], :] = np.array([127, 127, 127])

    # Draw interior area
    area = spa.get_interior_area()
    image[area] = np.array([0, 127, 0])

    # Draw result room shape
    size = corners_uv.shape[0]
    for i in range(size):
        x_u, x_v = corners_uv[i, :]
        y_u, y_v = corners_uv[(i+1) % size, :]

        cv2.line(image, (x_u, x_v), (y_u, y_v), (255, 255, 255), 1)
        cv2.circle(image, (x_u, x_v), 1, (255, 0, 0), -1)
        cv2.circle(image, (y_u, y_v), 1, (255, 0, 0), -1)
    if start_uv is not None:
        # Pink
        cv2.circle(image, (start_uv[0], start_uv[1]), 1, (254, 140, 173), -1)
    if end_uv is not None:
        # Green
        cv2.circle(image, (end_uv[0], end_uv[1]), 1, (0, 255, 0), -1)
    return image

def plot_room_result(spa, corners_uv, start_uv=None, end_uv=None):
    image_spa = visualize_room_result(spa, corners_uv, start_uv, end_uv, draw_plane=False)
    image_plane = visualize_room_result(spa, corners_uv, start_uv, end_uv, draw_plane=True)
    plt.figure("Room shape result")
    plt.clf()
    plt.subplot(121)
    plt.imshow(image_spa)
    plt.subplot(122)
    plt.imshow(image_plane)
    plt.draw()
    plt.waitforbuttonpress(0.1)


import numpy as np
import pytest

from pandas_profiling.report.formatters import (
    fmt_array,
    fmt_bytesize,
    fmt_class,
    fmt_color,
    fmt_numeric,
)

@pytest.mark.parametrize(
    "text, color, expected",
    [("This is a warning", "red", 'This is a warning')],
)
def test_fmt_color(text, color, expected):
    assert fmt_color(text, color) == expected

@pytest.mark.parametrize(
    "text, cls, expected",
    [
        (
            "This text is muted",
            "text-muted",
            'This text is muted',
        )
    ],
)
def test_fmt_class(text, cls, expected):
    assert fmt_class(text, cls) == expected

@pytest.mark.parametrize(
    "num, fmt, expected",
    [
        (0, None, "0.0 B"),
        (100, "Bytes", "100.0 Bytes"),
        (1024, None, "1.0 KiB"),
        (1024.0, None, "1.0 KiB"),
        (1024 ** 4, "Bytes", "1.0 TiBytes"),
        (1024 ** 3 * 7.5, None, "7.5 GiB"),
        (1024 ** 8, None, "1.0 YiB"),
    ],
)
def test_fmt_bytesize(num, fmt, expected):
    if fmt is None:
        assert fmt_bytesize(num) == expected
    else:
        assert fmt_bytesize(num, fmt) == expected

@pytest.mark.parametrize(
    "array, threshold, expected",
    [
        (np.array([1, 2, 3], dtype=np.int16), 3, "[1 2 3]"),
        (
            np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=np.float32),
            1,
            "[ 1. ... 10.]",
        ),
        (
            np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], dtype=np.float32),
            3,
            "[ 1.  2.  3. ...  8.  9. 10.]",
        ),
    ],
)
def test_fmt_array(array, threshold, expected):
    assert fmt_array(array, threshold) == expected

@pytest.mark.parametrize(
    "value, precision, expected",
    [
        (79.999531, 3, "80"),
        (79.999531, 10, "79.999531"),
        (81.000000, 10, "81"),
        (81, 10, "81"),
        (81.999861123123123123, 10, "81.99986112"),
    ],
)
def test_fmt_numeric(value, precision, expected):
    assert fmt_numeric(value, precision) == expected


from sklearn.neighbors import BallTree, DistanceMetric
import json
import os
import numpy as np

def RGB_to_HEX(rgb):
    return '%02x%02x%02x' % rgb

# Special metric based on human's sensitivity to red, green and blue
def dist(rgbx, rgby):
    dR = rgbx[0] - rgby[0]
    dG = rgbx[1] - rgby[1]
    dB = rgbx[2] - rgby[2]

    if dR < 128:
        return 2 * dR * dR + 4 * dG * dG + 3 * dB * dB
    else:
        return 3 * dR * dR + 4 * dG * dG + 2 * dB * dB

# Extracts all the threads stored in dmc.json as the dictionary
def get_dmcs():
    src_path = os.path.dirname(__file__)
    with open(os.path.join(src_path, 'dmc.json'), 'r') as to_read:
        dmcs = json.load(to_read)
        to_read.close()
        return dict(dmcs)

# Creates the Ball Tree with already inserted threads
def get_dmc_tree():
    metric = DistanceMetric.get_metric('pyfunc', func=dist)

    dmcs = get_dmcs()
    idxs = [thread['rgb'] for key, thread in dmcs.items()]
    tree = BallTree(np.array(idxs), leaf_size=5, metric=metric)
    return tree, dmcs, idxs

# Returns the most appropriate thread using the Ball Tree created above
def get_thread(rgb, tree, idxs, dmcs):
    idx = tree.query(np.array([rgb]), return_distance=False)
    rgb = idxs[idx[0][0]]
    hex_color = RGB_to_HEX(tuple(rgb))
    return dmcs[hex_color], rgb


from collections import defaultdict #나중에 제거
from random import uniform
from math import sqrt
import numpy as np
import matplotlib.pyplot as plt
from copy import deepcopy
from player_1 import Player, Record_Hitter, Record_Pitcher
import csv

#make list of players19
f = open('player_record/hitter19.csv', 'rt', encoding='UTF8')
lines = csv.reader(f)
record19 = [line for line in lines]
f.close()

player_list = [Player(record[1], record[2], 'Hitter') for record in record19[1:]]
player_names = [record[1] for record in record19[1:]]

for player in player_list:
    for record in record19:
        if player.name == record[1]:
            player.update(Record_Hitter(record[0], record[1], record[2], record[3], record[4], record[5], record[6], record[7], record[8], record[9], record[10], record[11], record[12], record[13], 2019))

X =[[player.record[i].AVG] for i in range(len(player.record)) for player in player_list]

#k-means clustering
def point_avg(points):
    """
    Accepts a list of points, each with the same number of dimensions.
    NB. points can have more dimensions than 2

    Returns a new point which is the center of all the points.
    """
    if len(points)==1:
        new_center= np.mean(points)
    else:
        new_center= [np.mean([x[y] for x in points]) for y in range(len(points[0]))]
    return new_center

def update_centers(data_set, assignments):
    """
    Accepts a dataset and a list of assignments; the indexes
    of both lists correspond to each other.
    Compute the center for each of the assigned groups.
    Return `k` centers where `k` is the number of unique assignments.
    """
    new_means = defaultdict(list)
    for assignment, point in zip(assignments, data_set):
        new_means[assignment].append(point)
    centers = [point_avg(points) for points in new_means.values()]
    return centers

def assign_points(data_points, centers):
    """
    Given a data set and a list of points betweeen other points,
    assign each point to an index that corresponds to the index
    of the center point on it's proximity to that point.
    Return an array of indexes of centers that correspond to
    an index in the data set; that is, if there are N points
    in `data_set` the list we return will have N elements. Also
    If there are Y points in `centers` there will be Y unique
    possible values within the returned list.
    """
    assignments = []
    for point in data_points:
        shortest = 10000  # positive infinity
        shortest_index = 0
        for i in range(len(centers)):
            val = distance(point, centers[i])
            if val < shortest:
                shortest = val
                shortest_index = i
        assignments.append(shortest_index)
    return assignments

def distance(a, b):
    difference_sq= [(a[x]-b[x])**2 for x in range(len(a))]
    _sum=sum(difference_sq)
    return sqrt(_sum)

def generate_k(data_set, k):
    """
    Given `data_set`, which is an array of arrays,
    find the minimum and maximum for each coordinate, a range.
    Generate `k` random points between the ranges.
    Return an array of the random points within the ranges.
    """
    centers = []
    dimensions = len(data_set[0])
    min_max = defaultdict(int)

    for point in data_set:
        for i in range(dimensions):
            val = point[i]
            min_key = 'min_{0}d'.format(i)
            max_key = 'max_{0}d'.format(i)
            if min_key not in min_max or val < min_max[min_key]:
                min_max[min_key] = val
            if max_key not in min_max or val > min_max[max_key]:
                min_max[max_key] = val

    for _k in range(k):
        rand_point = []
        for i in range(dimensions):
            min_val = min_max['min_{0}d'.format(i)]
            max_val = min_max['max_{0}d'.format(i)]

            rand_point.append(uniform(min_val, max_val))
        centers.append(rand_point)
    return centers

def k_means(dataset, k):
    k_points = generate_k(dataset, k)
    assignments = assign_points(dataset, k_points)
    old_assignments = None
    while assignments != old_assignments:
        new_centers = update_centers(dataset, assignments)
        old_assignments = assignments
        assignments = assign_points(dataset, new_centers)
    return zip(assignments, dataset)

Player_kmeans = k_means(X, 3)
Player_kmeans = list(Player_kmeans)
k_values=[i[0] for i in Player_kmeans]
player_values=[i[1] for i in Player_kmeans]

fig=plt.figure(figsize=(5,5))
axis = fig.add_subplot(111)
axis.set_title('k-means clustering with AVG of KBO players in 2019')
axis.grid(True)

plt.scatter(k_values, player_values, c = k_values, edgecolor = 'none', s=100)
plt.show()