from management.utils import validate_and_get_key
from rest_framework import status
from rest_framework.response import Response
from rest_framework.views import APIView

from api.common.pagination import StandardResultsSetPagination
from .proxy import PrincipalProxy
from ..permissions.principal_access import PrincipalAccessPermission

USERNAMES_KEY = "usernames"
EMAIL_KEY = "email"
SORTORDER_KEY = "sort_order"
VALID_SORTORDER_VALUE = ["asc", "desc"]
MATCH_CRITERIA_KEY = "match_criteria"
VALID_MATCH_VALUE = ["partial", "exact"]
STATUS_KEY = "status"
VALID_STATUS_VALUE = ["enabled", "disabled", "all"]
ADMIN_ONLY_KEY = "admin_only"
VALID_BOOLEAN_VALUE = ["true", "false"]
USERNAME_ONLY_KEY = "username_only"

class PrincipalView(APIView):
    """Obtain the list of principals for the tenant."""

    """
    @api {get} /api/v1/principals/   Obtain a list of principals
    @apiName getPrincipals
    @apiGroup Principal
    @apiVersion 1.0.0
    @apiDescription Obtain a list of principals

    @apiHeader {String} token User authorization token

    @apiParam (Query) {Number} offset Parameter for selecting the start of data (default is 0).
    @apiParam (Query) {Number} limit Parameter for selecting the amount of data (default is 10).

    @apiSuccess {Object} meta The metadata for pagination.
    @apiSuccess {Object} links  The object containing links of results.
    @apiSuccess {Object[]} data  The array of results.

    @apiSuccessExample {json} Success-Response:
        HTTP/1.1 200 OK
        {
            'meta': {
                'count': 2
            }
            'links': {
                'first': /api/v1/principals/?offset=0&limit=10,
                'next': None,
                'previous': None,
                'last': None
            },
            'data': [
                            {
                                "username": "jsmith",
                                "email": "jsmith@company.com"
                            },
                            {
                                "username": "ksmith",
                                "email": "ksmith@company.com"
                            }
                        ]
        }

    """

    permission_classes = (PrincipalAccessPermission,)

    def get(self, request):
        """List principals for account."""
        user = request.user
        path = request.path
        query_params = request.query_params
        default_limit = StandardResultsSetPagination.default_limit
        usernames_filter = ""
        options = {}
        try:
            limit = int(query_params.get("limit", default_limit))
            offset = int(query_params.get("offset", 0))
            options["sort_order"] = validate_and_get_key(query_params, SORTORDER_KEY, VALID_SORTORDER_VALUE, "asc")
            options["status"] = validate_and_get_key(query_params, STATUS_KEY, VALID_STATUS_VALUE, "enabled")
        except ValueError:
            error = {
                "detail": "Values for limit and offset must be positive numbers.",
                "source": "principals",
                "status": str(status.HTTP_400_BAD_REQUEST),
            }
            errors = {"errors": [error]}
            return Response(status=status.HTTP_400_BAD_REQUEST, data=errors)

        previous_offset = 0
        if offset - limit > 0:
            previous_offset = offset - limit

        resp, usernames_filter = self.users_from_proxy(user, query_params, options, limit, offset)

        status_code = resp.get("status_code")
        response_data = {}
        if status_code == status.HTTP_200_OK:
            data = resp.get("data", [])
            if isinstance(data, dict):
                count = data.get("userCount")
                data = data.get("users")
            elif isinstance(data, list):
                count = len(data)
            else:
                count = None
            response_data["meta"] = {"count": count}
            response_data["links"] = {
                "first": f"{path}?limit={limit}&offset=0{usernames_filter}",
                "next": f"{path}?limit={limit}&offset={offset + limit}{usernames_filter}",
                "previous": f"{path}?limit={limit}&offset={previous_offset}{usernames_filter}",
                "last": None,
            }
            response_data["data"] = data
        else:
            response_data = resp
            del response_data["status_code"]

        return Response(status=status_code, data=response_data)

    def users_from_proxy(self, user, query_params, options, limit, offset):
        """Format principal request for proxy and return prepped result."""
        proxy = PrincipalProxy()
        usernames = query_params.get(USERNAMES_KEY)
        email = query_params.get(EMAIL_KEY)
        match_criteria = validate_and_get_key(query_params, MATCH_CRITERIA_KEY, VALID_MATCH_VALUE, "exact")
        options["username_only"] = validate_and_get_key(query_params, USERNAME_ONLY_KEY, VALID_BOOLEAN_VALUE, "false")

        if not usernames and not email:
            options["admin_only"] = validate_and_get_key(query_params, ADMIN_ONLY_KEY, VALID_BOOLEAN_VALUE, "false")
            resp = proxy.request_principals(
                account=user.account, org_id=user.org_id, limit=limit, offset=offset, options=options
            )
            return resp, ""
        proxyInput = {}
        resp = None
        if usernames:
            principals = usernames.split(",")
            if match_criteria != "partial":
                resp = proxy.request_filtered_principals(
                    principals,
                    account=user.account,
                    org_id=user.org_id,
                    limit=limit,
                    offset=offset,
                    options=options,
                )
                usernames_filter = f"&usernames={usernames}"
                return resp, usernames_filter
            else:
                proxyInput["principalStartsWith"] = principals[0]
        if email:
            if match_criteria == "partial":
                proxyInput["emailStartsWith"] = email
            else:
                proxyInput["primaryEmail"] = email
        resp = proxy.request_principals(
            account=user.account, org_id=user.org_id, input=proxyInput, limit=limit, offset=offset, options=options
        )
        return resp, ""


import os
import numpy as np
import torch
import pytorch_lightning as pl
from collections import defaultdict
import cem.train.training as cem_train
import logging
import io
from contextlib import redirect_stdout

################################################################################
## HELPER FUNCTIONS
################################################################################

def load_trained_model(
    config,
    n_tasks,
    result_dir,
    n_concepts,
    split=0,
    imbalance=None,
    task_class_weights=None,
    train_dl=None,
    sequential=False,
    independent=False,
    gpu=int(torch.cuda.is_available()),
    intervention_policy=None,
    intervene=False,
):
    arch_name = config.get('c_extractor_arch', "")
    if not isinstance(arch_name, str):
        arch_name = "lambda"
    if split is not None:
        full_run_name = (
            f"{config['architecture']}{config.get('extra_name', '')}_"
            f"{arch_name}_fold_{split + 1}"
        )
    else:
        full_run_name = (
            f"{config['architecture']}{config.get('extra_name', '')}_"
            f"{arch_name}"
        )
    selected_concepts = np.arange(n_concepts)
    if sequential:
        extra = "Sequential"
    elif independent:
        extra = "Independent"
    else:
        extra = ""
    model_saved_path = os.path.join(
        result_dir or ".",
        f'{extra}{full_run_name}.pt'
    )

    if (
        ((intervention_policy is not None) or intervene) and
        (train_dl is not None) and
        (config['architecture'] == "ConceptBottleneckModel") and
        (not config.get('sigmoidal_prob', True))
    ):
        # Then let's look at the empirical distribution of the logits in order
        # to be able to intervene
        model = cem_train.construct_model(
            n_concepts=n_concepts,
            n_tasks=n_tasks,
            config=config,
            imbalance=imbalance,
            task_class_weights=task_class_weights,
        )
        model.load_state_dict(torch.load(model_saved_path))
        trainer = pl.Trainer(
            gpus=gpu,
        )
        batch_results = trainer.predict(model, train_dl)
        out_embs = np.concatenate(
            list(map(lambda x: x[1], batch_results)),
            axis=0,
        )
        active_intervention_values = []
        inactive_intervention_values = []
        for idx in range(n_concepts):
            active_intervention_values.append(
                np.percentile(out_embs[:, idx], 95)
            )
            inactive_intervention_values.append(
                np.percentile(out_embs[:, idx], 5)
            )
        if gpu:
            active_intervention_values = torch.cuda.FloatTensor(
                active_intervention_values
            )
            inactive_intervention_values = torch.cuda.FloatTensor(
                inactive_intervention_values
            )
        else:
            active_intervention_values = torch.FloatTensor(
                active_intervention_values
            )
            inactive_intervention_values = torch.FloatTensor(
                inactive_intervention_values
            )
    else:
        active_intervention_values = inactive_intervention_values = None

    if sequential:
        _, c2y_model = cem_train.construct_sequential_models(
            n_concepts=n_concepts,
            n_tasks=n_tasks,
            config=config,
            imbalance=imbalance,
            task_class_weights=task_class_weights,
        )
    elif independent:
        _, c2y_model = cem_train.construct_sequential_models(
            n_concepts=n_concepts,
            n_tasks=n_tasks,
            config=config,
            imbalance=imbalance,
            task_class_weights=task_class_weights,
        )
    else:
        c2y_model = None
    model = cem_train.construct_model(
        n_concepts=n_concepts,
        n_tasks=n_tasks,
        config=config,
        imbalance=imbalance,
        task_class_weights=task_class_weights,
        active_intervention_values=active_intervention_values,
        inactive_intervention_values=inactive_intervention_values,
        intervention_policy=intervention_policy,
        c2y_model=c2y_model,
    )

    model.load_state_dict(torch.load(model_saved_path))
    return model

################################################################################
## CONCEPT INTERVENTION SELECTION POLICIES
################################################################################

def random_int_policy(num_groups_intervened, concept_group_map, config=None):
    selected_groups_for_trial = np.random.choice(
        list(concept_group_map.keys()),
        size=num_groups_intervened,
        replace=False,
    )
    intervention_idxs = []
    for selected_group in selected_groups_for_trial:
        intervention_idxs.extend(concept_group_map[selected_group])
    return sorted(intervention_idxs)

class InterventionPolicyWrapper(object):

    def __init__(self, policy_fn, num_groups_intervened, concept_group_map):
        self.policy_fn = policy_fn
        self.num_groups_intervened = num_groups_intervened
        self.concept_group_map = concept_group_map

    def __call__(self, x, y, c):
        intervention_idxs = self.policy_fn(
            num_groups_intervened=self.num_groups_intervened,
            concept_group_map=self.concept_group_map,
        )
        return intervention_idxs, c

class IndependentRandomMaskIntPolicy(object):

    def __init__(self, num_groups_intervened, concept_group_map):
        self.num_groups_intervened = num_groups_intervened
        self.concept_group_map = concept_group_map

    def __call__(self, x, y, c):
        # We have to split it into a list contraction due to the
        # fact that we can't afford to run a np.random.choice
        # that does not allow replacement between samples...
        selected_groups_for_trial = np.array([
            np.random.choice(
                list(self.concept_group_map.keys()),
                size=self.num_groups_intervened,
                replace=False,
            ) for _ in range(x.shape[0])
        ])
        mask = np.zeros((x.shape[0], c.shape[-1]), dtype=np.int64)
        for sample_idx in range(selected_groups_for_trial.shape[0]):
            for selected_group in selected_groups_for_trial[sample_idx, :]:
                mask[sample_idx, self.concept_group_map[selected_group]] = 1
        return mask, c

################################################################################
## MAIN INTERVENTION FUNCTION
################################################################################

def intervene_in_cbm(
    config,
    test_dl,
    n_tasks,
    n_concepts,
    result_dir,
    imbalance=None,
    task_class_weights=None,
    adversarial_intervention=False,
    train_dl=None,
    sequential=False,
    independent=False,
    concept_group_map=None,
    intervened_groups=None,
    gpu=int(torch.cuda.is_available()),
    split=0,
    concept_selection_policy=random_int_policy,
    rerun=False,
    old_results=None,
    batch_size=None,
):
    if batch_size is not None:
        # Then overwrite the config's batch size
        test_dl = torch.utils.data.DataLoader(
            dataset=test_dl.dataset,
            batch_size=batch_size,
            num_workers=test_dl.num_workers,
        )
    intervention_accs = []
    # If no concept groups are given, then we assume that all concepts
    # represent a unitary group themselves
    concept_group_map = concept_group_map or dict(
        [(i, [i]) for i in range(n_concepts)]
    )
    groups = intervened_groups or list(range(0, len(concept_group_map) + 1, 1))

    if (not rerun) and (old_results is not None):
        for j, num_groups_intervened in enumerate(groups):
            print(
                f"\tTest accuracy when intervening with {num_groups_intervened} "
                f"concept groups is {old_results[j] * 100:.2f}%."
            )
        return old_results
    config["shared_prob_gen"] = config.get("shared_prob_gen", False)
    config["per_concept_weight"] = config.get(
        "per_concept_weight",
        False,
    )

    model = load_trained_model(
        config=config,
        n_tasks=n_tasks,
        n_concepts=n_concepts,
        result_dir=result_dir,
        split=split,
        imbalance=imbalance,
        task_class_weights=task_class_weights,
        intervene=True,
        train_dl=train_dl,
        sequential=sequential,
        independent=independent,
    )

    for j, num_groups_intervened in enumerate(groups):
        logging.debug(
            f"Intervening with {num_groups_intervened} out of "
            f"{len(concept_group_map)} concept groups"
        )
        n_trials = config.get('intervention_trials', 1)
        avg = []
        for trial in range(n_trials):

            logging.debug(
                f"\tFor trial {trial + 1}/{n_trials} for split {split} with "
                f"{num_groups_intervened} groups intervened"
            )

            ####
            # Set the model's intervention policy
            ####

            # Example of how to use an index-generating policy!
            #model.intervention_policy = InterventionPolicyWrapper(
            #    policy_fn=concept_selection_policy,
            #    num_groups_intervened=num_groups_intervened,
            #    concept_group_map=concept_group_map,
            #)
            # Example of how to use a mask-generating policy!
            model.intervention_policy = IndependentRandomMaskIntPolicy(
                num_groups_intervened=num_groups_intervened,
                concept_group_map=concept_group_map,
            )

            trainer = pl.Trainer(
                gpus=gpu,
            )
            f = io.StringIO()
            with redirect_stdout(f):
                [test_results] = trainer.test(model, test_dl, verbose=False,)
            acc = test_results['test_y_accuracy']
            avg.append(acc)
            logging.debug(
                f"\tFor model at split {split}, intervening with "
                f"{num_groups_intervened} groups (trial {trial + 1}) gives "
                f"test task accuracy {acc * 100:.2f}%."
            )
        print(
            f"\tTest accuracy when intervening with {num_groups_intervened} "
            f"concept groups is "
            f"{np.mean(avg) * 100:.2f}% ± {np.std(avg)* 100:.2f}%."
        )
        intervention_accs.append(np.mean(avg))
    return intervention_accs


#!/usr/bin/env python

import os.path
import sys

import agate

from csvkit.cli import CSVKitUtility, make_default_headers

class CSVStack(CSVKitUtility):
    description = 'Stack up the rows from multiple CSV files, optionally adding a grouping value. Files are assumed to have the same columns in the same order.'
    # Override 'f' because the utility accepts multiple files.
    override_flags = ['f', 'L', 'blanks', 'date-format', 'datetime-format']

    def add_arguments(self):
        self.argparser.add_argument(metavar='FILE', nargs='*', dest='input_paths', default=['-'],
                                    help='The CSV file(s) to operate on. If omitted, will accept input as piped data via STDIN.')
        self.argparser.add_argument('-g', '--groups', dest='groups',
                                    help='A comma-separated list of values to add as "grouping factors", one for each CSV being stacked. These will be added to the stacked CSV as a new column. You may specify a name for the grouping column using the -n flag.')
        self.argparser.add_argument('-n', '--group-name', dest='group_name',
                                    help='A name for the grouping column, e.g. "year". Only used when also specifying -g.')
        self.argparser.add_argument('--filenames', dest='group_by_filenames', action='store_true',
                                    help='Use the filename of each input file as its grouping value. When specified, -g will be ignored.')

    def main(self):
        if sys.stdin.isatty() and not self.args.input_paths:
            sys.stderr.write('No input file or piped data provided. Waiting for standard input:\n')

        has_groups = self.args.groups is not None or self.args.group_by_filenames

        if self.args.groups is not None and not self.args.group_by_filenames:
            groups = self.args.groups.split(',')

            if len(groups) != len(self.args.input_paths):
                self.argparser.error('The number of grouping values must be equal to the number of CSV files being stacked.')
        else:
            groups = None

        group_name = self.args.group_name if self.args.group_name else 'group'

        output = agate.csv.writer(self.output_file, **self.writer_kwargs)

        for i, path in enumerate(self.args.input_paths):
            f = self._open_input_file(path)

            if isinstance(self.args.skip_lines, int):
                skip_lines = self.args.skip_lines
                while skip_lines > 0:
                    f.readline()
                    skip_lines -= 1
            else:
                raise ValueError('skip_lines argument must be an int')

            rows = agate.csv.reader(f, **self.reader_kwargs)

            if has_groups:
                if groups:
                    group = groups[i]
                else:
                    group = os.path.basename(f.name)

            # If we have header rows, use them
            if not self.args.no_header_row:
                headers = next(rows, [])

                if i == 0:
                    if has_groups:
                        headers.insert(0, group_name)

                    output.writerow(headers)
            # If we don't generate simple column names based on first row
            else:
                row = next(rows, [])

                headers = make_default_headers(len(row))

                if i == 0:
                    if has_groups:
                        headers.insert(0, group_name)

                    output.writerow(headers)

                if has_groups:
                    row.insert(0, group)

                output.writerow(row)

            for row in rows:
                if has_groups:
                    row.insert(0, group)

                output.writerow(row)

            f.close()

def launch_new_instance():
    utility = CSVStack()
    utility.run()

if __name__ == '__main__':
    launch_new_instance()


from django.conf import settings
from django.urls import reverse

import pytest

from bbbs.common.choices import TagChoices

from . import factories

pytestmark = pytest.mark.django_db
PAGE_SIZE = settings.REST_FRAMEWORK.get('PAGE_SIZE')

def test_rights_endpoint_available(client):
    tags = factories.TagFactory.create_batch(3, model=TagChoices.RIGHTS)
    factories.RightFactory.create_batch(PAGE_SIZE + 1, tags=tags)
    url = reverse('rights-list')
    response = client.get(url)
    response_data = response.json()

    assert response.status_code == 200
    assert 'count' in response_data
    assert 'next' in response_data
    assert 'previous' in response_data
    assert 'results' in response_data
    assert type(response_data['results']) == list
    assert response_data['count'] == PAGE_SIZE + 1
    assert len(response_data['results']) == PAGE_SIZE

def test_rights_detail_endpoint_avilable(client):
    tags = factories.TagFactory.create_batch(3, model=TagChoices.RIGHTS)
    right = factories.RightFactory.create(tags=tags)
    url = reverse('rights-detail', kwargs={'pk': right.id})
    expected = {
        'title': right.title,
        'description': right.description,
        'text': right.text,
        'color': right.color,
        'image': right.image,
        'tags': [{'id': tag.id, 'name': tag.name, 'slug': tag.slug} for tag in tags]
    }
    response = client.get(url)

    assert response.status_code == 200
    assert expected == response.json()

def test_right_endpoint_readonly(mentor_client):
    right = factories.RightFactory.create()
    url_list = reverse('rights-list')
    url_detail = reverse('rights-detail', kwargs={'pk': right.id})

    response = mentor_client.post(url_list, data={})
    assert response.status_code == 405

    response = mentor_client.patch(url_detail, data={})
    assert response.status_code == 405

    response = mentor_client.delete(url_detail)
    assert response.status_code == 405

def test_rights_extra_tag_action_available(client):
    tags = factories.TagFactory.create_batch(5, model=TagChoices.RIGHTS)
    url = reverse('rights-tags')
    response = client.get(url)
    assert response.status_code == 200
    assert len(response.json()) == len(tags)

OTHER_TAGS = [
    TagChoices.EVENTS,
    TagChoices.MOVIES,
    TagChoices.PLACES,
    TagChoices.QUESTIONS,
    TagChoices.BOOKS,
]

@pytest.mark.parametrize('model', OTHER_TAGS)
def test_rights_tag_action_returns_only_tags_for_rights(client, model):
    rights_tags = factories.TagFactory.create_batch(5, model=TagChoices.RIGHTS)
    factories.TagFactory.create_batch(10, model=model)
    url = reverse('rights-tags')
    response = client.get(url)
    data = response.json()

    assert response.status_code == 200
    assert isinstance(data, list)
    assert len(data) == len(rights_tags)


import tkinter as tk
from tkinter import *
from tkinter import messagebox
from tkinter.ttk import *
import re
import openpyxl

# Hàm kiểm tra mã số sinh viên hợp lệ
def is_valid_student_id(student_id):
    return bool(re.match(r'^\d{7}$', student_id))

# Hàm kiểm tra email hợp lệ
def is_valid_email(email):
    return bool(re.match(r'^[\w\.-]+@[\w\.-]+$', email))

# Hàm kiểm tra số điện thoại hợp lệ
def is_valid_phone(phone):
    return bool(re.match(r'^\d{10}$', phone))

# Hàm kiểm tra học kỳ hợp lệ
def is_valid_semester(semester):
    return semester in ['1', '2', '3']

# Hàm kiểm tra định dạng ngày sinh hợp lệ
def is_valid_date_of_birth(date_of_birth):
    return bool(re.match(r'^\d{2}/\d{2}/\d{4}$', date_of_birth))

# Hàm lưu thông tin vào tệp Excel
def save_to_excel(student_id, full_name, email, phone, semester, date_of_birth, academic_year, selected_courses):
    wb = openpyxl.load_workbook('C:\\Python\\Lab04\\Cau6\\student_info.xlsx')
    sheet = wb.active
    row = [student_id, full_name, email, phone, semester, date_of_birth, academic_year] + selected_courses
    sheet.append(row)
    wb.save('C:\\Python\\Lab04\\Cau6\\student_info.xlsx')
    messagebox.showinfo("Thông báo", "Đăng ký thành công!")

# Hàm xử lý sự kiện khi nhấn nút Đăng ký
def register():
    student_id = student_id_entry.get()
    full_name = full_name_entry.get()
    email = email_entry.get()
    phone = phone_entry.get()
    semester = semester_var.get()
    date_of_birth = date_of_birth_entry.get()
    academic_year = academic_year_var.get()

    selected_courses = []
    for course_var in course_vars:
        if course_var.get():
            selected_courses.append(course_var.get())

    if not student_id or not full_name or not email or not phone or not semester or not date_of_birth or not academic_year:
        messagebox.showerror("Lỗi", "Vui lòng điền đầy đủ thông tin!")
        return

    if not is_valid_student_id(student_id):
        messagebox.showerror("Lỗi", "Mã số sinh viên không hợp lệ!")
        return

    if not is_valid_email(email):
        messagebox.showerror("Lỗi", "Email không hợp lệ!")
        return

    if not is_valid_phone(phone):
        messagebox.showerror("Lỗi", "Số điện thoại không hợp lệ!")
        return

    if not is_valid_semester(semester):
        messagebox.showerror("Lỗi", "Học kỳ không hợp lệ!")
        return

    if not is_valid_date_of_birth(date_of_birth):
        messagebox.showerror("Lỗi", "Ngày sinh không hợp lệ! (dd/mm/yyyy)")
        return

    if not selected_courses:
        messagebox.showerror("Lỗi", "Vui lòng chọn ít nhất một môn học!")
        return

    save_to_excel(student_id, full_name, email, phone, semester, date_of_birth, academic_year, selected_courses)

# Tạo giao diện
root = tk.Tk()
root.title("Đăng ký sinh viên")
root.geometry("600x450")

# Mã số sinh viên
student_id_label = tk.Label(root, text="THÔNG TIN SINH VIÊN")
student_id_label.grid(row=0, column=1, sticky= W, pady = 2)
student_id_label = tk.Label(root, text="Mã số sinh viên:")
student_id_label.grid(row=1, column=0, sticky = W, pady = 2)
student_id_entry = tk.Entry(root)
student_id_entry.grid(row=1, column=1, ipadx=100)

# Họ tên sinh viên
full_name_label = tk.Label(root, text="Họ tên sinh viên:")
full_name_label.grid(row=2, column=0, sticky = W, pady = 2)
full_name_entry = tk.Entry(root)
full_name_entry.grid(row=2, column=1, ipadx=100)

# Email
email_label = tk.Label(root, text="Email:")
email_label.grid(row=3, column=0, sticky = W, pady = 2)
email_entry = tk.Entry(root)
email_entry.grid(row=3, column=1, ipadx=100)

# Số điện thoại
phone_label = tk.Label(root, text="Số điện thoại:")
phone_label.grid(row=4, column=0, sticky = W, pady = 2)
phone_entry = tk.Entry(root)
phone_entry.grid(row=4, column=1, ipadx=100)

# Học kỳ
semester_label = tk.Label(root, text="Học kỳ:")
semester_label.grid(row=5, column=0, sticky = W, pady = 2)
semester_var = tk.StringVar()
semester_var.set("1")
semester_radio1 = tk.Radiobutton(root, text="1", variable=semester_var, value="1")
semester_radio2 = tk.Radiobutton(root, text="2", variable=semester_var, value="2")
semester_radio3 = tk.Radiobutton(root, text="3", variable=semester_var, value="3")
semester_radio1.grid(row=5, column=1, sticky = W, pady = 2)
semester_radio2.grid(row=6, column=1, sticky = W, pady = 2)
semester_radio3.grid(row=7, column=1, sticky = W, pady = 2)

# Ngày sinh
date_of_birth_label = tk.Label(root, text="Ngày sinh (dd/mm/yyyy):")
date_of_birth_label.grid(row=8, column=0, sticky = W, pady = 2)
date_of_birth_entry = tk.Entry(root)
date_of_birth_entry.grid(row=8, column=1, ipadx=100)

# Năm học
academic_year_label = tk.Label(root, text="Năm học:")
academic_year_label.grid(row=9, column=0, sticky = W, pady = 2)
academic_year_var = tk.StringVar()
academic_year_var.set("2022-2023")
academic_year_optionmenu = tk.OptionMenu(root, academic_year_var, "2022-2023", "2023-2024", "2024-2025")
academic_year_optionmenu.grid(row=9, column=1, sticky = W, pady = 2)

# Môn học
courses_label = tk.Label(root, text="Chọn môn học:")
courses_label.grid(row=10, column=0, sticky = W, pady = 2)
course_vars = []
course_checkboxes = []
course_names = ["Lập trình Java", "Lập trình Python", "Công nghệ phần mềm", "Phát triển ứng dụng web"]
for i, course_name in enumerate(course_names):
    course_var = tk.StringVar()
    course_var.set("")
    course_vars.append(course_var)
    course_checkbox = tk.Checkbutton(root, text=course_name, variable=course_var, onvalue=course_name, offvalue="")
    course_checkbox.grid(row=10 + i, column=1, sticky = W, pady = 2)
    course_checkboxes.append(course_checkbox)

# Nút Đăng ký
register_button = tk.Button(root, text="Đăng ký", command=register)
register_button.grid(row=14, column=0, columnspan=4)

root.mainloop()


"""Coordinate utilities used by the interpolation program"""
import logging

import numpy as np
from scipy.spatial import KDTree

logger = logging.getLogger(__name__)

def align_path(path):
    """Rotate and translate images to minimize RMSD movements along the path.
    Also moves the geometric center of all images to the origin.
    """
    path = np.array(path)
    path[0] -= np.mean(path[0], axis=0)
    max_rmsd = 0
    for g, nextg in zip(path, path[1:]):
        rmsd, nextg[:] = align_geom(g, nextg)
        max_rmsd = max(max_rmsd, rmsd)
    return max_rmsd, path

def align_geom(refgeom, geom):
    """Find translation/rotation that moves a given geometry to maximally overlap
    with a reference geometry. Implemented with Kabsch algorithm.

    Args:
        refgeom:    The reference geometry to be rotated to
        geom:       The geometry to be rotated and shifted

    Returns:
        RMSD:       Root-mean-squared difference between the rotated geometry
                    and the reference
        new_geom:   The rotated geometry that maximumally overal with the reference
    """
    center = np.mean(refgeom, axis=0)   # Find the geometric center
    ref2 = refgeom - center
    geom2 = geom - np.mean(geom, axis=0)
    cov = np.dot(geom2.T, ref2)
    v, sv, w = np.linalg.svd(cov)
    if np.linalg.det(v) * np.linalg.det(w) < 0:
        sv[-1] = -sv[-1]
        v[:, -1] = -v[:, -1]
    u = np.dot(v, w)
    new_geom = np.dot(geom2, u) + center
    rmsd = np.sqrt(np.mean((new_geom - refgeom) ** 2))
    return rmsd, new_geom

ATOMIC_RADIUS = dict(H=0.31, He=0.28,
                     Li=1.28, Be=0.96, B=0.84, C=0.76, N=0.71, O=0.66, F=0.57, Ne=0.58,
                     Na=1.66, Mg=1.41, Al=1.21, Si=1.11, P=1.07, S=1.05, Cl=1.02, Ar=1.06)

def get_bond_list(geom, atoms=None, threshold=4, min_neighbors=4, snapshots=30, bond_threshold=1.8,
                  enforce=()):
    """Get the list of all the important atom pairs.
    Samples a number of snapshots from a list of geometries to generate all
    distances that are below a given threshold in any of them.

    Args:
        atoms:      Symbols for each atoms.
        geom:       One or a list of geometries to check for pairs
        threshold:  Threshold for including a bond in the bond list
        min_neighbors: Minimum number of neighbors to include for each atom.
                    If an atom has smaller than this number of bonds, additional
                    distances will be added to reach this number.
        snapshots:  Number of snapshots to be used in the generation, useful
                    for speeding up the process if the path is long and
                    atoms numerous.

    Returns:
        List of all the included interatomic distance pairs.
    """
    # Type casting and value checks on input parameters
    geom = np.asarray(geom)
    if len(geom.shape) < 3:
        # If there is only one geometry or it is flattened, promote to 3d
        geom = geom.reshape(1, -1, 3)
    min_neighbors = min(min_neighbors, geom.shape[1] - 1)

    # Determine which images to be used to determine distances
    snapshots = min(len(geom), snapshots)
    images = [0, len(geom) - 1]
    if snapshots > 2:
        images.extend(np.random.choice(range(1, snapshots - 1), snapshots - 2, replace=False))
    # Get neighbor list for included geometry and merge them
    rijset = set(enforce)
    for image in images:
        tree = KDTree(geom[image])
        pairs = tree.query_pairs(threshold)
        rijset.update(pairs)
        bonded = tree.query_pairs(bond_threshold)
        neighbors = {i: {i} for i in range(geom.shape[1])}
        for i, j in bonded:
            neighbors[i].add(j)
            neighbors[j].add(i)
        for i, j in bonded:
            for ni in neighbors[i]:
                for nj in neighbors[j]:
                    if ni != nj:
                        pair = tuple(sorted([ni, nj]))
                        if pair not in rijset:
                            rijset.add(pair)
    rijlist = sorted(rijset)
    # Check neighbor count to make sure `min_neighbors` is satisfied
    count = np.zeros(geom.shape[1], dtype=int)
    for i, j in rijlist:
        count[i] += 1
        count[j] += 1
    for idx, ct in enumerate(count):
        if ct < min_neighbors:
            _, neighbors = tree.query(geom[-1, idx], k=min_neighbors + 1)
            for i in neighbors:
                if i == idx:
                    continue
                pair = tuple(sorted([i, idx]))
                if pair in rijset:
                    continue
                else:
                    rijset.add(pair)
                    rijlist.append(pair)
                    count[i] += 1
                    count[idx] += 1
    if atoms is None:
        re = np.full(len(rijlist), 2.0)
    else:
        radius = np.array([ATOMIC_RADIUS.get(atom.capitalize(), 1.5) for atom in atoms])
        re = np.array([radius[i] + radius[j] for i, j in rijlist])
    logger.debug("Pair list contain %d pairs", len(rijlist))
    return rijlist, re

def compute_rij(geom, rij_list):
    """Calculate a list of distances and their derivatives

    Takes a set of cartesian geometries then calculate selected distances and their
    cartesian gradients given a list of atom pairs.

    Args:
        geom: Cartesian geometry of all the points.  Must be 2d numpy array or list
            with shape (natoms, 3)
        rij_list: list of indexes of all the atom pairs

    Returns:
        rij (array): Array of all the distances.
        bmat (3d array): Cartesian gradients of all the distances."""
    nrij = len(rij_list)
    rij = np.zeros(nrij)
    bmat = np.zeros((nrij, len(geom), 3))
    for idx, (i, j) in enumerate(rij_list):
        dvec = geom[i] - geom[j]
        rij[idx] = r = np.sqrt(dvec[0] * dvec[0] +
                               dvec[1] * dvec[1] + dvec[2] * dvec[2])
        grad = dvec / r
        bmat[idx, i] = grad
        bmat[idx, j] = -grad
    return rij, bmat

def compute_wij(geom, rij_list, func):
    """Calculate a list of scaled distances and their derivatives

    Takes a set of cartesian geometries then calculate selected distances and their
    cartesian gradients given a list of atom pairs.  The distances are scaled with
    a given scaling function.

    Args:
        geom: Cartesian geometry of all the points.  Must be 2d numpy array or list
            with shape (natoms, 3)
        rij_list: 2d numpy array of indexes of all the atom pairs
        func: A scaling function, which returns both the value and derivative.  Must
            qualify as a numpy Ufunc in order to be broadcasted to array elements.

    Returns:
        wij (array): Array of all the scaled distances.
        bmat (2d array): Cartesian gradients of all the scaled distances, with the
            second dimension flattened (need this to be used in scipy.optimize)."""
    geom = np.asarray(geom).reshape(-1, 3)
    nrij = len(rij_list)
    rij, bmat = compute_rij(geom, rij_list)
    wij, dwdr = func(rij)
    for idx, grad in enumerate(dwdr):
        bmat[idx] *= grad
    return wij, bmat.reshape(nrij, -1)

def morse_scaler(re=1.5, alpha=1.7, beta=0.01):
    """Returns a scaling function that determines the metric of the internal
    coordinates using morse potential

    Takes an internuclear distance, returns the scaled distance, and the
    derivative of the scaled distance with respect to the unscaled one.
    """
    def scaler(x):
        ratio = x / re
        val1 = np.exp(alpha * (1 - ratio))
        val2 = beta / ratio
        dval = -alpha / re * val1 - val2 / x
        return val1 + val2, dval
    return scaler

def elu_scaler(re=2, alpha=2, beta=0.01):
    """Returns a scaling function that determines the metric of the internal
    coordinates using morse potential

    Takes an internuclear distance, returns the scaled distance, and the
    derivative of the scaled distance with respect to the unscaled one.
    """
    def scaler(x):
        val1 = (1 - x / re) * alpha + 1
        dval = np.full(x.shape, -alpha / re)
        large = x > re
        v1l = np.exp(alpha * (1 - x[large] / re))
        val1[large] = v1l
        dval[large] = -alpha / re * v1l
        val2 = beta * re / x
        return val1 + val2, dval - val2 / x
    return scaler


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri May  1 11:51:40 2020

@author: apolat
"""

import requests
import time
import csv
import pandas as pd
import datetime

'''
The key.txt file contains the consumer key, account id and refresh_token linebreak separated.
'''

class getData:
     def __init__(self):
         keys=[]
         with open('/Users/apolat/bin/key.txt') as csv_file:
             csv_reader = csv.reader(csv_file, delimiter=',')
             for row in csv_reader:
                 keys.append(row[1])
         self.consumer_key=keys[0]
         self.api_url='https://api.tdameritrade.com/v1/'
         self.mkt='marketdata/'
         self.account_id=keys[1]
         self.authorization=None
         self.last_refresh_time=None
         self.refresh_token=keys[2]

     def refresh(self):
         payload={'grant_type':'refresh_token',
                  'refresh_token':self.refresh_token,
                  'client_id':self.consumer_key
             }
         url=self.api_url+'oauth2/token'

         self.authorization=requests.post(url,data=payload).json()['access_token']
         self.last_refresh_time=time.time()

     def getPrice(self,ticker,periodType='day',period=None,frequencyType='minute',frequency=1,endDate=None,startDate=None,needExtendedHoursData=True, df=False):
         """

         Parameters
         ----------
         ticker : str
         periodType : str, optional
             The type of period to show. Valid values are day, month, year, or ytd (year to date). The default is 'day'.
         period : int, optional
             The number of periods to show.

            Example: For a 2 day / 1 min chart, the values would be:

            period: 2
            periodType: day
            frequency: 1
            frequencyType: min

            Valid periods by periodType (defaults marked with an asterisk):

            day: 1, 2, 3, 4, 5, 10*
            month: 1*, 2, 3, 6
            year: 1*, 2, 3, 5, 10, 15, 20
            ytd: 1*
         frequencyType : str, optional
             The type of frequency with which a new candle is formed.
            Valid frequencyTypes by periodType (defaults marked with an asterisk):

            day: minute*
            month: daily, weekly*
            year: daily, weekly, monthly*
            ytd: daily, weekly*
         frequency : int, optional
             The number of the frequencyType to be included in each candle.

            Valid frequencies by frequencyType (defaults marked with an asterisk):

            minute: 1*, 5, 10, 15, 30
            daily: 1*
            weekly: 1*
            monthly: 1*
         endDate : str, optional
             End date as milliseconds since epoch. If startDate and endDate are provided, period should not be provided. Default is previous trading day.
         startDate : TYPE, optional
             Start date as milliseconds since epoch. If startDate and endDate are provided, period should not be provided.
         needExtendedHoursData : str, optional
             true to return extended hours data, false for regular market hours only. Default is true

         Returns
         -------
         dict

         help: https://developer.tdameritrade.com/price-history/apis/get/marketdata/%7Bsymbol%7D/pricehistory#
         """
         if self.authorization is None or time.time()-self.last_refresh_time>1800:
             self.refresh()
         payload={'apikey':self.consumer_key,
                  'periodType':periodType,
                  'period':period,
                  'frequencyType':frequencyType,
                  'frequency':frequency,
                  'endDate':endDate,
                  'startDate':startDate,
                  'needExtendedHoursData':needExtendedHoursData}

         headers={'Authorization':'Bearer '+self.authorization}

         url=self.api_url+self.mkt+'{}/pricehistory'.format(ticker)
         ret = requests.get(url,params=payload, headers=headers).json()

         if df:
             ret = pd.DataFrame(ret['candles'])
             ret['date'] = ret['datetime'].apply(lambda x: datetime.datetime.fromtimestamp(x/1000).strftime('%Y-%m-%d %H-%M'))
             ret = ret.drop(columns=['datetime'])

         return ret

     def getQuote(self,ticker):
         """
         help: https://developer.tdameritrade.com/quotes/apis/get/marketdata/%7Bsymbol%7D/quotes
         """
         if self.authorization is None or time.time()-self.last_refresh_time>1800:
             self.refresh()
         payload={'apikey':self.consumer_key}

         url=self.api_url+self.mkt+'{}/quotes'.format(ticker)

         return requests.get(url,params=payload).json()

     def getQuotes(self,ticker_list):
         """
         help: https://developer.tdameritrade.com/quotes/apis/get/marketdata/%7Bsymbol%7D/quotes
         """
         if self.authorization is None or time.time()-self.last_refresh_time>1800:
             self.refresh()
         tickers=''
         for ticker in ticker_list:
             tickers+=ticker+','
         tickers=tickers[:-1]

         payload={'apikey':self.consumer_key,
                  'symbol':tickers}

         url=self.api_url+self.mkt+'/quotes'

         return requests.get(url,params=payload).json()

     def getPositions(self,fields=None):
         if self.authorization is None or time.time()-self.last_refresh_time>1800:
             self.refresh()
         url=self.api_url+'accounts/'+self.account_id
         payload={}
         if fields is not None:
             payload={'fields':fields}
         headers={'Authorization':'Bearer '+self.authorization}

         return requests.get(url,params=payload,headers=headers).json()

     def searchInstruments(self,symbol,projection):
         """
         Parameters
         ----------
         symbol : String
             Value to pass to the search.
         projection : TYPE
             The type of request:
                 symbol-search: Retrieve instrument data of a specific symbol or cusip
                 symbol-regex: Retrieve instrument data for all symbols matching regex. Example: symbol=XYZ.* will return all symbols beginning with XYZ
                 desc-search: Retrieve instrument data for instruments whose description contains the word supplied. Example: symbol=FakeCompany will return all instruments with FakeCompany in the description.
                 desc-regex: Search description with full regex support. Example: symbol=XYZ.[A-C] returns all instruments whose descriptions contain a word beginning with XYZ followed by a character A through C.
                 fundamental: Returns fundamental data for a single instrument specified by exact symbol.'
         Returns
         -------
         TYPE dict
             see https://developer.tdameritrade.com/instruments/apis/get/instruments

         """
         if self.authorization is None or time.time()-self.last_refresh_time>1800:
             self.refresh()

         url=self.api_url+'instruments/'
         payload={'apikey':self.consumer_key,
                  'symbol':symbol,
                  'projection':projection}
         headers={'Authorization':'Bearer '+self.authorization}

         return requests.get(url,params=payload,headers=headers).json()

     def getInstruments(self,cusip):
         if self.authorization is None or time.time()-self.last_refresh_time>1800:
             self.refresh()

         url=self.api_url+'instruments/'+cusip
         payload={'apikey':self.consumer_key}
         headers={'Authorization':'Bearer '+self.authorization}

         return requests.get(url,params=payload,headers=headers).json()

     def OptionChain(self, symbol, contractType='ALL', strikeCount=5, includeQuotes='TRUE', strategy='SINGLE', interval=None, strike=None, range_='ALL', fromDate=None, toDate=None, volatility=None, underlyingPrice=None, interestRate=None, daysToExpiration=None, expMonth='ALL', optionType='ALL'):
         """

         Parameters
         ----------
         symbol : str
             Enter one symbol.
         contractType : str, optional
             Type of contracts to return in the chain. Can be CALL, PUT, or ALL. The default is 'ALL'.
         strikeCount : int, optional
             The number of strikes to return above and below the at-the-money price.. The default is 5.
         includeQuotes : bool str, optional
             Include quotes for options in the option chain. Can be TRUE or FALSE. The default is 'TRUE'.
         strategy : str, optional
             Passing a value returns a Strategy Chain. Possible values are SINGLE, ANALYTICAL (allows use of the volatility, underlyingPrice, interestRate, and daysToExpiration params to calculate theoretical values), COVERED, VERTICAL, CALENDAR, STRANGLE, STRADDLE, BUTTERFLY, CONDOR, DIAGONAL, COLLAR, or ROLL. . The default is 'SINGLE'.
         interval : str, optional
             Strike interval for spread strategy chains (see strategy param). The default is None.
         strike : int, optional
             Provide a strike price to return options only at that strike price. The default is None.
         range_ : TYPE, optional
             Returns options for the given range. Possible values are:
             'ITM': In-the-money
             'NTM'': Near-the-money
             'OTM': Out-of-the-money
             'SAK': Strikes Above Market
             'SBK': Strikes Below Market
             'SNK': Strikes Near Market
             'ALL': All Strikes.
             The default is 'ALL'.
         fromDate : TYPE, optional
             Only return expirations after this date. For strategies, expiration refers to the nearest term expiration in the strategy. Valid ISO-8601 formats are: yyyy-MM-dd and yyyy-MM-dd'T'HH:mm:ss. The default is None.
         toDate : TYPE, optional
             Only return expirations before this date. For strategies, expiration refers to the nearest term expiration in the strategy. Valid ISO-8601 formats are: yyyy-MM-dd and yyyy-MM-dd'T'HH:mm:ss. The default is None.
         volatility : TYPE, optional
             Volatility to use in calculations. Applies only to ANALYTICAL strategy chains (see strategy param). The default is None.
         underlyingPrice : TYPE, optional
             Underlying price to use in calculations. Applies only to ANALYTICAL strategy chains (see strategy param). The default is None.
         interestRate : TYPE, optional
             Interest rate to use in calculations. Applies only to ANALYTICAL strategy chains (see strategy param). The default is None.
         daysToExpiration : TYPE, optional
             Days to expiration to use in calculations. Applies only to ANALYTICAL strategy chains (see strategy param). The default is None.
         expMonth : TYPE, optional
             'Return only options expiring in the specified month. Month is given in the three character format. The default is 'ALL'.
         optionType : TYPE, optional
             'Type of contracts to return. Possible values are:

             S: Standard contracts
             NS: Non-standard contracts
             ALL: All contracts
             The default is 'ALL'.

         Returns
         -------
         a dict
         see https://developer.tdameritrade.com/option-chains/apis/get/marketdata/chains

         """
         if self.authorization is None or time.time()-self.last_refresh_time>1800:
             self.refresh()
         url=self.api_url+'marketdata/chains'
         payload = {'apikey':self.consumer_key,
                    'symbol':symbol,
                    'contractType':contractType,
                    'strikeCount':strikeCount,
                    'includeQuotes':includeQuotes,
                    'strategy':strategy,
                    'interval':interval,
                    'strike':strike,
                    'range':range_,
                    'fromDate':fromDate,
                    'toDate':toDate,
                    'volatility':volatility,
                    'underlyingPrice':underlyingPrice,
                    'interestRate':interestRate,
                    'daysToExpiration':daysToExpiration,
                    'expMonth':expMonth,
                    'optionType':expMonth}
         headers={'Authorization':'Bearer '+self.authorization}

         return requests.get(url,params=payload,headers=headers).json()


import pygame
import field
"""Обрабатывает события в игре"""

class EventHandler:
    def __init__(self):
        self.evolution_tick = 6

    def event_handler(self, events_, mode_, is_done_, gamefield_, menu_):
        """Обработчик событий игры"""

        done_ = is_done_

        # Должен вернуть пару из mode, done

        for event in events_:
            if event.type == pygame.MOUSEBUTTONDOWN:
                mouse_pos = pygame.mouse.get_pos()

                if (mode_ == 'draw' or mode_ == 'pause') \
                        and mouse_pos[1] < gamefield_.field_size[1] - field.Field.menu_height:
                    gamefield_.click(mouse_pos)

                if mouse_pos[1] > gamefield_.field_size[1] - field.Field.menu_height:
                    mode_ = menu_.click(mouse_pos, mode_)
            if event.type == pygame.QUIT or mode_ == 'quit':
                done_ = True
            elif event.type == pygame.KEYDOWN and mode_ == 'draw':
                if event.key == pygame.K_RETURN:
                    mode_ = 'evolution'  # начало эволюции
                    print('evolution mode on')

            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_ESCAPE:
                    done_ = True
                if event.key == pygame.K_SPACE:
                    if mode_ == 'evolution':
                        mode_ = 'pause'
                    elif mode_ == 'pause':
                        mode_ = 'evolution'

                # Управление скорростью игры клавишами
                if event.key == pygame.K_UP:
                    self.evolution_tick = min(30, self.evolution_tick + 1)
                if event.key == pygame.K_DOWN:
                    self.evolution_tick = max(1, self.evolution_tick - 1)

        return mode_, done_


from unsupervised_llamas.label_scripts.spline_creator import get_horizontal_values_for_four_lanes
import tensorflow as tf
import numpy as np
import json
import os
import cv2
import matplotlib.pyplot as plt
from scipy.interpolate import interp1d

from utils.grid import generate_grid

LLAMAS_SHAPE = (717, 1276)
DTYPE = tf.float32

def resize_img(img, shape):
    w = img.shape[1]
    h = img.shape[0]

    ratio = shape[1] / shape[0]

    tgt = img[h-int(w/ratio):h,0:w]
    tgt = cv2.resize(tgt, (shape[1], shape[0]))
    return tgt.astype(np.float32) / 255
    #return tgt

class LlamasProcessor:
    def __init__(self, cls_shape, image_shape):
        self.cls_shape = cls_shape
        self.image_shape = image_shape
        self.llamas_shape = LLAMAS_SHAPE

    def load_image(self, file):
        file = file.numpy().decode("utf-8")

        with open(file, 'r') as fp:
            meta = json.load(fp)
        image_path = os.path.dirname(file)
        image_path = image_path.replace('/labels/', '/color_images/')
        img_name = meta['image_name'] + '_color_rect.png'
        fname = os.path.join(image_path, img_name)
        img = cv2.imread(fname)

        return img

    def read_image(self, file):
        img = self.load_image(file)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

        return resize_img(img, self.image_shape)

    def generate_grid_llamas(self, file):
        #img = self.load_image(file)
        file = file.numpy().decode("utf-8")
        lanes = get_horizontal_values_for_four_lanes(file)

        return generate_grid(lanes, self.cls_shape, self.llamas_shape, delete_lanes=(0,3))

    def process_json(self, json):
        img = tf.py_function(self.read_image, [json], Tout=DTYPE)
        grid = tf.py_function(self.generate_grid_llamas, [json], Tout=DTYPE)

        img = tf.reshape(img, shape=self.image_shape)
        grid = tf.reshape(grid, shape=self.cls_shape)
        return img, grid

class LabelmeProcessor:
    def __init__(self, cls_shape, image_shape):
        self.cls_shape = cls_shape
        self.image_shape = image_shape

    # Creates points for the left and right lane based on spline points
    # that are read from the json input file
    def get_points(self, file):
        x_points = [[], []]
        y_points = [[], []]

        with open(file, 'r') as fp:
            meta = json.load(fp)

        if meta['shapes'][0]['label'] == 'L':
            l = 0
            r = 1
        else:
            l = 1
            r = 0

        for x, y in meta['shapes'][l]['points']:
            x_points[l].append(x)
            y_points[l].append(y)

        for x, y in meta['shapes'][r]['points']:
            x_points[r].append(x)
            y_points[r].append(y)

        max_val = [0] * 2
        low_val = [0] * 2
        max_val[l] = np.max(y_points[l])
        low_val[l] = np.min(y_points[l])
        max_val[r] = np.max(y_points[r])
        low_val[r] = np.min(y_points[r])

        points = [[], []]
        f = [0] * 2
        f[l] = interp1d(y_points[l], x_points[l])
        f[r] = interp1d(y_points[r], x_points[r])

        for i in range(2):
            for j in range(meta['imageHeight']):
                if j < low_val[i] or j > max_val[i]:
                    points[i].append(-1)
                else:
                    points[i].append(f[i](j).item(0))
        return [points[l], points[r]]

    def read_image(self, file):
        file = file.numpy().decode("utf-8")
        with open(file, 'r') as fp:
            meta = json.load(fp)

        image_path = os.path.dirname(file)
        img_name = meta['imagePath']
        fname = os.path.join(image_path, img_name)
        img = cv2.imread(fname)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

        return resize_img(img, self.image_shape).astype(np.float32)

    def generate_grid_custom(self, file):
        file = file.numpy().decode("utf-8")
        with open(file, 'r') as fp:
            meta = json.load(fp)
        shape = (meta['imageHeight'], meta['imageWidth'])
        lines = self.get_points(file)

        return generate_grid(lines, self.cls_shape, shape)

    def process_json(self, json):
        img = tf.py_function(self.read_image, [json], Tout=DTYPE)
        grid = tf.py_function(self.generate_grid_custom, [json], Tout=DTYPE)

        img = tf.reshape(img, shape=self.image_shape)
        grid = tf.reshape(grid, shape=self.cls_shape)
        return img, grid

def load_json_dataset(json_file_pattern, processor, max_records=None, shuffle_size=100000, premap_func=None):
    ds = tf.data.Dataset.list_files(json_file_pattern)
    ds = ds.shuffle(shuffle_size)

    if max_records is not None:
        ds = ds.take(max_records)

    if premap_func is not None:
        ds = premap_func(ds)

    ds = ds.map(processor.process_json)
    return ds

def llamas_dataset(json_file_pattern, cls_shape, image_shape, max_records=None, shuffle_size=100000, premap_func=None):
    processor = LlamasProcessor(cls_shape, image_shape)

    ds = load_json_dataset(json_file_pattern, processor, max_records, shuffle_size, premap_func)

    return ds

def labelme_dataset(json_file_pattern, cls_shape, image_shape, max_records=None, shuffle_size=100000, premap_func=None):
    processor = LabelmeProcessor(cls_shape, image_shape)

    ds = load_json_dataset(json_file_pattern, processor, max_records, shuffle_size, premap_func)

    return ds


import torch
import torch.nn as nn
import torch.nn.functional as F
import matplotlib.pyplot as plt
import numpy as np

class VAE(nn.Module):
    def __init__(self, latent_dim):
        super(VAE, self).__init__()

        self.latent_dim = latent_dim

        # Encoder
        self.encoder_conv = nn.Sequential(
            nn.Conv2d(3, 32, kernel_size=4, stride=2, padding=1),
            nn.ReLU(),
            nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=1),
            nn.ReLU(),
            nn.Conv2d(64, 128, kernel_size=4, stride=2, padding=1),
            nn.ReLU(),
            nn.Conv2d(128, 256, kernel_size=4, stride=2, padding=1),
            nn.ReLU()
        )

        self.fc_mu = nn.Linear(256 * 16 * 16, latent_dim)
        self.fc_logvar = nn.Linear(256 * 16 * 16, latent_dim)

        # Decoder
        self.decoder_fc = nn.Linear(latent_dim, 256 * 16 * 16)

        self.decoder_conv = nn.Sequential(
            nn.ConvTranspose2d(256, 128, kernel_size=4, stride=2, padding=1),
            nn.ReLU(),
            nn.ConvTranspose2d(128, 64, kernel_size=4, stride=2, padding=1),
            nn.ReLU(),
            nn.ConvTranspose2d(64, 32, kernel_size=4, stride=2, padding=1),
            nn.ReLU(),
            nn.ConvTranspose2d(32, 3, kernel_size=4, stride=2, padding=1),
            nn.Sigmoid()  # To ensure outputs are in [0, 1]
        )

    def reparameterize(self, mu, logvar):
        std = torch.exp(0.5*logvar)
        eps = torch.randn_like(std)
        return mu + eps*std

    def encode(self, x):
        x = self.encoder_conv(x)
        x = x.view(x.size(0), -1)
        mu = self.fc_mu(x)
        logvar = self.fc_logvar(x)
        return mu, logvar

    def decode(self, z):
        z = self.decoder_fc(z)
        z = z.view(z.size(0), 256, 16, 16)
        x = self.decoder_conv(z)
        return x

    def forward(self, x):
        mu, logvar = self.encode(x)
        z = self.reparameterize(mu, logvar)
        recon_x = self.decode(z)
        return recon_x, mu, logvar


'''
date: 2020/2/7
author: 流氓兔233333
content: 注意力机制融合双向LSTM
model架构参考  NLP_融合主题模型和注意力机制的政策文本分类模型_胡吉明
'''

import numpy as np
import pandas as pd
from tqdm import tqdm
import time
import random
import os, warnings, pickle
warnings.filterwarnings('ignore')

import torch
import torch.utils.data as Data
# 导入nn及优化器
import torch.nn.functional as F
import torch.optim as optim
from torch import nn
import sklearn.metrics as ms

data_path = './data_raw/'
save_path = './temp_results/'

# load word2idx tar2idx
word2idx_dict = pickle.load(open(save_path+'word2idx_dict.pkl', 'rb'))
tar2idx_dict = pickle.load(open(save_path+'target2idx_dict.pkl', 'rb'))

# load tensor data
train_tensor_tuple = pickle.load(open(save_path+'train_tensor_tuple.pkl', 'rb'))
val_tensor_tuple = pickle.load(open(save_path+'val_tensor_tuple.pkl', 'rb'))
x_train_tensor, y_train_tensor = train_tensor_tuple[0], train_tensor_tuple[1]
x_val_tensor, y_val_tensor = val_tensor_tuple[0], val_tensor_tuple[1]

del train_tensor_tuple, val_tensor_tuple

batch_size = 128
dataset_trn = Data.TensorDataset(x_train_tensor, y_train_tensor)
loader_trn = Data.DataLoader(dataset_trn, batch_size, True)
dataset_val = Data.TensorDataset(x_val_tensor, y_val_tensor)
loader_val = Data.DataLoader(dataset_val, len(dataset_val), True)

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
device

def set_seed(seed):
    """ Set all seeds to make results reproducible """
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False
    np.random.seed(seed)
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)

class BiLSTM_Attention(nn.Module):
    def __init__(self, embed_dim, hid_dim, vocab_size, tar_size):
        super(BiLSTM_Attention, self).__init__()
        self.hid_dim = hid_dim
        self.word_embedding = nn.Embedding(vocab_size, embed_dim)
        self.lstm = nn.LSTM(embed_dim, hid_dim, bidirectional=True, batch_first=True)
        # attention, self.attn 的 out_dim 是任意的，这里直接设成 tar_dim
        self.attn = nn.Linear(self.hid_dim, tar_size)
        self.v = nn.Linear(tar_size, 1, bias=False)

        self.fc = nn.Linear(self.hid_dim, tar_size)

    def forward(self, sequence):
        batch_size = sequence.shape[0]

        # nn.Embedding 要求输入的数据为 [batch_size, seq_len]
        # 输出为 [batch_szie, seq_len, embed_dim]
        embeds = self.word_embedding(sequence)
        seq_len = embeds.shape[1]

        # nn.LSTM 要求输入数据为 [batch_size, seq_len, embed_dim]
        # hidden_0 未给出时会被默认设置为0
        # lstm_out  [batch_size, seq_len, hid_dim*num_directions]
        # h_n  [n_layers*num_directions, batch_size, embed_dim]
        # h_c 用于生成 h_n, 不用于输出
        lstm_out, (h_n, h_c) = self.lstm(embeds)
        # [bs, seq_len, hid_size]
        lstm_out = lstm_out[:, :, :self.hid_dim] + lstm_out[:, :, self.hid_dim:]

        lstm_out = F.tanh(lstm_out)

        attention = torch.zeros(seq_len, batch_size)   # [seq_len, batch_size]
        for t in range(seq_len):

            # attention_iuput [batch_size, hid_dim*2]
            # h_n[0, :, :] shape [batch_size, hid_dim]
            attention_iuput = lstm_out[:, t, :]
            # mt [batch_size, tar_size]
            mt = self.attn(attention_iuput)
            attention_t = self.v(mt)  # [batch_size, 1]
            attention[t] = attention_t.squeeze(1)

        attention = F.softmax(attention, dim=0)
        attention = attention.transpose(0,1).unsqueeze(1) # [batch_size, 1, seq_len]
        # lstm_out [batch_size, seq_len, hid_dim*2]
        c = torch.bmm(attention, lstm_out) # [batch_size, 1, hid_dim*2]
        c = c.squeeze(1) # [batch_size, hid_dim*2]

        output = self.fc(c) # [batch_size, tar_size]

        return output

def save(model, optimizer):
    # save
    torch.save({
        'model_state_dict': model.state_dict(),
        'optimizer_state_dict': optimizer.state_dict()
    }, './BiLSTM_Attention.pth')
    print('The best model has been saved')

def train_eval(model, criterion, optimizer, train_loader, val_loader, epochs=2):
    try:
        checkpoint = torch.load('./model_bert.pth', map_location='cpu')
        model.load_state_dict(checkpoint['model_state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
        print('-----Continue Training-----')
    except:
        print('No Pretrained model!')
        print('-----Training-----')

    model.to(device)
    for epoch in range(epochs):
        model.train()
        print('eopch: %d/%d'% (epoch, epochs))
        for i, batch in enumerate(tqdm(train_loader)):
            batch = tuple(t.to(device) for t in batch)
            output = model(batch[0])
            loss = criterion(output, batch[-1])
            if((i+1) % 8)==0:
                optimizer.step()        # 反向传播，更新网络参数
                optimizer.zero_grad()   # 清空梯度

        if epoch % 1 == 0:
            print(loss.item())
            eval(model, optimizer, val_loader)

best_score = 0
def eval(model, optimizer, val_loader):
    model.eval()
    _, batch_A = next(enumerate(val_loader))
    batch = tuple(t.to(device) for t in batch)
    with torch.no_grad():
        output = model(batch[0])
        label_ids = batch[-1].numpy()

        _, prediction = torch.max(F.softmax(output, dim=1), 1)
        pred_val = prediction.data.numpy().squeeze()
        acc_val = ms.accuracy_score(label_ids, pred_val)

    print("Validation Accuracy: {}".format(acc_val))
    global best_score
    if best_score < acc_val:
        best_score = acc_val
        save(model, optimizer)

tar_size = len(tar2idx_dict)
vocab_size = len(word2idx_dict)
embed_size = 2 * int(np.floor(np.power(vocab_size, 0.25)))

model = BiLSTM_Attention(embed_dim=embed_size, hid_dim=64, vocab_size=vocab_size, tar_size=tar_size)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=1e-3)

set_seed(1)
train_eval(model, criterion, optimizer, loader_trn, loader_val, epochs=2)

loss_his = []
step, (batch_x, batch_y) = next(enumerate(loader))
for epoch in range(20):
    model = model.train()
    for step, (batch_x, batch_y) in enumerate(tqdm(loader)):
        output = model(batch_x)

        loss = loss_fun(output, batch_y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    loss_his.append(loss.item())

    if epoch % 1 == 0:
        model = model.eval()
        output = model(x_val_tensor)

        _, prediction = torch.max(F.softmax(output, dim=1), 1)
        pred_val = prediction.data.numpy().squeeze()
        acc_test = ms.accuracy_score(y_val_tensor.numpy(), pred_val)
        print('EPOCH: ', epoch,   '         loss:', loss.item())
        print('  test_acc:', acc_test)

# loss 曲线图
import matplotlib.pyplot as plt
plt.plot(loss_his)
plt.ylabel('loss', fontsize=20)
plt.xlabel('eopch', fontsize=20)
plt.tick_params(labelsize=20)
plt.title('EOPCH_LOSS_BiLSTM_Attention', fontsize=20)
plt.grid()
plt.show()

# model save
torch.save(model.state_dict(), save_path+'BiLSTM_Attention.pkl')

x = [1, 2, 3, 5]

[1 if i >2 else 0 for i in x ]


from django.urls import path, re_path
from . import views

urlpatterns = [
    path('', views.webtoon_list, name='webtoon-list'),
    # re_path(r'(?P<pk>\d+)/$', views.post_detail),
    # path('', views.webtoon_detail(), name='webtoon-detail'),

    path('<int:pk>/', views.episode_detail, name='episode-detail'),
    path('webtoon/', views.episode_list, name='episode-list'),
]


import datetime

from flask_script import Command, Option
from sqlalchemy.orm import load_only

from app import db
from app.resources.users.models import User
from app.resources.user_settings.models import UserSettings
from app.resources.guest_user.schemas import GuestUserSchema
from app.resources.user_settings.helpers import create_default_user_settings
from app.resources.sectors.models import Sector
from app.resources.industries.models import Industry

class AddUserSettings(Command):
    """
    Command to add user settings for missing users

    :arg verbose:        print progress
    :arg dry:
        dry run
    """

    option_list = [
        Option('--verbose', '-v', dest='verbose', action='store_true',
               default=False),
        Option('--dry', '-dry', dest='dry', action='store_true',
               default=False),
    ]

    def run(self, verbose, dry):
        if verbose:
            print('---' + str(datetime.datetime.utcnow()) + '---')
            print('Adding user settings ...')
        try:
            user_all_data = User.query.join(UserSettings,
                UserSettings.user_id == User.row_id, isouter=True).filter(
                UserSettings.user_id == None
            ).all()

            remain_users = list(user_all_data)
            for user in remain_users:
                user = create_default_user_settings(user)
                db.session.add(user)
                db.session.commit()

        except Exception as e:
            raise e
            db.session.rollback()
            print(e)
            exit(1)

        print('---' + str(datetime.datetime.utcnow()) + '---')
        print('Done')


from flask import Flask, request, jsonify
import json
import sys
import os
import traceback

sys.path.append("./opennmt-inspection/")
from online_translator import translate
import getClassifierData

app = Flask(__name__)

# make flask load index.html by default
@app.route("/")
def index():
    return app.send_static_file("index.html")

# for loading css, js
@app.route("/<path:filename>")
def static_file(filename):
    return app.send_static_file(filename)

# frontend sends requests to http://localhost:5000/model
@app.route("/model", methods=["POST"])
def model():
    modelPath = os.path.join('models', request.form["modelPath"])
    modifications = json.loads(request.form["modifications"])
    tokensPath = os.path.join('modelInputs', request.form["tokensPath"])
    labelsPath = os.path.join('modelInputs', request.form["labelsPath"])

    with open(tokensPath, "r") as file:
        trainTokens = file.read().splitlines()

    with open(labelsPath, "r") as file:
        trainLabels = file.read().splitlines()
        labels = [line.split(" ") for line in trainLabels]

    numLines = len(trainTokens)
    modsList = [[] for _ in range(numLines)]

    for mod in modifications:
        sentenceIndex = mod[0]
        modsList[sentenceIndex].append(mod[1:])

    try:
        rawActivations, text, preds = getData(modelPath, trainTokens, modsList)

        # get rid of second dimension
        # to keep format activations[sentence][token][layer][neuron]
        # instead of activations[sentence][0][token][layer][neuron]
        activations = [sentence[0] for sentence in rawActivations]

        # scale activations linearly so that abs(largest_activation) == 1
        norm = max(abs(value) for value in flatten(activations))
        activations = listify(activations, norm=norm)

        trainingActivations = [torch.stack([torch.cat(token) for token in sentence[0]]) for sentence in rawActivations]
        topNeurons, topNeuronsByCategory = getClassifierData.topNeurons(tokensPath, labelsPath, trainingActivations)

    except Exception as err:
        # catch python exception, throw http 500 error with error message
        print(traceback.format_exc())
        return str(err), 500

    # save data to cache (helpful for fast UI debugging)
    with open("cache.json", "w") as file:
        file.write(json.dumps([activations, text, preds, labels, topNeurons, topNeuronsByCategory]))

    return jsonify([activations, text, preds, labels, topNeurons, topNeuronsByCategory])

@app.route("/cache", methods=["POST"])
def cache():
    with open("cache.json", "r") as file:
        data = json.loads(file.read())

    return jsonify(data)

# --------------------------------------------------------------------------------

import torch

# adapted from anthony's code
# normalize a nested list of values
# convert and tensors to regular lists
def listify(x, norm=1):
    if type(x) == torch.Tensor:
        return (x / norm).tolist()
    elif type(x) == list or type(x) == tuple:
        return [listify(y, norm=norm) for y in x]
    else:
        return x

# convert nested list of lists / tensors
# to a flat element stream
def flatten(x):
    if type(x) == torch.Tensor:
        yield from x.tolist()
    elif type(x) == list or type(x) == tuple:
        for y in x:
            yield from flatten(y)
    else:
        yield x

# modelPath: path to model in the ./models directory
# inputText: string of input text to model
# currently, input text must already be tokenized with spaces
def getData(modelPath, inputText, modifications):
    pred, rawActivations = translate(
        model         = modelPath,
        sentences     = inputText,
        modifications = modifications
    )

    text = [sentence.strip().split(" ") for sentence in inputText]
    preds = [sentence.split(" ") for sentence in pred]

    return rawActivations, text, preds

# --------------------------------------------------------------------------------

if __name__ == "__main__":
    app.run(host="0.0.0.0", debug=True)


import sys
from io import BytesIO
import socket
from time import monotonic as time

from message import *
from udpserver import *

# message的request封包测试
def testEncodeRequestMessage():
    h = Header(id=0x9ace, answer=0, opCode=0, recDes=0,
               recAv=1, authority=0, rCode=OK, trunc=0,
               qdCount=1, anCount=0, nsCount=0, arCount=0)
    q = Query(b'www.baidu.com', A, IN)
    m = Message(h)
    m.addQuery(q)
    strio = BytesIO()
    m.encode(strio)
    print(strio.getvalue())
    return m

# message的request拆包测试
def testDecodeRequestMessage():
    m = Message()
    message = b'\x9a\xce\x00\x80\x00\x01\x00\x00\x00\x00\x00\x00\x03www\x05baidu\x03com\x00\x00\x01\x00\x01'
    strio = BytesIO(message)
    m.decode(strio)
    print(m.header.id, m.header.answer, m.header.opCode, m.header.recDes,
          m.header.recAv, m.header.authority, m.header.rCode, m.header.trunc,
          m.header.qdCount, m.header.anCount, m.header.nsCount, m.header.arCount)
    print(m.queries[0].name, m.queries[0].type, m.queries[0].cls)

# message的response包封包测试
def testEncodeResponseMessage():
    h = Header(id=0x9ace, answer=1, opCode=0, recDes=0,
               recAv=1, authority=0, rCode=OK, trunc=0,
               qdCount=1, anCount=1, nsCount=0, arCount=0)
    q = Query(b'www.baidu.com', A, IN)
    rr = ResourceRecord(b'www.baidu.com', A, IN, 201, address='127.0.0.1')
    m = Message(h)
    m.addQuery(q)
    m.addAnswer(rr)
    strio = BytesIO()
    m.encode(strio)
    print(strio.getvalue())
    return m

# message的response包拆包测试
def testDecodeResponseMessage():
    m = Message()
    message = b'\x9a\xce\x80\x80\x00\x01\x00\x01\x00\x00\x00\x00\x03www\x05baidu\x03com\x00\x00\x01\x00\x01\xc0\x0c\x00\x01\x00\x01\x00\x00\x00\xc9\x00\x04\x7f\x00\x00\x01'
    strio = BytesIO(message)
    m.decode(strio)
    print(m.header.id, m.header.answer, m.header.opCode, m.header.recDes,
        m.header.recAv, m.header.authority, m.header.rCode, m.header.trunc,
        m.header.qdCount, m.header.anCount, m.header.nsCount, m.header.arCount)
    print(m.queries[0].name, m.queries[0].type, m.queries[0].cls)
    print(m.answers[0].name, m.answers[0].cls, m.answers[0].type, m.answers[0].ttl, m.answers[0].rdlength, m.answers[0].rdata.address)

# header封包测试
def testEncodeHeader():
    h = Header(id=0x9ace, answer=0, opCode=0, recDes=0,
               recAv=1, authority=0, rCode=OK, trunc=0,
               qdCount=1, anCount=0, nsCount=0, arCount=0)
    strio = BytesIO()
    h.encode(strio)
    print(strio.getvalue())

# header拆包测试
def testDecodeHeader():
    h = Header()
    header = b'\x9a\xce\x00\x80\x00\x01\x00\x00\x00\x00\x00\x00'
    strio = BytesIO(header)
    h.decode(strio)
    print(h.id, h.answer, h.opCode, h.recDes, h.recAv, h.authority, h.rCode, h.trunc, h.qdCount, h.anCount, h.nsCount, h.arCount)

# query封包测试
def testEncodeQuery():
    q = Query(b'www.baidu.com', A, IN)
    strio = BytesIO()
    q.encode(strio)
    print(strio.getvalue())

# query拆包测试
def testDecodeQuery():
    query = b'\x03www\x05baidu\x03com\x00\x00\x01\x00\x01'
    q = Query()
    strio = BytesIO(query)
    q.decode(strio)
    print(q.name, q.type, q.cls)

# name封包测试
def testEncodeName():
    name = b'www.baidu.com'
    n = Name(name)
    strio = BytesIO()
    nameDict = {}
    n.encode(strio, nameDict)  # 测试名字未在dict中出现
    n.encode(strio, nameDict)  # 测试名字在dict中出现
    print(strio.getvalue())

# name拆包测试
def testDecodeName():
    name = b'\x03www\x05baidu\x03com\x00'
    n = Name()
    strio = BytesIO(name)
    n.decode(strio)
    print(n.name)

# Arecord封包测试
def testEncodeARecord():
    testRR = ResourceRecord(b'www.baidu.com', A, IN, 0, address='127.0.0.1')
    strio = BytesIO()
    testRR.encode(strio)
    print(strio.getvalue())

# Arecord拆包测试
def testDecodeARecord():
    record = b'\x03www\x05baidu\x03com\x00\x00\x01\x00\x01\x00\x00\x00\x00\x00\x04\x7f\x00\x00\x01'
    strio = BytesIO(record)
    testRR = ResourceRecord()
    testRR.decode(strio)
    print(testRR.name, testRR.cls, testRR.type, testRR.ttl, testRR.rdlength, testRR.rdata.address)

# MXrecord

# request包发送测试
def testSendRequest():
    strio = BytesIO()
    m = testEncodeRequestMessage()
    m.encode(strio)
    addr = ('127.0.0.1', 60000)
    testClient = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
    testClient.sendto(strio.getvalue(), addr)
    testClient.close()

# request包接收测试
def testRecvRequest():
    '''
        sendRequest的结果发往本地拆包，在TestServer中被拆包
    '''
    pass

# response包发送测试
def testSendResponse():
    strio = BytesIO()
    m = testEncodeResponseMessage()
    m.encode(strio)
    addr = ('123.125.81.6', PORT)
    testClient = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, 0)
    testClient.sendto(strio.getvalue(), addr)
    testClient.close()

# response包接收测试
def testRecvResponse():
    '''
        sendRequest的结果发往本地拆包，在TestServer中被拆包
    '''
    pass

# testUDPAsyncServer 测试并发性
def testConcurrency():
    strio = BytesIO()
    m = testEncodeRequestMessage()
    m.encode(strio)
    addr = ('0.0.0.0', 53)
    testClient = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    # 连续发100次测试，观察活跃线程数来测试并发效果
    for i in range(1):
        testClient.sendto(strio.getvalue(), addr)
    testClient.close()

if __name__ == "__main__":
    testConcurrency()


import requests

url = "http://122.51.248.121:8910/"
header = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36"
}
proxies = {
    "http": "http://47.113.180.76:81"
}
# while True:
resp = requests.get(url, proxies=proxies)
resp.encoding = "utf-8"
print(resp.text)


from google.cloud import bigquery

def load_to_bigquery(event, context):
    client = bigquery.Client()
    dataset_id = 'googleplaystore'
    bucket = event['bucket']
    file = event['name']
    url = 'gs://{0}/{1}'.format(bucket, file)
    dataset_ref = client.dataset(dataset_id)
    job_config = bigquery.LoadJobConfig()
    job_config.skip_leading_rows = 1
    job_config.max_bad_records = 1
    # The source format defaults to CSV, so the line below is optional.
    job_config.source_format = bigquery.SourceFormat.CSV
    load_job = client.load_table_from_uri(
        url,
        dataset_ref.table('play_store'),
        job_config=job_config)  # API request
    print('Starting job {}'.format(load_job.job_id))

    result = load_job.result()  # Waits for table load to complete.
    print(result)
    print('Job finished.')


import json
import boto3
import os
import cfnresponse
import json
import logging
import signal
import time

LOGGER = logging.getLogger()
LOGGER.setLevel(logging.INFO)

timeout_seconds = 300

def handler(event, context):

    # setup alarm for remaining runtime minus a second
    signal.alarm(timeout_seconds)

    try:
        LOGGER.info('REQUEST RECEIVED: %s', event)
        LOGGER.info('REQUEST Context: %s', context)

        # set up env vars
        client = boto3.client("kinesisanalyticsv2")
        app_name = os.environ["app_name"]
        execution_role = os.environ["execution_role"]
        bootstrap_string = os.environ["bootstrap_string"]
        bootstrap_stack_name = os.environ["bootstrapStackName"]
        subnet1 = os.environ["subnet_1"]
        source_topic = os.environ["source_topic_name"]
        security_group = os.environ["security_group"]
        glue_db_arn = os.environ["glue_db_arn"]
        log_stream_arn = os.environ["log_stream_arn"]
        zep_flink_version = os.environ["RuntimeEnvironment"]
        blueprint_name = os.environ["blueprintName"]
        stack_id = os.environ["stackId"]

        if event['RequestType'] == 'Create' or event['RequestType'] == 'Update':
            LOGGER.info('In Create')

            create_app(client, app_name, execution_role, bootstrap_string, bootstrap_stack_name, subnet1,
                       source_topic, security_group, glue_db_arn, log_stream_arn, zep_flink_version,
                       blueprint_name, stack_id)
            cfnresponse.send(event, context, cfnresponse.SUCCESS, {
                             "Message": "Successfully Created Application"})
        if event['RequestType'] == 'Delete':
            delete_app(client, app_name)
            cfnresponse.send(event, context, cfnresponse.SUCCESS, {
                             "Message": "Successfully Deleted Application"})
    except Exception as e:
        LOGGER.info('FAILED!')
        LOGGER.info(str(e))
        cfnresponse.send(event, context, cfnresponse.FAILED,
                         {"Message": str(e)})

def create_app(client, app_name, execution_role, bootstrap_string, bootstrap_stack_name, subnet1,
               source_topic, security_group, glue_db_arn, log_stream_arn, zep_flink_version,
               blueprint_name, stack_id):

    # check if app already exists
    try:
        describe_response = client.describe_application(
            ApplicationName=app_name)
        LOGGER.info("App already exists %s", describe_response)
        return
    except Exception as e:
        LOGGER.info("App doesn't exist yet so I am creating it")

    code_content = generate_code_content(app_name, execution_role, bootstrap_string,
                                         subnet1, source_topic, security_group, glue_db_arn, log_stream_arn)

    response = client.create_application(
        ApplicationName=app_name,
        ApplicationDescription="blueprint studio application",
        RuntimeEnvironment=zep_flink_version,
        ServiceExecutionRole=execution_role,
        ApplicationConfiguration={
            "FlinkApplicationConfiguration": {
                "ParallelismConfiguration": {
                    "ConfigurationType": "CUSTOM",
                    "Parallelism": 4,
                    "ParallelismPerKPU": 1,
                    "AutoScalingEnabled": False,
                }
            },
            'EnvironmentProperties': {
                'PropertyGroups': [
                    {
                        'PropertyGroupId': 'BlueprintMetadata',
                        'PropertyMap': {
                            'StackId': stack_id,
                            'BlueprintName': blueprint_name,
                            'BootstrapStackName': bootstrap_stack_name
                        }
                    },
                ]
            },
            "ApplicationCodeConfiguration": {
                "CodeContent": {
                    "TextContent": code_content,
                },
                "CodeContentType": "PLAINTEXT",
            },
            "VpcConfigurations": [
                {"SubnetIds": [subnet1],
                    "SecurityGroupIds": [security_group]},
            ],
            "ZeppelinApplicationConfiguration": {
                "MonitoringConfiguration": {"LogLevel": "INFO"},
                "CatalogConfiguration": {
                    "GlueDataCatalogConfiguration": {"DatabaseARN": glue_db_arn}
                },
                "CustomArtifactsConfiguration": [
                    {
                        'ArtifactType': "DEPENDENCY_JAR",
                        'MavenReference': {
                            'GroupId': "org.apache.flink",
                            'ArtifactId': "flink-connector-kafka",
                            'Version': "1.15.4",
                        },
                    },
                    {
                        'ArtifactType': "DEPENDENCY_JAR",
                        'MavenReference': {
                            'GroupId': "org.apache.flink",
                            'ArtifactId': "flink-sql-connector-kinesis",
                            'Version': "1.15.4",
                        },
                    },
                    {
                        'ArtifactType': "DEPENDENCY_JAR",
                        'MavenReference': {
                            'GroupId': "software.amazon.msk",
                            'ArtifactId': "aws-msk-iam-auth",
                            'Version': "1.1.6",
                        },
                    },
                ],
            },
        },
        CloudWatchLoggingOptions=[
            {"LogStreamARN": log_stream_arn},
        ],
        ApplicationMode="INTERACTIVE")

    LOGGER.info("Create response %s", response)

def delete_app(client, app_name):
    LOGGER.info("Request to delete app")

    # check if app already deleted
    describe_response = ""
    try:
        describe_response = client.describe_application(
            ApplicationName=app_name)

        LOGGER.info("App exists, going to delete it %s", describe_response)
    except Exception as e:
        LOGGER.info("App doesn't exist or already deleted %s", e)
        return

    app_version = describe_response["ApplicationDetail"]["ApplicationVersionId"]
    vpc_configuration = describe_response["ApplicationDetail"][
        "ApplicationConfigurationDescription"]["VpcConfigurationDescriptions"][0]
    vpc_id = vpc_configuration["VpcConfigurationId"]

    LOGGER.info("Attempting to delete VPC from MSF App, VPC ID %s", vpc_id)

    delete_vpc_response = client.delete_application_vpc_configuration(
        ApplicationName=app_name,
        CurrentApplicationVersionId=app_version,
        VpcConfigurationId=vpc_id)

    LOGGER.info("Deleted VPC from App %s", delete_vpc_response)

    create_timestamp = describe_response["ApplicationDetail"]["CreateTimestamp"]

    while (True):
        desc_response = client.describe_application(ApplicationName=app_name)
        status = desc_response['ApplicationDetail']['ApplicationStatus']
        if status == "UPDATING":
            # wait until status is not updating
            LOGGER.info("Status is still UPDATING, sleeping until it is not.")
            time.sleep(1)
        else:
            LOGGER.info("App is done updating, proceeding with delete.")
            break

    delete_response = client.delete_application(
        ApplicationName=app_name, CreateTimestamp=create_timestamp)
    LOGGER.info("Delete response %s", delete_response)

def generate_code_content(app_name, execution_role, bootstrap_string, subnet1, source_topic, security_group, glue_db_arn, log_stream_arn):

    code_content = {}
    code_content["paragraphs"] = []

    create_udf_content = {}
    create_udf_content["text"] = """%flink \n\nclass RandomTickerUDF extends ScalarFunction {\n  private val randomStrings: List[String] = List(\"AAPL\", \"AMZN\", \"MSFT\", \"INTC\", \"TBV\")\n    private val random: scala.util.Random = new scala.util.Random(System.nanoTime())\n\n  \n  override def isDeterministic(): Boolean = {\n      return false;\n  }\n  \n  \n  def eval(): String = {\n        val randomIndex = random.nextInt(randomStrings.length)\n        randomStrings(randomIndex)\n    }\n}\n\nstenv.registerFunction(\"random_ticker_udf\", new RandomTickerUDF())"""
    create_udf_content["title"] = """
1) User Defined Function for generating stock ticker data
"""
    create_udf_content["config"] = {}
    create_udf_content["config"]["title"] = "true"

    create_stock_table_content = {}
    create_stock_table_content["text"] = """%flink.ssql(type=update)\nDROP TABLE IF EXISTS stock_table;\nCREATE TABLE stock_table (\n  ticker STRING,\n  event_time TIMESTAMP(3),\n  price DOUBLE,\n  WATERMARK for event_time as event_time - INTERVAL '15' SECONDS\n) WITH (\n  'connector' = 'kafka',\n  'topic' = 'sourceTopic',\n  'properties.bootstrap.servers' = '""" + bootstrap_string + """',\n  'properties.security.protocol' = 'SASL_SSL',\n  'properties.sasl.mechanism' = 'AWS_MSK_IAM',\n  'properties.sasl.jaas.config' = 'software.amazon.msk.auth.iam.IAMLoginModule required;',\n  'properties.sasl.client.callback.handler.class' = 'software.amazon.msk.auth.iam.IAMClientCallbackHandler',\n  'properties.group.id' = 'myGroup',\n  'scan.startup.mode' = 'earliest-offset',\n  'format' = 'json'\n);\n\n\nSELECT * FROM stock_table;"""
    create_stock_table_content["title"] = """
2) Defining a source table to source MSK topic and querying data
"""
    create_stock_table_content["config"] = {}
    create_stock_table_content["config"]["title"] = "true"

    insert_datagen_content = {}
    insert_datagen_content["text"] = """%flink.ssql(parallelism=1)\nDROP TABLE IF EXISTS generate_stock_data;\nCREATE TABLE generate_stock_data(\n  ticker STRING,\n  event_time TIMESTAMP(3),\n  price DOUBLE\n)\nWITH (\n    'connector' = 'datagen',\n    'fields.price.kind' = 'random',\n    'fields.price.min' ='0.00',\n    'fields.price.max' = '1000.00'\n\n\n);\n\n\nINSERT INTO stock_table \nSELECT random_ticker_udf() as ticker, event_time, price from generate_stock_data;"""
    insert_datagen_content["title"] = """
3) Please run this paragraph before running any additional event_time based queries in order to generate new data into the MSK topic
"""
    insert_datagen_content["config"] = {}
    insert_datagen_content["config"]["title"]= "true"

    code_content["paragraphs"].append(create_udf_content)
    code_content["paragraphs"].append(create_stock_table_content)
    code_content["paragraphs"].append(insert_datagen_content)

    code_content["name"] = app_name
    code_content["id"] = "ABCDEFGHI"
    code_content["defaultInterpreterGroup"] = "flink"
    code_content["version"] = "0.9.0-rc1-msf1"
    code_content["path"] = "/" + app_name

    return json.dumps(code_content)

def timeout_handler(_signal, _frame):
    '''Handle SIGALRM'''
    raise Exception('Operation timed out')

signal.signal(signal.SIGALRM, timeout_handler)


import matplotlib.pyplot as plt

def plot_loss(loss_file):
    with open(loss_file, 'r') as file:
        losses = []
        avg_losses = []
        avg = 0
        index = 1
        for line in file:
            loss = float(line.strip())
            losses.append(loss)
            avg += loss
            index += 1
            if index % 100 == 0:
                avg_losses.append(avg / 100)
                avg = 0

    plt.plot(list(range(len(losses))), losses, 'bp')
    plt.title('Train loss  (Binary-Cross-Entropy)')
    plt.xlabel('number of examples')
    plt.ylabel('loss')
    plt.show()

    plt.plot(list(range(len(avg_losses))), avg_losses)
    plt.title('Train average loss  (Binary-Cross-Entropy)')
    plt.xlabel('number of examples / 100')
    plt.ylabel('average loss')
    plt.show()

def plot_auc(train_auc, test_auc):
    with open(train_auc, 'r') as file:
        tr_auc = []
        for line in file:
            tr_auc.append(float(line.strip()))
    with open(test_auc, 'r') as file:
        te_auc = []
        for line in file:
            te_auc.append(float(line.strip()))

    plt.plot(list(range(len(tr_auc))), tr_auc, label='train')
    plt.plot(list(range(len(te_auc))), te_auc, label='test')

    plt.title('edited cancer data AUC score per number of epochs')
    plt.xlabel('epoch')
    plt.ylabel('auc score')
    plt.legend()
    plt.show()

def plot_mul_auc(auc_files, labels, title):
    for auc_file, label in zip(auc_files, labels):
        with open(auc_file, 'r') as file:
            aucs = []
            for line in file:
                aucs.append(float(line.strip()))
            plt.plot(list(range(len(aucs))), aucs, label=label)
    plt.title(title)
    plt.xlabel('epoch')
    plt.ylabel('auc score')
    plt.legend()
    plt.show()

# plot_loss('loss_file1')
# plot_loss('loss_file2')
# plot_loss('loss_file3')
# plot_loss('loss_file100_n')

# plot_auc('train_auc', 'test_auc')
# plot_auc('train_auc_b50', 'test_auc_b50')
# plot_auc('train_auc_b100', 'test_auc_b100')
'''
plot_mul_auc(['test_auc_lr0.001_wd0', 'test_auc_lr0.001_wd0.001', 'test_auc_lr0.001_wd0.0001', 'test_auc_lr0.001_wd1e-05',
              'test_auc_lr0.0001_wd0', 'test_auc_lr0.0001_wd0.001', 'test_auc_lr0.0001_wd0.0001', 'test_auc_lr0.0001_wd1e-05',
              'test_auc_lr1e-05_wd0', 'test_auc_lr1e-05_wd0.001', 'test_auc_lr1e-05_wd0.0001', 'test_auc_lr1e-05_wd1e-05'],
             ['lr=1e-3, wd=0', 'lr=1e-3, wd=1e-3', 'lr=1e-3, wd=1e-4', 'lr=1e-3, wd=1e-5',
              'lr=1e-4, wd=0', 'lr=1e-4, wd=1e-3', 'lr=1e-4, wd=1e-4', 'lr=1e-4, wd=1e-5',
              'lr=1e-5, wd=0', 'lr=1e-5, wd=1e-3', 'lr=1e-5, wd=1e-4', 'lr=1e-5, wd=1e-5'])

plot_mul_auc(['train_auc_lr0.01_wd0', 'train_auc_lr0.01_wd0.001', 'train_auc_lr0.01_wd0.0001', 'train_auc_lr0.01_wd1e-05',
              'train_auc_lr0.1_wd0', 'train_auc_lr0.1_wd0.001', 'train_auc_lr0.1_wd0.0001', 'train_auc_lr0.1_wd1e-05'],
             ['lr=1e-2, wd=0', 'lr=1e-2, wd=1e-3', 'lr=1e-2, wd=1e-4', 'lr=1e-2, wd=1e-5',
              'lr=1e-1, wd=0', 'lr=1e-1, wd=1e-3', 'lr=1e-1, wd=1e-4', 'lr=1e-1, wd=1e-5'])
'''
'''
plot_mul_auc(['train_auc_s', 'train_auc_d', 'test_auc_s', 'test_auc_d'],
             ['siamese model train', 'double model train', 'siamese model test', 'double model test'],
             'Train and test AUC for different models')
'''

'''
lrs = [1e-4, 1e-3, 1e-2, 1e-1]
wds = [1, 1e-1, 1e-2, 1e-3, 1e-4, 1e-5]
models = ['siamese', 'double']
for model in models:
    for option in [3]:
        files = []
        labels = []
        for lr in lrs:
            for wd in wds:
                if model == 'siamese':
                    key = 's'
                elif model == 'double':
                    key = 'd'
                files.append('cancer_grid_o3/cancer_grid_o3_auc/test_auc2_' + key + str(option) + '_lr' + str(lr) + '_wd' + str(wd))
                labels.append('lr=' + str(lr) + ', wd=' + str(wd))
        title = 'Test auc score, ' + model + ' model, option ' + str(option)
        plot_mul_auc(files, labels, title)
'''

def plot_auc_option_2(train, test_w, test_c):
    with open(train, 'r') as file:
        tr_auc = []
        for line in file:
            tr_auc.append(float(line.strip()))
    with open(test_w, 'r') as file:
        te_auc_w = []
        for line in file:
            te_auc_w.append(float(line.strip()))
    with open(test_c, 'r') as file:
        te_auc_c = []
        for line in file:
            te_auc_c.append(float(line.strip()))

    plt.plot(list(range(len(tr_auc))), tr_auc, label='train')
    plt.plot(list(range(len(te_auc_w))), te_auc_w, label='weizmann test')
    plt.plot(list(range(len(te_auc_c))), te_auc_c, label='cancer test')

    plt.title('Option 2 AUC score per number of epochs')
    plt.xlabel('epoch')
    plt.ylabel('auc score')
    plt.legend()
    plt.show()

#plot_auc_option_2('train_auc_2', 'test_auc_2_w', 'test_auc_2_c')
#plot_auc('train_auc_3000', 'test_auc_3000')
#plot_auc('train_auc_3_10tcr1000', 'test_auc_3_10tcr1000')

'''
# weizmann, shugay, cancer, different regularization (dropout=0.1 hdim=30)
plot_mul_auc(['train_w_d01_hdim30', 'test_w_d01_hdim30', 'train_w_d01_hdim30_wd-5', 'test_w_d01_hdim30_wd-5'],
             ['train, wd=0', 'test, wd=0', 'train, wd=1e-5', 'test, wd=1e-5'],
             'LSTM based model with different regularization, Weizmann data')

plot_mul_auc(['train2_s_d01_hdim30', 'test2_s_d01_hdim30', 'train2_s_d01_hdim30_wd-5', 'test2_s_d01_hdim30_wd-5'],
             ['train, wd=0', 'test, wd=0', 'train, wd=1e-5', 'test, wd=1e-5'],
             'LSTM based model with different regularization, Shugay data')

plot_mul_auc(['train_c_d05_hdim30_wd-5', 'test_c_d05_hdim30_wd-5', 'train_c_d01_hdim30_wd-5', 'test_c_d01_hdim30_wd-5'],
             ['train, dropout=0.5', 'test, dropout=0.5', 'train, dropout=0.1', 'test, dropout=0.1'],
             'LSTM based model with different dropouts, cancer data')
'''


import random
import sys
import time

from gurobi import solve
from MTM_EXTENDED_recursive import MTM_EXTENDED_recursive
from MMKAI_recursive import MMKAI_recursive
from src.MTM_EXTENDED_iterative import MTM_EXTENDED_iterative
from src.models.knapsack import Knapsack
from src.models.item_class import ItemClass

import gurobipy as grb

seed = random.randrange(sys.maxsize)
# seed = 42
random.seed(seed)
print(f"Seed: {seed}")

topend = 10_000

numer_of_knapsacks = 5
print(f"Number of knapsacks: {numer_of_knapsacks}")

knapsacks = [Knapsack(random.randint(1, topend * 10)) for _ in range(numer_of_knapsacks)]

number_of_weightclasses = 2
print(f"Number of weight classes: {number_of_weightclasses}")

weightclasses = [ItemClass(1, random.randint(1, topend // 2)) for _ in range(number_of_weightclasses)]

number_of_items = topend
print(f"Number of items: {number_of_items}")

items = [random.choice(weightclasses).add_item(knapsacks) for _ in
         range(number_of_items)]

if __name__ == '__main__':
    from pprint import pprint
    print("Solving...")

    class_to_use = input("Class to use (1) MMKAI, (2) MTM_EXTENDED: ")

    if "1" in class_to_use or "MMKAI" in class_to_use:
        solver = MMKAI_recursive(weightclasses, knapsacks, items)
    elif "2" in class_to_use or "MTM_EXTENDED" in class_to_use:
        iterative = input("Iterative? (y/n): ")
        if iterative in ["y", "Y", "j", "J", "yes", "Yes", "YES", "1", "Ja", "ja", "JA"] or bool(iterative):
            solver = MTM_EXTENDED_iterative(items=items, knapsacks=knapsacks, item_classes=None)
        else:
            solver = MTM_EXTENDED_recursive(items=items, knapsacks=knapsacks, item_classes=None)
    else:
        raise Exception("Wrong class")

    start = time.time()
    val, sol = solver.solve()
    end = time.time()

    print("Solved")

    start_gurobi = time.time()
    gurobisol, gurobisolution = solve(knapsacks, items, threads=1)
    end_gurobi = time.time()

    print(f"\033[92mValue: {val}")
    print(f"Time: {end - start}")
    print(f"Solution: {sol}")

    pprint(solver.best_solution)

    print(f"Gurobi Value: {gurobisol}")
    print(f"Gurobi Time: {end_gurobi - start_gurobi}\033[0m")

    # transform gurobisol to a match_vec
    match_vec = {i: [0 for _ in range(len(knapsacks))] for i in weightclasses}
    for item, knapsack in gurobisolution.items():
        match_vec[item.item_class][knapsacks.index(knapsack)] += 1

    pprint(match_vec)

    from pprint import pprint

    # pprint(_)
    model = grb.Model()

    # We will be maximizing
    model.modelSense = grb.GRB.MAXIMIZE

    # create variables
    x = grb.tupledict()
    for item in items:
        for knapsack in item.restrictions:
            x[item, knapsack] = model.addVar(vtype=grb.GRB.BINARY,
                                             name=f'x[{item.identifier},{knapsack.identifier}]',
                                             obj=item.profit)

    # add constraints
    model.addConstrs((x.sum(item, '*') <= 1 for item in items))

    model.addConstrs(
        (grb.quicksum(x[item, knapsack] * item.weight
                      for item in items if knapsack in item.restrictions)
         <= knapsack.capacity for knapsack in knapsacks), name="knapsack capacity")

    model.update()
    print("\033[94mModel created\033[0m")

    it = set(items)

    for k, j in sol.items():
        for i in j:
            it.remove(i)
            x[i, k].LB = 1

    for i in it:
        for k in knapsacks:
            x[i, k].UB = 0

    model.optimize()

    if model.status == grb.GRB.Status.OPTIMAL:
        if abs(model.objVal - val) > 0.1:
            raise Exception("Wrong solution value")

        if abs(gurobisol - val) > 0.1:
            raise Exception("Wrong solution")

    elif model.status == grb.GRB.Status.INFEASIBLE:
        model.computeIIS()
        model.write("model.ilp")
        raise Exception("Non feasible solution")