from django.test import LiveServerTestCase

from djangae.tasks import (
    cloud_tasks_parent_path,
    cloud_tasks_queue_path,
    ensure_required_queues_exist,
    get_cloud_tasks_client,
)
from google.api_core.exceptions import GoogleAPIError

class TaskFailedBehaviour:
    DO_NOTHING = 0
    RETRY_TASK = 1
    RAISE_ERROR = 2

class TaskFailedError(Exception):
    def __init__(self, task_name, status_code, original_exception=None):
        self.task_name = task_name
        self.status_code = status_code
        self.original_exception = original_exception

        super(TaskFailedError, self).__init__(
            "Task {} failed with status code: {}. \n\nMessage was: {}".format(
                task_name, status_code, original_exception
            )
        )

class TestCaseMixin(LiveServerTestCase):
    """
        A TestCase base class that manages task queues
        during testing. Ensures that required queues
        are created and paused, and manually runs the
        queued tasks in them to check their responses
    """
    def __init__(self, *args, **kwargs):
        self.max_task_retry_count = 100
        super().__init__(*args, **kwargs)

    def setUp(self):
        # Create all the queues required by this application

        super().setUp()

        # Find the port we were allocated
        self._server_port = self.live_server_url.rsplit(":")[-1]

        ensure_required_queues_exist()

        self.task_client = get_cloud_tasks_client()

        parent = cloud_tasks_parent_path()

        for queue in self.task_client.list_queues(parent=parent):
            # Make sure the queue is paused
            self.task_client.pause_queue(queue.name)

            # Make sure it's empty
            self.task_client.purge_queue(queue.name)

    def _get_queues(self, queue_name=None):
        if queue_name:
            path = cloud_tasks_queue_path(queue_name)
            queue = self.task_client.get_queue(path)
            queues = [queue]
        else:
            parent = cloud_tasks_parent_path()
            queues = self.task_client.list_queues(parent)

        return queues

    def flush_task_queues(self, queue_name=None):
        for queue in self._get_queues(queue_name=queue_name):
            self.task_client.purge_queue(queue.name)

    def get_task_count(self, queue_name=None):
        count = 0
        for queue in self._get_queues(queue_name=queue_name):
            path = queue.name
            count += len(list(self.task_client.list_tasks(path)))

        return count

    def assertNumTasksEquals(self, num, queue_name=None):
        self.assertEqual(num, self.get_task_count(queue_name=queue_name))

    def _get_all_tasks_for_queues(self, queue_names):
        tasks = []
        for path in queue_names:
            tasks += [x for x in self.task_client.list_tasks(path)]
        return tasks

    def process_task_queues(self, queue_name=None, failure_behaviour=TaskFailedBehaviour.RAISE_ERROR):
        queue_names = [q.name for q in self._get_queues(queue_name)]

        tasks = self._get_all_tasks_for_queues(queue_names)
        task_failure_counts = {}

        while tasks:
            task = tasks.pop(0)

            try:
                # Appending the port number to the task name
                # is a special-case that does not exist
                # on the live server and it exists so that
                # local development servers can direct a task
                # to run on a particular port.
                response = self.task_client.run_task(task.name + "?port=%s" % self._server_port)

                # If the returned status wasn't a success then
                # drop into the except block below to handle the
                # failure
                status = response.last_attempt.response_status.code
                if str(status)[0] != "2":
                    raise GoogleAPIError("Task returned bad status: %s" % status)

            except GoogleAPIError as e:
                if failure_behaviour == TaskFailedBehaviour.RETRY_TASK:
                    if task.name not in task_failure_counts:
                        task_failure_counts[task.name] = 1
                    else:
                        task_failure_counts[task.name] += 1

                    if task_failure_counts[task.name] >= self.max_task_retry_count:
                        # Make sure we don't get an infinite loop while retrying
                        raise

                    tasks.append(task)  # Add back to the end of the queue
                    continue
                elif failure_behaviour == TaskFailedBehaviour.RAISE_ERROR:
                    raise TaskFailedError(task.name, str(e))
                else:
                    # Do nothing, ignore the failure
                    pass

            if not tasks:
                tasks = self._get_all_tasks_for_queues(queue_names)


from dataclasses import dataclass
from typing import List
from app.db.entities.transport_worker import TransportWorker

@dataclass
class EmployeesPosition:
    id: int
    name: str
    description: str
    available_workers: List[TransportWorker]

    def to_dict(self) -> dict:
        transport_dict = self.__dict__
        transport_dict['available_workers'] = list(
            map(lambda x: x.to_dict(), self.available_workers)
        )
        return transport_dict

    @classmethod
    def from_json(cls, item_dict: dict) -> 'EmployeesPosition':
        return cls(
            item_dict.get("id"),
            item_dict.get("name"),
            item_dict.get("description"),
            list(
                map(
                    lambda x: TransportWorker(
                        x.get("id"),
                        x.get("full_name"),
                        x.get("phone"),
                        x.get("birth_date"),
                    ),
                    item_dict.get("available_workers"),
                )
            ),
        )


import os
import csv
import json
import pandas as pd
from itertools import islice

from math import sqrt
#test

def make_json(csvFilePath, jsonFilePath):
    data = {}

    with open(csvFilePath, encoding='utf-8-sig') as csvf:
        csvReader = csv.DictReader(csvf)

        for row in islice(csvReader, 0, None, 3):
            # print(row)
            row["point"] = row["point"].replace("淡", "")
            row["system"] = row["system"].replace(" ", "")

            #strat the process of Left Side of the River
            row["L-mode"] = row["L-mode"].replace(" ", "")
            if row["L-mode"] == "支距":
                row["L-mode"] = "distance"
            elif row["L-mode"] == "轉換":
                row["L-mode"] = "transform"

            row["L-Ordinate-N(Y)"] = row["L-Ordinate-N(Y)"].replace(" ", "")
            row["L-Ordinate-N(Y)"] = row["L-Ordinate-N(Y)"].replace(",", "")
            row["L-Abscissa-E(X)"] = row["L-Abscissa-E(X)"].replace(" ", "")
            row["L-Abscissa-E(X)"] = row["L-Abscissa-E(X)"].replace(",", "")

            row["L-Rel-0-points"] = row["L-Rel-0-points"].replace(" ", "")
            row["L-Rel-0-points"] = row["L-Rel-0-points"].replace(",", "")

            row["L-Elevation"] = row["L-Elevation"].replace(" ", "")
            row["L-Elevation"] = row["L-Elevation"].replace(",", "")

            #strat the process of Right Side of the River
            row["R-mode"] = row["R-mode"].replace(" ", "")
            if row["R-mode"] == "支距":
                row["R-mode"] = "distance"
            elif row["R-mode"] == "轉換":
                row["R-mode"] = "transform"

            row["R-Ordinate-N(Y)"] = row["R-Ordinate-N(Y)"].replace(" ", "")
            row["R-Ordinate-N(Y)"] = row["R-Ordinate-N(Y)"].replace(",", "")
            row["R-Abscissa-E(X)"] = row["R-Abscissa-E(X)"].replace(" ", "")
            row["R-Abscissa-E(X)"] = row["R-Abscissa-E(X)"].replace(",", "")

            row["R-Rel-0-points"] = row["R-Rel-0-points"].replace(" ", "")
            row["R-Rel-0-points"] = row["R-Rel-0-points"].replace(",", "")

            row["R-Elevation"] = row["R-Elevation"].replace(" ", "")
            row["R-Elevation"] = row["R-Elevation"].replace(",", "")

            key = row['point']
            data[key] = row
            dx = pow((float(row["R-Abscissa-E(X)"]) - float(row["L-Abscissa-E(X)"])), 2)
            dy = pow((float(row["R-Ordinate-N(Y)"]) - float(row["L-Ordinate-N(Y)"])), 2)
            data[key]['distance'] = pow((dx + dy), 0.5)

    with open(jsonFilePath, 'w', encoding='utf-8-sig') as jsonf:
        jsonf.write(json.dumps(data, indent=4))

def readjson2Dict(jsonFilePath):
    global tamsui_json
    with open(jsonFilePath, 'r', encoding='utf-8-sig') as jsonf:
        tamsui_json = json.load(jsonf)

def print_preprocessed_data(x_ext, y_ext):
    print(f" -> after data processing: x_ext: {x_ext}")
    print(f" -> after data processing: y_ext: {y_ext}")

def fix_extdata(file_name, x_ext, y_ext):
    isAcceptable = True

    if len(x_ext) != len(y_ext):
        print(f" -> {file_name} data is not complete.")
        isAcceptable = False
    else:
        for list_index in range(len(x_ext)):
            x_ext[list_index] = round(x_ext[list_index], 2)
            y_ext[list_index] = round(y_ext[list_index], 2)

    return x_ext, y_ext, isAcceptable

def validate_data(new_x, ex_data, ny_data):
    if len(ex_data) != len(ny_data):
        print("Data is not complete.")
        return False # if the data is not complete

    for list_index in range(1, len(ex_data)):
        check_value = sqrt(pow((ex_data[list_index] - ex_data[0]), 2) + pow((ny_data[list_index] - ny_data[0]), 2))
        # print(f"The delta is {check_value} - {new_x[list_index]} = {abs(check_value - new_x[list_index])} at pos: {list_index}")
        if abs(check_value - new_x[list_index]) > 0.2:
            print("Data is not correct. ")
            # print(f"The delta is {check_value} - {new_x[list_index]} = {abs(check_value - new_x[list_index])}")
            return False # if the value is not correct

    return True #pass the validation

def axis_predict_formula(curr_rel_0, last_rel_0, first_axis, last_axis):
    r = (round(curr_rel_0, 2) * (last_axis - first_axis) + (last_rel_0 * first_axis)) / last_rel_0
    # print(f"({round(curr_rel_0, 2)} * ({last_axis} - {first_axis}) + ({last_rel_0} * {first_axis})) / {last_rel_0}")
    # print(f"The result of predict: {r}")
    return round((round(curr_rel_0, 2) * (last_axis - first_axis) + (last_rel_0 * first_axis)) / last_rel_0, 4) # the formula to predict next value of axis(x or y)

def tr_write_csv(filename, elevation, ex_data, ny_data, year, month, riversection):
    transform_df = pd.DataFrame()

    elevation.insert(0, year)
    ex_data.insert(0, month)
    ny_data.insert(0, riversection)

    transform_df['elevation'] = elevation
    transform_df['ex_data'] = ex_data
    transform_df['ny_data'] = ny_data

    filename = "/Users/alecwu/TRANSFORM_NEW/" + "TRANSFORM_" + filename

    pd.set_option('display.float_format', '{:.5f}'.format)

    print(transform_df)
    transform_df.to_csv(filename, index=False)

def pos_transform(raw_data_path):
    if not os.path.exists(raw_data_path):
        print(f"The directory '{raw_data_path}' does not exist.")
        return

    if not os.path.isdir(raw_data_path):
        print(f"'{raw_data_path}' is not a directory.")
        return

    files = os.listdir(raw_data_path)
    print(f"Files in the directory '{raw_data_path}':")

    month_id = ["120", "100", "050"]
    status = "pass"
    declined_list = []
    Correct_list = []
    not_found_rel0points = []
    for file_name in files:
        #message for processing the file
        print(f"Processing '{file_name}'...", end="")

        for month in month_id:
            if file_name.find(month) == -1:
                continue

            mid_index = file_name.find(month)
            year = file_name[1:mid_index]
            river_section = file_name[mid_index+2:]
            river_section = river_section.replace("csv", "")
            river_section = river_section.replace(".", "")

            print(f" -> year: {year}, month: {month}, river_section: {river_section}")

            if river_section in list(tamsui_json.keys()):
                print(f" -> {river_section} is ACCEPTED.")

                to_open = raw_data_path + "/" + file_name
                with open(to_open, 'r', encoding='utf-8-sig') as csvf:
                    print(f" -> {file_name} is opened.")

                    #read the csv file
                    csv_reader = pd.read_csv(csvf)

                    # Define the column to extract
                    extracted_x = "x"
                    extracted_y = "y"

                    # Extract data using a list comprehension
                    x_ext = csv_reader[extracted_x].tolist()
                    y_ext = csv_reader[extracted_y].tolist()

                    # print(f" -> Extracted data: {extracted_data}")
                    if len(x_ext) == 0 or len(y_ext) == 0:
                        print(f" -> {file_name} data is empty.")
                        break
                    elif len(x_ext) != len(y_ext):
                        print(f" -> {file_name} data is not complete.")
                        break
                    else:
                        print(f" -> {file_name} data is complete.")

                    if x_ext[0] > x_ext[-1]:
                        print(f" -> range check failed. [{x_ext[0]}, {x_ext[-1]}]")

                    print(f" -> x_ext: {x_ext} len: {len(x_ext)}")
                    print(f" -> y_ext: {y_ext} len: {len(y_ext)}")

                    R_rel_0_points = tamsui_json[river_section]["R-Rel-0-points"] # checking -96.55
                    R_rel_0_points = float(R_rel_0_points)
                    R_elevation = tamsui_json[river_section]["R-Elevation"]
                    R_elevation = float(R_elevation)
                    L_rel_0_points = tamsui_json[river_section]["L-Rel-0-points"] # checking -96.55
                    L_rel_0_points = float(L_rel_0_points)
                    L_elevation = tamsui_json[river_section]["L-Elevation"]
                    L_elevation = float(L_elevation)
                    found = False
                    pos = -1

                    if R_rel_0_points in x_ext: # find the value (Right side)
                        print(f" -> {x_ext[x_ext.index(R_rel_0_points)]} is the R-Rel-0-points.")
                        found = True
                        pos = x_ext.index(R_rel_0_points)

                        for list_index in range(len(x_ext)-1):
                            if(x_ext[list_index] == x_ext[pos] and y_ext[list_index] == R_elevation):
                                pos = list_index # the currect position of rel_0
                                continue
                            elif(x_ext[list_index] == x_ext[pos]): # remove the duplicated rel_0
                                x_ext.remove(x_ext[list_index])
                                y_ext.remove(y_ext[list_index])

                            if(x_ext[list_index] > x_ext[pos]):
                                break

                        point_found = False # the flag of finding the correct index with correct rel_0 and elevation
                        for list_value in x_ext[:]:
                            list_index = x_ext.index(list_value)
                            print(f" -> check {list_index}'s {list_value} {R_rel_0_points} at {y_ext[list_index]} {round(R_elevation,2)}")
                            if(list_value == R_rel_0_points):
                                if(y_ext[list_index] == round(R_elevation,2)):
                                    pos = list_index # the currect position of rel_0
                                    point_found = True # find the correct index
                                    print(f" -> {list_value} at {list_index} is the corrert value")
                                    continue
                                else: # remove the duplicated rel_0
                                    x_ext.remove(list_value)
                                    y_ext.remove(y_ext[list_index])

                            if(list_value > R_rel_0_points):
                                break

                        if(not point_found): # not find the correct index -> using forced insertion
                            print(f" -> Forced inserting {R_rel_0_points} {round(R_elevation,2)} to list")
                            x_ext.insert(pos,R_rel_0_points)
                            y_ext.insert(pos,round(R_elevation,2))
                            point_found = True

                        x_ext = x_ext[:pos+1]
                        y_ext = y_ext[:pos+1]

                        print(f" -> pos: {pos}")
                        # print_preprocessed_data(x_ext, y_ext)

                    if not found: # Insert values (Right side)
                        not_found_dict = {file_name: R_rel_0_points}
                        not_found_rel0points.append(not_found_dict)

                        print(f" -> {R_rel_0_points} is NOT found.")

                        for list_index in range(len(x_ext)-1):
                            if x_ext[list_index] < R_rel_0_points and R_rel_0_points < x_ext[list_index+1]:
                                pos = list_index
                                found = True

                                print(f" -> {file_name} insert value.")
                                x_ext.insert(pos+1, R_rel_0_points)
                                y_ext.insert(pos+1, R_elevation)

                                x_ext = x_ext[:pos+2]
                                y_ext = y_ext[:pos+2]

                                break

                        if found:
                            pos = x_ext.index(R_rel_0_points)
                            print(f" -> pos: {pos+1}")
                            # print_preprocessed_data(x_ext, y_ext)
                        else:
                            print(f" -> {R_rel_0_points} is NOT between {x_ext[0]} and {x_ext[-1]}.")
                            print(f" -> Appending {R_rel_0_points}, {R_elevation} to the end of x_ext and y_ext list ...")
                            x_ext.append(R_rel_0_points)
                            y_ext.append(R_elevation)

                    found = False # Reset found to false for left side

                    if L_rel_0_points in x_ext: # find the value (Left side)
                        print(f" -> {x_ext[x_ext.index(L_rel_0_points)]} is the L-Rel-0-points.")
                        found = True
                        pos = x_ext.index(L_rel_0_points)

                        point_found = False # the flag of finding the correct index with correct rel_0 and elevation
                        for list_value in x_ext[:]:
                            list_index = x_ext.index(list_value)
                            print(f" -> check {list_index}'s {list_value} {L_rel_0_points} at {y_ext[list_index]} {round(L_elevation,2)}")
                            if(list_value == L_rel_0_points):
                                if(y_ext[list_index] == round(L_elevation,2)):
                                    pos = list_index # the currect position of rel_0
                                    point_found = True # find the correct index
                                    print(f" -> {list_value} at {list_index} is the corrert value")
                                    continue
                                else: # remove the duplicated rel_0
                                    x_ext.remove(list_value)
                                    y_ext.remove(y_ext[list_index])

                            if(list_value > L_rel_0_points):
                                break

                        if(not point_found): # not find the correct index -> using forced insertion
                            print(f" -> Forced inserting {L_rel_0_points} {round(L_elevation,2)} to list")
                            x_ext.insert(pos,L_rel_0_points)
                            y_ext.insert(pos,round(L_elevation,2))
                            point_found = True

                        x_ext = x_ext[pos:]
                        y_ext = y_ext[pos:]

                        print(f" -> pos: {pos}")

                    if not found: # Insert values (Left side)
                        not_found_dict = {file_name: L_rel_0_points}
                        not_found_rel0points.append(not_found_dict)

                        print(f" -> {L_rel_0_points} is NOT found.")
                        # print(f" -> x_ext : {x_ext}")
                        for list_index in range(len(x_ext)-1):
                            if x_ext[list_index] < L_rel_0_points and L_rel_0_points < x_ext[list_index+1]:
                                pos = list_index
                                found = True

                                print(f" -> {file_name} insert value.")
                                x_ext.insert(pos+1, L_rel_0_points)
                                y_ext.insert(pos+1, L_elevation)

                                x_ext = x_ext[pos+1:]
                                y_ext = y_ext[pos+1:]
                                break

                        if found:
                            pos = x_ext.index(L_rel_0_points)
                            print(f" -> pos: {pos+1}")
                        else:
                            print(f" -> {L_rel_0_points} is NOT between {x_ext[0]} and {x_ext[-1]}.")
                            print(f" -> Inserting {L_rel_0_points}, {L_elevation} to the head of x_ext and y_ext list ...")
                            x_ext.insert(0,L_rel_0_points)
                            y_ext.insert(0,L_elevation)

                    print(f" -> x_ext: {x_ext} len: {len(x_ext)}")
                    print(f" -> y_ext: {y_ext} len: {len(y_ext)}")

                    print(f" -> {file_name} is doing fixing.")
                    x_ext, y_ext, isAcceptable = fix_extdata(file_name, x_ext, y_ext)

                    if not isAcceptable:
                        print(f" -> ERROR occurred in {file_name}.")
                        break

                    # print(f" -> after fixing - round(numebr, 2): x_ext: {x_ext}")
                    # print(f" -> after fixing - round(numebr, 2): y_ext: {y_ext}")

                    print(f" -> removing duplicated data in {file_name}.")
                    xtemp = []
                    ytemp = []
                    now = x_ext[0]
                    xtemp.append(x_ext[0])
                    ytemp.append(y_ext[0])
                    for list_index in range(1, len(x_ext)-1):
                        if x_ext[list_index] != now:
                            xtemp.append(x_ext[list_index])
                            ytemp.append(y_ext[list_index])
                        now = x_ext[list_index]
                    xtemp.append(x_ext[-1])
                    ytemp.append(y_ext[-1])

                    if len(xtemp) != len(ytemp):
                        print(f" -> ERROR occurred in the process of removing duplicated data in {file_name}.")
                        break

                    x_ext = xtemp
                    y_ext = ytemp

                    del xtemp
                    del ytemp

                    '''
                    This is a exmple for using the function of validate_data(new_x, ex_data, ny_data)

                    90A:
                    test_new_x = [0.0000, 4.7300, 12.1500, 14.3600]
                    test_ex_data = [275470.255, 275474.1711, 275480.3151, 275482.1451]
                    test_ny = [2746480.522, 2746477.8695, 2746473.7094, 2746472.4703]

                    if validate_data(test_new_x, test_ex_data, test_ny):
                        print(" -> Data is correct.")
                    else:
                        print(f" -> A validation error occurred in the process of removing duplicated data in {file_name}.")
                        break
                    '''

                    print(f" -> after removing duplicated data: x_ext: {x_ext} len: {len(x_ext)}")
                    print(f" -> after removing duplicated data: y_ext: {y_ext} len: {len(y_ext)}")

                    L_Ordinate_N_Y = tamsui_json[river_section]["L-Ordinate-N(Y)"]
                    L_Ordinate_N_Y = float(L_Ordinate_N_Y) # first Y
                    L_Abscissa_E_X = tamsui_json[river_section]["L-Abscissa-E(X)"]
                    L_Abscissa_E_X = float(L_Abscissa_E_X) # first X
                    R_Ordinate_N_Y = tamsui_json[river_section]["R-Ordinate-N(Y)"]
                    R_Ordinate_N_Y = float(R_Ordinate_N_Y) # last Y
                    R_Abscissa_E_X = tamsui_json[river_section]["R-Abscissa-E(X)"]
                    R_Abscissa_E_X = float(R_Abscissa_E_X) # last X
                    axis_x_predict_list = []
                    axis_y_predict_list = []
                    print(f" -> Computing the value of axis in {file_name}.")
                    axis_x_predict_list.append(L_Abscissa_E_X)
                    axis_y_predict_list.append(L_Ordinate_N_Y)
                    x_ext[-1] -= x_ext[0]

                    for index in range(1,len(x_ext)-1):
                        x_ext[index] -= x_ext[0]
                        # parameter => curr_rel_0, last_rel_0, first_axis, last_axis
                        predict_x = axis_predict_formula(x_ext[index],x_ext[-1],L_Abscissa_E_X,round(R_Abscissa_E_X,3))
                        predict_y = axis_predict_formula(x_ext[index],x_ext[-1],L_Ordinate_N_Y,round(R_Ordinate_N_Y,3))
                        axis_x_predict_list.append(predict_x)
                        axis_y_predict_list.append(predict_y)
                    axis_x_predict_list.append(R_Abscissa_E_X)
                    axis_y_predict_list.append(R_Ordinate_N_Y)

                    x_ext[0] = 0
                    isVaild = validate_data(x_ext, axis_x_predict_list, axis_y_predict_list)
                    if(isVaild):
                        Correct_list.append(river_section)
                        print(" -> Data is correct.")
                        print(f" -> Writing the data to csv file in {file_name}.")
                        tr_write_csv(file_name, y_ext, axis_x_predict_list, axis_y_predict_list, year, month, river_section)

                        print(f" -> {file_name} is all done.\n")

                    print(f" -> Process of {file_name} is all done.\n")
            else:
                print(f" -> {river_section} is DECLINED.\n")
                declined_list.append(river_section)

            break

    print(f"Correct list: {Correct_list}")
    print(f"Total correct: {len(Correct_list)}")
    print()
    print(f"Declined list: {declined_list}")
    print(f"Total declined: {len(declined_list)}") # 4898
    print()
    #print Not found Rel-0-points line by line
    # for not_found in not_found_rel0points:
    #     print(f"Not found Rel-0-points: {not_found}")
    # print(f"Total not found Rel-0-points: {len(not_found_rel0points)}")

    # print(f"\n{len(files)} files in total.")

def merge_csv(files_path): #sorry我把妳ㄉcode改了一點點，"一點點"
    files = os.listdir(files_path)

    river_section = tamsui_json.keys()

    river_section = list(river_section)
    river_section.reverse()

    '''
    the dictionary should be like this:
    river_dict = {
        "58":{   ------> year
            "90A": [[elevation, ex, ny], [elevation, ex, ny], [elevation, ex, ny]], ------> each of river section
            "90": [................],
            "89": [], ------> may be empty
            ....
            "00": [................],
        },
        "60":{
            "90A":[................],
            "90":[................],
            ....
        },
        ...
        "109":{
            "90A":[................],
            "90":[................],
            ....
        }
    }
    '''

    # create a dictionary to store the data
    river_dict = {}
    for year in range(58, 110):
        river_dict[str(year)] = {}
        for river in river_section:
            river_dict[str(year)][river] = []

    # print the river_dict line by line
    for year in river_dict:
        print(f" -> {year}: {river_dict[year]}")
        print()

    ll = []

    print(f" -> River section: {river_section}")

    month_id = ["120", "100", "050"]

    for file_name in files:
        if file_name.find("TRANSFORM") == -1:
            continue

        print(f" -> Checking {files_path + '/' + file_name}...")

        for month in month_id:
            if file_name.find(month) == -1:
                continue

            mid_index = file_name.find(month)
            year = file_name[11:mid_index]
            river_section = file_name[mid_index+2:]
            river_section = river_section.replace("csv", "")
            river_section = river_section.replace(".", "")

            print(f" -> year: {year}, month: {month}, river_section: {river_section}")

            print(f" -> Opening {files_path + '/' + file_name}...")
            to_open = files_path + "/" + file_name
            df = pd.read_csv(to_open)
            print(f" -> [v] {files_path + '/' + file_name} is opened.")

            # print(f" -> test df: {df['ny_data'][1]}")

            # add the data to the dictionary
            print(" -> Adding data to the dictionary...")
            for index in range(len(df['ny_data'])):
                river_dict[year][river_section].append([df['elevation'][index], df['ex_data'][index], df['ny_data'][index]])
            print(" -> [v] Data is added to the dictionary.")

            print()
            break

    print("\n\n")
    # merge the data of each year to a csv file
    print(" -> Merging the data of each year to a csv file...")
    for year in river_dict:
        print(f" -> year: {year}")
        df = pd.DataFrame()
        elevation = []
        ex_data = []
        ny_data = []
        for river in river_dict[year]:
            print(f" -> river: {river}")
            for index in range(len(river_dict[year][river])):
                if index == 0: continue
                elevation.append(river_dict[year][river][index][0])
                ex_data.append(river_dict[year][river][index][1])
                ny_data.append(river_dict[year][river][index][2])
            print(f" -> [v] {river} is done.")
        print(f" -> [v] {year} data is all collected.")

        df["elevation"] = elevation
        df["ex_data"] = ex_data
        df["ny_data"] = ny_data

        # save the data to a csv file
        print(f" -> Saving the data to a csv file in /Users/alecwu/TRANSFORM_eachyear/{year}.csv...")
        df.to_csv(f"/Users/alecwu/TRANSFORM_eachyear/{year}.csv", index=False)

        print(f" -> [v] {year} is done.")
        print()

if __name__ == "__main__":
    dataFilePath = r"./data/"
    csvFilePath = r"tamsui_new.csv"
    jsonFilePath = r"tamsui.json"
    mergeCsvPath = r"./merge/"

    raw_data_path = "/Users/alecwu/result_all" #Today is a good day xD

    nonduplicated_data_path = "/Users/wufangyi/result_non-duplicated"
    save_data_path = "/Users/wufangyi/result_non-duplicated" #commit

    transformed_data_path = "/Users/alecwu/TRANSFORM"

    # make_json(dataFilePath + csvFilePath, jsonFilePath) #make json file
    # readjson2Dict(jsonFilePath) #read json

    # os.mkdir("./TRANSFORM")
    # pos_transform(raw_data_path)

    # os.mkdir("./MERGE")
    # # merge the csv files of each year
    # merge_csv(transformed_data_path)

    # print(tamsui_json)


import Variable_sus as vs
import util
import os
import numpy as np
import openpyxl
import sys

alpha = 0.8 # 怀疑度系数
beta = 0.8  # 输出系数
data_path = r'D:\fault_loc\ITSP-data'
res_file = r''
res_file_temp = os.path.join('result', 'vsbfl_BUCT_sus.xlsx')  # 请先自己创建这个文件

def find_pair_by_tag(dir_path):
    '''
    根据数据集寻找每份错误代码对应的正确代码
    '''
    pair_info = {}
    tag_files = os.listdir(dir_path)
    for i in tag_files:
        tag_file = os.path.join(dir_path, i)
        lines = util.read_file(tag_file)
        wa_file = lines[0].replace('\n', '')
        ac_file = lines[1].replace('\n', '')
        pair_info[wa_file] = ac_file
    # for file in pair_info:
    #     print(file, pair_info[file])
    return pair_info

def find_pair_by_res(file_path):
    '''
    根据匹配结果寻找每份错误代码对应的正确代码
    '''
    pair_info = {}
    if file_path.split('.')[-1] != 'xlsx' and file_path.split('.')[-1] != 'xls':
        print('not an execl file')
        return
    wb = openpyxl.load_workbook(file_path)
    for i, sheet in enumerate(wb):
        if i == 0:
            continue
        # print(sheet.title)
        buggy_name = ''
        for j, row in enumerate(sheet.rows):
            if j == 0:
                continue
            if row[0].value != None:
                buggy_name = row[0].value
            if row[1].value.find('[') >= 0:
                ac_name_list = eval(row[1].value)
                pair_info[buggy_name] = ac_name_list[0]
    #     break
    # print(pair_info)
    return pair_info

def find_possiable_correct_line(wa_res, ac_res):
    '''
    寻找配对的变量
    '''
    # print(wa_res)
    weight, weight_LCS = vs.add_weight(wa_res, ac_res)
    vars_pair = util.cal_KM(weight) #二分图最大完备匹配
    # print(vars_pair)
    # print(weight_LCS)
    possiable_correct_line = {}
    for var in vars_pair:
        possiable_correct_line[var] = weight_LCS[var][vars_pair[var]['var']]
    # possiable_correct_line分为n个列表，n是测试用例的数量，每个列表中的数字为
    # 错误代码变量值序列中能和正确代码匹配的值在整个值序列中的位置
    # print(possiable_correct_line)
    return possiable_correct_line, vars_pair

def cal_sus(wa_file, ac_file, test_dir_path, language):
    '''
    计算怀疑度
    '''
    wa_res, ac_res = vs.get_sequences(wa_file, ac_file, test_dir_path, language)
    # print(wa_res)
    # print(ac_res)
    possiable_correct_line, vars_pair = find_possiable_correct_line(wa_res, ac_res)
    # print(possiable_correct_line)
    lines = util.read_file(wa_file)
    file_length = len(lines)
    line_sus = list(np.ones(file_length + 1))
    for var in possiable_correct_line:
        line_nums = possiable_correct_line[var]
        # print(var, line_nums)
        for i, test_case in enumerate(wa_res):
            # print(i, test_case['info'])
            if var not in test_case['info']:
                continue
            info = test_case['info'][var]
            line_num = line_nums[i]
            possiable_fault_line = range(0, len(info))
            possiable_fault_line = list(filter(lambda x: x not in line_num, possiable_fault_line))
            possiable_fault_line = list(map(lambda x: info[x]['line'], possiable_fault_line))
            line_num = list(map(lambda x: info[x]['line'], line_num))
            # print(var, i, line_num, len(ac_res[i]['info'][var]) - len(line_nums[i]))
            if test_case['res'] == False:
                for j in possiable_fault_line:
                    line_sus[int(j)] *= alpha
                # 这个补丁是为了解决计算不足的问题
                # 这里列表直接用-1还是有危险的，因为可能列表长度为0
                if(len(possiable_fault_line) == 0 and info[-1]['value'] != ac_res[i]['info'][vars_pair[var]['var']][-1]['value']):
                    print(var, i, info)
                    line_sus[int(info[-1]['line'])] *= alpha
            else:
                for j in line_num:
                    line_sus[int(j)] *= (2-alpha)
    # print(line_sus)
    if line_sus.count(min(line_sus)) >= file_length * beta:
        for i, line in enumerate(lines):
            flag = False
            if language == 'cpp' or language == 'c':
                flag = (util.find_pos('cin', line) or util.find_pos('printf', line))
            elif language == 'py':
                flag = util.find_pos('print', line)
            # print(line)
            # print(util.find_pos('printf', line))
            if flag == True:
                # print(line, i)
                line_sus[i + 1] *= alpha
    print(line_sus)
    return line_sus

def run_file(file_path, ac_file, test_dir_path, language):
    '''
    计算程序的最后怀疑度列表
    '''
    print(file_path, ac_file)
    line_sus = cal_sus(file_path, ac_file, test_dir_path, language)
    line_rank = []
    for i,sus in enumerate(line_sus):
        if i == 0:
            continue
        line_rank.append({
            'sus': sus,
            'no': i
        })
    line_rank.sort(key=lambda s: s['sus'])
    # print(line_rank)
    final_rank = []
    tmp_list = []
    for i in range(len(line_rank)):
        if i != 0 and abs(line_rank[i]['sus'] - line_rank[i-1]['sus']) > 0.0001:
            final_rank.append(tmp_list)
            tmp_list = []
        tmp_list.append(line_rank[i]['no'])
    if len(tmp_list) != 0:
        final_rank.append(tmp_list)
    print(final_rank)
    return final_rank, line_sus

def run_dir(file_dir_path, pair_info, test_dir_path):
    '''
    计算某个文件夹内所有文件的最后怀疑度列表
    '''
    file_list = os.listdir(file_dir_path)
    wb = openpyxl.load_workbook(res_file)
    if sys.platform == "linux":
        problem_id = file_dir_path.split('/')[-2]
    else:
        problem_id = file_dir_path.split('\\')[-2]
    wb.create_sheet(problem_id)
    ws = wb[problem_id]
    ws.append({'a':'name', 'b':'suspicion_rank', 'c':'suspicion'})
    for file in file_list:
        file_type = file.split('.')[-1]
        if file_type == 'c' or file_type == 'cpp' or file_type == 'py':
            wa_file_path = os.path.join(file_dir_path, file)
            ac_file_path = os.path.join(data_path, str(problem_id), 'AC_'+file_type, pair_info[file])
            # print(wa_file_path, ac_file_path)
            try:
                final_rank, line_sus = run_file(wa_file_path, ac_file_path, test_dir_path, file_type)
                ws.append({'a':file, 'b':str(final_rank), 'c': str(line_sus)})
                # util.add_file(res_file, file + '    ' + str(final_rank) + '    ' + str(VSBFL_rank) + '\n')
            except Exception:
                print(Exception)
                ws.append({'a':file, 'b':'[]', 'c':'[]'})
                continue
                # util.add_file(res_file, file + '    ' + 'contains error\n')
        # break
    wb.save(res_file)
    return

if __name__ == "__main__":
    # wa_file = r'C:\Users\ShenJitao\Desktop\cppsnooper\instrumentation\test\WA_c\wa.c'
    # ac_file = r'C:\Users\ShenJitao\Desktop\cppsnooper\instrumentation\test\AC_c\ac.c'
    # test_dir_path = r'C:\Users\ShenJitao\Desktop\cppsnooper\instrumentation\test\TEST_DATA_TCG1'
    # wa_file = r'D:\fault_loc\ITSP-data\2871\WA_c\278496_buggy.c'
    # ac_file = r'D:\fault_loc\ITSP-data\2871\AC_c\278496_correct.c'
    # test_dir_path = r'D:\fault_loc\ITSP-data\2871\TEST_DATA_TCG1'
    # rank = run_file(wa_file, ac_file, test_dir_path, 'cpp')

    res_file = res_file_temp
    # problem_list = [2810, 2811, 2812, 2813, 2824, 2825, 2827, 2828, 2830, 2831, 2832, 2833, 2864, 2865, 2866, 2867, 2868, 2869, 2870, 2871]
    problem_list = ['2174', '3310', '3904', '3905', '3919', '3922', '3924']

    for problem in problem_list:
         # problem = 2810
        print(problem)
        pair_info = find_pair_by_tag(os.path.join(data_path, str(problem), 'Tag_c'))
        # pair_info = find_pair_by_res(r'D:\fault_loc\VSFL-TCG\result\cluster_op2.xlsx')
        dir_path = os.path.join(data_path, str(problem), 'WA_c')
        test_dir_path = os.path.join(data_path, str(problem), 'TEST_DATA_TCG1')
        run_dir(dir_path, pair_info, test_dir_path)
        # cal_time(dir_path)
    #     break

'''
[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0736, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
[[1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21], [7]]
'''


# -*- coding: utf-8 -*-
from PyQt5 import QtCore, QtWidgets

class StartDialogWidget(object):
    def setupUi(self, Dialog):
        Dialog.setObjectName("Dialog")
        Dialog.resize(400, 245)
        self.buttonBox = QtWidgets.QDialogButtonBox(Dialog)
        self.buttonBox.setGeometry(QtCore.QRect(30, 210, 341, 32))
        self.buttonBox.setOrientation(QtCore.Qt.Horizontal)
        self.buttonBox.setStandardButtons(QtWidgets.QDialogButtonBox.Cancel|QtWidgets.QDialogButtonBox.Ok)
        self.buttonBox.setObjectName("buttonBox")
        self.size_x = QtWidgets.QSpinBox(Dialog)
        self.size_x.setGeometry(QtCore.QRect(240, 10, 61, 21))
        self.size_x.setMinimum(300)
        self.size_x.setMaximum(1024)
        self.size_x.setObjectName("size_x")
        self.size_y = QtWidgets.QSpinBox(Dialog)
        self.size_y.setGeometry(QtCore.QRect(330, 10, 61, 21))
        self.size_y.setMinimum(300)
        self.size_y.setMaximum(1024)
        self.size_y.setObjectName("size_y")
        self.label = QtWidgets.QLabel(Dialog)
        self.label.setGeometry(QtCore.QRect(310, 10, 16, 16))
        self.label.setAlignment(QtCore.Qt.AlignCenter)
        self.label.setObjectName("label")
        self.label_2 = QtWidgets.QLabel(Dialog)
        self.label_2.setGeometry(QtCore.QRect(66, 10, 161, 20))
        self.label_2.setAlignment(QtCore.Qt.AlignRight|QtCore.Qt.AlignTrailing|QtCore.Qt.AlignVCenter)
        self.label_2.setObjectName("label_2")
        self.basic_color = QtWidgets.QLabel(Dialog)
        self.basic_color.setGeometry(QtCore.QRect(10, 10, 61, 41))
        self.basic_color.setText("")
        self.basic_color.setObjectName("basic_color")
        self.extra_color = QtWidgets.QLabel(Dialog)
        self.extra_color.setGeometry(QtCore.QRect(10, 90, 61, 41))
        self.extra_color.setText("")
        self.extra_color.setObjectName("extra_color")
        self.extr_button = QtWidgets.QPushButton(Dialog)
        self.extr_button.setGeometry(QtCore.QRect(10, 140, 75, 23))
        self.extr_button.setObjectName("extr_button")
        self.basic_button = QtWidgets.QPushButton(Dialog)
        self.basic_button.setGeometry(QtCore.QRect(10, 60, 75, 23))
        self.basic_button.setObjectName("basic_button")
        self.file_path = QtWidgets.QLineEdit(Dialog)
        self.file_path.setGeometry(QtCore.QRect(220, 60, 131, 20))
        self.file_path.setReadOnly(True)
        self.file_path.setObjectName("file_path")
        self.choose_file = QtWidgets.QPushButton(Dialog)
        self.choose_file.setGeometry(QtCore.QRect(360, 60, 21, 23))
        self.choose_file.setObjectName("choose_file")
        self.label_3 = QtWidgets.QLabel(Dialog)
        self.label_3.setGeometry(QtCore.QRect(130, 60, 71, 20))
        self.label_3.setAlignment(QtCore.Qt.AlignRight|QtCore.Qt.AlignTrailing|QtCore.Qt.AlignVCenter)
        self.label_3.setObjectName("label_3")
        self.view = QtWidgets.QLabel(Dialog)
        self.view.setGeometry(QtCore.QRect(220, 90, 160, 90))
        self.view.setText("")
        self.view.setObjectName("view")

        self.retranslateUi(Dialog)
        self.buttonBox.accepted.connect(Dialog.accept)
        self.buttonBox.rejected.connect(Dialog.reject)
        QtCore.QMetaObject.connectSlotsByName(Dialog)

    def retranslateUi(self, Dialog):
        _translate = QtCore.QCoreApplication.translate
        Dialog.setWindowTitle(_translate("Dialog", "Dialog"))
        self.label.setText(_translate("Dialog", "X"))
        self.label_2.setText(_translate("Dialog", "Size of canvas"))
        self.extr_button.setText(_translate("Dialog", "Extra color"))
        self.basic_button.setText(_translate("Dialog", "Base color"))
        self.choose_file.setText(_translate("Dialog", "..."))
        self.label_3.setText(_translate("Dialog", "Custom image"))


# -*- encoding: utf8 -*-
import os
import json
from setuptools import setup, find_packages

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
version_info = json.load(
    open(os.path.join(BASE_DIR, 'simple_loggers', 'version', 'version.json')))

setup(
    name='simple-loggers',
    version=version_info['version'],
    author=version_info['author'],
    author_email=version_info['author_email'],
    description=version_info['desc'],
    long_description=open(os.path.join(BASE_DIR, 'README.md')).read(),
    long_description_content_type="text/markdown",
    url='https://github.com/suqingdong/simple-loggers',
    project_urls={
        'Tracker': 'https://github.com/suqingdong/simple-loggers/issues',
    },
    license='BSD License',
    install_requires=[
        'colorama',
        'coloredlogs',
    ],
    packages=find_packages(),
    include_package_data=True,
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Operating System :: OS Independent',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.6',
        'Topic :: Software Development :: Libraries'
    ]
)


import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np

class LinearRegressionMultivariable:
    def __init__(self, alpha, n):
        self.thetas = np.zeros((n, 1), dtype= float)
        self.m = 0
        self.n = n
        self.cost = np.zeros((n, 1), dtype= float)
        self.alpha = alpha

    def hypothesis(self, x):
        return self.thetas.T.dot(x.T)

    def costs(self, x, y):
        self.cost = ((self.hypothesis(x).T - y).T.dot(x)).T

    def GD(self):
        self.thetas -= self.alpha * (1/self.m) * self.cost

    def calculate(self, x, y):
        self.costs(x, y)
        self.GD()
        print("costs:", end=" ")
        for cos in self.cost:
            print(cos[0], ",", end=" ")
        print(" ")
        self.cost = np.zeros((self.n, 1), dtype= float)
        print("thetas:", end=" ")
        for theta in self.thetas:
            print(theta[0], ",", end=" ")
        print('\n')

    def run(self, i, x, y):
        self.m = len(y)
        for _ in range(i):
            self.calculate(x, y)
        if self.n == 3:
            self.display3D(x, y)
        elif self.n == 2:
            self.display2D(x, y)
        else:
            pass

    def predict(self, x):
        return sum(self.hypothesis(x))

    def resFunction(self, x1, x2):
        return self.thetas[0][0] + self.thetas[1][0] * x1 + self.thetas[2][0] * x2

    def display3D(self, x1, y1):
        x = y = np.arange(-10.0, 10.0, 1)
        X, Y = np.meshgrid(x, y)
        zs = np.array([self.resFunction(x, y) for x, y in zip(np.ravel(X), np.ravel(Y))])
        Z = zs.reshape(X.shape)
        X1 = []
        Y1 = []
        Z1 = y1.tolist()
        for row in x1:
            X1.append(row[1])
            Y1.append(row[2])

        fig = plt.figure()
        ax = Axes3D(fig)
        ax.scatter(X1, Y1, Z1, color='red')
        ax.plot_surface(X, Y, Z, alpha=0.5, color='grey')
        plt.show()

    def display2D(self, x, y):
        X1 = []
        Y1 = y.T.tolist()[0]
        for row in x:
            X1.append(row[1])
        X = [0, X1[self.m - 1] + 1]
        Y = [self.thetas[0], self.thetas[0] + self.thetas[1] * (X1[self.m - 1] + 1)]
        plt.plot(X1, Y1, "ro", label="data")
        plt.plot(X, Y, label="best fit line")
        plt.legend()
        plt.show()


import requests
from urllib.parse import urlencode
from spyderpro.port_connect.internet_connect import Connect

"""股票模块"""
'''
该模块包括港股和泸深A股实时数据
'''

class Stock(Connect):

    def __init__(self, user_agent: str = None):
        """请求参数初始化
        :type user_agent: str
        :param:浏览器
        """

        self.request = requests.Session()

        self.headers = dict()

        if user_agent is None:
            self.headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 ' \
                                         '(KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'

        else:
            self.headers['User-Agent'] = user_agent
        self.query_string_parameters = {
            'block': '',
            'commodityid': 0,
            'title': 15,
            'direction': 0,
            'start': '',
            'number': '',  # 一次返回对数据量，最大100，即两页
        }

    def get_real_time_a_stock(self, page: int, block: int = 2, number: int = 50) -> list:
        """获取泸深A股实时数据
        :rtype: iterable[dict]
        :param page: 第几页,范围为1-43
        :param block: 请求类型，默认2，暂且支持2
        :param number: 一次请求返回多少条数据，建议默认值50
        :return:iterable[{"代码": code, '名称': name, "最新价": price, '涨跌幅': updownrate, '昨收': lastclose, '今开': todayopen,
                   '最高': high, '最低': low, '成交量': volume, '成交额': priceweight, '换手': exchangeratio,
                   '振幅': vibration_ratio, '量比': volumeratio
                   },,,,,]
        """

        assert isinstance(page, int)
        if page < 1 or page > 43:
            raise TypeError("page参数有误")
        self.headers['Host'] = 'webstock.quote.hermes.hexun.com'
        pre_url = 'http://webstock.quote.hermes.hexun.com/a/sortlist?'
        parameters = self.query_string_parameters
        parameters['block'] = block
        parameters['number'] = number
        parameters['start'] = (page - 1) * 50
        parameters[
            'column'] = "code ,name,price,updownrate,LastClose,open,high,low,volume,priceweight,amount,exchangeratio," \
                        "VibrationRatio,VolumeRatio"

        url = pre_url + urlencode(parameters)
        par = "\((.*?)\);"
        g: dict = self.connect(par, url)

        for value in g['Data'][0]:
            code = value[0]  # 代码
            name = value[1]  # 名称
            price = float(value[2]) / 100  # 最新价
            updownrate = float(value[3]) / 100  # 涨跌幅
            lastclose = float(value[4]) / 100  # 昨收
            todayopen = float(value[5]) / 100  # 今开
            high = float(value[6]) / 100  # 最高
            low = float(value[7]) / 100  # 最低
            volume = float(value[8]) / 100  # 成交量
            priceweight = value[10]  # 成交额
            exchangeratio = float(value[11]) / 100  # 换手
            vibration_ratio = float(value[12]) / 100  # 振幅
            volumeratio = float(value[13]) / 100  # 量比

            yield {"代码": code, '名称': name, "最新价": price, '涨跌幅': updownrate, '昨收': lastclose, '今开': todayopen,
                   '最高': high, '最低': low, '成交量': volume, '成交额': priceweight, '换手': exchangeratio,
                   '振幅': vibration_ratio, '量比': volumeratio
                   }

    def get_real_time_gb_hk_stock(self, page: int, block: int = 259, number: int = 50) -> list:
        """获取泸深港股实时数据
        :rtype: iterable
        :param page:
        :param block:
        :param number:
        :return:iterable[{"代码": , '名称': , "最新价": , '涨跌幅': , '昨收': , '今开': ,
                            '最高': , '最低': , '成交额': ,
                            },,,,,,]
        """
        '''page::表示第几页数据，page范围为1-41
         返回迭代器字典数据：{"代码": , '名称': , "最新价": , '涨跌幅': , '昨收': , '今开': ,
                            '最高': , '最低': , '成交额': ,
                            }'''
        assert isinstance(page, int)
        if page < 1 or page > 41:
            raise TypeError("page参数有误")
        self.headers['Host'] = 'webhkstock.quote.hermes.hexun.com'
        pre_url = 'http://webhkstock.quote.hermes.hexun.com/gb/hk/sortlist?'
        parameters = self.query_string_parameters
        parameters['block'] = block
        parameters['start'] = (page - 1) * 50
        parameters['number'] = number
        url = pre_url + urlencode(
            parameters) + "&column=code,name,price,open,LastClose,updownrate,volume,high,low,priceweight"

        par = "\((.*?)\);"
        g: dict = self.connect(par, url)

        for value in g['Data'][0]:
            code = value[0]  # 代码
            name = str(value[1]).encode('utf-8').decode("utf-8")  # 名称
            price = float(value[2]) / 1000  # 最新价
            todayopen = float(value[3]) / 1000  # 开盘价
            lastclose = float(value[4]) / 1000  # 前收盘
            updownrate = float(value[5]) / 100  # 涨跌幅
            priceweight = float(value[6]) / 10000  # 成交量
            high = float(value[7]) / 1000  # 最高价
            low = float(value[8]) / 1000  # 最低价

            yield {"代码": code, '名称': name, "最新价": price, '涨跌幅': updownrate, '昨收': lastclose, '今开': todayopen,
                   '最高': high, '最低': low, '成交额': priceweight,
                   }


import pandas as pd
import yfinance as yf
from datetime import datetime, timedelta, timezone
import random

def last_trading_day():
    rightnow = datetime.now(timezone(timedelta(hours=-5), 'EST'))

    # US Markets close at 4pm, but afterhours trading ends at 8pm.
    # yFinance stubbornly only gives the day's data after 8pm, so we will wait until 9pm to pull data from
    # the current day.
    market_close = rightnow.replace(hour=21, minute=0, second=0, microsecond=0)
    if rightnow < market_close:
        DELTA = 1
    # If it is saturday or sunday
    elif rightnow.weekday() >= 5:
        DELTA = 1
    else:
        DELTA = 0

    start_date = (datetime.now() - timedelta(days=15)).strftime("%Y-%m-%d")
    end_date = (datetime.now() - pd.tseries.offsets.BDay(DELTA)).strftime("%Y-%m-%d")
    MarketIndex = "^GSPC" # We can use the S&P 500's data to see the last day where we have data

    market_hist = yf.Ticker(MarketIndex).history(start=start_date, end=end_date).filter(like="Close").dropna()

    latest_day = market_hist.index[-1]
    return latest_day.strftime("%Y-%m-%d")

# get_stock_beta produces the beta of a specified ticker
# Inputs:
# stock_returns_series is of type Series, represents the column of a dataframe of the returns of the price

def get_weight_list(k):
    # We will pick k random numbers from a range of 250 numbers
    # random.sample is O(n), so we want to minimize this number
    random_num_list = random.sample(range(250), k)

    listsum = sum(random_num_list)
    for i in range(len(random_num_list)):
        random_num_list[i] /= listsum

    return random_num_list

def get_stock_beta(stock_returns_series, marketVar, market_hist):
    binary_portfolio = market_hist[["Returns"]].copy()
    binary_portfolio["stock"] = stock_returns_series
    return (binary_portfolio.cov() / marketVar)['stock'].loc["Returns"]

#get_all_betas takes a list of tickers and outputs a list of all the betas of every
# ticker in the list.

def get_all_betas(ticker_list, ticker_hist, marketVar, market_hist):
    betas = {}
    for ticker in ticker_list:
        ticker_returns = ticker_hist[ticker][['Close']].pct_change()*100
        betas[ticker] = get_stock_beta(ticker_returns['Close'], marketVar, market_hist)
    return betas

def get_option_interest(ticker):
    stock = yf.Ticker(ticker)
    options = stock.option_chain(stock.options[0])
    calls = pd.DataFrame().append(options.calls)
    puts = pd.DataFrame().append(options.puts)
    option_interest = calls.loc[calls['inTheMoney']]['openInterest'].sum() + puts.loc[puts['inTheMoney']]['openInterest'].sum()
    return option_interest

def get_daily_volume(ticker, start, end, ticker_hist):
    filled_in_hist = ticker_hist.copy()
    filled_in_hist.fillna(0, inplace=True)
    volume = ticker_hist[ticker].loc[pd.to_datetime(start) : pd.to_datetime(end)].Volume.mean()
    return volume

def import_options(tickerlist, opt_start, opt_end, tickerhist):
    option_interest_dict = {}
    for ticker in tickerlist:
        try:
            open_interest = get_option_interest(ticker)
            daily_vol = get_daily_volume(ticker, opt_start, opt_end, tickerhist)
            option_interest_dict[ticker] = open_interest / daily_vol
        # yFinance's historical data is sometimes incomplete. This try except will catch any tickers that have insufficient data and drops them from our analysis.
        except IndexError as error:
            print(f'Dropped {ticker} - no data')
    return option_interest_dict

def generate_risky_portfolio(tickerlist: list, totalspend: int):
    ticker_hist = yf.download(
                tickers = " ".join(tickerlist),
                # Download Data From the past 6 months
                period = "6mo",
                interval = "1d",
                group_by = 'ticker',
                threads = True,
            )

    # Data Cleanup
    ticker_hist.dropna(how='all', inplace=True)
    ticker_hist.fillna(method='ffill', inplace=True)
    ticker_hist.fillna(method='bfill', inplace=True)

    start_date = (datetime.now() - timedelta(days=60)).strftime("%Y-%m-%d")
    end_date = last_trading_day()

    MarketIndex = "^GSPC" # We will compare each stock's covariance to the S&P 500

    market_hist = yf.Ticker(MarketIndex).history(start=start_date, end=end_date).filter(like="Close")
    market_hist["Returns"] = market_hist["Close"].pct_change()*100

    market_var = market_hist["Returns"].var()

    beta_dict = get_all_betas(tickerlist, ticker_hist, market_var, market_hist)
    beta_df = pd.DataFrame.from_dict(beta_dict, orient='index')
    beta_df.columns = ['Beta']
    beta_df.index.name = 'Ticker'

    #Sorts the dictionary from lowest beta to highest beta
    sorted_beta_df = beta_df.sort_values('Beta').copy()

    if len(sorted_beta_df) > 10:
        #Creates a dataframe with the 10 lowest betas and 10 highest betas.
        lower_bound = sorted_beta_df.iloc[:10]
        upper_bound = sorted_beta_df.iloc[-10:]
        # Gets the final stock list we want

        #Calculates the mean of the upper and lower list of betas
        avg_upper = upper_bound['Beta'].mean()
        avg_lower = lower_bound['Beta'].mean()

        # Compares the averages of the two and makes the one with the higher absolute value a
        # candidate for our final list of stocks.
        if abs(avg_upper) >= abs(avg_lower):
            beta_final_ticker_list = list(upper_bound.index)
        else:
            beta_final_ticker_list = list(lower_bound.index)
    else:
        beta_final_ticker_list = list(sorted_beta_df.index)

    # Options Analysis

    opt_start = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d")
    opt_end = end_date

    option_interest_dict = import_options(tickerlist, opt_start, opt_end, ticker_hist)

    if len(option_interest_dict.keys()) >= len(beta_final_ticker_list):
        # Creates a dataframe that corresponds tickers with the option interest over the daily volume
        option_interest_df = pd.DataFrame.from_dict(option_interest_dict, orient='index')
        option_interest_df.columns = ['Option Interest / dailyVol']
        sorted_option_interest_df = option_interest_df.sort_values('Option Interest / dailyVol').copy()
        sorted_option_interest_df.dropna(inplace=True)
        final_option_df = sorted_option_interest_df.tail(10)
        opt_tickers = list(final_option_df.index)
        print('Found an option portfolio')
    else:
        option_interest_df = False
        print('Error - Insufficient options data for inputted tickers, skipping options analysis...')

    # Monte Carlo Analysis
    trials = 250

    # We will collect the risk coefficients of all portfolios in this dictionary
    # Ultimately, we are comparing the best stocks obtained by the beta and options method,
    # and will take the best one from whichever method.
    risk_coefficients = {}

    # Don't mind the O(4n^2), it's O(e^2ln(2n)) :^)
    for i in range(trials):
        portfolio = pd.DataFrame(index=pd.date_range(start=start_date, end=end_date))
        portfolio_shares = {}
        weight_list = get_weight_list(len(beta_final_ticker_list))
        total = 0
        for i in range(len(beta_final_ticker_list)):
            allocation = weight_list[i] * totalspend
            # Simulate purchasing $100,000 worth of shares at the beginning of the time period
            initial_ticker_price = ticker_hist[beta_final_ticker_list[i]].Close.iloc[0]
            shares = allocation / initial_ticker_price

            # Alongside this, create a portfolio of shares with the same weightings, but priced in the present.
            present_ticker_price = ticker_hist[beta_final_ticker_list[i]].Close.loc[end_date]
            shares_present = allocation / present_ticker_price
            portfolio_shares[beta_final_ticker_list[i]] = shares_present
            total += present_ticker_price * shares_present

            # Simulate Purchasing of the shares and perform analysis on the past data
            portfolio[beta_final_ticker_list[i]] = shares * ticker_hist[beta_final_ticker_list[i]].loc[pd.to_datetime(start_date) : pd.to_datetime(end_date)].Close
        portfolio.dropna(how='all', inplace=True)
        portfolio.fillna(method='ffill', inplace=True)
        portfolio.fillna(method='bfill', inplace=True)
        portfolio['Total Value'] = portfolio.sum(axis=1)
        portfolio['Percent Return'] = portfolio['Total Value'].pct_change() * 100

        # Calculate risk coefficient
        std = portfolio['Percent Return'].std()
        avg_return = portfolio['Percent Return'].mean()
        risk_coefficient = (avg_return * std) ** 2

        # Save into risk_coefficients dict
        risk_coefficients[risk_coefficient] = portfolio_shares

        if option_interest_df is not False:
            portfolio = pd.DataFrame(index=pd.date_range(start=start_date, end=end_date))
            portfolio_shares = {}
            weight_list = get_weight_list(len(opt_tickers))
            total = 0
            for i in range(len(opt_tickers)):
                allocation = weight_list[i] * totalspend
                # Simulate purchasing $100,000 worth of shares at the beginning of the time period
                initial_ticker_price = ticker_hist[opt_tickers[i]].Close.iloc[0]
                shares = allocation / initial_ticker_price

                # Alongside this, create a portfolio of shares with the same weightings, but priced in the present.
                present_ticker_price = ticker_hist[opt_tickers[i]].Close.loc[end_date]
                shares_present = allocation / present_ticker_price
                portfolio_shares[opt_tickers[i]] = shares_present
                total += present_ticker_price * shares_present

                # Simulate Purchasing of the shares and perform analysis on the past data
                portfolio[opt_tickers[i]] = shares * ticker_hist[opt_tickers[i]].loc[pd.to_datetime(start_date) : pd.to_datetime(end_date)].Close
            portfolio.dropna(how='all', inplace=True)
            portfolio.fillna(method='ffill', inplace=True)
            portfolio.fillna(method='bfill', inplace=True)
            portfolio['Total Value'] = portfolio.sum(axis=1)
            portfolio['Percent Return'] = portfolio['Total Value'].pct_change() * 100

            # Calculate risk coefficient
            std = portfolio['Percent Return'].std()
            avg_return = portfolio['Percent Return'].mean()
            risk_coefficient = (avg_return * std) ** 2

            # Save into risk_coefficients dict
            risk_coefficients[risk_coefficient] = portfolio_shares

    optimal_weight = risk_coefficients[max(risk_coefficients.keys())]

    # Create final output portfolio
    final_stocks_df = pd.DataFrame.from_dict(optimal_weight, orient='index')
    final_stocks_df.reset_index(inplace=True)
    final_stocks_df.columns = ['Ticker', 'Shares']
    final_stocks_df.set_index('Ticker', inplace=True)
    return (final_stocks_df, end_date)


#!/usr/bin/env python

import os
import re
import yaml
from datetime import datetime
from jinja2 import Environment, FileSystemLoader, PackageLoader

def render_email(data_file, j2template, output_file):
    with open(data_file, 'r') as file:
        try:
            data = yaml.safe_load(file.read())
        except Exception as err:
            print(err)

    env = Environment(
        loader=FileSystemLoader(searchpath="."),
        trim_blocks=True,
        lstrip_blocks=True
    )

    template = env.get_template(name=j2template)
    rendered = template.render(data)

    with open(output_file, "w") as file:
        file.write(rendered)

output_date = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
print(output_date)

files = [f for f in os.listdir('devopsdays-web/data/events') if
         re.match('202[3-4]+.*\.yml', f)]

print("*********")
print("---- Getting open CFP -----")
print("*********")

with open("email.yml", "w") as f:
    f.write("---\n")
    f.write("cfp:\n")

counter = 1
for file in files:
    with open(f'devopsdays-web/data/events/{file}', "r") as stream:
        try:

            yaml_file = yaml.safe_load(stream)
            try:
                if yaml_file['cfp_open'] != "false" or "no":
                    try:
                        date_object = datetime.strptime(str(yaml_file['enddate']), '%Y-%m-%d %H:%M:%S%z')
                        if date_object.date() < datetime.today().date():
                            continue
                        else:
                            with open("email.yml", "a") as f:
                                f.write(f"  - name: {yaml_file['name']}\n")
                                f.write(f"    count: {counter}\n")
                                f.write(f"    main_link: https://devopsdays.org/events/{yaml_file['name']}/welcome/\n")
                                f.write(f"    cfp_link: {yaml_file['cfp_link']}\n")
                                f.write(f"    reg_link: {yaml_file['registration_link']}\n")
                                counter += 1
                    except:
                        continue

            except KeyError:
                continue

        except yaml.YAMLError as exc:
            print(exc)

print("*********")
print("---- Getting open sponsorship -----")
print("*********")

with open("email.yml", "a") as f:
    f.write("sponsorship:\n")

counter = 1
for file in files:
    with open(f'devopsdays-web/data/events/{file}', "r") as stream:
        try:

            yaml_file = yaml.safe_load(stream)

            try:
                if yaml_file['sponsors_accepted'] != "false" or "no":
                    try:
                        date_object = datetime.strptime(str(yaml_file['enddate']), '%Y-%m-%d %H:%M:%S%z')
                        if date_object.date() < datetime.today().date():
                            continue
                        else:
                            with open("email.yml","a") as f:
                                f.write(f"  - name: {yaml_file['name']}\n")
                                f.write(f"    count: {counter}\n")
                                f.write(f"    main_link: https://devopsdays.org/events/{yaml_file['name']}/welcome/\n")
                                f.write(f"    cfp_link: {yaml_file['cfp_link']}\n")
                                f.write(f"    reg_link: {yaml_file['registration_link']}\n")
                                counter += 1
                    except:
                        continue

            except KeyError:
                continue

        except yaml.YAMLError as exc:
            print(exc)

if __name__ == "__main__":
    render_email(data_file="email.yml", j2template="email.html.j2", output_file="index.html")


# _*_ encoding: utf-8 _*_

from parser_helper import HandlerBase
from endpoint_helper import EndPoint
from config import CONTENTS
import os

class Download(HandlerBase):
    """
    Allows download a  file from OSDR BLOB store.
    osdr download {remote-id} [{metadata}]
    Command: download
    """
    info = '''
            name: download
            help: Allows to download an OSDR file.
            params:
                -
                    names:
                        - container
                    help: >
                          Remote OSDR file id.
                          OSDR file can be choosed by its
                          full id system wide or by substring in
                          current OSDR folder.
                          Substring compared to filename starting
                          from the beggining or to file id ending.
                -
                    names:
                        - -o
                        - --output
                    help: Path to file or directory to save.
                -
                    names:
                        - -f
                        - --force
                    action: store_true
                    dest: overwrite
                    help: Force overwrite if file exists.

    '''

    def __call__(self):
        # super().__call__()
        ep = EndPoint()
        session = ep.connect()

        list_url = CONTENTS.format(session['cwd'])

        record = ep.get_container_by_id(self.container)
        if not record:
            records = ep.get_containers(list_url)
            record = ep.get_uniq_container(records, self.container)

        folder = '.'
        file_name = record['name']
        if self.output:
            if os.path.isdir(self.output):
                folder = self.output
            elif not os.path.dirname(self.output):
                file_name = self.output
            else:
                file_name = os.path.basename(self.output)
                folder = os.path.dirname(self.output)
                assert os.path.isdir(folder), 'No such folder: {}'.format(folder)

        path = os.path.join(folder, file_name)
        if not self.overwrite:
            assert not os.path.isfile(path), "File {path} exists. To overwrite use -f key.".format(path=path)
        print(record)
        ep.download(record, path)


# -*- coding: utf-8 -*-
from setuptools import setup, find_packages

import sys

with open('README.rst') as f:
    description = f.read()

setup(
    name='seamless',
    url='http://github.com/emulbreh/seamless/',
    version='0.2.0-dev',
    packages=find_packages(),
    license=u'BSD',
    author=u'Johannes Dollinger',
    author_email=u'emulbreh@googlemail.com',
    long_description=description,
    include_package_data=True,
    install_requires=[
        'click',
        'itsdangerous',
        'requests',
    ],
    entry_points={
        'console_scripts': ['seamless = seamless.cli:main'],
    },
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: BSD License',
        'Operating System :: POSIX :: Linux',
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3'
    ]
)


from django.urls import path
from PrimeraApp import views

urlpatterns = [

    path('', views.InicioApp, name="inicioApp"),
    #path('crear_curso/', views.crear_curso ),
    path('Cursos/' , views.Cursos, name="cursos"),
    path('Profesores/' , views.Profesores, name="profesores"),
    path('Estudiantes/' , views.Estudiantes,name="estudiantes"),
    path('Entregables/', views.Entregables, name="entregables" ),

]


import boto3
from pathlib import Path
from tqdm import tqdm

def get_data():
    # Download data from S3
    local_data_folder = Path("darknet/yolo_data")
    local_data_folder.mkdir(parents=True, exist_ok=True)
    s3 = boto3.resource("s3")
    s3_client = boto3.client("s3")
    bucket_name = "broker-reports"
    my_bucket = s3.Bucket(bucket_name)
    for obj in tqdm(my_bucket.objects.all()):
        # There should be about 2000++ files
        if (local_data_folder / obj.key).suffix in [".png", ".txt"]:
            fp = obj.key.replace("labels-yolo/", "")
            (local_data_folder / fp).parent.mkdir(parents=True, exist_ok=True)
            s3_client.download_file(bucket_name, obj.key, str(local_data_folder / fp))
    # Update paths to input files
    for datalist in ["train.txt", "valid.txt", "test.txt"]:
        filenames = []
        with open(local_data_folder / datalist, 'r') as file:
            for line in file:
                path = local_data_folder / "data" / Path(line.strip()).name
                if path.exists(): filenames.append(str(path))
        with open(local_data_folder / datalist, 'w') as file:
            for path in filenames: file.write(path + '\n')

if __name__ == "__main__":
    get_data()


from dataclasses import dataclass, is_dataclass, fields, MISSING
from ruka.util.json import XJSONSerializable, State
from typing import Dict, List, Optional

class Migrating(XJSONSerializable):
    """
    USAGE
        Migrating provides mechanism of backward-compatibility of serialized
        data. Specifically, newer version of code will be able to load a state,
        saved by the older version of code.

        Lets take this dataclass:

            @dataclass
            class Foo(Migrating):
                x: int
                y: int

            >> Foo(1, 2).__getstate__()
            {'x': 1, 'y': 2, 'class': {'name': Foo, 'module': 'foo', 'version': 1}}

        And create version 2 by adding a field:

            @dataclass
            class Foo(Migrating):
                x: int
                y: int
                z: int = 0

                CHANGES: typing.ClassVar = {
                    2: [Add('z')]
                }

            >>> Foo(1, 2).__getstate__()
            {'x': 1, 'y': 2, 'z': 0, 'class': {'name': Foo, 'module': 'foo', 'version': 2}}

        We've created version 2 of the class. When you add a new field, you
        must always specify either default= or default_factory=.

        Next, let's create version 3 by removing one field and by renaming
        another:

            @dataclass
            class Foo(Migrating):
                y1: int
                z: int = 0

                CHANGES: typing.ClassVar = {
                    2: [Add('z')],
                    3: [Remove('x'), Rename('y', 'y1')]
                }

            >>> Foo(2, 3).__getstate__()
            {'y1': 2, 'z': 3, 'class': {'name': Foo, 'module': 'foo', 'version': 3}}

    NOTES
        - Currently, there are three types of changes available: Add, Remove,
          and Rename. More might be available later.

        - Warning: only dataclass fields are saved or loaded in __getstate__ /
          __setstate__:

              @dataclass
              class Bar(Migrating):
                  x: int

              >>> b = Bar(1)
              >>> b.__getstate__()
              >>> {'x': 1, 'class': {'name': 'Bar', 'module': 'bar', 'version': 1}}
              >>> b.y = 2
              >>> b.__getstate__()
              >>> {'x': 1, 'class': {'name': 'Bar', 'module': 'bar', 'version': 1}}  # <--- No y!

        - Migrating is XJSONSerializable. This means that if all its fields are
          [X]JSONSerializable, the dataclass itself will also be serializable
          through ruka.json.x{dump,load}*
    """

    def __new__(cls, *args, **kwargs):
        validate_class(cls)
        return super().__new__(cls)

    def xjson_getstate(self) -> State:
        state = {f.name: getattr(self, f.name) for f in fields(self)}
        state['class'] = self.xjson_getclass()
        return state

    def xjson_setstate(self, state: State):
        validate_state(state, type(self))
        migrate_state_to_current_version(type(self), state)
        for k, v in state.items():
            if k == 'class':
                continue
            setattr(self, k, v)

    @classmethod
    def xjson_getclass(cls):
        j = super().xjson_getclass()
        j['version'] = get_current_version(cls)
        return j

class Change:
    pass

@dataclass
class Add(Change):
    name: str

@dataclass
class Remove(Change):
    name: str

@dataclass
class Rename(Change):
    name_old: str
    name_new: str

# --------------------------------------------------------------------- Util --

def validate_class(cls):
    """
    Check that class is a well-formed Migrating.
    """
    # Is Migrating.
    assert issubclass(cls, Migrating)

    # Class is a dataclass.
    assert is_dataclass(cls)

    # Versions are consecutive.
    changes = _get_changes(cls)
    if changes:
        current_version = get_current_version(cls)
        assert list(changes.keys()) == list(range(2, current_version + 1))

    # Have required defaults.
    should_have_defaults = set()

    for version in sorted(changes.keys()):
        for change in changes[version]:
            if isinstance(change, Add):
                should_have_defaults.add(change.name)
            elif isinstance(change, Remove):
                should_have_defaults.discard(change.name)
            elif isinstance(change, Rename):
                if change.name_old in should_have_defaults:
                    should_have_defaults.remove(change.name_old)
                    should_have_defaults.add(change.name_new)
            else:
                assert 0

    field_map = {field.name: field for field in fields(cls)}
    for key in should_have_defaults:
        field = field_map[key]
        if field.default is MISSING and field.default_factory is MISSING:
            assert 0, f'field {key} requires default= or default_factory='

def validate_state(state: State, cls: Optional[type] = None):
    """
    Check that state is well-formed.
    If cls is not None, check that state corresponds to cls.
    """
    # Check version.
    version_from = state['class']['version']
    if version_from <= 0:
        raise RuntimeError(f"invalid version: {state!r}")

    # Check that state corresponds to class.
    if cls is not None:
        if (
            state['class']['name'] != cls.__qualname__ or
            state['class']['module'] != cls.__module__
        ):
            raise RuntimeError(f'invalid state for {cls}: {state!r}')

    # Chech state version against class version.
    if cls is not None:
        current_version = get_current_version(cls)
        if not (0 < version_from <= current_version):
            raise RuntimeError(
                f"invalid version: {state!r}; expected 0..{current_version}")

def get_current_version(cls: type) -> int:
    assert issubclass(cls, Migrating)
    changes = _get_changes(cls)
    if not changes:
        return 1
    return max(changes.keys())

def migrate_state_to_current_version(cls: type, state: State):
    assert issubclass(cls, Migrating)
    class _Stub: pass

    version_from = state['class']['version']
    current_version = get_current_version(cls)

    # Sequentially migrate to a higher version.
    changes = _get_changes(cls)
    while version_from < current_version:
        for change in changes[version_from + 1]:
            if isinstance(change, Add):
                state[change.name] = _Stub
            elif isinstance(change, Remove):
                if change.name not in state:
                    raise RuntimeError(
                        f'invalid state: {state!r}, '
                        f'missing field {change.name}')
                state.pop(change.name)
            elif isinstance(change, Rename):
                if change.name_old not in state:
                    raise RuntimeError(
                        f'invalid state: {state!r}, '
                        f'missing field {change.name}')
                state[change.name_new] = state.pop(change.name_old)
            else:
                assert 0
        version_from += 1

    # Fill in defaults.
    field_map = {field.name: field for field in fields(cls)}
    for key in state.keys():
        if key == 'class':
            continue
        field = field_map[key]
        if state[key] is _Stub:
            if field.default is not MISSING:
                state[key] = field.default
            elif field.default_factory is not MISSING:
                state[key] = field.default_factory()
            else:
                assert 0

    # Fix version.
    state['class']['version'] = current_version

    # Check for extra fields.
    expected_keys = set(field_map.keys())
    real_keys = set(state.keys()) - {'class'}
    missing_keys = expected_keys - real_keys
    extra_keys = real_keys - expected_keys
    if missing_keys:
        raise RuntimeError(
            f'invalid state: {state!r}; '
            f'following keys are missing: {missing_keys}')
    if extra_keys:
        raise RuntimeError(
            f'invalid state: {state!r}; '
            f'following keys shouldn\'t be here: {extra_keys}')

def _get_changes(cls) -> Dict[str, List['Change']]:
    return getattr(cls, 'CHANGES', {})


#!/usr/bin/env python
# coding: utf-8

# In[2]:

# These imports enhance Python2/3 compatibility.
from __future__ import print_function, absolute_import, division, unicode_literals, with_statement

# In[3]:

# General imports
import argparse
import numpy as np
import os
import sys

# Use PyTorch/torchvision for dataloading (more reliable/faster)
from torchvision import datasets

# In[1]:

# Keras modules
from keras.preprocessing import image

# Keras models
from keras.applications import (
    DenseNet121,
    DenseNet169,
    DenseNet201,
    InceptionResNetV2,
    InceptionV3,
    MobileNet,
    MobileNetV2,
    NASNetLarge,
    NASNetMobile,
    ResNet50,
    VGG16,
    VGG19,
    Xception,
)

# Import preprocess_inputs parent modules
from keras.applications import (
    densenet,
    inception_resnet_v2,
    inception_v3,
    mobilenet,
    mobilenet_v2,
    nasnet,
    resnet50,
    vgg16,
    vgg19,
    xception,
)

# # Helps with compatibilty with CUDA version 10 and the RTX 2080 GPU line. Uncomment if relevant.
# import tensorflow as tf
# from keras.backend.tensorflow_backend import set_session
# gpu_options = tf.GPUOptions2(allow_growth=True)
# config = tf.ConfigProto(gpu_options=gpu_options)
# set_session(tf.Session(config=config))

# In[ ]:

keras_models = {
    "densenet121" : DenseNet121,
    "densenet169" : DenseNet169,
    "densenet201" : DenseNet201,
    "mobilenet" : MobileNet,
    "mobilenetV2" : MobileNetV2,
    "nasnetmobile" : NASNetMobile,
    "resnet50" : ResNet50,
    "vgg16" : VGG16,
    "vgg19" : VGG19,
    "xception" : Xception,
    "inceptionresnetv2" : InceptionResNetV2,
    "inceptionv3" : InceptionV3,
    "nasnetlarge" : NASNetLarge,
}

models_preprocessing = {
    "densenet121" : densenet.preprocess_input,
    "densenet169" : densenet.preprocess_input,
    "densenet201" : densenet.preprocess_input,
    "mobilenet" : mobilenet.preprocess_input,
    "mobilenetV2" : mobilenet_v2.preprocess_input,
    "nasnetmobile" : nasnet.preprocess_input,
    "resnet50" : resnet50.preprocess_input,
    "vgg16" : vgg16.preprocess_input,
    "vgg19" : vgg19.preprocess_input,
    "xception" : xception.preprocess_input,
    "inceptionresnetv2" : inception_resnet_v2.preprocess_input,
    "inceptionv3" : inception_v3.preprocess_input,
    "nasnetlarge" : nasnet.preprocess_input,
}

models_img_size = {
    "densenet121" : (224, 224),
    "densenet169" : (224, 224),
    "densenet201" : (224, 224),
    "mobilenet" : (224, 224),
    "mobilenetV2" : (224, 224),
    "nasnetmobile" : (224, 224),
    "resnet50" : (224, 224),
    "vgg16" : (224, 224),
    "vgg19" : (224, 224),
    "xception" : (299, 299),
    "inceptionresnetv2" : (299, 299),
    "inceptionv3" : (299, 299),
    "nasnetlarge" : (331, 331),
}

# In[ ]:

# Set up argument parser
parser = argparse.ArgumentParser(description='Keras ImageNet Inference')
parser.add_argument('val_dir', metavar='DIR',
                    help='path to imagenet val dataset folder')
parser.add_argument('-m', '--model', metavar='MODEL', default=None,
                    choices=keras_models.keys(),
                    help='model architecture: ' +
                        ' | '.join(keras_models.keys()) +
                        ' (example: resnet50)' +
                        ' (default: Runs across all Keras models)')
parser.add_argument('-g', '--gpu', metavar='MODEL', default=0,
                    help='int of GPU to use. Only uses single GPU.')
parser.add_argument('-o', '--output-dir', metavar='OUTPUT_DIR', default="keras_imagenet/",
                    help='directory folder to store output results in.')
parser.add_argument('--save-all-probs',  action='store_true', default = False,
                    help='Store entire softmax output for all examples (100 MB)')

# In[ ]:

def main(args = parser.parse_args()):
    '''Select GPU and set up data loader for ImageNet val set.'''
    os.environ["CUDA_VISIBLE_DEVICES"]=str(args.gpu)

    # Create output directory if it does not exist
    if not os.path.exists(args.output_dir):
        os.makedirs(args.output_dir)

    # Grab imagenet data
    val_dataset = datasets.ImageFolder(args.val_dir)
    img_paths, labels = (list(t) for t in zip(*val_dataset.imgs))

    # Run forward pass inference on all models for all examples in val set.
    models = keras_models if args.model is None else [args.model]
    for model in models:
        process_model(model, img_paths, labels, args.output_dir, args.save_all_probs)

# In[6]:

# def images2data(img_paths, img_size = (224, 224)):
#     result = []
#     for i, img_path in enumerate(img_paths):
#         if i % 50 == 0:
#             print("\rComplete: {:.1%}".format(i / len(img_paths)), end = "")
#         img = image.load_img(img_path, target_size=img_size)
#         result.append(np.expand_dims(image.img_to_array(img), axis=0))
#     print()
#     return result

# In[9]:

def process_model(
    model_name,
    img_paths,
    labels,
    out_dir="keras_imagenet/",
    save_all_probs = False,
):
    '''Actual work is done here. This runs inference on a Keras model,
    by computing the output of a forward pass, individually for each example.
    Running examples in batches or using vectorized operations results in
    random outputs and lack of reproducibility in Keras (this is a bug).
    This method will avoid those issues by running the forward pass on each
    example, one at a time. This is slower, but accurate.

    Top5 predictions and probabilities for each example for the model
    are stored in the keras_imagenet/ output directory.'''

    preprocess_model = models_preprocessing[model_name]
    img_size = models_img_size[model_name]
    Model = keras_models[model_name]
    wfn_base = os.path.join(out_dir, model_name + "_keras_imagenet_")

    # Create Keras model
    model = Model(weights='imagenet')

    # Preprocessing and Forward pass through validation set.
    probs = []
    for i, img_path in enumerate(img_paths):
        if i % 32 == 0:
            print("\r{} completed: {:.2%}".format(model_name, i / len(img_paths)), end="")
            sys.stdout.flush()
        img = image.load_img(img_path, target_size=img_size)
        img = np.expand_dims(image.img_to_array(img), axis=0)
        probs.append(model.predict(preprocess_model(img)))
    probs = np.vstack(probs)
    if save_all_probs:
        np.save(wfn_base + "probs.npy", probs.astype(np.float16))

    # Extract top 5 predictions for each example
    n = 5
    top = np.argpartition(-probs, n, axis = 1)[:,:n]
    top_probs = probs[np.arange(probs.shape[0])[:, None], top]
    acc1 = sum(top[range(len(top)), np.argmax(top_probs, axis = 1)] == labels) / float(len(labels))
    acc5 = sum([labels[i] in row for i, row in enumerate(top)]) / float(len(labels))
    print('\n{}: acc1: {:.2%}, acc5: {:.2%}'.format(model_name, acc1, acc5))

    # Save top 5 predictions and associated probabilities
    np.save(wfn_base + "top5preds.npy", top)
    np.save(wfn_base + "top5probs.npy", top_probs.astype(np.float16))

# In[ ]:

if __name__ == '__main__':
    main()


import argparse
import matplotlib.pyplot as plt
import os
import sys
import tempfile
import time

from tensorflow.examples.tutorials.mnist import input_data

import tensorflow as tf

FLAGS = None

# For running on CPU
# os.environ['CUDA_VISIBLE_DEVICES'] = '-1'

# For opening tensorboard
# tensorboard --logdir=/home/<username>/lab/tensorflow/
learn_rate = 1e-2

def deepnn(x):
  """deepnn builds the graph for a deep net for classifying digits.
  Args:
    x: an input tensor with the dimensions (N_examples, 784), where 784 is the
    number of pixels in a standard MNIST image.
  Returns:
    A tuple (y, keep_prob). y is a tensor of shape (N_examples, 10), with values
    equal to the logits of classifying the digit into one of 10 classes (the
    digits 0-9). keep_prob is a scalar placeholder for the probability of
    dropout.
  """

  # Reshape to use within a convolutional neural net.
  # Last dimension is for "features" - there is only one here, since images are
  # grayscale -- it would be 3 for an RGB image, 4 for RGBA, etc.
  with tf.name_scope('reshape'):
    x_image = tf.reshape(x, [-1, 28, 28, 1])

  # First convolutional layer
  with tf.name_scope('conv1'):
    W_conv1 = weight_variable([3, 3, 1, 8])
    b_conv1 = bias_variable([8])
    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)

  # Pooling layer - downsamples by 2X.
  with tf.name_scope('pool1'):
    h_pool1 = max_pool_2x2(h_conv1)

  # Second convolutional layer
  with tf.name_scope('conv2'):
    W_conv2 = weight_variable([3, 3, 8, 4])
    b_conv2 = bias_variable([4])
    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)

  # Second pooling layer.
  with tf.name_scope('pool2'):
    h_pool2 = max_pool_2x2(h_conv2)

  # Third convolutional layer
  with tf.name_scope('conv3'):
    W_conv3 = weight_variable([3, 3, 4, 2])
    b_conv3 = bias_variable([2])
    h_conv3 = tf.nn.relu(conv2d(h_pool2, W_conv3) + b_conv3)

  # First transposed convolution layer
  with tf.name_scope('transconv1'):
    h_trans1 = transpose_convolution(h_conv3, 4)

  # Fourth convolutional layer
  with tf.name_scope('conv4'):
    W_conv4 = weight_variable([3, 3, 4, 4])
    b_conv4 = bias_variable([4])
    h_conv4 = tf.nn.relu(conv2d(h_trans1, W_conv4) + b_conv4)

  # Second transposed convolution layer
  with tf.name_scope('transconv2'):
    h_trans2 = transpose_convolution(h_conv4, 8)

  # Fifth convolutional layer
  with tf.name_scope('conv5'):
    W_conv5 = weight_variable([3, 3, 8, 8])
    b_conv5 = bias_variable([8])
    h_conv5 = tf.nn.relu(conv2d(h_trans2, W_conv5) + b_conv5)

  with tf.name_scope('outputconv'):
    W_outputconv = weight_variable([1, 1, 8, 1])
    b_outputconv = bias_variable([1])
    h_outputconv = conv2d(h_conv5, W_outputconv) + b_outputconv

  return h_outputconv

def conv2d(x, W):
  """conv2d returns a 2d convolution layer with full stride."""
  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

def max_pool_2x2(x):
  """max_pool_2x2 downsamples a feature map by 2X."""
  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
                        strides=[1, 2, 2, 1], padding='SAME')

def transpose_convolution(inputs, filters):
  return tf.layers.conv2d_transpose(inputs, filters, [2, 2], strides=[2, 2],
                                    padding='SAME')

def weight_variable(shape):
  """weight_variable generates a weight variable of a given shape."""
  initial = tf.truncated_normal(shape, stddev=0.1)
  return tf.Variable(initial)

def bias_variable(shape):
  """bias_variable generates a bias variable of a given shape."""
  initial = tf.constant(0.1, shape=shape)
  return tf.Variable(initial)

def main(_):
  # Import data
  mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)

  # Create the model
  x = tf.placeholder(tf.float32, [None, 784])

  # Build the graph for the deep net
  y_conv = deepnn(x)
  img_autoenc = tf.summary.image("autoencoder/rate" + str(learn_rate), y_conv, max_outputs=2)
  y_conv = tf.reshape(y_conv, [64, 784])

  with tf.name_scope('loss'):
    loss = tf.reduce_mean(tf.square(y_conv - x))
  tf.summary.scalar('loss', loss)

  with tf.name_scope('AdamOptimizer'):
    train_step = tf.train.AdamOptimizer(learn_rate).minimize(loss)

  graph_location = tempfile.mkdtemp()
  print('Saving graph to: %s' % graph_location)
  writer = tf.summary.FileWriter(graph_location)
  writer.add_graph(tf.get_default_graph())

  with tf.Session() as sess:
    # start_time = time.time()
    sess.run(tf.global_variables_initializer())
    sumv = tf.summary.scalar('loss', loss)
    writer = tf.summary.FileWriter(FLAGS.log_dir + '/learningRate'
                                   + str(learn_rate), sess.graph)
    for i in range(100000):
      # Don't need the labels anymore
      batch, _ = mnist.train.next_batch(64)
      if i % 100 == 0:
        summ, train_loss, img = sess.run([sumv, loss, img_autoenc],
                                          feed_dict={x: batch})
        writer.add_summary(summ, i)
        writer.add_summary(img, i)
        print('step %d, training loss %g' % (i, train_loss))
      train_step.run(feed_dict={x: batch})
    writer.close()
    # end_time = time.time()

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument('--data_dir', type=str,
                      default='/tmp/tensorflow/mnist/input_data',
                      help='Directory for storing input data')
  parser.add_argument('--log_dir', type=str,
                      default=os.path.join(os.getenv('HOME', '/home'),
                                           'lab/tensorflow'),
                      help='Summaries log directory')
  FLAGS, unparsed = parser.parse_known_args()
  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)


from flask import Flask, request, redirect, render_template, session, flash
from mysqlconnection import MySQLConnector
app = Flask(__name__)
mysql = MySQLConnector(app,'friendsdb')

@app.route('/')
def index():
    query = "SELECT * FROM friends"
    friends = mysql.query_db(query)
    return render_template('index.html', all_friends=friends)

# Action of adding new friend, returning to homepage
@app.route('/friends', methods=['POST'])
def create():
    query = "INSERT INTO friends (first_name, last_name, occupation, created_at, updated_at) VALUES (:first_name, :last_name, :occupation, NOW(), NOW())"
    data = {
             'first_name': request.form['first_name'],
             'last_name':  request.form['last_name'],
             'occupation': request.form['occupation']
           }
    mysql.query_db(query, data)
    return redirect('/')

# Edit page
@app.route('/friends/<id>/edit')
def edit(id):
    query = "SELECT * FROM friends WHERE id = :specific_id"
    data = {'specific_id': id}
    friends = mysql.query_db(query, data)
    return render_template('edit.html', all_friends=friends)

# Action of updating a friend, returning to homepage
@app.route('/friends/<id>', methods=['POST'])
def update(id):
    query = "UPDATE friends SET first_name = :first_name, last_name = :last_name, occupation = :occupation WHERE id = :specific_id"
    data = {
             'specific_id': id,
             'first_name': request.form['first_name'],
             'last_name':  request.form['last_name'],
             'occupation': request.form['occupation']
           }
    mysql.query_db(query, data)
    return redirect('/')

# Action of deleting a friend, returning to homepage
@app.route('/friends/<id>/delete', methods=['GET', 'POST'])
def delete(id):
    query = "DELETE FROM friends WHERE id = :specific_id"
    data = {'specific_id': id}
    mysql.query_db(query, data)
    return redirect('/')

app.run(debug=True)


import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np
from PIL import Image, ImageDraw
import PIL
import math
from math import sqrt
import random

def saltAndPeper(img):
	input_image = img
	input_pixels = input_image.load()

	# Create output image
	output_image = Image.new("RGB", input_image.size)
	draw = ImageDraw.Draw(output_image)

	#add salt and pepper noise
	for x in range(input_image.width):
		for y in range(input_image.height):
			acc = [0, 0, 0]
			pixel = input_pixels[x, y]
			rnd = random.randint(1,1000) #take a random integer between 1 and 1000 for every pixel
			if(rnd>950):  #if random int >950 make pepper that pixel
				acc[0] = 0
				acc[1] = 0
				acc[2] = 0
			elif(rnd<50):  #if random int <50 make salt that pixel
				acc[0] = 255
				acc[1] = 255
				acc[2] = 255
			else:		#if random int between 50 and 950 then do not any changes on pixel
				acc[0] = pixel[0]
				acc[1] = pixel[1]
				acc[2] = pixel[2]
			draw.point((x, y), (int(acc[0]), int(acc[1]), int(acc[2])))

	output_image.save('SaltAndPapper.png')

def medianFilter(input_image):
	img = input_image
	input_pixels = img.load()
	members = [(0,0)] * 9

	# Create output image
	output_image = Image.new("RGB", img.size)
	draw = ImageDraw.Draw(output_image)

	for i in range(1,img.width-1):
		for j in range(1,img.height-1):
			acc = [0, 0, 0]
			members[0] = img.getpixel((i-1,j-1))  # It takes 9 pixels value
			members[1] = img.getpixel((i-1,j))
			members[2] = img.getpixel((i-1,j+1))
			members[3] = img.getpixel((i,j-1))
			members[4] = img.getpixel((i,j))
			members[5] = img.getpixel((i,j+1))
			members[6] = img.getpixel((i+1,j-1))
			members[7] = img.getpixel((i+1,j))
			members[8] = img.getpixel((i+1,j+1))
			members.sort()  	#sort the pixel values
			pixel = members[4] 	#replace the median pixel to other 9 pixels
			acc[0] = pixel[0]
			acc[1] = pixel[1]
			acc[2] = pixel[2]
			draw.point((i,j),(int(acc[0]), int(acc[1]), int(acc[2])))
	output_image.save('MedianFilterForSaltAndPepper.png')

def main():

	img=Image.open('resimOrjinal.png')
	saltAndPeper(img)
	img2 = Image.open('SaltAndPapper.png')
	medianFilter(img2)

if __name__ == "__main__":
    main()