#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright (С) ABBYY (BIT Software), 1993 - 2019. All rights reserved.
"""
Классы для вычисления метрик качества для задачи поиска объектов
немного модифицированный код из FindText
- добавлены метрики для классификации объектов по типам
- оценка для произвольных многоугольников вместо четырехугольников
"""

import logging
import math

import cv2
import numpy as np
import pandas as pd
from shapely.geometry import Polygon

from semantic_segmentation import utils

class FtMetrics:
    def __init__(self, all_type_names=None, compute_classification_metrics=False):
        """
        Note: Второй параметр нужен, т.к. в этом месте мы не знаем есть ли классификация на самом деле, несмотря на
        all_object_types, который, если определен, не гарантирует, что классификация на самом деле есть
        :param all_type_names: список всех строк-имен типов/классов/языков объектов, которые детектятся
        :param compute_classification_metrics: нужно ли считать метрики для классификации
        """
        self.tp = 0
        self.fp = 0
        self.fn = 0

        self.one_to_one = 0  # те из tp, которые были сматчены идеально
        self.one_to_many = 0  # те из tp, которые были сматчены несколькими областями
        self.many_to_one = 0  # те из tp, которые были сматчены в группах по несколько

        self.average_iou = 0
        self.matched_boxes_count = 0  # число пар сматченных областей, по которым вычисляется average_iou

        self.average_iou_by_area = 0
        self.average_precision_by_area = 0
        self.average_recall_by_area = 0
        self.matched_images_count = 0  # число пар сматченных изображений, по которым вычисляются метрики ...by_area

        self.detection_rate = 0  # количество изображений на которых iou больше порога

        self.all_type_names = all_type_names
        self.confusion_matrix = None
        if all_type_names is not None and compute_classification_metrics:
            self.confusion_matrix = np.zeros((len(all_type_names), len(all_type_names)), dtype=np.float32)

    def append(self, other):
        self.tp += other.tp
        self.fp += other.fp
        self.fn += other.fn

        self.one_to_one += other.one_to_one
        self.one_to_many += other.one_to_many
        self.many_to_one += other.many_to_one

        self.average_iou, self.matched_boxes_count = self.__sum_up(
            self.average_iou,
            self.matched_boxes_count,
            other.average_iou,
            other.matched_boxes_count
        )

        self.average_iou_by_area, matched_images_count = self.__sum_up(
            self.average_iou_by_area,
            self.matched_images_count,
            other.average_iou_by_area,
            other.matched_images_count
        )
        self.average_precision_by_area, matched_images_count = self.__sum_up(
            self.average_precision_by_area,
            self.matched_images_count,
            other.average_precision_by_area,
            other.matched_images_count
        )
        self.average_recall_by_area, matched_images_count = self.__sum_up(
            self.average_recall_by_area,
            self.matched_images_count,
            other.average_recall_by_area,
            other.matched_images_count
        )
        self.detection_rate, matched_images_count = self.__sum_up(
            self.detection_rate,
            self.matched_images_count,
            other.detection_rate,
            other.matched_images_count
        )

        self.matched_images_count = matched_images_count

        if self.confusion_matrix is not None:
            self.confusion_matrix += other.confusion_matrix

    def get_metrics(self):
        precision = self.tp / (self.tp + self.fp) if (self.tp + self.fp) > 0 else 0
        recall = self.tp / (self.tp + self.fn) if (self.tp + self.fn) > 0 else 0
        f1 = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0
        return precision, recall, f1

    def get_report(self):
        precision, recall, f1 = self.get_metrics()

        report_str = 'pr = {:.4f}, r = {:.4f}, f1 = {:.4f} ' \
                     '[tp = {} = {} (1-1) + {} (1-m) + {} (m-1); fp = {}; fn = {}];' \
                     ' iou boxes = {:.2f}  >>> by area: pr = {:.4f}, r = {:.4f}, iou = {:.4f}, rate = {:.4f}' \
            .format(
                precision, recall, f1,
                self.tp, self.one_to_one, self.one_to_many, self.many_to_one,
                self.fp, self.fn, self.average_iou,
                self.average_precision_by_area, self.average_recall_by_area, self.average_iou_by_area,
                self.detection_rate
            )
        return report_str

    def get_confusion_matrix_report(self):
        assert self.confusion_matrix is not None and self.all_type_names is not None, \
            "confusion matrix undefined in report"

        acc_per_type = np.diag(self.confusion_matrix) / np.maximum(np.sum(self.confusion_matrix, axis=1), 1)
        acc_as_df = pd.DataFrame(acc_per_type.reshape(-1, 1), columns=['Accuracy'], index=self.all_type_names)
        pd.set_option('precision', 3)
        pd.set_option("display.max_columns", self.confusion_matrix.shape[0])
        pd.set_option("display.max_rows", self.confusion_matrix.shape[1])
        report_str = str(acc_as_df)

        report_str += '\nAverage accuracy: {:.3f}\n\n'.format(
            np.diag(self.confusion_matrix).sum() / np.maximum(self.confusion_matrix.sum(), 1))

        report_str += 'Confusion matrix (predicted \\ actual):'
        confusion_as_df = pd.DataFrame(
            # не смотря на то что матрица вещественная можно округлить до интов
            # во имя форматирования, ошибка округления не должна быть большой
            self.confusion_matrix.T.astype(int),
            index=self.all_type_names, columns=self.all_type_names)
        report_str += f'\n{confusion_as_df}'

        return report_str

    def get_types_acc(self):
        assert self.confusion_matrix is not None and self.all_type_names is not None, \
            "confusion matrix undefined in report"
        eps = 1e-5
        accs = np.diag(self.confusion_matrix) / (np.sum(self.confusion_matrix, axis=1) + eps)
        type_to_acc = dict((self.all_type_names[i], accs[i]) for i in range(len(accs)))
        return type_to_acc

    def get_average_acc(self):
        assert self.confusion_matrix is not None and self.all_type_names is not None, \
            "confusion matrix undefined in report"
        return np.diag(self.confusion_matrix).sum() / np.maximum(self.confusion_matrix.sum(), 1)

    @staticmethod
    def __sum_up(average_left, count_left, average_right, count_right):
        """
        Вычисляет сумму двух накопленных статистик
        """
        count = count_left + count_right
        accumulator = average_left * count_left + average_right * count_right
        average = accumulator / count if count > 0 else 0
        return average, count

class FtMetricsCalculator:
    """
    Класс для вычисления метрик качества по целевой разметке gt_boxes и
    найденным многоугольниками found_boxes
    """

    def __init__(self, gt_boxes, found_boxes,
                 gt_object_types=None, found_object_types=None, all_object_types=None,
                 compute_classification_metrics=True):
        """
        если хотя бы один из необязательных параметров None, метрики по типам считаться не будут
        :param gt_boxes: список (выпуклых) многоугольников
        :param found_boxes: список (выпуклых) многоугольников
        :param gt_object_types: список типов(классов/языков) объектов, соответствующих gt_boxes
        :param found_object_types: список типов(классов/языков) объектов, соответствующих found_boxes
        :param all_object_types: список (всех) строк-имен типов/классов/языков объектов, которые детектятся
        :param compute_classification_metrics:
        """
        self.__gt_boxes = np.array(gt_boxes)
        self.__found_boxes = np.array(found_boxes)

        # считаем метрики для классификации (confusion matrix) только в этом случае
        # здесь чтобы понять, есть ли классификация, достаточно посмотреть на все эти объекты:
        # действительно, если в разметке и предсказаниях указаны типы - значит, модель с классификацией
        self.__are_classification_metrics_computed = \
            gt_object_types is not None \
            and found_object_types is not None \
            and all_object_types is not None \
            and compute_classification_metrics

        self.__gt_object_types = gt_object_types
        self.__found_object_types = found_object_types
        self.__all_object_types = all_object_types
        self.__gt_boxes_count = len(gt_boxes)
        self.__found_boxes_count = len(found_boxes)

        if self.__are_classification_metrics_computed:
            assert len(gt_object_types) == len(gt_boxes) and len(found_object_types) == len(found_boxes), "uneq len"
            assert all(obj_type in all_object_types for obj_type in np.unique(gt_object_types)), "other types"
            assert all(obj_type in all_object_types for obj_type in np.unique(found_object_types)), "other types"
            self.__typename_id = dict((obj_type, i) for i, obj_type in enumerate(self.__all_object_types))

        # вычисляем площади областей
        self.__gt_boxes_areas = [self.__calc_area(box) for box in gt_boxes]
        self.__found_boxes_areas = [self.__calc_area(box) for box in found_boxes]

        # вычисляем площади пересечений
        # intersection_table: первый индекс относится к GT, второй к тестируемому алгоритму
        self.__intersections_table = np.zeros([self.__gt_boxes_count, self.__found_boxes_count])
        self.__iou_table = np.zeros([self.__gt_boxes_count, self.__found_boxes_count])
        for i, gt_box in enumerate(gt_boxes):
            for j, found_box in enumerate(found_boxes):
                try:
                    intersection = self.__calc_intersection_area(gt_box, found_box)
                except Exception as e:
                    logging.error(f"error while __calc_intersection_area in metrics {str(e)}")
                    intersection = 0
                self.__intersections_table[i, j] = intersection
                self.__iou_table[i, j] = self.__calc_iou(
                    self.__gt_boxes_areas[i], self.__found_boxes_areas[j], intersection)

    def analyze(self, iou_threshold):
        """
        Вычисление метрик качества при заданном пороге на IoU
        если IoU двух областей меньше iou_precision_threshold, то считаем, что они не пересекаются
        """

        iou_precision_threshold = 0.05
        gt_to_found = [np.nonzero(self.__iou_table[i] > iou_precision_threshold)[0] for i in
                       range(self.__gt_boxes_count)]
        found_to_gt = [np.nonzero(self.__iou_table[:, j] > iou_precision_threshold)[0] for j in
                       range(self.__found_boxes_count)]

        one_to_ones = []
        one_to_manys = []  # 1 gt <-> много found
        many_to_ones = []  # много gt <-> 1 found
        for gt_index, indices in enumerate(gt_to_found):
            if len(indices) == 1:
                found_area_index = indices[0]
                inverse_indices = found_to_gt[found_area_index]
                if len(inverse_indices) == 1:
                    # соответствие 1 к 1
                    one_to_ones.append([gt_index, found_area_index])
            elif len(indices) > 1:
                # проверим возможность того, что это соответствие 1 <-> много
                if all(len(found_to_gt[index]) == 1 for index in indices):
                    one_to_manys.append([gt_index, indices])

        for found_area_index, inverse_indices in enumerate(found_to_gt):
            if len(inverse_indices) > 1:
                # проверим возможность того, что это соответствие много <-> 1
                if all(len(gt_to_found[index]) == 1 for index in inverse_indices):
                    many_to_ones.append([inverse_indices, found_area_index])

        metrics = FtMetrics(all_type_names=self.__all_object_types,
                            compute_classification_metrics=self.__are_classification_metrics_computed)
        matched_gt = 0
        matched_found = 0
        iou_summed = 0

        # проверим все соответствия 1 к 1 - это либо tp, либо fn (если пересечение слишком мало)
        one_to_ones_iou = [self.__calc_iou(self.__gt_boxes_areas[gt_index], self.__found_boxes_areas[found_index],
                                           self.__intersections_table[gt_index][found_index])
                           for [gt_index, found_index] in one_to_ones]
        match_iou = [(gt_found, iou) for gt_found, iou in zip(one_to_ones, one_to_ones_iou) if iou >= iou_threshold]
        if match_iou:
            one_to_ones, one_to_ones_iou = list(zip(*match_iou))
        else:
            one_to_ones, one_to_ones_iou = [], []

        one_to_ones_count = len(one_to_ones_iou)
        metrics.one_to_one = one_to_ones_count
        matched_gt += one_to_ones_count
        matched_found += one_to_ones_count
        iou_summed += sum(one_to_ones_iou)
        metrics.matched_boxes_count += one_to_ones_count

        if self.__are_classification_metrics_computed:
            # для соответствий 1 к 1
            confusion_matrix = np.zeros((len(self.__all_object_types), len(self.__all_object_types)), dtype=np.float32)
            for gt_index, found_index in one_to_ones:
                self.__update_confusion_matrix([gt_index], [found_index], confusion_matrix)

        # проверим все соответствия 1 <-> много
        for [gt_index, found_indices] in one_to_manys:
            iou = self.__get_group_to_box_iou(self.__gt_boxes[gt_index], self.__found_boxes[found_indices])
            if iou >= iou_threshold:
                matched_gt += 1
                metrics.one_to_many += 1
                matched_found += len(found_indices)
                iou_summed += iou
                metrics.matched_boxes_count += 1
                if self.__are_classification_metrics_computed:
                    self.__update_confusion_matrix([gt_index], found_indices, confusion_matrix)

        # проверим все соответствия много <-> 1
        for [gt_indices, found_index] in many_to_ones:
            iou = self.__get_group_to_box_iou(self.__found_boxes[found_index], self.__gt_boxes[gt_indices])
            if iou >= iou_threshold:
                matched_gt += len(gt_indices)
                metrics.many_to_one += len(gt_indices)
                matched_found += 1
                iou_summed += iou
                metrics.matched_boxes_count += 1
                if self.__are_classification_metrics_computed:
                    self.__update_confusion_matrix(gt_indices, [found_index], confusion_matrix)

        metrics.tp = matched_gt
        metrics.fn = self.__gt_boxes_count - matched_gt
        metrics.fp = self.__found_boxes_count - matched_found

        metrics.average_iou = iou_summed / metrics.matched_boxes_count if metrics.matched_boxes_count > 0 else 0
        metrics.average_precision_by_area, metrics.average_recall_by_area, metrics.average_iou_by_area = \
            self.__calc_iou_by_area()
        metrics.detection_rate = 1 if metrics.average_iou_by_area > iou_threshold else 0
        metrics.matched_images_count = 1

        if self.__are_classification_metrics_computed:
            metrics.confusion_matrix = confusion_matrix
            metrics.all_type_names = self.__all_object_types
        return metrics

    def __update_confusion_matrix(self, gt_indices, found_indices, confusion_matrix):
        """
        Обноляет значения confusion_matrix, сответствующие переданным наборам индексов
        при len(gt_indices) == len(found_indices) == 1 - one-to-one matching
        len(gt_indices) == 1, len(found_indices) != 1 - one-to-many
        len(gt_indices) != 1,  len(found_indices) == 1 -many-to-one
        :param gt_indices: list индексов
        :param found_indices: list индексов
        :param confusion_matrix:
        :return:
        """
        if len(gt_indices) == 1 == len(found_indices):
            # 1 <-> 1
            actual_type = self.__gt_object_types[gt_indices[0]]
            predicted_type = self.__found_object_types[found_indices[0]]
            confusion_matrix[self.__typename_id[actual_type], self.__typename_id[predicted_type]] += 1
        elif len(gt_indices) == 1:
            # 1 <-> many
            gt_index = gt_indices[0]
            found_intersections = self.__intersections_table[gt_index][found_indices]
            relative_weights = found_intersections / np.sum(found_intersections)
            actual_type = self.__gt_object_types[gt_index]
            predicted_types = [self.__found_object_types[found_index] for found_index in found_indices]
            for predicted_type, relative_weight in zip(predicted_types, relative_weights):
                confusion_matrix[self.__typename_id[actual_type], self.__typename_id[predicted_type]] += relative_weight
        elif len(found_indices) == 1:
            # many <-> 1
            actual_types = [self.__gt_object_types[gt_index] for gt_index in gt_indices]
            predicted_type = self.__found_object_types[found_indices[0]]
            for actual_type in actual_types:
                confusion_matrix[self.__typename_id[actual_type], self.__typename_id[predicted_type]] += 1
        else:
            raise ValueError("At least one of gt/predicted indices must be of length 1")

    @staticmethod
    def __union_polygons(boxes):
        """
        Составляет полигон, являющийся объединением всех поданных полигонов
        :param boxes: массив, в котором каждый элемент это список координат многоугольника [x1, y1, x2, y2, ..., xN, yN]
        """
        poly = Polygon()
        for box_to_add in boxes:
            poly_to_add = Polygon(np.reshape(box_to_add, [-1, 2]))
            try:
                poly = poly.union(poly_to_add)
            except Exception as e:
                logging.error(f"error while union polygons in metrics {str(e)}")
        return poly

    def __get_group_to_box_iou(self, box, boxes_group):
        """
        Объединяет многоугольники из группы в один новый полигон, и вычисляет iou
        """
        poly1 = self.__union_polygons(boxes_group)
        poly2 = Polygon(np.reshape(box, [-1, 2]))

        intersection = poly1.intersection(poly2).area
        return self.__calc_iou(poly1.area, poly2.area, intersection)

    def __calc_iou_by_area(self):
        """
        Объединяет все многоугольники на двух изображениях, и вычисляет
        precision, recall и iou двух полученных областей (на всё изображение)
        """
        gt_poly = self.__union_polygons(self.__gt_boxes)
        found_poly = self.__union_polygons(self.__found_boxes)
        try:
            intersection = gt_poly.intersection(found_poly).area
        except Exception as e:
            logging.error(f"error while poligons intersection in metrics {str(e)}")
            intersection = 0
        precision = intersection / found_poly.area if found_poly.area > 0 else 0
        recall = intersection / gt_poly.area if gt_poly.area > 0 else 0
        iou = self.__calc_iou(gt_poly.area, found_poly.area, intersection)
        return [precision, recall, iou]

    @staticmethod
    def __calc_area(box):
        """
        Вычисление площади многоугольника
        """
        poly = Polygon(np.reshape(box, [-1, 2]))
        return poly.area

    @staticmethod
    def __calc_intersection_area(box1, box2):
        """
        Вычисление площади пересечения для двух многоугольников
        """
        poly1 = Polygon(np.reshape(box1, [-1, 2]))
        poly2 = Polygon(np.reshape(box2, [-1, 2]))
        return poly1.intersection(poly2).area

    @staticmethod
    def __calc_iou(area1, area2, intersection):
        """
        Вычисление IoU
        """
        union = area1 + area2 - intersection
        return intersection / union if union > 0 else 0

class DatasetMetricCalculator:
    """
    Подсчет метрик по всему датасету
    """
    # пороги iou при которых считаются целевые метрики (precision, recall, f1)
    IOU_THRESHOLDS = np.arange(0.4, 1, 0.05)
    # значение порога, при котором ошибки будут отрисовываться
    ERROR_IOU_THRESHOLD = 0.5

    def __init__(self, net_config):
        """
        :param net_config:
        """
        self._net_config = net_config
        self.__reset_accumulators()

    def __reset_accumulators(self):
        self._n_correct_pixels = 0
        self._n_total_pixels = 0
        self._n_objects = 0
        self._curr_mean_object_acc = 0
        self._per_iou_metrics = dict(
            (k, FtMetrics(all_type_names=self._net_config.get_class_names(),
                          compute_classification_metrics=self._net_config.is_classification_supported()))
            for k in DatasetMetricCalculator.IOU_THRESHOLDS)

    def __update_accumulators(self, n_correct_pixels, n_total_pixels, n_objects, mean_object_acc):
        self._n_correct_pixels += n_correct_pixels
        self._n_total_pixels += n_total_pixels
        self._curr_mean_object_acc = \
            (self._curr_mean_object_acc * self._n_objects + mean_object_acc * n_objects) / (self._n_objects + n_objects)
        self._n_objects += n_objects

    def evaluate_batch(self, gt_objects, found_objects, gt_segmap, classification_logits, meta_infos):
        """
        Подсчет метрик для текущего батча с добавлением их к уже накопленным метрикам
        :param gt_objects:
        :param found_objects:
        :param gt_segmap:
        :param classification_logits:
        :param meta_infos:
        :return: selected_threshold_image_metrics, pixel_classification_mask
            selected_threshold_image_metrics - метрики для каждой картинки отдельно
            pixel_classification_mask - маска маску из -1 (неверный ответ),
                0 (на этом месте нет объекта), 1 (верно предсказанный класс)
        """
        selected_threshold_image_metrics = []
        classification_mode = self._net_config.is_classification_supported()
        # считаем целевые метрики
        for image_idx, (image_gt_objects, image_found_objects) in enumerate(zip(gt_objects, found_objects)):
            assert len(image_gt_objects) > 0, "empty gt bboxes (it should contain at least one bbox)"
            image_gt_bboxes, image_gt_box_types = \
                utils.extract_bboxes_and_object_types(image_gt_objects, self._net_config)
            image_pr_bboxes, image_pr_box_types = \
                utils.extract_bboxes_and_object_types(image_found_objects, self._net_config)
            metrics_calculator = FtMetricsCalculator(
                image_gt_bboxes,
                image_pr_bboxes,
                gt_object_types=image_gt_box_types,
                found_object_types=image_pr_box_types,
                all_object_types=self._net_config.get_class_names(),
                compute_classification_metrics=classification_mode
            )
            for iou_threshold in DatasetMetricCalculator.IOU_THRESHOLDS:
                image_metrics = metrics_calculator.analyze(iou_threshold=iou_threshold)
                self._per_iou_metrics[iou_threshold].append(image_metrics)
                if math.isclose(DatasetMetricCalculator.ERROR_IOU_THRESHOLD, iou_threshold):
                    selected_threshold_image_metrics.append(image_metrics)

        pixel_classification_correctness_mask = None
        if self._net_config.is_classification_supported():
            # pixel classification accuracy считается по gt
            pixel_classification_correctness_mask, k_correct_pixels, k_total_pixels, mean_acc, n_objects = \
                DatasetMetricCalculator._calc_pixel_classification_correctness_mask(classification_logits, gt_segmap)
            self.__update_accumulators(k_correct_pixels, k_total_pixels, n_objects, mean_acc)
        return selected_threshold_image_metrics, pixel_classification_correctness_mask

    def get_metrics(self):
        """
        Возвращает посчитанные метрики по всем прошедшим картинкам
        :return: scalar_logs - dict
        """
        scalar_logs = dict()
        for iou_threshold in DatasetMetricCalculator.IOU_THRESHOLDS:
            save_accs = self._net_config.is_classification_supported() and math.isclose(iou_threshold, 0.5)
            DatasetMetricCalculator._fill_log(scalar_logs, self._per_iou_metrics[iou_threshold],
                                              iou_threshold=iou_threshold, save_accs=save_accs)
        scalar_logs["average_iou_by_area"] = \
            self._per_iou_metrics[DatasetMetricCalculator.IOU_THRESHOLDS[0]].average_iou_by_area
        scalar_logs["average_precision_by_area"] = \
            self._per_iou_metrics[DatasetMetricCalculator.IOU_THRESHOLDS[0]].average_precision_by_area
        scalar_logs["average_recall_by_area"] = \
            self._per_iou_metrics[DatasetMetricCalculator.IOU_THRESHOLDS[0]].average_recall_by_area

        if self._net_config.is_classification_supported():
            # количество правильно угаданных пикселей / количество пикселей всего в ground truth объектах
            scalar_logs["classification_pixel_acc_total"] = self._n_correct_pixels / self._n_total_pixels
            # среднее по всем объектам средних точностей классификации на объекте (инвариантно к размеру)
            scalar_logs["classification_pixel_acc_object"] = self._curr_mean_object_acc
        return scalar_logs

    @staticmethod
    def _fill_log(logs_dict, metrics, iou_threshold, save_accs=False):
        pr, r, f1 = metrics.get_metrics()
        logs_dict["pr_iou{:.2f}".format(iou_threshold)] = pr
        logs_dict["recall_iou{:.2f}".format(iou_threshold)] = r
        logs_dict["f1_iou{:.2f}".format(iou_threshold)] = f1
        logs_dict["detection_rate_iou{:.2f}".format(iou_threshold)] = metrics.detection_rate
        if save_accs:
            logs_dict["types_avg_acc_iou{:.2f}".format(iou_threshold)] = metrics.get_average_acc()
            type_to_acc = metrics.get_types_acc()
            for object_type, acc in type_to_acc.items():
                logs_dict[f"acc_{object_type}_iou{iou_threshold:.2f}"] = acc

    @staticmethod
    def _calc_pixel_classification_correctness_mask(classification_logits, targets):
        """
        Возвращает маску из -1 (неверный ответ), 0 (на этом месте нет объекта), 1 (верно предсказанный класс)
        :param classification_logits: логиты (или вероятности) для каждого пикселя для всех классов
        :param targets: метки классов для каждого пикселя на изображении
        :return:
        """
        mask = (targets > 0).astype(np.float)
        labels_true = (targets - 1) * mask
        labels_pred = np.expand_dims(np.argmax(classification_logits, axis=-1), axis=-1)
        correct = np.equal(labels_true, labels_pred).astype(np.float)
        n_correct_pixels = np.sum(correct * mask)
        n_total_pixels = np.sum(mask)

        object_classification_accs = []
        for i in range(len(targets)):
            contours, _ = utils.get_contours_and_boxes(np.squeeze(mask[i].astype(np.uint8), -1), min_area=-1)
            for cnt in contours:
                object_mask = np.zeros(targets.shape[1:3], dtype=np.uint8)
                cv2.drawContours(object_mask, [cnt], -1, 1, -1)
                acc = correct[i][object_mask.astype(np.bool)].mean()
                object_classification_accs.append(acc)
        mean_acc = np.mean(object_classification_accs)
        n_objects = len(object_classification_accs)

        pixel_correct = correct * 2 - 1  # 0, 1 -> -1, 1
        pixel_correct = np.where(mask, pixel_correct, 0)
        pixel_correct = np.squeeze(pixel_correct, axis=-1)
        return pixel_correct, n_correct_pixels, n_total_pixels, mean_acc, n_objects

# TODO нуждается в рефакторинге
class ImageResultCategories:
    """
    Класс отвечающий за подсчета различных категорий, с точки зрения которых, результат на изображении
    может быть в каком-то смысле интересным
    """
    RECALL_ERROR = 'errors/recall'
    PRECISION_ERROR = 'errors/precision'
    DETECTION_RATE_ERROR = 'errors/detection_rate'
    ALL = 'all'

    # что считается важным в данный момент
    SUITABLE_CATEGORIES = (
        RECALL_ERROR,
        DETECTION_RATE_ERROR,
        ALL
    )

    @staticmethod
    def _is_suitable(category):
        return category in ImageResultCategories.SUITABLE_CATEGORIES

    @staticmethod
    def get_categories(metrics):
        """
        Подсчет различных категорий, с точки зрения которых, результат на изображении
        может быть в каком-то смысле интересным
        :param metrics:
        :return:
        """
        categories = []
        if metrics.detection_rate < 1 and ImageResultCategories._is_suitable(
                ImageResultCategories.DETECTION_RATE_ERROR):
            categories.append(ImageResultCategories.DETECTION_RATE_ERROR)
        precision, recall, f1 = metrics.get_metrics()
        if precision < 1 and ImageResultCategories._is_suitable(ImageResultCategories.PRECISION_ERROR):
            categories.append(ImageResultCategories.PRECISION_ERROR)
        if recall < 1 and ImageResultCategories._is_suitable(ImageResultCategories.RECALL_ERROR):
            categories.append(ImageResultCategories.RECALL_ERROR)
        if ImageResultCategories._is_suitable(ImageResultCategories.ALL):
            categories.append(ImageResultCategories.ALL)
        return categories

    @staticmethod
    def get_errors(visualization_categories):
        return [vis_category for vis_category in visualization_categories
                if vis_category != ImageResultCategories.ALL]

    @staticmethod
    def get_folders():
        return ImageResultCategories.SUITABLE_CATEGORIES


import argparse
import datetime
import imutils
import time
from datetime import datetime
import cv2
import numpy as np
import os
from videoreader import VideoReader
import tensorflow as tf
import json
import tweepy
import random

THRESHOLD = 0.55
MODEL_FILE = "models/norppabot.tflite"
INPUT_MEAN = 127.5
INPUT_STD = 127.5
WIDTH = 1280
HEIGHT = 720

MESSAGES_FILE = 'messages.json'
TWITTER_CONFIG = 'twitter_config.json'
TWEET_TIME_MINUTES = 30

class Detector:
    def __init__(self, camera):
        self.initModel()
        self.initTwitter()
        self.frameTime = 0
        self.frames = 0
        self.lastImageTime = 0
        self.fps = 30
        self.lastTweetTime = 0
        self.vr = VideoReader(camera, self.fps)

    def initModel(self):
        self.interpreter = tf.lite.Interpreter(model_path=MODEL_FILE)
        self.interpreter.allocate_tensors()
        self.input_details = self.interpreter.get_input_details()
        self.output_details = self.interpreter.get_output_details()

        self.model_height = self.input_details[0]['shape'][1]
        self.model_width = self.input_details[0]['shape'][2]

        self.floating_model = (self.input_details[0]['dtype'] == np.float32)

    def initTwitter(self):
        with open(TWITTER_CONFIG) as f:
            data = json.load(f)
        auth = tweepy.OAuthHandler(data['apiKey'], data['apiSecret'])
        auth.set_access_token(data['token'], data['tokenSecret'])
        self.twitter = tweepy.API(auth)

    def detect(self, image):
        image = image.copy()

        # Test model on random input data.
        input_shape = self.input_details[0]['shape']
        image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        image_resized = cv2.resize(image_rgb, (self.model_width, self.model_height))
        input_data = np.expand_dims(image_resized, axis=0)
        if self.floating_model:
            input_data = (np.float32(input_data) - INPUT_MEAN) / INPUT_STD
        self.interpreter.set_tensor(self.input_details[0]['index'], input_data)

        # Run the network
        self.interpreter.invoke()

        boxes = self.interpreter.get_tensor(self.output_details[0]['index'])[0]
        classes = self.interpreter.get_tensor(self.output_details[1]['index'])[0]
        scores = self.interpreter.get_tensor(self.output_details[2]['index'])[0]

        return boxes, scores, classes

    def triggerDetection(self, frame, orig_frame):
        self.log('Norppa detected!')

        self.lastDetectTime = time.time()

        # Only react every 30 seconds
        if (time.time() - self.lastImageTime) < 30:
            return

        # Save the original image and frame with detection boxes
        filename = "images/frame%s.jpg" % datetime.now().strftime("%Y%m%d-%H%M%S")
        o_filename = "images/orig_frame%s.jpg" % datetime.now().strftime("%Y%m%d-%H%M%S")
        cv2.imwrite(filename, frame)
        cv2.imwrite(o_filename, orig_frame)
        self.log("Wrote %s" % filename)

        # Tweet
        self.tweetNorppaIsLive(frame)

        self.lastImageTime = time.time()

    def tweetNorppaIsLive(self, frame):
        # only tweet every TWEET_TIME_MINUTES minutes
        if time.time() - self.lastTweetTime < TWEET_TIME_MINUTES * 60:
            return

        # Get random message from messages.json
        with open(MESSAGES_FILE, 'r') as f:
            messages = json.load(f)

        randomMessage = random.choices(messages)[0].replace('#', 'hashtag-')

        filename = 'uploaded_image.jpg'
        cv2.imwrite(filename, frame)
        self.twitter.update_with_media(filename, status=randomMessage)
        self.log('Tweeted: %s' % randomMessage)
        os.remove(filename)
        self.lastTweetTime = time.time()

    def checkCoordinates(self, x, y, x2, y2):
        # You can add coordinate & width/height checks here
        width = x2 - x
        height = y2 - y
        area = width * height
        return area < 350000 and area > 10000

    def log(self, msg):
        date = datetime.now().strftime("%Y-%m-%d-%H:%M:%S")
        print(date, msg)

    def run(self):
        print("Ready")
        self.vr.start()
        self.startTime = time.time()
        frameCounter = 0
        lastFrameTime = 0
        detected = False
        detections = 0

        # Wait for the stream to start
        while not self.vr.more() or time.time() - self.startTime > 10:
            time.sleep(0.5)

        frame = None
        while True:
            if not self.vr.more():
                print('asd')
                break

            # Read the latest frame
            frame = np.copy(self.vr.read())
            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
            frameCounter += 1

            lastFrameTime = time.time()

            orig_frame = frame.copy()
            detected = False

            # Get detection boxes and scores from tensorflow
            boxes, scores, classes = self.detect(frame)

            # Debug logging
            if scores[0] > 0.4:
                self.log(scores[0])

            # Go through the boxes
            for i in range(len(scores)):
                score = scores[i]
                box = boxes[i]

                y = int(box[0] * HEIGHT)
                x = int(box[1] * WIDTH)
                y2 = int(box[2] * HEIGHT)
                x2 = int(box[3] * WIDTH)

                if score < THRESHOLD:
                    continue

                # Some sanity checks
                if not self.checkCoordinates(x,y,x2,y2):
                    continue

                # Norppa detected here!
                detected = True
                self.log(score)
                print((x,y), (x2,y2))

                # Draw the detection box to the frame
                cv2.rectangle(frame, (x, y), (x2, y2), (0, 255, 0), 2)
                label = '%s: %d%%' % ('norppa', int(score * 100))
                labelSize, baseLine = cv2.getTextSize(
                    label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2)
                label_ymin = max(y, labelSize[1] + 10)
                # Draw white box to put label text in
                cv2.rectangle(frame, (x, label_ymin-labelSize[1]-10), (
                    x+labelSize[0], label_ymin+baseLine-10), (255, 255, 255), cv2.FILLED)
                cv2.putText(frame, label, (x, label_ymin-7),
                            cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2)
            if detected:
                detections += 1
            else:
                detections = 0

            # Only detect after the second frame in a row that breaks the threshold
            if detections >= 2:
                self.triggerDetection(frame, orig_frame)

            if frameCounter % 1000 == 0:
                self.log('1000 frames')
                frameCounter = 0
            frameTime = time.time() - lastFrameTime
            if frameTime < 1 / self.fps:
                time.sleep(1 / self.fps - frameTime)
        self.vr.stop()
        time.sleep(1)

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("-v", "--video", help="path to the video file")
    args = vars(ap.parse_args())

    video = args["video"]

    detector = Detector(video)
    detector.run()

if __name__ == "__main__":
    main()


from django.apps import AppConfig
from django.conf import settings
import tensorflow as tf
import numpy as np

class Predictor(AppConfig):
    """
    Predictor app
    """
    name = 'ML'
    model = tf.keras.models.load_model(settings.MODEL_PATH)


import numpy as np
import pyopencl as cl
import pyopencl.tools
import pyopencl.array
import base64
import hashlib
import time
import os, sys, inspect
import requests
import threading
import logging
import json
import hashlib
import traceback
import queue
from client import PoolClient
from sha256 import SHA256
from hdata import get_hdata, get_hdata_prefixed

mf = cl.mem_flags
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logging.getLogger('pyopencl').setLevel(logging.FATAL)
start_time = time.time()

# Config
batchSize = 512 * 64
repeats = 1
internal_iterations = 500000
threads = 1
double_ring = False
wallet = sys.argv[1] if len(sys.argv) >= 2 else None
pool_addr = sys.argv[2] if len(sys.argv) >= 3 else 'https://server1.whalestonpool.com'
APP_VERSION = '1.1.4' # keep in sync with hive config

def check_wallet(pool_address, wallet):
    try:
        res = requests.get(f'{pool_address}/wallet/{wallet}').json()
    except:
        print('Connection error. Check pool address.')
        sys.exit(0)
    if not res['registered']:
        print("- whales miner version " + APP_VERSION)
        print(">")
        print("> To use miner you should register via @WhalesPoolBot")
        print(">")
        try:
            sys.exit(0)
        except SystemExit:
            os._exit(0)

check_wallet(pool_addr, wallet)

# Resolve unique id
if os.path.isfile('state.json'):
    with open('state.json', 'r') as json_file:
        data = json.load(json_file)
else:
    with open('state.json', 'w') as json_file:
        json.dump({'wallet': wallet}, json_file)

pool_client = PoolClient(pool_addr, wallet)
if not wallet:
    print('Invalid wallet.')
    sys.exit(1)

logger.info("Starting miner...")
logger.info("Pool address: " + pool_addr)
logger.info("Wallet: " + wallet)

# Mining config
def load_config():
    while True:
        try:
            task = pool_client.load_next_task()
            task['lock'] = threading.Lock()
            task['offset'] = 0
            return task
        except Exception as e:
            logger.error(traceback.format_exc())
            logger.warn("Config load error")
            time.sleep(5)
            continue

def get_kernel_config(params):
    hdata = get_hdata_prefixed(params['wallet'], params['prefix'], params['expire'], params['seed'])
    hash = SHA256()
    hash.update(bytes(hdata[:64]))
    head = np.fromiter(hash.h, np.uint32)
    tail = hdata[64:]
    complexity = np.frombuffer(params['complexity'], np.uint8)
    return (head, tail, complexity, params['expire'], params['seed'], params['giver'])

def validate(config, random, value):
    m = hashlib.sha256()
    m.update(get_hdata(config['wallet'], random, config['expire'], config['seed']).tobytes())
    if m.digest().hex() != value.hex():
        return False
    else:
        return True

def apply_speed(src, value):
    src.append(value)
    if len(src) > 5:
        del src[0]

def resolve_speed(src):
    if len(src) == 0:
        return 0
    res = 0
    for i in range(0, len(src)):
        res = res + src[i]
    res = res / len(src)
    return res

reportQueue = queue.Queue()
def report(wallet, giver, random, expire, seed):
    global shipId
    global shipName
    while True:
        try:
            solution = { 'giver': giver, 'input': get_hdata(wallet, random, expire, seed).tobytes().hex()  }
            pool_client.report_solution(solution)
        except Exception as e:
            logger.error(traceback.format_exc())
            logger.warn("Reporting error")
            time.sleep(5)
            continue
        break
def reportQueueJob():
    while True:
        try:
            item = reportQueue.get()
            report(item['wallet'], item['giver'], item['random'], item['expire'], item['seed'])
            reportQueue.task_done()
        except Exception as e:
            logging.warn(traceback.format_exc())
            logging.warn("Monitoring error")
            time.sleep(5)
            continue
def startReportQueue():
    threading.Thread(target=reportQueueJob).start()

def postReport(wallet, giver, random, expire, seed):
    reportQueue.put({'wallet': wallet, 'giver': giver, 'random': random, 'expire': expire, 'seed': seed})

# Code
platforms = cl.get_platforms()
devices = []
for p in cl.get_platforms():
    devices = devices + p.get_devices()
ctx = cl.Context(devices)
src = ''
# with open(os.path.join(os.path.dirname(__file__),"kernels/sha256_util.cl"), "r") as rf:
#     src += rf.read()
# src += '\n'
# with open(os.path.join(os.path.dirname(__file__),"kernels/sha256_impl.cl"), "r") as rf:
#     src += rf.read()
# src += '\n'
# with open(os.path.join(os.path.dirname(__file__),"kernels/miner.cl"), "r") as rf:
#     src += rf.read()
with open(os.path.join(os.path.dirname(__file__),"opencl_sha256.cl"), "r") as rf:
    src += rf.read()

program = cl.Program(ctx, src).build()
sha256_kernel = program.hash_main

queue = []
for i in range(0, len(devices)):
    queue.append(cl.CommandQueue(ctx, devices[i]))
def compare(a, b):
    i = 0
    while (i < 32):
        if (a[i] != b[i]):
            if a[i] < b[i]:
                return -1
            else:
                return 1
        i += 1
    return 0

#
# Config
#

logging.info("Loading initial config...")
latest_config = load_config()
logging.info("Config loaded")

def config_refresh_job():
    global latest_config
    latest_config = load_config()

#
# Mining
#

mined = 0
mined_dev = {}
rate = 0
speed_average = []
speed_average_dev = {}
def miner_job(index, deviceId):
    global mined
    global mined_dev
    global repeats
    global latest_config
    global speed_average

    config = latest_config
    start = time.time()
    # logging.info("[" + str(index) + "/"+str(deviceId)+"]: Job started")
    # random = os.urandom(32)
    random = bytes(np.random.bytes(32))
    data = get_hdata(config['wallet'], random, config['expire'], config['seed'])
    m = SHA256()
    m.update(bytes(data[0:64]))
    initial_vector = np.fromiter(m._h, np.uint32)

    tail = data[64:]
    tail = np.pad(tail, (0, 64 - len(tail)))
    head_g = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=initial_vector)
    tail_g = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=tail)
    complexity_g = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=np.frombuffer(config['complexity'], dtype=np.uint8))

    output_1 = np.zeros(32, dtype=np.uint8)
    output_1_random = np.zeros(32, dtype=np.uint8)
    output_2 = np.zeros(32, dtype=np.uint8)
    output_2_random = np.zeros(32, dtype=np.uint8)
    res = None
    res_random = None
    cl_output_1 = cl.Buffer(ctx, cl.mem_flags.WRITE_ONLY | cl.mem_flags.COPY_HOST_PTR, output_1.nbytes, hostbuf=output_1)
    cl_output_1_random = cl.Buffer(ctx, cl.mem_flags.WRITE_ONLY | cl.mem_flags.COPY_HOST_PTR, output_1_random.nbytes, hostbuf=output_1_random)
    cl_output_2 = cl.Buffer(ctx, cl.mem_flags.WRITE_ONLY | cl.mem_flags.COPY_HOST_PTR, output_2.nbytes, hostbuf=output_2)
    cl_output_2_random = cl.Buffer(ctx, cl.mem_flags.WRITE_ONLY | cl.mem_flags.COPY_HOST_PTR, output_2_random.nbytes, hostbuf=output_2_random)
    found = False
    for i in range(0, repeats):
        # print("[" + str(index) +"/"+str(deviceId)+ "]: Iteration " + str(i))

        # Assign offset
        config['lock'].acquire()
        offset = config['offset']
        config['offset'] += internal_iterations * batchSize
        config['lock'].release()

        params = np.fromiter([offset, internal_iterations], np.uint64)
        params_g = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=params)

        event1 = sha256_kernel(queue[deviceId], (batchSize,), None,
            head_g,
            tail_g,
            params_g,
            complexity_g,
            cl_output_1_random,
            cl_output_1,
        )

        # event1.wait()
        # elapsed = 1e-9*(event1.profile.end - event1.profile.start)
        # logging.info("[" + str(index) + "/"+str(deviceId)+"]: Elapsed  " + str(elapsed))
        if double_ring:

            # Assign offset
            config['lock'].acquire()
            offset = config['offset']
            config['offset'] += internal_iterations * batchSize
            config['lock'].release()

            params = np.fromiter([offset, internal_iterations], np.uint64)
            params_g2 = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=params)

            event2 = sha256_kernel(queue[deviceId], (batchSize,), None,
                head_g,
                tail_g,
                params_g2,
                complexity_g,
                cl_output_2_random,
                cl_output_2,
            )
        wait_start = time.time()
        # event1.wait()
        # logging.info("[" + str(index) + "/"+str(deviceId)+"]: Waited (1) " + str(( time.time() - wait_start)))
        # wait_start = time.time()
        read1 = cl.enqueue_copy(queue[deviceId], output_1, cl_output_1, is_blocking=False)
        read2 = cl.enqueue_copy(queue[deviceId], output_1_random, cl_output_1_random, is_blocking=False)
        queue[deviceId].flush()

        while read1.get_info(cl.event_info.COMMAND_EXECUTION_STATUS) != cl.command_execution_status.COMPLETE or read2.get_info(cl.event_info.COMMAND_EXECUTION_STATUS) != cl.command_execution_status.COMPLETE:
            time.sleep(0.05)

        mined += internal_iterations * batchSize
        mined_dev[str(deviceId)] = mined_dev.get(str(deviceId),0) + internal_iterations * batchSize
        logging.info("[" + str(index) + "/"+str(deviceId)+"]: Processed in " + str(( time.time() - wait_start)))

        if double_ring:
            wait_start = time.time()
            event2.wait()
            # logging.info("[" + str(index) + "/"+str(deviceId)+"]: Waited (2) " + str(( time.time() - wait_start)))
            wait_start = time.time()
            read1 = cl.enqueue_copy(queue[deviceId], output_2, cl_output_2)
            read2 = cl.enqueue_copy(queue[deviceId], output_2_random, cl_output_2_random)
            queue[deviceId].flush()

            while read1.get_info(cl.event_info.COMMAND_EXECUTION_STATUS) != cl.command_execution_status.COMPLETE or read2.get_info(cl.event_info.COMMAND_EXECUTION_STATUS) != cl.command_execution_status.COMPLETE:
                time.sleep(0.05)
            # logging.info("[" + str(index) + "/"+str(deviceId)+"]: Read (2) " + str(( time.time() - wait_start)))
            mined += internal_iterations * batchSize
            mined_dev[str(deviceId)] = mined_dev.get(str(deviceId),0) + internal_iterations * batchSize

        res = config['complexity']
        if np.sum(output_1) > 0 and np.sum(output_1_random) > 0:
            if compare(output_1.tobytes(), res) < 0:
                found = True
                res = output_1.tobytes()
                res_random = output_1_random.tobytes()
        if double_ring:
            if np.sum(output_2) > 0 and np.sum(output_2_random) > 0 and compare(output_2.tobytes(), res) < 0:
                found = True
                res = output_2.tobytes()
                res_random = output_2_random.tobytes()
#        if found_new:
#            print("[" + str(index) +"/"+str(deviceId)+ "]: Intermediate result: " + res.hex())
    params_g.release()
    tail_g.release()
    head_g.release()
    complexity_g.release()
    cl_output_1.release()
    cl_output_1_random.release()
    cl_output_2.release()
    cl_output_2_random.release()
    if found:
        logging.info("[" + str(index) +"/"+str(deviceId)+ "]: Job result: " + res.hex())
        if not validate(config, res_random, res):
            logging.info("[" + str(index) +"/"+str(deviceId)+ "]: Invalid job result")
        else:
            postReport(config['wallet'], config['giver'], res_random, config['expire'], config['seed'])
    else:
        logging.debug("[" + str(index) +"/"+str(deviceId)+ "]: Job not found valid share")
    # logging.info("[" + str(index) +"/"+str(deviceId)+ "]: Job ended")

def buildHash(data):
    m = hashlib.sha256()
    m.update(data)
    return m.digest()

def miner_thread(index, deviceId):
    logging.info("[" + str(index) + "/"+str(deviceId)+"]: Thread started")
    while True:
        try:
            miner_job(index, deviceId)
        except:
            logging.warn(traceback.format_exc())
            logging.info("[" + str(index) + "/"+str(deviceId)+"]: Miner error")
            time.sleep(5)
            continue

def start_miner_thread(index, deviceId):
    threading.Thread(target=miner_thread, args=(index,deviceId)).start()

def miner_config_thread():
    while True:
        time.sleep(1)
        try:
            config_refresh_job()
            # logging.info("Config updated")
        except Exception as e:
            logging.warn(traceback.format_exc())
            logging.warn("Config error")
            time.sleep(20)
            continue
def start_miner_config_thread():
    threading.Thread(target=miner_config_thread).start()

def miner_mon(count):
    global mined
    global mined_dev
    global rate
    global speed_average
    global speed_average_dev

    # Start Timer
    start = time.time()

    # Reset mined state
    mined = 0
    for i in range(0, count):
        mined_dev[str(i)] = 0
        speed_average_dev[str(i)] = []

    while True:
        time.sleep(10)
        try:

            # Resolve time
            time_delta = time.time() - start
            start = time.time()

            # Resolve speed
            delta = mined / (time_delta * 1000 * 1000)
            mined = 0

            # Update average speed
            apply_speed(speed_average, delta)
            total_average = resolve_speed(speed_average)
            rate = total_average

            # Calculate thread speed
            rates = []
            for i in range(0, count):

                # Resolve speed
                dev_delta = mined_dev.get(str(i), 0) / (time_delta * 1000 * 1000)
                mined_dev[str(i)] = 0

                # Update average speed
                apply_speed(speed_average_dev[str(i)], dev_delta)
                rates.append(resolve_speed(speed_average_dev[str(i)]))

            # Logging
            logging.info("Performance " + str(total_average) + " MH/s")

            # Persisting
            with open('stats.json', 'w') as json_file:
                json.dump({
                    'total': total_average * 1000, # in khs
                    'rates': rates,
                    'uptime': time.time() - start_time
                }, json_file)
        except Exception as e:
            logging.warn(traceback.format_exc())
            logging.warn("Monitoring error")
            time.sleep(5)
            continue

def start_miner_mon(count):
    threading.Thread(target=miner_mon, args=(count,)).start()

#
# Start
#

# Start monitoring
start_miner_mon(len(devices))

# Start config loader
start_miner_config_thread()

# Start reporting queue
startReportQueue()

# Start miners
for i in range(0, len(devices)):
    for t in range(0, threads):
        start_miner_thread(i * threads + t, i)

logger.info("Miner started")


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 31 02:00:37 2018

@author: cbkeller
"""
#def configure_isobaric(perplexdir, scratchdir, composition, elements = ['SIO2','TIO2','AL2O3','FEO','MGO','CAO','NA2O','K2O','H2O'], index = 1, P = 10000, T_range = [500+273.15, 1500+273.15], dataset = 'hp11ver.dat', solution_phases = 'O(HP)\nOpx(HP)\nOmph(GHP)\nGt(HP)\noAmph(DP)\ncAmph(DP)\nT\nB\nChl(HP)\nBio(TCC)\nMica(CF)\nCtd(HP)\nIlHm(A)\nSp(HP)\nSapp(HP)\nSt(HP)\nfeldspar_B\nDo(HP)\nF\n', excludes = 'ts\nparg\ngl\nged\nfanth\ng\n'):

# Import some useful packages
import os # os.system lets us access the command line
import numpy as np # For np.array
import pandas as pd # Pandas, for importing PerpleX text file output as data frames

def configure(meltspath, scratchdir, composition, elements, batchstring, T_range, P_range, dT=-10, dP=0, index=1, version='pMELTS',mode='isobaric',fo2path='FMQ',fractionatesolids='!'):

    ############################ Default Settings ###############################
    ##MELTS or pMELTS
    #version='pMELTS';
    ##Mode (isothermal, isobaric, isentropic, isenthalpic, isochoric, geothermal or PTPath)
    #mode='isobaric';
    ## Set fO2 constraint, i.e. 'IW','COH','FMQ','NNO','HM','None' as a string
    #fo2path='FMQ';
    ## Fractionate all solids? ('!' for no, '' for yes)
    #fractionatesolids='!';
    # Mass retained during fractionation
    massin=0.001;
    # Ouptut temperatures in celcius? ('!' for no, '' for yes)
    celciusoutput='';
    # Save all output? ('!' for no, '' for yes)
    saveall='!';
    # Fractionate all water? ('!' for no, '' for yes)
    fractionatewater='!';
    # Fractionate individual phases (specify as strings in cell array, i.e. {'olivine','spinel'})
    fractionate=[];
    # Supress individual phases (specify as strings in cell array, i.e. {'leucite'})
    supress=[];
    # Coninuous (fractional) melting? ('!' for no, '' for yes)
    continuous='!';
    # Threshold above which melt is extracted (if fractionation is turned on)
    minf=0.005;
    # Do trace element calculations
    dotrace='!';
    # Treat water as a trace element
    dotraceh2o='!';
    # Initial trace compositionT
    tsc=[];
    # Initial trace elements
    telements=[];
    # Default global constraints
    Pmax=90000;
    Pmin=2;
    Tmax=3000;
    Tmin=450;
    # Simulation number (for folder, etc)

    ########################## end Default Settings ############################

    # Guess if intention is for calculation to end at Tf or Pf as a min or max
    if T_range[1]<T_range[0]:
        Tmin=T_range[1]
    if T_range[1]>T_range[0]:
        Tmax=T_range[1];
    if P_range[1]<P_range[0]:
        Pmin=P_range[1];
    if P_range[1]>P_range[0]:
        Pmax=P_range[1];

    # Normalize starting composition
    composition = np.array(composition)/sum(composition)*100;

    # output prefixectory name
    prefix = scratchdir + 'out%i' %(index);
    os.system('rm -rf %s; mkdir -p %s' %(prefix,prefix)); # Ensure directory is empty

    # Make .melts file containing the starting composition you want to run
    # simulations on
    fp=open(prefix + '/sc.melts','w');
    for i in range(len(elements)):
        fp.write('Initial Composition: %s %.4f\n' %(elements[i],composition[i]));
    for i in range(len(telements)):
        fp.write('Initial Trace: %s %.4f\n' %(telements[i],tsc[i]));

    fp.write('Initial Temperature: %.2f\nInitial Pressure: %.2f\nlog fo2 Path: %s\n' %(T_range[0],P_range[0],fo2path));

    for i in range(len(fractionate)):
        fp.write('Fractionate: %s\n' %(fractionate[i]));
    for i in range(len(supress)):
        fp.write('Suppress: %s\n' %(supress[i]));
    fp.close();

    # Make melts_env file to specify type of MELTS calculation
    fp=open(prefix + '/melts_env.txt','w');
    fp.write('! *************************************\n!  Python-generated environment file\n! *************************************\n\n' + \
        '! this variable chooses MELTS or pMELTS; for low-pressure use MELTS\n' + \
        'ALPHAMELTS_VERSION		%s\n\n' %(version) + \
        '! do not use this unless fO2 anomalies at the solidus are a problem\n' + \
        '!ALPHAMELTS_ALTERNATIVE_FO2	true\n\n' + \
        '! use this if you want to buffer fO2 for isentropic, isenthalpic or isochoric mode\n! e.g. if you are doing isenthalpic AFC\n' + \
        '!ALPHAMELTS_IMPOSE_FO2		true\n\n' + \
        '! use if you want assimilation and fractional crystallization (AFC)\n' + \
        '!ALPHAMELTS_ASSIMILATE		true\n\n' + \
        '! isothermal, isobaric, isentropic, isenthalpic, isochoric, geothermal or PTPath\n' + \
        'ALPHAMELTS_MODE			%s\n' %(mode) + \
        '!ALPHAMELTS_PTPATH_FILE		ptpath.txt\n\n' + \
        '! need to set DELTAP for polybaric paths; DELTAT for isobaric paths\nALPHAMELTS_DELTAP	%.0f\n' %(dP) + \
        'ALPHAMELTS_DELTAT	%.0f\n' %(dT) + \
        'ALPHAMELTS_MAXP		%.0f\n' %(Pmax) + \
        'ALPHAMELTS_MINP		%.0f\n' %(Pmin) + \
        'ALPHAMELTS_MAXT		%.0f\n' %(Tmax) + \
        'ALPHAMELTS_MINT		%.0f\n\n' %(Tmin) + \
        '! this one turns on fractional crystallization for all solids\n! use Fractionate: in the melts file instead for selective fractionation\n' + \
        '%sALPHAMELTS_FRACTIONATE_SOLIDS	true\n' %(fractionatesolids) + \
        '%sALPHAMELTS_MASSIN		%g\n\n' %(fractionatesolids,massin) + \
        '! free water is unlikely but can be extracted\n' + \
        '%sALPHAMELTS_FRACTIONATE_WATER	true\n' %(fractionatewater) + \
        '%sALPHAMELTS_MINW			0.005\n\n' %(fractionatewater) + \
        '! the next one gives an output file that is always updated, even for single calculations\n' + \
        '%sALPHAMELTS_SAVE_ALL		true\n' %(saveall)+ \
        '!ALPHAMELTS_SKIP_FAILURE		true\n\n' + \
        '! this option converts the output temperature to celcius, like the input\n' + \
        '%sALPHAMELTS_CELSIUS_OUTPUT	true\n\n' %(celciusoutput) + \
        '! the next two turn on and off fractional melting\n' + \
        '%sALPHAMELTS_CONTINUOUS_MELTING	true\n' %(continuous) + \
        '%sALPHAMELTS_MINF			%g\n' %(continuous,minf) + \
        '%sALPHAMELTS_INTEGRATE_FILE	integrate.txt\n\n' %(continuous) + \
        '! the next two options refer to the trace element engine\n' + \
        '%sALPHAMELTS_DO_TRACE		true\n' %(dotrace)+ \
        '%sALPHAMELTS_DO_TRACE_H2O		true\n' %(dotraceh2o));
    fp.close();

    # Make a batch file to run the above .melts file starting from the liquidus
    fp = open(prefix + '/batch.txt','w');
    fp.write(batchstring);
    fp.close();

    # Run the command
    # Edit the following line(s to make sure you have a correct path to the 'run_alphamelts.command' perl script
    os.system('cd ' + prefix  + '; ' + meltspath + ' -f melts_env.txt -b batch.txt');
    return;

# Get modal phase proportions, return as pandas DataFrame
def query(scratchdir, index=1):
    prefix = scratchdir + 'out%i/' %(index); # path to data files
    # n_header_lines = 1;

    # Read results and return them if possible
    try:
        # Returns results as text string
        fp = open(prefix + 'Phase_main_tbl.txt','r');
        data = fp.read();
        fp.close();
    except:
        data = '';
    return data;

# Get modal phase proportions, return as pandas DataFrame
def query_modes(scratchdir, index=1):
    prefix = scratchdir + 'out%i/' %(index); # path to data files
    n_header_lines = 1;

    # Read results and return them if possible
    try:
        data = pd.read_csv(prefix + 'Phase_mass_tbl.txt', delim_whitespace=True, header=n_header_lines);
                # Ensure columns are numeric

        for c in data.columns:
            if data[c].dtype!='float64':
                data[c] = np.genfromtxt(data[c])
    except:
        data = 0;
    return data;

# Get liquid composition, return as pandas DataFrame
def query_liquid(scratchdir, index=1):
    prefix = scratchdir + 'out%i/' %(index); # path to data files
    n_header_lines = 1;

    # Read results and return them if possible
    try:
        data = pd.read_csv(prefix + 'Liquid_comp_tbl.txt', delim_whitespace=True, header=n_header_lines);
        # Ensure columns are numeric
        for c in data.columns:
            if data[c].dtype!='float64':
                data[c] = np.genfromtxt(data[c])
    except:
        data = 0;
    return data;

# Read solid composition, return as pandas DataFrame
def query_solid(scratchdir, index=1):
    prefix = scratchdir + 'out%i/' %(index); # path to data files
    n_header_lines = 1;

    # Read results and return them if possible
    try:
        data = pd.read_csv(prefix + 'Solid_comp_tbl.txt', delim_whitespace=True, header=n_header_lines);
        # Ensure columns are numeric
        for c in data.columns:
            if data[c].dtype!='float64':
                data[c] = np.genfromtxt(data[c])
    except:
        data = 0;
    return data;

# Read system thermodynamic data, return as pandas DataFrame
def query_system(scratchdir, index=1):
    prefix = scratchdir + 'out%i/' %(index); # path to data files
    n_header_lines = 1;

    # Read results and return them if possible
    try:
        data = pd.read_csv(prefix + 'System_main_tbl.txt', delim_whitespace=True, header=n_header_lines);
        # Ensure columns are numeric
        for c in data.columns:
            if data[c].dtype!='float64':
                data[c] = np.genfromtxt(data[c])
    except:
        data = 0;
    return data;


#!/usr/bin/env python
"""
Simple Python interface to fft_fit.c.

Hazen 10/17
"""

import ctypes
import numpy
from numpy.ctypeslib import ndpointer
import os
import sys

import storm_analysis.sa_library.ia_utilities_c as utilC
import storm_analysis.sa_library.loadclib as loadclib
import storm_analysis.sa_library.dao_fit_c as daoFitC

def loadFFTFitC():
    fft_fit = loadclib.loadCLibrary("fft_fit")

    # From sa_library/multi_fit.c
    fft_fit.mFitGetFitImage.argtypes = [ctypes.c_void_p,
                                        ndpointer(dtype=numpy.float64)]

    fft_fit.mFitGetNError.argtypes = [ctypes.c_void_p]
    fft_fit.mFitGetNError.restype = ctypes.c_int

    fft_fit.mFitGetPeakPropertyDouble.argtypes = [ctypes.c_void_p,
                                                  ndpointer(dtype=numpy.float64),
                                                  ctypes.c_char_p]

    fft_fit.mFitGetPeakPropertyInt.argtypes = [ctypes.c_void_p,
                                               ndpointer(dtype=numpy.int32),
                                               ctypes.c_char_p]

    fft_fit.mFitGetResidual.argtypes = [ctypes.c_void_p,
                                        ndpointer(dtype=numpy.float64)]

    fft_fit.mFitGetUnconverged.argtypes = [ctypes.c_void_p]
    fft_fit.mFitGetUnconverged.restype = ctypes.c_int

    fft_fit.mFitIterateLM.argtypes = [ctypes.c_void_p]

    fft_fit.mFitNewBackground.argtypes = [ctypes.c_void_p,
                                          ndpointer(dtype=numpy.float64)]

    fft_fit.mFitNewImage.argtypes = [ctypes.c_void_p,
                                     ndpointer(dtype=numpy.float64)]

    fft_fit.mFitRemoveErrorPeaks.argtypes = [ctypes.c_void_p]

    fft_fit.mFitRemoveRunningPeaks.argtypes = [ctypes.c_void_p]

    fft_fit.mFitSetPeakStatus.argtypes = [ctypes.c_void_p,
                                          ndpointer(dtype=numpy.int32)]

    # From psf_fft/fft_fit.c
    fft_fit.ftFitCleanup.argtypes = [ctypes.c_void_p]

    fft_fit.ftFitInitialize.argtypes = [ctypes.c_void_p,
                                        ndpointer(dtype=numpy.float64),
                                        ndpointer(dtype=numpy.float64),
                                        ctypes.c_double,
                                        ctypes.c_int,
                                        ctypes.c_int]
    fft_fit.ftFitInitialize.restype = ctypes.POINTER(daoFitC.fitData)

    fft_fit.ftFitNewPeaks.argtypes = [ctypes.c_void_p,
                                      ndpointer(dtype=numpy.float64),
                                      ctypes.c_char_p,
                                      ctypes.c_int]

    return fft_fit

#
# Classes.
#
class CFFTFit(daoFitC.MultiFitterArbitraryPSF):

    def __init__(self, psf_fn = None, **kwds):
        super(CFFTFit, self).__init__(**kwds)
        self.psf_fn = psf_fn

        self.clib = loadFFTFitC()

    def cleanup(self, spacing = "  ", verbose = True):
        super(CFFTFit, self).cleanup(spacing = spacing,
                                     verbose = verbose)
        if self.mfit is not None:
            self.clib.ftFitCleanup(self.mfit)
            self.mfit = None
        self.psf_fft = None

    def getSize(self):
        return self.psf_fn.getSize()

    def initializeC(self, image):
        """
        This initializes the C fitting library.
        """
        super(CFFTFit, self).initializeC(image)

        self.mfit = self.clib.ftFitInitialize(self.psf_fn.getCPointer(),
                                              self.rqe,
                                              self.scmos_cal,
                                              self.default_tol,
                                              self.scmos_cal.shape[1],
                                              self.scmos_cal.shape[0])

    def newPeaks(self, peaks, peaks_type):
        """
        Pass new peaks to the C library.
        """
        c_peaks = self.formatPeaks(peaks, peaks_type)
        self.clib.ftFitNewPeaks(self.mfit,
                                c_peaks,
                                ctypes.c_char_p(peaks_type.encode()),
                                c_peaks.shape[0])

    def rescaleZ(self, z):
        return self.psf_fn.rescaleZ(z)


import numpy as np
import pandas as pd
from hanziconv import HanziConv
import re

# Prior and Likelihood
place_prior = 0.3
likelihood = {"地名": 0.80,
              "方向": 0.75,
              "城鎮": 0.7,
              "姓氏": 0.2,
              "單位": 0.2,
              "數字": 0.2,
              "年號": 0.3,
              "官名": 0.2,
              "經驗字": 0.4,
              "不相干的字": 0.45}

# loading our own bag-of-words for place name
lsperson_excel = pd.ExcelFile("../SongShu/SongshuDH/lsperson - 法鼓山劉宋相關人名地名.xlsx")
df_place1 = pd.read_excel(lsperson_excel, "place")
place_excel = pd.ExcelFile("../SongShu/SongshuDH/劉宋地名與官名 2017-11-14.xlsx")
df_place2 = pd.read_excel(place_excel, "Places")
df_bureaucracy = pd.read_excel(place_excel, "Offices")
place_set = set(df_place2.Name.values.tolist() + df_place1.placeName.values.tolist())

# cleaning place names
place_set = set([p for p in place_set if len(re.findall(r"[a-zA-Z.\-]", p)) == 0])

# loading emperical bag-of-words
direction_bag = "至于於向破奔鎮入征"
place_unit = "鄉市縣鎮郡城州"
unit = "年載月日初中末閏"
number    = "是元正𨳝一二三四五六七八九十廿卅" # MARKUS 大概是用這行判斷數字
year_name = {"建國","元統","元貞","大德","天曆","天順","延祐","泰定","皇慶","至元","至大","至正","至治","至順","致和","永興","太清","永樂","元璽","光壽","建熙","壽光","延初","皇始","乾德","光天","廣大","通正","佐初","光初","元徽","大明","孝建","昇明","景和","景平","泰豫","保定","大成","大象","天和","宣政","建德","武成","乾興","元祐","元符","元豐","咸平","嘉祐","大中祥符","大觀","天聖","宣和","寶元","崇寧","康定","建中靖國","慶曆","政和","明道","景德","景祐","治平","淳化","熙寧","皇祐","端拱","紹聖","至和","至道","重和","雍熙","靖康","康應","康曆","應安","曆應","永德","觀應","承光","承和","承平","承玄","玄始","神璽","義和","乾佑","天會","天福","建福","德興","神曆","大延","天安","天興","天賜","太平真君","太延","太昌","始光","孝昌","延和","延昌","承明","普泰","景明","正光","正平","武泰","泰常","熙平","登國","皇興","神䴥","神瑞","神龜","興光","興安","乾明","大寧","天統","德昌","武平","河清","皇建","隆化","乾貞","天祚","武義","順義","頒義","交泰","保大","升元","昇元","乾道","咸淳","嘉定","嘉泰","嘉熙","寶慶","寶祐","建炎","德祐","慶元","景定","景炎","淳熙","淳祐","祥興","端平","紹定","紹熙","開慶","開禧","隆興","庚寅","弘光","永曆","紹武","隆武","興國","中大同","中大通","大同","大寶","大通","天成","天正","天監","天鑑","承聖","普通","紹泰","弘昌","乾亨","乾和","大有","應乾","白龍","太上","燕王","延興","永明","永泰","隆昌","保貞","太平興國","寶大","寶太","寶正","寶貞","廣初","建隆","正明","開寶","上元","中和","乾元","乾寧","乾封","乾甯","乾符","儀鳳","先天","光化","光啟","光宅","光慶","咸亨","咸通","唐元","唐安","唐興","唐隆","嗣聖","垂拱","大中","大和","大曆","大順","天寶","天復","天祐","太極","寶應","寶曆","寶歷","廣德","廣明","建中","弘道","文德","文明","明慶","景福","景雲","景龍","會昌","正觀","武德","永崇","永徽","永淳","永貞","永隆","神龍","總章","興元","調露","貞元","貞觀","載初","長慶","開元","開成","開耀","顯慶","麟德","龍朔","龍紀","嘉靖","天啟","宣德","崇禎","建文","弘治","成化","景泰","正統","泰昌","洪武","洪熙","萬曆","隆慶","建陽","乾隆","光緒","同治","咸豐","嘉慶","宣統","崇德","康熙","道光","雍正","順治","天眷","天輔","崇慶","承安","收國","明昌","正大","正豐","正隆","泰和","皇統","至寧","興定","貞祐","開興","光武","隆熙","乾統","保寧","咸雍","壽昌","壽隆","大平","大康","天慶","天祿","天贊","天顯","應曆","會同","泰安","清寧","神冊","統和","重熙","開泰","嘉禾","天冊","天璽","天紀","太元","太平","寶鼎","建衡","永安","神鳳","赤烏","鳳凰","黃武","廣順","顯德","同光","應順","淸泰","清泰","長興","開運","乾化","貞明","開平","鳳曆","龍德","承康","龍飛","乾祐","光始","建始","燕元","長樂","弘始","洪始","白雀","皇初","廣政","明德","泰寧","天命","天聰","政開","水德萬歲","聖冊","虎泰","嘉寧","宣平","晏平","漢興","玉恒","玉衡","仁平","大昌","開國","鴻濟","地皇","天鳳","始建國","咸熙","太和","景元","景初","正元","正始","青龍","黃初","升平","咸和","咸安","咸康","大亨","太寧","太興","寧康","崇和","崇安","昇平","永昌","義熙","興寧","隆和","隆安","中平","元初","元和","元嘉","元興","光和","光熹","初平","和平","延光","延平","延康","延熹","建初","建和","建安","建寧","建康","建武中元","昭寧","本初","永元","永初","永和","永嘉","永壽","永寧","永平","永康","永建","永憙","永漢","永熹","漢安","熹平","章和","興平","陽嘉","元象","天平","武定","興和","永始","久視","大足","天冊萬歲","天授","如意","延載","神功","聖曆","萬歲登封","萬歲通天","證聖","長壽","長安","民國","元熙","光興","嘉平","建元","永鳳","河瑞","漢昌","麟嘉","定鼎","建光","神鼎","勝光","承陽","昌武","真興","鳳翔","龍升","中統","延熙","建興","景耀","炎興","章武","乾定","人慶","元德","光定","大慶","天佑垂聖","天佑民安","天儀治平","天安禮定","天授禮法延祚","天盛","天賜禮盛國慶","奲都","寶義","延嗣甯國","應天","拱化","正德","福聖承道","雍寧","顯道","光熙","咸寧","大安","太安","太康","太熙","永熙","泰始","大定","天保","廣運","嘉興","五鳳","元光","元壽","元始","元封","元平","元康","元延","元朔","元狩","元鳳","元鼎","初元","初始","地節","天漢","太初","太初元將","太始","始元","始建","居攝","建昭","征和","本始","永光","河平","甘露","神爵","竟寧","綏和","陽朔","鴻嘉","黃龍","中興","建平","建明","建武","昌平","更始","燕興","建宏","建弘","建義","永宏","永弘","永洪","咸清","天禧","崇福","康國","延慶","紹興","大統","天德","通文","龍啟","光大","天嘉","天康","太建","永定","禎明","至德","仁壽","大業","義寧","開皇","延壽","光德"}
emperical_chars = "彼令生北哀帝王國士歸夫已以之大為"
bureaucracy_set = set(df_bureaucracy.Name.values.tolist())

# exclude the rare family names
family_name = pd.read_excel("Chinese_Family_Name（1k）.xlsx")
family_set = {HanziConv.toTraditional(n) for n,ref in zip(family_name.Nb, family_name.RealFre) if ref > 30}

# construct a dict for iterations
iterables  = {"地名": place_set,
              "方向": direction_bag,
              "城鎮": place_unit,
              "姓氏": family_set,
              "單位": unit,
              "數字": number,
              "年號": year_name,
              "官名": bureaucracy_set,
              "經驗字": emperical_chars, }

def update_feature_vector(phrase, feature_vector, feature_list, feature_name):
    """update the feature vector based on the list of features and a given phrase and
    a given feature name"""
    for x in feature_list:
        if x in phrase:
            feature_vector[feature_name] += len(x)

    return feature_vector

def calc_posterior(feature_vector, likelihood, prior):
    """Calc the posterior using bayes theorem.
    The assumption here is the naive bayes, so the prob of features
    are independent. The joint prob of likelihood is a prod of individual
    feature prob."""
    positive_likelihood = prior * np.prod([
        likelihood[key]**hits for key, hits in feature_vector.items()
    ])
    negative_likelihood = (1 - prior) * np.prod([
        (1 - likelihood[key])**hits for key, hits in feature_vector.items()
    ])

    # Bayes theorem
    posterior = positive_likelihood / (positive_likelihood + negative_likelihood)
    return posterior

def update_place_feature_vector(place_phrase, feature_vector, iterables, ):
    """
    Update the place feature vector based on a given place phrase and a dict of
    time phrase feature iterables. The definiation of feature_vector refer to `calc_place_posterior`.
    """
    irrelevant_phrase = place_phrase

    for key, iterable in iterables.items():
        update_feature_vector(
            place_phrase, feature_vector, iterable, key
        )

        # measure the length irrelevant phrase
        for x in iterable:
            if x in irrelevant_phrase:
                irrelevant_phrase = irrelevant_phrase.replace(x, "")

    feature_vector['不相干的字'] = len(irrelevant_phrase)
    return feature_vector

def calc_place_posterior(phrase, iterables=iterables, likelihood=likelihood, place_prior=place_prior):
    """Calc the posterior of a place phrase based on a given feature iterables and
    a pre-defined likelehood model and a time words frequency prior.
    Current feature for time:
        { "地名", "方向","城鎮", "姓氏", "單位", "年號", "數字", "經驗字", "不相干的字", "官名" }
    """
    feature_vector = {
        "地名": 0,
        "方向": 0,
        "城鎮": 0,
        "姓氏": 0,
        "單位": 0,
        "年號": 0,
        "數字": 0,
        "官名": 0,
        "經驗字":0,
        "不相干的字": 0}

    feature_vector = update_place_feature_vector(
        phrase, feature_vector, iterables
    )
    return calc_posterior(feature_vector, likelihood, place_prior)


# In this tutorial, you will learn how to write functions in python
# Lets write a function that will return the square, square root and the cube of the number that is passed to it

import numpy as np

# This is how a function is defined:
def my_func(val): # the colon works the same way it did in for loops, the indenting begins after that (Do not delete the indenting)
    sq = val*val
    sqrt = np.sqrt(val)
    cube = val*val*val
    return sq,sqrt,cube

# you press shift+tab to come out of the indent (or loop)

v =100
print("The output from the first technique is:")
print(my_func(v))

# or you can also do:
val1, val2, val3 = my_func(v) # in this case, val1 contains sq, val2 contains sqrt and val3 contains cube
print("The output from the second technique is:")
print("value of val1 is {0}".format(val1))
print("value of val2 is {0}".format(val2))
print("value of val3 is {0}".format(val3))

# assignments

# write a function that will determine if a given number is even or odd
# write a function to find the sin of a number using taylors expansion
# write a function to find e^x of a number
# write a function that do all the above in a single go


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @File  : brain.py
# @Author: zixiao
# @Date  : 2019-04-02
# @Desc  :
from double_dqn.model import CNN
from double_dqn.memory import Memory
import numpy as np
import torch
import torch.nn as nn
import torch.autograd as autograd
import os
from settings.conf import *

USE_CUDA = torch.cuda.is_available()
dtype = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor
save_q_eval_path = 'double_dqn/save_pkl/q_eval.pkl'
save_q_next_path = 'double_dqn/save_pkl/q_next.pkl'

class Variable(autograd.Variable):
    def __init__(self, data, *args, **kwargs):
        if USE_CUDA:
            data = data.cuda()
        super(Variable, self).__init__(data, *args, **kwargs)

class Brain:
    def __init__(self,
                 memory_size,
                 input_args,
                 num_actions,
                 shape,
                 learning_rate,
                 reward_decay,
                 e_greedy,
                 e_greedy_increment,
                 e_greedy_start,
                 batch_size,
                 replace_target_iter):

        self.q_eval = CNN(in_channels=input_args, num_action=num_actions).type(dtype)
        self.q_next = CNN(in_channels=input_args, num_action=num_actions).type(dtype)
        if os.path.isfile(save_q_eval_path):
            print('load q_eval ...')
            self.q_eval.load_state_dict(torch.load(save_q_eval_path))
        if os.path.isfile(save_q_next_path):
            print('load q_next ...')
            self.q_next.load_state_dict(torch.load(save_q_next_path))
        self.memory = Memory(memory_size, shape[0], shape[1], input_args)
        self.channels = input_args
        self.num_action = num_actions
        self.learning_rate = learning_rate
        self.gamma = reward_decay
        self.batch_size = batch_size
        self.replace_target_iter = replace_target_iter
        self.epsilon_max = e_greedy
        self.epsilon_increment = e_greedy_increment
        self.epsilon = e_greedy_start
        self.learn_step_count = 0

        self.op = torch.optim.Adam(self.q_eval.parameters(), lr=learning_rate)  # optimize all cnn parameters
        self.loss_func = nn.MSELoss()
        # self.loss_func = nn.CrossEntropyLoss()
        self.save_step = 1000
        self.learn_step = 0

        self.cost_his = []

    def choose_action(self, obs):
        if np.random.uniform() < self.epsilon and obs.shape[0] == self.channels:
            obs = torch.FloatTensor(obs)
            obs = obs.unsqueeze(0).type(dtype)
            out = self.q_eval(obs / 255.0)
            return np.argmax(out.detach().cpu()).item()
        return np.random.randint(0, self.num_action - 1)

    def store_transition(self, action, reward, obs_):
        self.memory.store_transition(action, reward, obs_)

    def learn(self):
        if self.learn_step_count == self.replace_target_iter:
            self.learn_step_count = 0
            self.q_next.load_state_dict(self.q_eval.state_dict())
        if self.learn_step == self.save_step:
            torch.save(self.q_eval.state_dict(), save_q_eval_path)
            torch.save(self.q_next.state_dict(), save_q_next_path)
            self.learn_step = 0
        obs_batch, act_batch, reward_batch, obs_batch_ = self.memory.get_memory(self.batch_size)
        obs_batch = Variable(torch.from_numpy(obs_batch).type(dtype))
        obs_batch_ = Variable(torch.from_numpy(obs_batch_).type(dtype))

        q_next = self.q_next(obs_batch_ / 255.0)
        q_eval = self.q_eval(obs_batch / 255.0)
        reward_batch = torch.from_numpy(reward_batch)

        if USE_CUDA:
            reward_batch = reward_batch.cuda()

        q_target = q_eval.clone().detach()
        batch_index = np.arange(self.batch_size)
        q_target[batch_index, act_batch] = reward_batch.float() + self.gamma * torch.max(q_next, dim=1)[0]
        loss = self.loss_func(q_eval, q_target)
        self.op.zero_grad()
        loss.backward()
        self.op.step()

        self.epsilon = self.epsilon + self.epsilon_increment if self.epsilon < self.epsilon_max else self.epsilon_max
        self.learn_step_count += 1
        self.learn_step += 1

    def get_last_memory(self):
        res = self.memory.get_last_frame()
        return res

    def store_start_frame(self, obs):
        self.memory.store_obs(obs)

    def double_learn(self):
        if self.learn_step_count % self.replace_target_iter == 0:
            self.q_next.load_state_dict(self.q_eval.state_dict())
        if self.learn_step == self.save_step:
            torch.save(self.q_eval.state_dict(), save_q_eval_path)
            torch.save(self.q_next.state_dict(), save_q_next_path)
            self.learn_step = 0
        obs_batch, act_batch, reward_batch, obs_batch_ = self.memory.get_memory(self.batch_size)
        obs_batch = Variable(torch.from_numpy(obs_batch).type(dtype))
        obs_batch_ = Variable(torch.from_numpy(obs_batch_).type(dtype))

        q_next = self.q_next(obs_batch_ / 255.0)
        q_eval_next = self.q_eval(obs_batch_ / 255.0)
        q_eval = self.q_eval(obs_batch / 255.0)

        reward_batch = torch.from_numpy(reward_batch)

        if USE_CUDA:
            reward_batch = reward_batch.cuda()

        q_target = q_eval.clone().detach()
        batch_index = np.arange(self.batch_size)
        max_act_q_eval_next = torch.argmax(q_eval_next, dim=1)
        print(max_act_q_eval_next)
        select_q_next = q_next[batch_index, max_act_q_eval_next]
        q_target[batch_index, act_batch] = reward_batch.float() + self.gamma * select_q_next
        loss = self.loss_func(q_eval, q_target)
        # self.cost_his.append(loss.float())
        self.op.zero_grad()
        loss.backward()
        self.op.step()

        self.epsilon = self.epsilon + self.epsilon_increment if self.epsilon < self.epsilon_max else self.epsilon_max
        self.learn_step_count += 1
        self.learn_step += 1


# -*- coding: utf-8 -*-
"""Running the Smile CNN in Real-time.

Access our webcam/video file and apply smile detection to each frame.

Example:
    $ python detect_smile.py --cascade haarcascade_frontalface_default.xml --model output/lenet.hdf5

Attributes:
    cascade (str):
        Path to a Haar cascade used to detect faces in images
    model (str):
        Path to our serialized LeNet weights on disk
    video (str, optional):
        Path to the (optional) video file. {Default is to read frames from a built-in/USB webcam}
"""
import argparse
import cv2
import numpy as np
import imutils
from keras.preprocessing.image import img_to_array
from keras.models import load_model

def main():
    """Run the real-time face detection.
    """
    # construct the argument parse and parse the arguments
    args = argparse.ArgumentParser()
    args.add_argument("-c", "--cascade", required=True, help="path to where the face cascade resides")
    args.add_argument("-m", "--model", required=True, help="path to pre-trained smile detector CNN")
    args.add_argument("-v", "--video", help="path to the (optional) video file")
    args = vars(args.parse_args())

    # load the face detector cascade and smile detector CNN
    detector = cv2.CascadeClassifier(args["cascade"])
    model = load_model(args["model"])
    # if a video path was not supplied, grab the reference to the webcam
    if not args.get("video", False):
        camera = cv2.VideoCapture(0)
    # otherwise, load the video file from the disk
    else:
        camera = cv2.VideoCapture(args["video"])

    # keep looping
    while True:
        # grab the current frame
        (grabbed, frame) = camera.read()
        # if we are viewing a video and we did not grab a frame, then we
        # have reached the end of the video
        if args.get("video") and not grabbed:
            break
        # resize the frame, convert it to grayscale, and then clone the
        # original frame so we can draw on it later in the program
        frame = imutils.resize(frame, width=300)
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        frame_clone = frame.copy()
        # detect faces in the input frame, then clone the frame so that we can draw on it
        rects = detector.detectMultiScale(
            gray, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE
        )
        # loop over the face bounding boxes
        for (x, y, width, height) in rects:
            # extract the ROI of the face from the grayscale image, resize it to a fixed 28x28
            # pixels, and then prepare the ROI for classification via the CNN
            roi = gray[y : y + height, x : x + width]
            roi = cv2.resize(roi, (28, 28))
            roi = roi.astype("float") / 255.0
            roi = img_to_array(roi)
            roi = np.expand_dims(roi, axis=0)
            # determine the probabilities of both "smiling" and "not
            # smiling", then set the label accordingly
            (not_smiling, smiling) = model.predict(roi)[0]
            label = "Smiling" if smiling > not_smiling else "Not Smiling"
            # display the label and bounding box rectangle on the output
            # frame
            cv2.putText(frame_clone, label, (x, y - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 0, 255), 2)
            cv2.rectangle(frame_clone, (x, y), (x + width, y + height), (0, 0, 255), 2)
        # show our detected faces along with smiling/not smiling labels
        cv2.imshow("Face", frame_clone)
        # if the 'q' key is pressed, stop the loop
        if cv2.waitKey(1) & 0xFF == ord("q"):
            break

    # cleanup the camera and close any open windows
    camera.release()
    cv2.destroyAllWindows()

if __name__ == "__main__":
    main()