#importing the model-building libraries
import numpy as np
import tensorflow
import keras
from keras.models import Sequential

#initializing the model
cnn_model=Sequential()

#preprocessing data
from keras.layers import Dropout,Dense,Conv2D,MaxPooling2D,Flatten
from keras.preprocessing.image import ImageDataGenerator
train_datagen=ImageDataGenerator(rescale=1./255,shear_range=0.2,zoom_range=0.2,horizontal_flip=True)
test_datagen=ImageDataGenerator(rescale=1./255)

#path of training data:D:\AAA\COURSES\AI\PROJECT\data
x_train=train_datagen.flow_from_directory(r"D:\Project_AI\data\data\trainset",target_size=(64,64),batch_size=32,class_mode='categorical')
x_test=train_datagen.flow_from_directory(r"D:\Project_AI\data\data\testset",target_size=(64,64),batch_size=32,class_mode='categorical')

#input layers
cnn_model.add(Conv2D(32,3,3,input_shape=(64,64,3),activation='relu'))
cnn_model.add(MaxPooling2D(pool_size=(2,2)))
#cnn_model.add(Dropout(0.25))
cnn_model.add(Flatten())

#hidden layers
cnn_model.add(Dense(128,activation='relu'))

#output layer
cnn_model.add(Dense(5,activation='softmax'))

#configuring the learning process
cnn_model.compile(loss="categorical_crossentropy",optimizer='adam',metrics=['accuracy'])

#training the model
cnn_model.fit_generator(x_train,
                         samples_per_epoch = 5000,
                         nb_epoch = 5,
                         validation_data = x_test,
                         nb_val_samples = 1500)

#save the model
cnn_model.save('D:\Project_AI\model_op\crop_protection_model.h5')  #use this location for prediction


import torch
import torch.nn.functional as F
from torch import nn

from mmdet.core import BitmapMasks
from mmdet.models.builder import LOSSES
from mmocr.utils import check_argument

@LOSSES.register_module()
class TextSnakeLoss(nn.Module):
    """The class for implementing TextSnake loss:
    TextSnake: A Flexible Representation for Detecting Text of Arbitrary Shapes
    [https://arxiv.org/abs/1807.01544].
    This is partially adapted from
    https://github.com/princewang1994/TextSnake.pytorch.
    """

    def __init__(self, ohem_ratio=3.0):
        """Initialization.

        Args:
            ohem_ratio (float): The negative/positive ratio in ohem.
        """
        super().__init__()
        self.ohem_ratio = ohem_ratio

    def balanced_bce_loss(self, pred, gt, mask):

        assert pred.shape == gt.shape == mask.shape
        positive = gt * mask
        negative = (1 - gt) * mask
        positive_count = int(positive.float().sum())
        gt = gt.float()
        if positive_count > 0:
            loss = F.binary_cross_entropy(pred, gt, reduction='none')
            positive_loss = torch.sum(loss * positive.float())
            negative_loss = loss * negative.float()
            negative_count = min(
                int(negative.float().sum()),
                int(positive_count * self.ohem_ratio))
        else:
            positive_loss = torch.tensor(0.0, device=pred.device)
            loss = F.binary_cross_entropy(pred, gt, reduction='none')
            negative_loss = loss * negative.float()
            negative_count = 100
        negative_loss, _ = torch.topk(negative_loss.view(-1), negative_count)

        balance_loss = (positive_loss + torch.sum(negative_loss)) / (
            float(positive_count + negative_count) + 1e-5)

        return balance_loss

    def bitmasks2tensor(self, bitmasks, target_sz):
        """Convert Bitmasks to tensor.

        Args:
            bitmasks (list[BitmapMasks]): The BitmapMasks list. Each item is
                for one img.
            target_sz (tuple(int, int)): The target tensor size HxW.

        Returns
            results (list[tensor]): The list of kernel tensors. Each
                element is for one kernel level.
        """
        assert check_argument.is_type_list(bitmasks, BitmapMasks)
        assert isinstance(target_sz, tuple)

        batch_size = len(bitmasks)
        num_masks = len(bitmasks[0])

        results = []

        for level_inx in range(num_masks):
            kernel = []
            for batch_inx in range(batch_size):
                mask = torch.from_numpy(bitmasks[batch_inx].masks[level_inx])
                # hxw
                mask_sz = mask.shape
                # left, right, top, bottom
                pad = [
                    0, target_sz[1] - mask_sz[1], 0, target_sz[0] - mask_sz[0]
                ]
                mask = F.pad(mask, pad, mode='constant', value=0)
                kernel.append(mask)
            kernel = torch.stack(kernel)
            results.append(kernel)

        return results

    def forward(self, pred_maps, downsample_ratio, gt_text_mask,
                gt_center_region_mask, gt_mask, gt_radius_map, gt_sin_map,
                gt_cos_map):

        assert isinstance(downsample_ratio, float)
        assert check_argument.is_type_list(gt_text_mask, BitmapMasks)
        assert check_argument.is_type_list(gt_center_region_mask, BitmapMasks)
        assert check_argument.is_type_list(gt_mask, BitmapMasks)
        assert check_argument.is_type_list(gt_radius_map, BitmapMasks)
        assert check_argument.is_type_list(gt_sin_map, BitmapMasks)
        assert check_argument.is_type_list(gt_cos_map, BitmapMasks)

        pred_text_region = pred_maps[:, 0, :, :]
        pred_center_region = pred_maps[:, 1, :, :]
        pred_sin_map = pred_maps[:, 2, :, :]
        pred_cos_map = pred_maps[:, 3, :, :]
        pred_radius_map = pred_maps[:, 4, :, :]
        feature_sz = pred_maps.size()
        device = pred_maps.device

        # bitmask 2 tensor
        mapping = {
            'gt_text_mask': gt_text_mask,
            'gt_center_region_mask': gt_center_region_mask,
            'gt_mask': gt_mask,
            'gt_radius_map': gt_radius_map,
            'gt_sin_map': gt_sin_map,
            'gt_cos_map': gt_cos_map
        }
        gt = {}
        for key, value in mapping.items():
            gt[key] = value
            if abs(downsample_ratio - 1.0) < 1e-2:
                gt[key] = self.bitmasks2tensor(gt[key], feature_sz[2:])
            else:
                gt[key] = [item.rescale(downsample_ratio) for item in gt[key]]
                gt[key] = self.bitmasks2tensor(gt[key], feature_sz[2:])
                if key == 'gt_radius_map':
                    gt[key] = [item * downsample_ratio for item in gt[key]]
            gt[key] = [item.to(device) for item in gt[key]]

        scale = torch.sqrt(1.0 / (pred_sin_map**2 + pred_cos_map**2 + 1e-8))
        pred_sin_map = pred_sin_map * scale
        pred_cos_map = pred_cos_map * scale

        loss_text = self.balanced_bce_loss(
            torch.sigmoid(pred_text_region), gt['gt_text_mask'][0],
            gt['gt_mask'][0])

        text_mask = (gt['gt_text_mask'][0] * gt['gt_mask'][0]).float()
        loss_center_map = F.binary_cross_entropy(
            torch.sigmoid(pred_center_region),
            gt['gt_center_region_mask'][0].float(),
            reduction='none')
        if int(text_mask.sum()) > 0:
            loss_center = torch.sum(
                loss_center_map * text_mask) / torch.sum(text_mask)
        else:
            loss_center = torch.tensor(0.0, device=device)

        center_mask = (gt['gt_center_region_mask'][0] *
                       gt['gt_mask'][0]).float()
        if int(center_mask.sum()) > 0:
            map_sz = pred_radius_map.size()
            ones = torch.ones(map_sz, dtype=torch.float, device=device)
            loss_radius = torch.sum(
                F.smooth_l1_loss(
                    pred_radius_map / (gt['gt_radius_map'][0] + 1e-2),
                    ones,
                    reduction='none') * center_mask) / torch.sum(center_mask)
            loss_sin = torch.sum(
                F.smooth_l1_loss(
                    pred_sin_map, gt['gt_sin_map'][0], reduction='none') *
                center_mask) / torch.sum(center_mask)
            loss_cos = torch.sum(
                F.smooth_l1_loss(
                    pred_cos_map, gt['gt_cos_map'][0], reduction='none') *
                center_mask) / torch.sum(center_mask)
        else:
            loss_radius = torch.tensor(0.0, device=device)
            loss_sin = torch.tensor(0.0, device=device)
            loss_cos = torch.tensor(0.0, device=device)

        results = dict(
            loss_text=loss_text,
            loss_center=loss_center,
            loss_radius=loss_radius,
            loss_sin=loss_sin,
            loss_cos=loss_cos)

        return results


import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.models as models

from .build import ModelFactory

class _vision_transformer_base(nn.Module):
    '''
        Add splitter / predictor support for the vit in torchvision.models
    '''
    def __init__(self,
                 include_label: int,
                 num_classes: int,
                 model_name: str,
                 weight_name: str):
        super().__init__()

        self.include_label = include_label
        self.num_classes = num_classes

        # Get the name of the weights
        weights = getattr(models, weight_name).DEFAULT

        # Load the pre-trained model
        self.vit = getattr(models, model_name)(weights=weights)

        # Get the last hidden dim size
        hidden = self.vit.heads[-1].in_features

        # Remove the last layer
        self.vit.heads[-1] = nn.Identity()

        # Redefine the last classifier
        self.fc = nn.Linear(hidden + self.include_label, self.num_classes)

    def forward(self, x, y=None):

        # Note that ViT requires the input to be batch_size x 3 x 224 x 224
        # Users need to resize the input correspondingly.
        x = self.vit(x)

        if self.include_label > 0:
            # Append the one hot label embedding
            x = torch.cat(
                [x, F.one_hot(y, num_classes=self.include_label).float()],
                dim=1
            )

        x = self.fc(x)

        return x

@ModelFactory.register('vit_b_16')
class vit_b_16(_vision_transformer_base):
    def __init__(self, **kwargs):
        super().__init__(**kwargs, model_name='vit_b_16',
                         weight_name='ViT_B_16_Weights')

@ModelFactory.register('vit_b_32')
class vit_b_32(_vision_transformer_base):
    def __init__(self, **kwargs):
        super().__init__(**kwargs, model_name='vit_b_32',
                         weight_name='ViT_B_32_Weights')

@ModelFactory.register('vit_l_16')
class vit_l_16(_vision_transformer_base):
    def __init__(self, **kwargs):
        super().__init__(**kwargs, model_name='vit_l_16',
                         weight_name='ViT_L_16_Weights')

@ModelFactory.register('vit_l_32')
class vit_l_32(_vision_transformer_base):
    def __init__(self, **kwargs):
        super().__init__(**kwargs, model_name='vit_l_32',
                         weight_name='ViT_L_32_Weights')

@ModelFactory.register('vit_h_14')
class vit_h_14(_vision_transformer_base):
    def __init__(self, **kwargs):
        super().__init__(**kwargs, model_name='vit_h_14',
                         weight_name='ViT_H_14_Weights')


from PySide6.QtCore import QCoreApplication
from PySide6.QtWidgets import (
                                QHBoxLayout,
                                QLabel,
                                QVBoxLayout,
                                QWidget
                            )

from qfluentwidgets import (DoubleSpinBox, PushButton, ComboBox, FluentIcon)

from .navigationInterface import NavigationBaseInterface
from .fileNameListViewInterface import FileNameListView
from .outputLabelLineEditButtonWidget import OutputGroupWidget

from .style_sheet import StyleSheet

from .config import STEMS

class DemucsParamGroupWidget(QWidget):
    def __tr(self, text):
        return QCoreApplication.translate(self.__class__.__name__, text)

    def __init__(self, parent: QWidget | None = ...) -> None:
        super().__init__(parent)

        self.vBoxLayout = QVBoxLayout()
        self.setLayout(self.vBoxLayout)
        self.mainWidget = QWidget()
        self.mainWidget.setObjectName("mainObject")

        self.mainLayout = QVBoxLayout()
        self.mainWidget.setLayout(self.mainLayout)

        self.setupUI()

    def addWidget(self, widget, alignment):
        self.mainLayout.addWidget(widget, alignment=alignment)

    def addLayout(self, layout):
        self.mainLayout.addLayout(layout)

    def setupUI(self):

        self.label_ = QLabel(self.__tr("Demucs 参数"))
        self.vBoxLayout.addWidget(self.label_)

        # =====================================================================

        self.vBoxLayout.addWidget(self.mainWidget)

        # =====================================================================
        self.param_hBoxLayout = QHBoxLayout()
        self.addLayout(self.param_hBoxLayout)
        self.param_hBoxLayout.setContentsMargins(10,10,10,10)

        self.vBoxLayout_overlap = QVBoxLayout()
        self.param_hBoxLayout.addLayout(self.vBoxLayout_overlap)

        self.spinBox_overlap = DoubleSpinBox()
        self.spinBox_overlap.setMaximum(1.0)
        self.spinBox_overlap.setMinimum(0.1)
        self.spinBox_overlap.setSingleStep(0.1)
        self.spinBox_overlap.setValue(0.10)
        self.spinBox_overlap.setDecimals(2)
        self.spinBox_overlap.setToolTip(self.tr("分段采样的段间重叠度，该值不宜过小，以免影响分段之间的分离结果的融合度"))

        self.label_overlap_param = QLabel(self.tr("采样重叠度"))

        self.vBoxLayout_overlap.addWidget(self.label_overlap_param)
        self.vBoxLayout_overlap.addWidget(self.spinBox_overlap)

        # =============================================================================
        self.vBoxLayout_segment = QVBoxLayout()
        self.param_hBoxLayout.addLayout(self.vBoxLayout_segment)

        self.spinBox_segment = DoubleSpinBox()
        self.spinBox_segment.setMaximum(100.0)
        self.spinBox_segment.setMinimum(1)
        self.spinBox_segment.setSingleStep(5)
        self.spinBox_segment.setValue(10)
        self.spinBox_segment.setDecimals(1)
        self.spinBox_segment.setToolTip(self.tr("Demucs 是一个极其消耗计算机内存和显存的模型，\n世界上没有任何计算机能够直接使用该模型处理长时音频数据\n因此数据将按照该值所示的秒数，分段处理，值越大将越消耗计算机资源，但效果可能会有提升"))

        self.label_segment_param = QLabel(self.tr("分段长度(s)"))

        self.vBoxLayout_segment.addWidget(self.label_segment_param)
        self.vBoxLayout_segment.addWidget(self.spinBox_segment)

        # =============================================================================
        self.vBoxLayout_stems = QVBoxLayout()
        self.param_hBoxLayout.addLayout(self.vBoxLayout_stems)

        self.comboBox_stems = ComboBox()
        self.comboBox_stems.addItems(STEMS)
        self.comboBox_stems.setCurrentIndex(1)

        self.label_stems_param = QLabel(self.tr("输出音轨"))

        self.vBoxLayout_stems.addWidget(self.label_stems_param)
        self.vBoxLayout_stems.addWidget(self.comboBox_stems)

class DemucsPageNavigation(NavigationBaseInterface):
    def __init__(self, parent=None):
        super().__init__(title="Demucs", subtitle=self.tr("使用 Demucs4.0 模型的 AVE（自动人声提取）方案"), parent=parent)
        # self.parent = parent

        self.setupUI()

        StyleSheet.DENUCE_INTERFACE.apply(self.demucs_param_widget)

    def setupUI(self):

        self.toolBar.modelStatusLabel.setVisible(False)

        self.fileListView = FileNameListView(self)
        self.addWidget(self.fileListView)

        # =============================================================================
        self.demucs_param_widget = DemucsParamGroupWidget(self)
        self.addWidget(self.demucs_param_widget)

        # =============================================================================
        self.outputGroupWidget = OutputGroupWidget(self)
        self.addWidget(self.outputGroupWidget)

        # =============================================================================
        self.process_button = PushButton()
        self.addWidget(self.process_button)
        self.process_button.setText(self.tr("提取"))
        self.process_button.setIcon(FluentIcon.IOT)

    def getParam(self) -> dict:
        param = {}
        param["overlap"] = self.demucs_param_widget.spinBox_overlap.value()
        param["segment"]= self.demucs_param_widget.spinBox_segment.value()
        param["tracks"] = self.demucs_param_widget.comboBox_stems.currentIndex()

        return param

    def setParam(self, param: dict) -> None:

        self.demucs_param_widget.spinBox_overlap.setValue(param["overlap"] )
        self.demucs_param_widget.spinBox_segment.setValue(param["segment"])
        self.demucs_param_widget.comboBox_stems.setCurrentIndex(param["tracks"])


import factory
from factory.django import DjangoModelFactory

from user.models import User, City

class CityFactory(DjangoModelFactory):
    class Meta:
        model = City

    name = factory.Faker('city', locale='ru_RU')

class UserFactory(DjangoModelFactory):
    class Meta:
        model = User

    email = factory.Faker('email')
    first_name = factory.Faker('first_name', locale='ru_RU')
    last_name = factory.Faker('last_name', locale='ru_RU')
    other_name = factory.Faker('middle_name', locale='ru_RU')
    phone = factory.Faker('phone_number')
    birthday = factory.Faker('past_date')
    city = factory.SubFactory(CityFactory)
    additional_info = factory.Faker('sentence', nb_words=10, variable_nb_words=True, locale='ru_RU')


from db import cursor
from db import db as mongodb
from pymongo import ASCENDING
import bson
import datetime
mongo_user = mongodb['user']
mongo_video = mongodb['video']
mongo_author = mongodb['author']

# 用户相关

INSERT_USER_SQL = """
INSERT INTO `user` (`name`, `password`, `credit`, `exp`, `gmt_create`, `role`)
VALUES (%(name)s, %(password)s, %(credit)s, %(exp)s, %(gen_time)s, %(role)s)
ON DUPLICATE KEY UPDATE `name` = VALUES(`name`), `exp` = VALUES(`exp`), `credit` = VALUES(`credit`), `password` = VALUES(`password`), `role` = VALUES(`role`);
"""

GET_USER_ID_SQL = """
SELECT `user_id` FROM `user` WHERE `name` = %s
"""

DELETE_USER_FOCUS_VIDEO_SQL = """
DELETE FROM biliob.user_focus_video
WHERE
    `user_id` = %s;
"""

DELETE_USER_FOCUS_AUTHOR_SQL = """
DELETE FROM biliob.user_focus_author
WHERE
    `user_id` = %s;
"""

INSERT_USER_FOCUS_VIDEO_SQL = """
INSERT INTO `user_focus_video` (`user_id`, `video_id`)
VALUES (%(user_id)s, %(video_id)s);
"""

INSERT_USER_FOCUS_AUTHOR_SQL = """
INSERT INTO `user_focus_author` (`user_id`, `author_id`)
VALUES (%(user_id)s, %(author_id)s)
"""

def translate_int64(item):
    for each_key in item:
        if type(item[each_key]) is bson.int64.Int64:
            item[each_key] = int(item[each_key])

def move_user():
    for each_doc in mongo_user.find().sort('_id', direction=ASCENDING):
        item = dict()
        item['gen_time'] = each_doc.pop('_id').generation_time
        item['name'] = each_doc['name']
        item['credit'] = each_doc['credit'] if 'credit' in each_doc else 0
        item['password'] = each_doc['password'] if 'password' in each_doc else 0
        item['exp'] = each_doc['exp'] if 'exp' in each_doc else 0
        item['role'] = each_doc['role'] if 'role' in each_doc else 0
        if len(item['name']) > 45:
            print(item['name'])
            continue
        cursor.execute(INSERT_USER_SQL, item)
        cursor.execute(GET_USER_ID_SQL, (each_doc['name']))

        user_id = cursor.fetchone()['user_id']
        cursor.execute(DELETE_USER_FOCUS_VIDEO_SQL, (user_id))
        cursor.execute(DELETE_USER_FOCUS_AUTHOR_SQL, (user_id))
        if 'favoriteAid' in each_doc:
            for each_aid in each_doc['favoriteAid']:
                if each_aid == None or each_aid > 4294967295:
                    continue
                item = {}
                item['user_id'] = int(user_id)
                item['video_id'] = int(each_aid)
                cursor.execute(INSERT_USER_FOCUS_VIDEO_SQL, item)
        if 'favoriteMid' in each_doc:
            for each_mid in each_doc['favoriteMid']:
                if each_mid == None or each_mid > 4294967295:
                    continue
                item = {}
                item['user_id'] = int(user_id)
                item['author_id'] = int(each_mid)
                cursor.execute(INSERT_USER_FOCUS_AUTHOR_SQL, item)

# 视频相关

INSERT_VIDEO_SQL = """
INSERT INTO `video` (`video_id`, `author_id`, `title`, `pic`, `is_observe`, `gmt_create`, `channel`, `subchannel`, `pub_datetime`)
VALUES (%(video_id)s, %(author_id)s, %(title)s, %(pic)s, %(is_observe)s, %(gen_time)s, %(channel)s, %(subchannel)s, %(pub_datetime)s)
ON DUPLICATE KEY UPDATE `title` = VALUES(`title`), `pic` = VALUES(`pic`), `is_observe` = VALUES(`is_observe`), `channel` = VALUES(`channel`), `subchannel` = VALUES(`subchannel`), `pub_datetime` = VALUES(`pub_datetime`);
"""

INSERT_VIDEO_RECORD_SQL = """
INSERT INTO `video_record` (`video_id`, `view`, `danmaku`, `favorite`, `coin`, `share`, `like`, `dislike`, `gmt_create`)
VALUES (%(video_id)s, %(view)s, %(danmaku)s, %(favorite)s, %(coin)s, %(share)s, %(like)s, %(dislike)s, %(gmt_create)s)
ON DUPLICATE KEY UPDATE
`video_id` = VALUES(`video_id`),
`view` = VALUES(`view`),
`danmaku` = VALUES(`danmaku`),
`favorite` = VALUES(`favorite`),
`coin` = VALUES(`coin`),
`share` = VALUES(`share`);
`like` = VALUES(`like`);
`dislike` = VALUES(`dislike`);
"""

def move_video():
    for each_doc in mongo_video.find().batch_size(8):
        translate_int64(each_doc)
        item = {}
        item['video_id'] = each_doc['aid'] if 'aid' in each_doc else None
        print(item['video_id'])
        item['author_id'] = each_doc['mid'] if 'mid' in each_doc else None
        item['title'] = each_doc['title'] if 'title' in each_doc else None
        item['pic'] = each_doc['pic'] if 'pic' in each_doc else None
        item['is_observe'] = each_doc['focus'] if 'focus' in each_doc else 1
        item['channel'] = each_doc['channel'] if 'channel' in each_doc else None
        item['subchannel'] = each_doc['subChannel'] if 'subChannel' in each_doc else None
        item['gen_time'] = each_doc.pop('_id').generation_time
        item['pub_datetime'] = each_doc['datetime'] if 'datetime' in each_doc else None
        cursor.execute(INSERT_VIDEO_SQL, item)
        if 'data' in each_doc:
            item_list = []
            for each_record in each_doc['data']:
                translate_int64(each_record)
                item = {}
                item['video_id'] = each_doc['aid'] if 'aid' in each_doc else None
                item['view'] = each_record['view'] if 'view' in each_record else None
                item['danmaku'] = each_record['danmaku'] if 'danmaku' in each_record else None
                item['favorite'] = each_record['favorite'] if 'favorite' in each_record else None
                item['coin'] = each_record['coin'] if 'coin' in each_record else None
                item['share'] = each_record['share'] if 'share' in each_record else None
                item['like'] = each_record['like'] if 'like' in each_record else None
                item['dislike'] = each_record['dislike'] if 'dislike' in each_record else None
                item['gmt_create'] = each_record['datetime'] if 'datetime' in each_record else None
                item_list.append(item)
            cursor.executemany(INSERT_VIDEO_RECORD_SQL, item_list)


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.stats import shapiro
from scipy.stats import chisquare
from scipy.stats import t
from scipy.stats import norm
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
import fileIO

DEFAULT_NUM = 10000
EPSILON = 1e-20
INFINITY = 1000000
NEG_INFINITY = -1000000
def almostEqual(a,b,epsilon = EPSILON):
    return np.abs(a-b) < epsilon

def makeDivisible(a, d = 2):
    if a % d != 0:
        return d - a % d + a
    else:
        return a

def findCommonInterval(list1, list2, as_series = True):
    """
    find the common interval of list  1 and list 2

    ----
    Parameters:
    ----

    as_series: bool
    boolean indicating whether list1 and list2 are arithmetic sequence, if so, more efficient method can be used

    return: low1,up1, low2,up2, s.t list1[low1:up1] ~ list2[low2:up2]

    ------
    To do:
    ------
    create version for non-as series, and check also whether the given series are arithmetic
    check also that the two lists can have an intersection, also the calculation for high_small and low_large is very sloppy
    """
    # find which one is larger
    if (list1[1] < list2[1]):
        larger = list2
        smaller = list1
        inOrder = True
    else:
        larger = list1
        smaller = list2
        inOrder = False
    # epsilon = INFINITY
    # for count in range(len(list1)):
    #     difference = abs(list(1) - )

    ## alternative
    ## assuming arithmetic_sequence
    if (as_series):
        dstep_small = smaller[1] - smaller[0]
        dstep_large = larger[1] - larger[0]
        low_small = int(dstep_large / dstep_small) - 1
        high_small = len(smaller)  ### sloppy
        low_large = 1
        high_large = int(smaller[-1] / dstep_large) + 1
        if (inOrder):
            return low_small, high_small, low_large, high_large
        else:
            return low_large, high_large, low_small, high_small
    else:

        raise Exception("Non arithmetic increase not supported yet!")

def findCommonPoints(list1, list2, as_series = True):
    """
    find the common points (in the sense of almostEqual) of two lists

    -----
    Parameters:
    -----

    return: index1List, index2List,

    ------
    To do:
    ---------
    The case for non as_series
    """
    # find which one is larger
    if (list1[1] < list2[1]):
        larger = list2
        smaller = list1
        inOrder = True
    else:
        larger = list1
        smaller = list2
        inOrder = False
    # epsilon = INFINITY
    # for count in range(len(list1)):
    #     difference = abs(list(1) - )

    ## alternative
    ## assuming arithmetic_sequence
    if (as_series):
        dstep_small = smaller[1] - smaller[0]
        dstep_large = larger[1] - larger[0]
        factor = int(dstep_large / dstep_small)

        low_large = 0
        low_small = int((larger[low_large] - smaller[0]) / dstep_small)
        high_small = len(smaller)  ### sloppy

        smallIndex = list(range(low_small, high_small, factor))

        high_large = int((smaller[-1] - larger[low_large]) / dstep_large) + 1
        largeIndex = list(range(low_large, high_large))
        if (inOrder):
            return smallIndex, largeIndex
        else:
            return largeIndex, smallIndex
    else:

        raise Exception("Non arithmetic increase not supported yet!")

def merge2TimeScale(df1, df2, columns = ["MSD_xy_mean","MSD_z"]):
    """
    given two dataframe saving the msd for two different time scales, merge the two together

    ------
    To do:
    ------
    Check first that df1 and df2 are of the same composition and molecule!

    ------
    Parameters:
    ----------

    df1: dataframe,

    df2: dataframe

    columns: list like, saving the columns to be merged

    return : dataframe with columns as specified in the variable columns

    ------
    Assuming
    -------
    dataframe has columns called "time"

    """

    ## To do, check if it is reasonable to merge these two

    ## find common interval
    l1, l2 = findCommonPoints(df1["time"], df2["time"], as_series = True)
    df1 = df1.iloc[l1]
    df2 = df2.iloc[l2]
    if (len(df1) != len(df2)):
        print(l1)
        print(l2)
        raise Exception("findCommonPoints error, findCommonPoints returned list1 and 2 not equal length!")
    for col in columns:
        minDiff = INFINITY
        minIndex = -1
        for i in range(len(df1)):
            difference = abs(df1.iloc[i][col] - df2.iloc[i][col])
            if (difference < minDiff):
                minDiff = difference
                minIndex = i
        timePt = df1.iloc[minIndex]["time"]

def chooseIndex(indices, num, lowPercentile = 0, upPercentile = 1):
    """
    choose indices from [lowPercentile * len, upPercentile * len]:

    ------
    Parameters:
    -------

    indices: array like

    num: int

    lowPercentile: float

    upPercentile: float

    """

    lenIndices = len(indices)
    lowerBound = int(np.floor(lenIndices * lowPercentile))
    upperBound = int(np.floor(lenIndices * upPercentile))
    # upperBound will not be reached so no need to add 1,
    # think for example upPercentile = 1, then upPercentile == lenIndices, which should not be reached as an index of the list
    # but lowerBound = 0 could be reached.
    totalLen = upperBound - lowerBound
    if (totalLen < 0):
        raise Exception("totalLen < 0!")
    if ( num >= totalLen):
        return list(range(lowerBound, upperBound))
    else:
        indexPrim = np.random.choice(range(lowerBound, upperBound), num , replace= False)
        indexPrim.sort()
        return indexPrim

def rvNormalityTest(rvSeries, alpha = "0.05"):
    """
    Test if a given series of random variables is normal distributed based on shapiro wilk

    ------
    Parameter
    -------

    rvSeries:
    array like structure, series of random variables

    alpha: float
    the alpha value for the hypothesis test

    """
    stat, p = shapiro(rvSeries)
    if p > alpha:
        return True
    else:
        return False

def rvTest(rvSeries, alpha = "0.05", distribution = "normal"):
    """
    Test if a given series of random variables is distributed based according to the scheme specified distribution

    ------
    Parameter
    -------

    rvSeries:
    array like structure, series of random variables

    alpha: float
    the alpha value for the hypothesis test

    """

    if (distribution == "normal"):
        stat, p = shapiro(rvSeries)
    elif (distribution == "chisquare"):
        stat, p = chisquare(rvSeries)

    if p > alpha:
        return True
    else:
        return False

def dfNormalityTest(df, num = DEFAULT_NUM, lowPercentile = 0.02, upPercentile = 0.1, alpha = 0.05, normalityTestCol = ["MSD_xy"]):
    """
    Test if a dataframe from the msd analysis, in particular the "MSD_xy"

    -------
    Parameter
    -------

    df: pandas.DataFrame

    num: int

    alpha: float

    """

    ## select only the MSD_xy and time
    ### maybe could insert at the beginning

    ### To do
    ### add concentration, molecule, info
    timeStep = df["samplingrate"].unique()
    dataFrameCol = ["concentration", "samplingrate", "molecule", "lowPercentile","upPercentile", "alpha"] + normalityTestCol
    testResultFinalDf = pd.DataFrame(columns = dataFrameCol)
    concentrationLists = df["concentration"].unique()
    for samplingRate in timeStep:
        df_samplingLevel = df[df["samplingrate"] == samplingRate]
        for concentration in concentrationLists:
            moleculeConcentrationDict = fileIO.getMoleculeConentration(concentration)
            columnConcentration = ["time"] + normalityTestCol + ["concentration", "molecule"]
            df_concentrationLevel = df_samplingLevel[columnConcentration]
            df_concentrationLevel = df_concentrationLevel[df_concentrationLevel["concentration"] == concentration]
            for molecule in moleculeConcentrationDict.keys():
                df_moleculeLevel = df_concentrationLevel[df_concentrationLevel["molecule"] == molecule]
                testResultDict = {}
                # initialize testResultDict
                for elem in normalityTestCol:
                    testResultDict[elem] = 0

                columns = ["time"] + normalityTestCol
                df_prune = df_moleculeLevel[columns]
                timeLag = df_prune["time"].unique()
                Indices = chooseIndex(list(range(len(timeLag))), num, lowPercentile = lowPercentile, upPercentile= upPercentile)
                # perhaps there should be more molecules, but this dataframe only include one
                # like POPC75_CHOL25, and only CHOL is present
                if (len(Indices) < 1):
                    continue
                for index in Indices:
                    tempDF = df_prune[almostEqual(df_prune["time"], timeLag[index])]
                    for elem in normalityTestCol:
                        rvSeries = tempDF[elem]
                        isNormal = rvNormalityTest(rvSeries, alpha)
                        if (isNormal):
                            testResultDict[elem] += 1
                # print("the testResultDict absolute: ")
                # print(testResultDict)
                totalNum = len(Indices)
                for key in testResultDict.keys():
                    testResultDict[key] /= totalNum
                # print("------------------------------")
                # print("after modification:")
                # print("the testResultDict absolute: ")
                # print(testResultDict)
                # print("-----------------------------")
                ## print result:
                print ("running normality test with time range :")
                print("({0:.4f}, {1:.4f})".format(timeLag[Indices[0]], timeLag[Indices[-1]]))
                print("total number of samplings: ")
                print(totalNum)
                print("alpha: " + str('{:.5f}'.format(alpha)))
                print("rate of passing:")
                # ["concentration", "samplingrate", "molecule", "lowPercentile","upPercentile"] + normalityTestCol
                df_testResult = pd.DataFrame({
                    "concentration": [concentration],
                    "samplingrate": [samplingRate],
                    "molecule": [molecule],
                    "lowPercentile": [lowPercentile],
                    "upPercentile":[upPercentile],
                    "alpha": [alpha]
                }
                )
                for key, value in testResultDict.items():
                    print("%-10s %10.5f"%(key, value))
                    df_testResult[key] = value
                testResultFinalDf = testResultFinalDf.append(df_testResult)
    return testResultFinalDf
def dfTest(df, num = DEFAULT_NUM, lowPercentile = 0.02, upPercentile = 0.1, alpha = 0.05, normalityTestCol = ["MSD_xy"], distribution = "normal"):
    """
    Test if a dataframe from the msd analysis, in particular the "MSD_xy"

    -------
    Parameter
    -------

    df: pandas.DataFrame

    num: int

    alpha: float

    """

    ## select only the MSD_xy and time
    ### maybe could insert at the beginning

    ### To do
    ### add concentration, molecule, info
    timeStep = df["samplingrate"].unique()
    dataFrameCol = ["concentration", "samplingrate", "molecule", "lowPercentile","upPercentile","alpha", "distribution"] + normalityTestCol
    testResultFinalDf = pd.DataFrame(columns = dataFrameCol)
    concentrationLists = df["concentration"].unique()
    for samplingRate in timeStep:
        df_samplingLevel = df[df["samplingrate"] == samplingRate]
        "df_samp"
        for concentration in concentrationLists:
            moleculeConcentrationDict = fileIO.getMoleculeConentration(concentration)
            columnConcentration = ["time"] + normalityTestCol + ["concentration", "molecule"]
            df_concentrationLevel = df_samplingLevel[columnConcentration]
            df_concentrationLevel = df_concentrationLevel[df_concentrationLevel["concentration"] == concentration]
            for molecule in moleculeConcentrationDict.keys():
                df_moleculeLevel = df_concentrationLevel[df_concentrationLevel["molecule"] == molecule]
                testResultDict = {}
                # initialize testResultDict
                for elem in normalityTestCol:
                    testResultDict[elem] = 0

                columns = ["time"] + normalityTestCol
                df_prune = df_moleculeLevel[columns]
                timeLag = df_prune["time"].unique()
                Indices = chooseIndex(list(range(len(timeLag))), num, lowPercentile = lowPercentile, upPercentile= upPercentile)
                # perhaps there should be more molecules, but this dataframe only include one
                # like POPC75_CHOL25, and only CHOL is present
                if (len(Indices) < 1):
                    continue
                for index in Indices:
                    tempDF = df_prune[almostEqual(df_prune["time"], timeLag[index])]
                    for elem in normalityTestCol:
                        rvSeries = tempDF[elem]
                        distriTrue = rvTest(rvSeries, alpha, distribution= distribution)
                        if (distriTrue):
                            testResultDict[elem] += 1
                # print("the testResultDict absolute: ")
                # print(testResultDict)
                totalNum = len(Indices)
                for key in testResultDict.keys():
                    testResultDict[key] /= totalNum
                # print("------------------------------")
                # print("after modification:")
                # print("the testResultDict absolute: ")
                # print(testResultDict)
                # print("-----------------------------")
                ## print result:
                print ("running test with time range :")
                print("({0:.4f}, {1:.4f})".format(timeLag[Indices[0]], timeLag[Indices[-1]]))
                print("total number of samplings: ")
                print(totalNum)
                print("alpha: " + str('{:.5f}'.format(alpha)))
                print("rate of passing:")
                # ["concentration", "samplingrate", "molecule", "lowPercentile","upPercentile"] + normalityTestCol
                df_testResult = pd.DataFrame({
                    "concentration": [concentration],
                    "samplingrate": [samplingRate],
                    "molecule": [molecule],
                    "lowPercentile": [lowPercentile],
                    "upPercentile":[upPercentile],
                    "alpha": [alpha],
                    "distribution": [distribution]
                }
                )
                for key, value in testResultDict.items():
                    print("%-10s %10.5f"%(key, value))
                    df_testResult[key] = value
                testResultFinalDf = testResultFinalDf.append(df_testResult)
    return testResultFinalDf

def histPlot(df, timePoints, timeNum , particleNum, bins = 50, normalityTestCol = ["MSD_xy"]):
    """
    plot the histogram of df

    -------
    Parameters:
    -------

    timePoints: array-like structure
    the time points from which to choose points to plot

    timeNum: int
    number of time points to choose

    particleNum: int
    number of particles to choose from

    """

    ## make sure the timePoints are unique
    timePoints = list(set(timePoints))
    concentrationLists = df["concentration"].unique()

    # select certain particles
    # calculate the total timeNum of trajectories
    numTrajectories = len(df["trajectory"].unique())
    numTrajectories = min(numTrajectories, particleNum)
    trajectoryIndex = chooseIndex(df["trajectory"],particleNum)
    trajectories = np.array(df["trajectory"])[trajectoryIndex]
    # subset the df
    df = df[df["trajectory"].isin(trajectories)]

    columns = ["time"] + normalityTestCol
    df_prune = df[columns]
    timePointsNum = len(timePoints)

    timeNum = makeDivisible(timeNum)
    numberOfPoints = min(timePointsNum, timeNum)
    if (numberOfPoints > timePointsNum):
        timeLists = np.random.choice(timePoints, size = numberOfPoints, replace = False)
    else:
        timeLists = timePoints

    timeLists.sort()
    # df_prune = df_prune[df_prune[almostEqual(df_prune["time"], timeLists)]]
    ## to do
    ## adapt plotting
    ncol = 2
    nrow = int(timeNum / ncol)

    fig,axes = plt.subplots(nrow,ncol)
    fig.tight_layout(pad = 5.0)
    fig.suptitle("total number of particles chosen: " + str(numTrajectories), fontsize = 20)
    count = 1
    for elem in normalityTestCol:
        for time in timeLists:
            rvVariables = df_prune[almostEqual(df_prune["time"], time)][elem]
            plt.subplot(nrow, ncol, count)
            rvVariables.plot.hist(bins = bins ) #ax = axes[count, 0]
            plt.title("time:{0:.2f}".format(time))
            plt.xlabel(elem)
            ## figSub.autofmt_xdate()
            count += 1

    plt.show()

def testHist(df, timeNum = 8, particleAverageNum = 8, normalityTestCol = ["MSD_xy"]):
    """

    given a dataframe, genearte a historam visualisation of the dataframe

    """

def diffArr(arr, method = "forward"):
    """

    calculates the difference between each elements of arr

    -------
    Parameters:
    ------

    method: str
    method used for diffArr, "forward", "central", "backward"

    -------
    To do:
    -------
    for central diff, how to treat the first element, here use upward diff
    """
# not sure if some arr starts indexing with 0
# for example, doing a slicing of a dataframe, will cause arr starts at index 1
# df_prune = df[1:]
# diffArr(df_prune["MSD_xy"])  <---- index starts at 1
    lenArr = len(arr)
    arTemp = [None] * (lenArr - 1)
    count = 0
    if (method == "forward"):
        for val in arr:
            if (count == 0):
                oldval = val
            else:
                arTemp[count - 1] = val - oldval
                oldval = val
            count += 1
    elif (method == "central"):
        for val in arr:
            if (count == 0):
                oldval = val
            elif (count == 1):
                arTemp[0] = val - oldval # the diff for the first element use forward
            elif(count >= 2):
                arTemp[count - 1] = val - oldval
                oldval = arr[count - 1]
            count += 1
    elif (method == "backward"):
        raise Exception("temporarily not supported for backward")
    else:
        raise Exception("unknown type of method for calculation diff: " + method)

    return np.array(arTemp)

def getSlope(xArr, yArr, method = "forward"):
    """
    get the slope of yArr w.r.t xArr,

    ------
    Parameters:
    ------

    method: str
    method use for calculating slope, "forward", "central", "backward"

    ---
    To do:
    ----
    1.  realize the "backward" scheme
    2.  does central makes sense, if x is not arithmetic sequence?
    3.  could relax np.array(diffArr(...)) because diffArrr already ensures the returned is ndarray
    """
    if (len(xArr) != len(yArr)) :
        raise Exception("x and y not equal length, cant calculate the slope")
    if (method == "forward"):
        xArr_diff = np.array(diffArr(xArr, method = "forward"))
        yArr_diff = np.array(diffArr(yArr, method = "forward"))
    elif (method == "central"):
        xArr_diff = np.array(diffArr(xArr, method = "central"))
        yArr_diff = np.array(diffArr(yArr, method = "central"))
    elif (method == "backward"):
        xArr_diff = np.array(diffArr(xArr, method = "backward"))
        yArr_diff = np.array(diffArr(yArr, method = "backward"))
    else:
        raise Exception("Unknown type of method for slope calculation: " + method)

    slopes = yArr_diff/xArr_diff
    return slopes
def eliminateZeroTime(df):
    """
    eliminate the 0 time row, (doing log will cause problem)

    -----
    To do
    ------
    figure out how to drop 0 that emerge at rows other than the first line
    """

    if (almostEqual(df.iloc[0,:]["time"], 0)):
        return df[1:]
    else:
        return df

def findLowUpIndex(arr, lowBound = NEG_INFINITY, upBound = INFINITY, startIndex = -1, endIndex = -1, userIsKing = True):
    """
    find the left, right index of an array, that has a continuous range lying between [lowBound, upBound]
    only count the first time this range is hit, and with consideration to the startIndex

    -------
    Parameters:
    -------
    arr: array like structure

    lowBound: float

    upBound: float

    startIndex: int

    endIndex: int, -1 default meaning no constraint

    userIsKing: boolean, default True :-)
    when there is collision of the lowBound and the available min, if False, use the available min, else the considering the user given and choose an optimal

    ------
    To do:
    ------
    optimise the code for lowBound == NEG_INFINITY, upBOUND == INFINITY
    """

    # make sure it is a numpy.array
    arr = np.array(arr)
    boolArr = (arr < upBound) & (arr > lowBound)
    leftIndex = -1
    rightIndex = -1
    leftFound = False
    rightFound = False
    continuous = True

    if (endIndex > 0):
        endIndex = min(endIndex, len(arr))
    else:
        endIndex = len(arr)

    if (startIndex < 0):
        startIndex = 0
    arrSlice = arr[startIndex : endIndex]
    arrSlice_low = min(arrSlice)
    arrSlice_max = max(arrSlice)

    if ( not userIsKing):
        lowBound = arrSlice_low
        upBound = arrSlice_max
    else:
        if (lowBound < arrSlice_max ):
            lowBound = max(lowBound, arrSlice_low)
        if (upBound > arrSlice_low):
            upBound = min(upBound, arrSlice_max)
    for i in range(len(boolArr)):
        if (i >= startIndex):
            if ((boolArr[i] == True) and (leftFound == False)):
                leftFound = True
                leftIndex = i
            elif (leftFound == True):
                # need to first check if the leftIndex is already found
                if((boolArr[i]) and (not rightFound) and (continuous)):
                    rightIndex = i
                elif (rightFound == False):
                    # boolArr[i] == False
                    rightFound = True
                    continuous = False
                    break
        if (i > endIndex):
            if (leftFound == False):
                raise Exception("could not find any index, leftIndex already beyond the array")
            else:
                break
    print("lowBound: " + str(lowBound))
    print("upBound: " + str(upBound))
    print("leftIndex: " + str(leftIndex))
    print("rightIndex: " + str(rightIndex))
    return leftIndex, rightIndex

def getLogLogSlope(df, yColumn = ["MSD_xy_mean"], xColumn = ["time"], method = "forward"):
    """
    find the slope k which is given by log(y) = k * log(x) + b

    ------
    Parameters:
    ------

    method: str
    method used for the calculation of slopes, "forward", "central", "backward"
    """
    ## eliminate possible zero rows

    if (len(yColumn) > 1 or len(xColumn) > 1):
        raise Exception("Only support for one dimensional array!")
    if (len(yColumn) < 1 or len(xColumn) < 1):
        raise Exception("no entry in yColumn or xColumn!")
    df_prune = eliminateZeroTime(df)

    for xCol in xColumn:
        for yCol in yColumn:
            tLog = np.log(df_prune[xCol])
            msdLog = np.log(df_prune[yCol])
            tLogDiff = diffArr(tLog, method = method)
            msdLogDiff = diffArr(msdLog, method = method)
            slopes = msdLogDiff / tLogDiff
            return slopes

def plotLogLog(df, yColumn = ["MSD_xy_mean"], label = ""):
    """
        print the loglog
    """
    df_prune = eliminateZeroTime(df)
    tLog = np.log(df["time"])
    for col in yColumn:
        yLog = np.log(df[col])
        plt.loglog(np.exp(tLog),np.exp(yLog), label = label)

def divide3Region(df, columns = ["MSD_xy_mean"],threshold2_low = 0.3,threshold2_high = 0.8, \
    threshold1_low = 1.6, threshold3_low = 0.8, region2start = -1, region3start = -1, secondDerivThreshold = 0.6, \
        indexJump = 20, method = "forward"):
    """

    very specific for MSD analysis, given the msd, divide the series into three regions
    assume df has "time" as one of its column name
    ------
    Parameters
    -------

    secondDerivThreshold: float
    set the threshold for the secondDerivative of the slopes

    return: three lists and the slopes
    example usage l1,l2,l3,slopes = divide3Region(df, columns = ["MSD_xy"])
    df_1 = df.iloc[l1]

    ------
    To do
    ------

    1.  make several columns possible, make the search adaptive, do not require user to guess which threshold to set, maybe could try to use second order difference
    2.  region3Start, need  to check not too large?

    """
    # threshold1_low = 1.6
    # threshold3_low = 0.8
    if (threshold3_low < threshold2_high + EPSILON):
        threshold3_low += 0.09
    threshold3_high = 1.2
    df_prune = eliminateZeroTime(df)
    if (len(columns) > 1):
        raise Exception("currenlty only division works for one column")

    for col in columns:
        # timeSeries = df_prune["time",col]
        # tLog = np.log(df_prune["time"])
        # msdLog = np.log(df_prune[col])
        # tLogDiff = diffArr(tLog)
        # msdLogDiff = diffArr(msdLog)
        # slopes = msdLogDiff / tLogDiff
        slopes = getLogLogSlope(df_prune, yColumn = [col], xColumn= ["time"], method = method)
        slopesDiff = diffArr(slopes)
        tLog = diffArr(df_prune["time"])[0:-1]
        slopesDeriv = slopesDiff / tLog
        print("slopesDeriv:")
        print(slopesDeriv)
        boolArr = np.abs(slopesDeriv) > secondDerivThreshold
        print("number > threshold : " + str(np.sum(boolArr)))
        print("indexArr:")
        # indexArr n * 1
        indexArr = np.where(boolArr)[0]
        print(indexArr)
        # find the region where the index jump occurs
        indexArrDiff = diffArr(indexArr)
        print("indexArrDiff:")
        print(indexArrDiff)
        boolDiffArr = np.abs(indexArrDiff) > indexJump
        index_indexArr = np.where(boolDiffArr)[0]
        print("index_indexArr:")
        print(index_indexArr)
        start2IndexCandidate = indexArr[index_indexArr[0]]
        print("startPoint for region2")
        print(start2IndexCandidate)

        start2Index = max(start2IndexCandidate, region2start)
        end2Index = indexArr[index_indexArr[0] + 1]

        ## find the first region
        l1Dummy, r1 = findLowUpIndex(slopes, lowBound= threshold1_low)
        l2,r2 = findLowUpIndex(slopes, lowBound=threshold2_low, upBound= threshold2_high, startIndex= start2Index, endIndex= end2Index)
        # have to make sure that l3 starts to count after the region 2

        region3start = max(region3start, r2)
        if (region3start >= len(df[col])):
            raise Exception("region3start too large, already exceeds the series length!")
        l3,r3Dummy = findLowUpIndex(slopes, lowBound= threshold3_low, upBound = threshold3_high, startIndex=region3start)
        list1 = list(range(0,r1 + 1))
        list2 = list(range(l2,r2 + 1))
        list3 = list(range(l3,len(slopes)))
        return list1, list2, list3, slopes

def simpleLinearRegression(xArr,yArr, confidence = 0.95):
    """
    do a linear regression s.t yArr_approx = k * xArr + b

    ----
    Parameters:
    ----

    return: dictionary which includes k,b,mean_squared_error,confidence level,  confidenceInterval of k, confidenceInterval of b
    """
    model = LinearRegression().fit(xArr,yArr)
    x = xArr.flatten()
    yPred = model.predict(xArr)
    n = len(x)
    if (n > 2):
        mean_sqre_error = mean_squared_error(yPred,yArr) * n / (n - 2)
    else :
        mean_sqre_error = mean_squared_error(yPred,yArr)
        print("less than 2 points, the estimator for variance sigma may be invalid")

    lxx = np.sum(np.square(x)) - np.square(np.mean(x)) * n
    if (lxx < 0):
        raise Exception("lxx < 0!")
    alpha = 1 - confidence
    alphaHalfQuantile = t(df = n).ppf(1 - alpha/2)
    k_interval_len = np.sqrt(mean_sqre_error/lxx)
    k = model.coef_[0]
    k_low = k - alphaHalfQuantile * k_interval_len
    k_high = k + alphaHalfQuantile * k_interval_len

    b = model.intercept_
    b_interval_len = np.sqrt(mean_sqre_error * (1/n + np.square(np.mean(x))/lxx))
    b_low = b - b_interval_len * alphaHalfQuantile
    b_high = b + b_interval_len * alphaHalfQuantile
    return {'k': k, 'b': b, 'mean_square_error': mean_sqre_error, 'confidence': confidence, 'k_interval': [k_low, k_high], 'b_interval': [b_low, b_high]}

def msdRegression(df, columns = ["MSD_xy_mean"], confidence = 0.95):
    """
    do a linear regression s.t log(columns) = k * log(time) + b

    ----
    Pay Attention:
    ----
    Assume df has no zero Time

    -----
    Parmaeters:
    -----

    return: dictionary which includes k,b,mean_squared_error,confidence level,  confidenceInterval of k, confidenceInterval of b

    ----
    To do:
    -----

    Making multiple columns possible

    """

    # pandas.Series do not support reshape function, and given a pandas.Series, the np.log() wont convert that to a ndarray
    # so need to convert to a ndarray first
    tLog = np.log(df["time"]).to_numpy().reshape((-1,1))

    for col in columns:
        msdLog = np.log(df[col])
        return simpleLinearRegression(tLog, msdLog, confidence)

def msdFittingPlot(df, columns = ["MSD_xy_mean"], threshold2_low = 0.3,threshold2_high = 0.8, \
    threshold1_low = 1.6, threshold3_low = 0.8, region2start = -1, region3start = -1, time_unit = "ps", msd_unit = "nm", outputUnit = "si", \
        potentKnown = True, confidence = 0.95, secondDerivThreshold = 0.6, method = "forward", title = "", directShow = True):
    """
    plot the msd and the curve fitting

    -----
    Parameters
    -----

    time_unit: str,
    indicate the units of the timescale, default ps

    ----
    To do:
    ----
    1. df should eliminate 0?
    2. making multiple columns possible
    3. annotate the curve with slope and intercept
    4. given units, figure out the constant to change to SI seconds
    """
    df = eliminateZeroTime(df)

    if (time_unit == "ps" and msd_unit == "nm"):
        # factor = 1e-12
        offset_alpha = 12 * np.log(10) # each intercept calculated need to add this offset * alpha, e.g MSD * 1e-18 = k (1e-12 * t)^a, logMSD = logk + a * log(t) - 12 a * log(10) + 18 log(10)
        offset_const = -18 * np.log(10)
    else:
        raise Exception("currently does not support other units")

    resultRegression = {}
    for col in columns:
        list1, list2, list3, slopes = divide3Region(df,columns = [col], threshold2_low = threshold2_low, \
            threshold2_high = threshold2_high, threshold1_low = threshold1_low, threshold3_low = threshold3_low, \
                region2start = region2start, region3start = region3start, secondDerivThreshold= secondDerivThreshold,\
                    method = method)
        indexLists = [list1,list2,list3]
        dfLists = [df.iloc[indexLists[var]] for var in range(3)]
        fig, axes = plt.subplots()
        # plot the whole msd
        legends = ["MSD","ballistic","subdiffusion","brownian"]
        tLog = np.log(df["time"])
        msdLog = np.log(df[col])
        plt.plot(np.exp(tLog),np.exp(msdLog), label = legends[0])
        # plt.plot(tLog,msdLog, label = legends[0])
        if (outputUnit == "si"):
            resultRegression['units'] = "si"
        else:
            resultRegression['units'] = "nm_ps"
        alpha_2Interval = []
        for i in range(3):
            regressionDict = msdRegression(dfLists[i], columns = [col], confidence = confidence)
            k = regressionDict['k']
            b = regressionDict['b']
            b_interval = np.array(regressionDict['b_interval'])
            b_siunit = b + offset_alpha * k + offset_const
            b_si_interval = b_interval + offset_alpha * k + offset_const
            ts = np.log(df["time"])
            ys = k * ts + b
            legend = legends[i + 1]
            if (i == 0):
                intercept_1 = np.exp(b_siunit)
                if (outputUnit == "si"):
                    b_1 = intercept_1
                else:
                    b_1 = np.exp(b)
                resultRegression['b1'] = b_1
                resultRegression['b1_interval'] = regressionDict['b_interval']
                k_1 = k
                resultRegression['k1'] = k_1
                resultRegression['k1_interval'] = regressionDict['k_interval']
            elif ( i == 1):
                intercept_2 = np.exp(b_siunit)
                alpha_2 = k
                alpha_2Interval = regressionDict['k_interval']
                resultRegression['alpha'] = k
                resultRegression['alpha_interval'] = alpha_2Interval
                legend += ":{0:.5f}".format(k)
            else:
                intercept_3 = np.exp(b_siunit)
                k_3 = k
                if (outputUnit == "si"):
                    print("for b_3 only nm_ps units")
                    # b_3 = np.exp(b)
                    if (potentKnown == False):
                        print("The result for the errorbar of 2dD may be inaccurate!")
                        b_3 = intercept_3
                        b_3_interval = b_si_interval
                    else:
                        b_3 = np.exp(b) * 1e-6
                        b_3_interval = b_interval # move the multiplication of 1e-6 at the end
                else:
                    b_3 = np.exp(b)
                    b_3_interval = b_interval
                resultRegression['b3'] = b_3
                resultRegression['b3_interval'] = np.exp(b_3_interval)
                if (outputUnit == "si" and potentKnown):
                    resultRegression['b3_interval'] *= 1e-6
                resultRegression['k3'] = k_3
                resultRegression['k3_interval'] = regressionDict['k_interval']
            # ts = np.log(dfLists[i]["time"])

            plt.loglog(np.exp(ts), np.exp(ys), label = legend, linestyle = "dashed")
            # plt.plot(ts, ys, label = legends[i + 1], linestyle = "dashed")
        # calculate diffusion * d
        if (outputUnit == "si"):
            si_factor = (1e-6)
            msd_factor = 1e-18
            t_factor = 1e-12
        else:
            si_factor = 1
            msd_factor = 1
            t_factor = 1
        D_from_raw_data = df.iloc[-101:-1][col]/(df.iloc[-101:-1]['time']) * si_factor
        variance_name = col.split("_")
        variance_name[-1] = "var"
        variance_name = "_".join(variance_name)
        # heuristic:
        D_varList = df.iloc[-101:-1][variance_name]
        Dcompact_var = np.sum(D_varList)/100 * msd_factor
        D_std = np.sqrt(Dcompact_var)/(2 * df.iloc[-50]['time'] * t_factor)
        sum =0
        for i in D_from_raw_data:
            sum = sum + i
        Dd = sum/100
        alpha = 1 - confidence
        alphaHalfQuantile = norm.ppf(1 - alpha/2)
        Dd_interval_len = D_std * alphaHalfQuantile
        resultRegression['Dd'] = Dd
        resultRegression['Dd_interval'] = [Dd - Dd_interval_len, Dd + Dd_interval_len]
        plt.legend()
        plt.title(title)
        if (directShow):
            plt.show()
    constant1 = df.iloc[0][col] * 1e6 / np.square(df.iloc[0]["time"])
    constant2 = df.iloc[1][col] * 1e6 / np.square(df.iloc[1]["time"])
    print("ballistic region, one point calculated const (d * kT/m):{0:.4e}, {1:.4e}".format(constant1, constant2))
    print("ballistic region, const (d * kT/m): {0:.4e}, potent: {1:.4f}".format(intercept_1, k_1))
    print("subdiffusion region, intercept: {0:.4e}, potent alpha: {1:.4f}".format(intercept_2, alpha_2))
    print("confidence: {0:.4f}".format(confidence))
    print("interval for alpha: ")
    print(alpha_2Interval)
    print("brownian region, const (2dD): {0:.4e}, potent: {1:.4f}".format(intercept_3, k_3))
    return resultRegression

if __name__ == "__main__":
    arr = np.array([2,3,4,5,6,8])
    arrT = diffArr(arr)
    print(arrT)

    charArr = ["hi","there","yo","yo1","haha"]
    tempIndx = chooseIndex(charArr,2, 0,0.5)
    print(tempIndx)

    ar1 = np.arange(4,20,4)
    ar2 = np.arange(1,20,1)

    l1, l2 = findCommonPoints(ar1,ar2)
    print(ar1)
    print(ar2)
    print(l1)
    print(l2)


import pandas as pd
import matplotlib.pyplot as plt

#import our data file
stock_prices = pd.read_csv('intel_amd_stock_prices.csv')

#print DataFrame
print(stock_prices)

#create y-columns
ycol = ['INTEL','AMD']
#create x-column
xcol =['Month']
#name and assign axis
stock_prices.plot()
#plot title
plt.title('Monthly Stock Prices for Top-2 Chip Manufacturers in the World')
#title for x-axis
plt.ylabel('PRICES ($USD)')
#show plot
plt.show()


import numpy as np
import cv2
import cvlib as cv

# Capturing video through webcam
webcam = cv2.VideoCapture(2)

# Start a while loop
while(1):

    # Reading the video from the
    # webcam in image frames
    _, imageFrame = webcam.read()

    imageFrame = cv2.resize(imageFrame, (1020, 600))
    bbox, label, conf = cv.detect_common_objects(imageFrame, enable_gpu=True)

    output_image = cv.object_detection.draw_bbox(imageFrame, bbox, label, conf)

    # Program Termination
    cv2.imshow("Object detection", imageFrame)
    if cv2.waitKey(10) & 0xFF == ord('q'):
        webcam.release()
        cv2.destroyAllWindows()
        break


from PIL import Image
import struct
import os
from pdb import set_trace as int3

def extract_alpha_chan(dib, width, height, is_bottom):
  ch=[]
  stride = width * 4
  if not is_bottom:
    line = stride * (height - 1)
    for i in range(height):
      p = line
      for j in range(width):
        ch.append(255 - dib[p + 3])
        p += 4
      line -= stride
  else:
    line = 0
    i = 0
    for i in range(height):
      p = line
      for j in range(width):
        ch.append(255 - dib[p + 3])
        p += 4
      line += stride
  return bytes(ch)

def read_bmp(fname):
  fs=open(fname,'rb')
  magic, file_size, _, _, off_bits = struct.unpack('<HIHHI',fs.read(14))
  self_size, width, height, planes, bit_count, comp, image_size, _,_,_,_ =\
             struct.unpack('<IiiHHIIIIII',fs.read(40))
  if bit_count != 32:
    fs.seek(0)
    im = Image.open(fs)
    im.load()
    fs.close()
    return im
  else:
    fs.seek(0)
    im = Image.open(fs)
    im.load()

    fs.seek(off_bits)
    stride = width * 4;
    is_bottom = False
    if height < 0:
      height = -height;
      is_bottom = True
    dibs = fs.read(stride * height)
    fs.close()
    #int3()
    r,g,b = im.split()
    alpha = extract_alpha_chan(dibs, width, height, is_bottom)
    a = Image.frombytes('L', (width, height), alpha)
    return Image.merge('RGBA',(r,g,b,a))

def cvt_all(folder1, folder2):
  for f in os.listdir(folder1):
    if not f.endswith('.BMP'):
      continue
    pic = read_bmp(os.path.join(folder1,f))
    pic.save(os.path.join(folder2,f.replace('.BMP','.png')))
    pic.close()

cvt_all(r'd:\galgame\fortissimo FA\advdata\GRP\SYSTEM_pic',
        r'd:\galgame\fortissimo FA\advdata\GRP\SYSTEM_png')


import sys
import math
import csv
import os
import argparse
import numpy as np
import pandas as pd
import json
import seaborn as sns
import matplotlib.pyplot as plt
from random import random

class Train:
    def __init__(self, data, epochs, lr, visu):
        self.data = data
        self.lr = lr
        self.visu = visu
        self.epochs = epochs

    def train(self, net, details):
        # sns.set(style="ticks", color_codes=True)
        # sns.pairplot(self.data, hue = 'M')
        # plt.tight_layout()
        # plt.savefig('pair_plot_selected.pdf')
        network = Network([23, 6, 4, 2], net).gardient_descent(self.data, self.epochs, self.lr, self.visu, details)
        np.save('network.npy', network)

class Network:
    def __init__(self, sizes, network):
        self.num_layers = len(sizes)
        self.sizes = sizes
        self.network = list()
        i = 1
        if network == False:
            while i < self.num_layers:
                layer = {'W' : np.random.randn(self.sizes[i], self.sizes[i - 1]), 'b' : np.zeros(shape=(self.sizes[i], 1))}
                self.network.append(layer)
                i += 1
            np.save('initial_random_network.npy', self.network)
        else:
            self.network = np.load(network)
        # print (self.network)

    def ft_get_stats(self, matrix):
        minmax = list()
        for column in matrix:
            stats = [matrix[column].min(), matrix[column].max()]
            minmax.append(stats)
        return minmax

    def normalize_dataset(self, dataset, minmax):
        normalized_data = (dataset - dataset.min()) / (dataset.max() - dataset.min())
        return normalized_data

    def build_excpected(self, data):
        expected = []
        for row in data.iterrows():
            index, batch = row
            inputs = batch.tolist()
            if inputs[0] == 'M':
                expected.append([0, 1])
            else:
                expected.append([1, 0])
        df_ex = pd.DataFrame(expected)
        return df_ex

    def sigmoid(self, x):
        return 1 / (1 + np.exp(-x))

    def sigmoid_prime(self, x):
        return x * (1 - x)

    def relu(self,x):
        return abs(x) * (x > 0)

    def relu_prime(self, x):
        return 1. * (x > 0)

    def softmax(self, x):
        e_x = np.exp(x - np.max(x))
        return e_x / np.column_stack((e_x.sum(axis=1),e_x.sum(axis=1)))

    def softmax_prime(self, x):
        return x

    def forward_propagation(self, data):
        inputs = data
        caches = []
        i = 0
        for layer in self.network:
            Z = np.dot(inputs, layer['W'].T) + layer['b'].T
            layer['Z'] = Z
            if i < len(self.network) - 1:
                layer['A'] = self.relu(Z)
            else:
                layer['A'] = self.softmax(Z)
            layer['A_prev'] = inputs
            inputs = layer['A']
            cache = (layer['A'], layer['W'], layer['b'], layer['A_prev'], layer['Z'])
            caches.append(cache)
            i += 1
        return inputs, caches

    def ft_cost_evaluation(self, output, y):
        m = y.shape[0]
        output = output + 1e-15
        cost = (-1 / m) * np.sum((np.multiply(y, np.log(output)) + np.multiply(1 - y, np.log(1 - output)))[:,0])
        return cost

    def ft_acc_evaluation(self, prediction, y):
        error = np.sum((np.argmax(prediction, axis=1) - np.argmax(y, axis=1)) ** 2)
        acc = (len(y) - error) / len(y)
        return acc

    def ft_precision_evaluation(self, prediction, y):
        tp = np.count_nonzero(np.argmax(prediction, axis=1) + np.argmax(y, axis=1) == 0)
        fp_tp = np.count_nonzero(np.argmax(prediction, axis=1) == 0)
        if fp_tp:
            precision = tp / (fp_tp)
        else:
            return 0
        return precision

    def backward_propagation(self, output, y, caches):
        gradients = []
        m = y.shape[0]
        dA = - (1 / m) * (np.divide(y, output) - np.divide(1 - y, 1 - output))
        cache = caches[-1]
        dZ = dA * self.softmax_prime(cache[0])
        grads = {}
        grads['dW'] = np.dot(dZ.T, cache[3]) / m
        grads['db'] = np.sum(np.squeeze(np.sum(dZ, axis=1))) / m
        dA_previous = np.dot(dZ, cache[1])
        gradients.append(grads)
        for i in reversed(range(len(self.network) - 1)):
            grads = {}
            cache = caches[i]
            dZ = dA_previous * self.relu_prime(cache[4])
            grads['dW'] = np.dot(dZ.T, cache[3]) / m
            grads['db'] = np.sum(np.squeeze(np.sum(dZ, axis=1))) / m
            dA_previous =  np.dot(dZ, cache[1])
            gradients.insert(0, grads)
        return gradients

    def update_weights(self, data, lr, gradient):
        for i in range(len(self.network)):
            self.network[i]['W'] = self.network[i]['W'] - lr * gradient[i]['dW']
            self.network[i]['b'] = self.network[i]['b'] - lr * gradient[i]['db']

    def gardient_descent(self, data, epochs, lr, visu, details):
        y = self.build_excpected(data)
        data = data.iloc[:, 1:]
        minmax = self.ft_get_stats(data)
        std_data = self.normalize_dataset(data, minmax)
        tr_y = y.values[:455, :]
        val_y = y.values[456:, :]
        tr_set = std_data.values[:455, :]
        val_set = std_data.values[456:, :]
        plot_tr_loss = []
        plot_val_loss = []
        plot_tr_acc = []
        val_cost = 0
        tr_prec = 0
        for epoch in range(epochs):
            tr_outputs, caches = self.forward_propagation(tr_set)
            cost = self.ft_cost_evaluation(tr_outputs, tr_y)
            gradients = self.backward_propagation(tr_outputs, tr_y, caches)
            self.update_weights(tr_set, lr, gradients)
            tr_acc = self.ft_acc_evaluation(tr_outputs, tr_y)
            val_outputs, _ = self.forward_propagation(val_set)
            val_cost_previous = val_cost
            val_cost = self.ft_cost_evaluation(val_outputs, val_y)
            tr_prec_previous = tr_prec
            tr_prec = self.ft_precision_evaluation(tr_outputs, tr_y)
            val_acc = self.ft_acc_evaluation(val_outputs, val_y)
            val_prec = self.ft_precision_evaluation(val_outputs, val_y)
            if math.isnan(val_cost) or (val_cost > val_cost_previous and epoch and tr_prec > tr_prec_previous):
                with open('minmax.json', 'w+') as json_file:
                    json.dump(minmax, json_file)
                    if visu:
                        plt.plot(plot_tr_loss)
                        plt.plot(plot_tr_acc)
                        plt.plot(plot_val_loss)
                        plt.show()
                    np.save('network.npy', self.network)
                exit()
            if visu:
                plot_tr_loss.append(cost)
                plot_tr_acc.append(tr_acc)
                plot_val_loss.append(val_cost)
            if details:
                print('>epoch=%d, lrate=%.4f, loss=%.3f, acc=%.3f, prec=%.3f, val_loss=%.3f, val_acc=%.3f, val_prec=%.3f' % (epoch, lr, cost, tr_acc, tr_prec, val_cost, val_acc, val_prec))
            else:
                print('>epoch=%d, lrate=%.4f, loss=%.3f, val_loss=%.3f' % (epoch, lr, cost, val_cost))
        with open('minmax.json', 'w+') as json_file:
            json.dump(minmax, json_file)
        if visu:
            plt.plot(plot_tr_loss)
            plt.plot(plot_tr_acc)
            plt.plot(plot_val_loss)
            plt.show()
        return self.network

args = argparse.ArgumentParser("Statistic description of your data file")
args.add_argument("file", help="File to descripte", type=str)
args.add_argument("-e", "--epoch", help="The number of iterations to go through the regression", default=5000)
args.add_argument("-l", "--learning", help="The learning rate to use during the regression", default=0.1, type=float)
args.add_argument("-v", "--visu", help="Visualize functions", action="store_true", default=False)
args.add_argument("-n", "--network", help="Specific network input", type=str, default=False)
args.add_argument("-d", "--details", help="Display more performance metrics", action="store_true", default=False)
args = args.parse_args()

if os.path.isfile(args.file):
    try:
        df = pd.read_csv(args.file, sep=',')
        df = df.dropna()
        df = df.iloc[:, [1, 2, 3, 6, 7, 8, 9, 10, 11, 12, 13, 16, 17, 18, 19, 20, 21, 22, 23, 26, 27, 28, 29, 30]]
        network = Train(df, args.epoch, args.learning, args.visu).train(args.network, args.details)

    except Exception as e:
        raise(e)
        sys.stderr.write(str(e) + '\n')
        sys.exit()
else:
    sys.stderr.write("Invalid input\n")
    sys.exit(1)


from dataclasses import dataclass, field

from canvasdl.utils.path import Path

from ..utils import config
from .base import Item

@dataclass(order=True)
class Course(Item):
    name: str
    id: str
    video_id: str = None
    drive_name: str = None
    piazza_id: str = None
    websites: dict = field(default_factory=dict)

    @property
    def sort_index(self):
        return -Path.content_path(self.name).size

    @property
    def abbreviation(self):
        removes = ("Topics in", "Introduction to", "and")
        name = self.name
        for r in removes:
            name = name.replace(r, "")
        return "".join(word[0].capitalize() for word in name.split(" ") if word)

    def assignment_name(self, name: str):
        return f"{self.abbreviation}: {name} due"

@dataclass
class Courses:
    courses: list[Course]

    @classmethod
    def from_dict(cls, courses):
        if config.one_course_nr:
            courses = [courses[config.one_course_nr - 1]]
        return Courses(sorted([Course(**c) for c in courses]))


import numpy as np
import sys
from datetime import datetime
import joblib
import torch
from loguru import logger

sys.path.append("../")
# from utils.data import data
from utils.evaluate import evaluate_sklearn
# from utils.evaluate import write_evaluation_result
from utils.config import config

DEVICE = torch.device('cpu')
normal_samples = torch.tensor([])
predict_label_file = "./normal_dataset_predict.label"
wordpress_svm_model = joblib.load('220513-1515_wordpress_albert_svm_joblib.model')

# 读取正常样本
for file in config["datasets"]["wordpress"][5]["albert_file"]:
    normal_samples = torch.cat((normal_samples, torch.load(file, map_location=DEVICE)))
logger.info("[Wordpress] Normal samples count: {}", normal_samples.shape)

# 创建真实标签
actual_label = np.array(len(normal_samples) * [5])
logger.info("Actual labels count: {}", np.shape(actual_label))

# 预测
logger.debug("Predicting ...")
start_time = datetime.now()
predict_label = wordpress_svm_model.predict(normal_samples.detach().numpy()) # 得到预测标签
end_time = datetime.now()
time_diff = end_time - start_time                                            # 时间差
milliseconds = time_diff.seconds * 1000 + time_diff.microseconds / 1000      # 毫秒级时间差
logger.info("Predict time: {} ms, {} s, {} min", milliseconds, time_diff.seconds, time_diff.seconds / 60)

# 保存预测标签
logger.info("Save predict label to file: {}", predict_label_file)
predict_label = np.array(predict_label, dtype=np.int8)
predict_label.tofile(predict_label_file)

# 评估
logger.debug("Evaluating ...")
evaluate_sklearn(predict_label, actual_label, 'test.png')


""" This script do the experiments of questions 13. """

import argparse

import numpy as np
import matplotlib.pyplot as plt

ITERS = 300

def get_data(train_path, test_path):
    """ Load and split data."""
    # load data
    train = np.genfromtxt(train_path)
    test = np.genfromtxt(test_path)

    # split x/y
    x_train = train[:, :-1]
    y_train = train[:, -1]
    x_test = test[:, :-1]
    y_test = test[:, -1]
    return x_train, y_train, x_test, y_test

def quick_stump_train(x_train, y_train, u): # pylint: disable=invalid-name
    """ Obtain best g by given u. """
    u_sum = u.sum()
    best_err = u_sum + 100
    best_para = {
        's': None,
        'i': None,
        'theta': None,
    }
    for i in range(x_train.shape[1]):
        sorted_idx = x_train[:, i].argsort()
        sorted_x = x_train[sorted_idx, i]
        sorted_y = y_train[sorted_idx]
        sorted_u = u[sorted_idx]

        # theta = -infty
        err = ((sorted_y != 1) * sorted_u).sum()
        if err < best_err:
            best_err = err
            best_para['s'] = 1
            best_para['i'] = i
            best_para['theta'] = sorted_x[0] - 1
        if u_sum - err < best_err:
            best_err = u_sum - err
            best_para['s'] = -1
            best_para['i'] = i
            best_para['theta'] = sorted_x[0] - 1

        # other thetas
        for theta_idx in range(x_train.shape[0] - 1):
            err += sorted_u[theta_idx] * sorted_y[theta_idx]
            if err < best_err:
                best_err = err
                best_para['s'] = 1
                best_para['i'] = i
                best_para['theta'] = (sorted_x[theta_idx] + sorted_x[theta_idx + 1]) / 2
            if u_sum - err < best_err:
                best_err = u_sum - err
                best_para['s'] = -1
                best_para['i'] = i
                best_para['theta'] = (sorted_x[theta_idx] + sorted_x[theta_idx + 1]) / 2

    return best_para

def stump_train(x_train, y_train, u): # pylint: disable=invalid-name
    """ Obtain best g by given u. """
    u_sum = u.sum()
    best_err = u_sum + 100
    best_para = {
        's': None,
        'i': None,
        'theta': None,
    }
    for i in range(x_train.shape[1]):
        sorted_idx = x_train[:, i].argsort()
        sorted_x = x_train[sorted_idx, i]
        sorted_y = y_train[sorted_idx]
        sorted_u = u[sorted_idx]

        # theta_idx = 0
        err = ((sorted_y != 1) * sorted_u).sum()
        if err < best_err:
            best_err = err
            best_para['s'] = 1
            best_para['i'] = i
            best_para['theta'] = sorted_x[0] - 1
        if u_sum - err < best_err:
            best_err = u_sum - err
            best_para['s'] = -1
            best_para['i'] = i
            best_para['theta'] = sorted_x[0] - 1

        for theta_idx in range(1, x_train.shape[0]):
            err = (((sorted_y[:theta_idx] != -1) * sorted_u[:theta_idx]).sum()
                   + ((sorted_y[theta_idx:] != 1) * sorted_u[theta_idx:]).sum())
            if err < best_err:
                best_err = err
                best_para['s'] = 1
                best_para['i'] = i
                best_para['theta'] = (sorted_x[theta_idx - 1] + sorted_x[theta_idx]) / 2
            if u_sum - err < best_err:
                best_err = u_sum - err
                best_para['s'] = -1
                best_para['i'] = i
                best_para['theta'] = (sorted_x[theta_idx - 1] + sorted_x[theta_idx]) / 2

    return best_para

def stump_test(x_test, para):
    """ Test the result by given para. """
    thres = np.ones(x_test.shape[0]) * para['theta']
    return para['s'] * ((x_test[:, para['i']] > thres) * 2 - 1)

def main():
    """ Main function. """
    # parse args
    parser = argparse.ArgumentParser()
    parser.add_argument('train', help='The path to the training data.')
    parser.add_argument('test', help='The path to the testing data.')
    args = parser.parse_args()

    # get data
    x_train, y_train, x_test, y_test = get_data(args.train, args.test)

    # train
    e_in_list = []
    alpha_list = []
    u = np.ones(x_train.shape[0]) / x_train.shape[0] # pylint: disable=invalid-name
    for _ in range(ITERS):
        para = stump_train(x_train, y_train, u)
        y_train_pred = stump_test(x_train, para)

        # update u
        epsilon = ((y_train_pred != y_train) * u).sum() / u.sum()
        scaling_factor = np.sqrt((1 - epsilon) / epsilon)
        u[y_train_pred != y_train] *= scaling_factor
        u[y_train_pred == y_train] /= scaling_factor

        # get e_in, e_out, alpha
        e_in = (y_train_pred != y_train).mean()
        e_in_list.append(e_in)
        alpha_list.append(np.log(scaling_factor))

    # plot
    plt.plot(e_in_list, label='Ein(g)')
    # plt.ylim((-0.1, 1))
    plt.xlabel('t')
    plt.legend()
    plt.show()
    print(e_in_list[0])
    print(alpha_list[0])
    print(e_in_list[-1])

if __name__ == '__main__':
    main()


import solver
import json
import sys
import os

from csvUtils.writer import writeToCsv
from mpl.plotter     import plot

from shapes.logSpiral import *
from shapes.triangle  import *
from shapes.point     import *
from shapes.line      import *

def prettify(d):
    '''Prettifies dictionary'''
    return json.dumps(
        d,
        sort_keys = False,
        indent = 4,
        separators = (
            ',',
            ': '
        )
    )

engine = 'mpl'
if len(sys.argv) > 1:
    if sys.argv[1] == '-manim':
        engine = 'manim'
    elif sys.argv[1] == '-noplot':
        engine = None
    else:
        raise ValueError('Argument should be -manim or -noplot')

f = solver.f

x1, y1, solver.a = map(
    float,
    input("Enter x1, y1 & a\t").split()
)

solver.init = list(map(
    int,
    input('Enter initial values\t').split()
))

assert len(solver.init) == 2 * solver.b, f"Length has to be {2 * b}"
assert 0 < solver.a, "a has to be greater than 0"
assert 0 < solver.b, "b has to be greater than 0"

n = int(input("Enter the number of points\t"))

assert n <= 1400,   "Too Large for computing f(n)"

if engine is not None:
    n1 = int(input("Enter the number of points to plot\t"))

I0 = Point(x1, y1)
I1 = Point(f(0) + x1, y1)
points.append(Point(f(0) + x1, f(1) + y1))
makePoints(n)

lines.append(Line(I0, points[0]))
lines.append(Line(I1, points[1]))
makeLines(n + 1)

T0 = Triangle(
    Line(I0, I1),
    Line(I1, points[0]),
    lines[0]
)
triangles.append(T0)
makeTriangles(n + 1)

print(prettify(
    dict([
        limitizeLine(),
        limitizeTriangle(),
        limitizeLogSpiral(I0, n)
    ])
))

I0.x, I0.y = 0, 0
I1.x, I1.y = f(0), 0
shiftPoints(x1, y1)

if engine is not None:
    if engine == 'mpl':
        plot(n1 - 2, [I0, I1])
    if engine == 'manim':
        writeToCsv(
            [I0, I1] + points,
            lines,
            triangles,
            Spiral()
        )
        os.system(
            f'manim ./manim/allAnim.py {n1} Shapes -pl'
        )


import pygame
import time

BG_COLOR = (50, 50, 50)
DARK_TILE_COLOR = (95, 120, 49)
LIGHT_TILE_COLOR = (223, 225, 196)
N = 6
SIZE = 110 + N * 100

knight = pygame.image.load('knight.png')
KNIGHT = pygame.transform.scale(knight, (45, 70))

X = 0
Y = 0
# Variable to keep track of each step of the tour
# This will allow us to visualize all the tour steps
track = 0

# All possible moves for a given position
moves_x = [1, 2, 2, 1, -1, -2, -2, -1]
moves_y = [-2, -1, 1, 2, 2, 1, -1, -2]

# DRAW FUNCTIONS
def drawKnight(win, x, y):
    win.blit(KNIGHT, (80 + 100 * x, 65 + 100 * y))
    pygame.display.update()

def deleteDrawing(win, x, y):
    if (x + y) % 2 == 0:
        pygame.draw.rect(win, LIGHT_TILE_COLOR, (50 + 100 * x, 50 + 100 * y, 100, 100))
    else:
        pygame.draw.rect(win, DARK_TILE_COLOR, (50 + 100 * x, 50 + 100 * y, 100, 100))
    pygame.display.update()

def drawTrack(win, x, y, i=0):
    font = pygame.font.SysFont('Arial Bold', 32)
    text = font.render(str(track - i), True, (255, 0, 0))
    win.blit(text, ((120 + 100 * x), (120 + 100 * y)))
    pygame.display.update()

def drawBoard(win, n):
    pygame.draw.rect(win, (0, 0, 0), (45, 45, (100 * n + 10), (100 * n + 10)), 5)

    for i in range(n):
        for j in range(n):
            if (i + j) % 2 == 0:
                pygame.draw.rect(win, LIGHT_TILE_COLOR, (50 + 100 * j, 50 + 100 * i, 100, 100))
            else:
                pygame.draw.rect(win, DARK_TILE_COLOR, (50 + 100 * j, 50 + 100 * i, 100, 100))
    pygame.display.update()

def drawPath(win, b):
    xi = X * 100 + 100
    yi = Y * 100 + 100
    PURPLE = (100, 40, 115)
    pygame.draw.circle(win, PURPLE, (xi, yi), 10)
    for i in range(0, track + 1):
        for sublist in b:
            if i in sublist:
                x = sublist.index(i)
                y = b.index(sublist)
                pygame.draw.line(win, PURPLE, (xi, yi), ((x * 100 + 100), (y * 100 + 100)), 6)
                pygame.draw.circle(win, PURPLE, ((x * 100 + 100), (y * 100 + 100)), 10)
                pygame.display.update()
                time.sleep(0.2)
                xi = x * 100 + 100
                yi = y * 100 + 100

# KNIGHT'S TOUR SOLVER
# Create nested list for the board
def createBoard(n):
    b = [[-1 for i in range (n)] for j in range (n)]
    return b
# Validates moves
def isValid(b, next_x, next_y):
    if ((next_x) >= 0 and (next_x) <= (N - 1)):
        if ((next_y) >= 0 and (next_y) <= (N - 1)):
            if ((b[next_y][next_x] == -1)):
                return (True)
            else:
                return (False)
        else:
            return (False)
    else:
        return (False)
# Recursive function to solve tour
def solveTour(board, pos_x, pos_y, win):
    if N <= 5:
        time.sleep(0.05)

    global track
    # Base Case: all cells visited, no more -1
    if not (any(-1 in sublist for sublist in board)):
        return (True)

    prev_x = pos_x
    prev_y = pos_y

    track = track + 1

    for i in range(8):
        next_x = pos_x + moves_x[i]
        next_y = pos_y + moves_y[i]
        if isValid(board, next_x, next_y):
            board[next_y][next_x] = track
            deleteDrawing(win, prev_x, prev_y)
            drawTrack(win, prev_x, prev_y, 1)
            drawKnight(win, next_x, next_y)
            drawTrack(win, next_x, next_y)
            if solveTour(board, next_x, next_y, win):
                return (True)
    track = track - 1
    board[prev_y][prev_x] = -1
    deleteDrawing(win, prev_x, prev_y)

    return (False)

def main():
    pygame.init()
    WIN = pygame.display.set_mode((SIZE, SIZE))
    pygame.display.set_caption("Knight's Tour Visualizer")
    WIN.fill(BG_COLOR)

    board = createBoard(N)
    board[Y][X] = track
    drawBoard(WIN, N)
    drawTrack(WIN, X, Y)
    drawKnight(WIN, X, Y)

    while True:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                return
            if event.type == pygame.KEYUP:
                solveTour(board, X, Y, WIN)
                drawPath(WIN, board)
if __name__ == '__main__':
    main()


import flask
from ..authenticated_blueprint_preparator import AuthenticatedBlueprintPreparator
from .decorators import rest_endpoint

name = 'confidence'
app = AuthenticatedBlueprintPreparator(name, __name__, template_folder=None)

@app.route('/confidence-values', role=['user', 'visitor'])
@rest_endpoint
def get_confidence_list(c):
    '''
    Get a list of all confidence values.

    @returns        application/json

    Example return value:

      ['false', 'uncertain', 'contested', 'probable', 'certain']

    '''
    c.execute('SELECT unnest(enum_range(NULL::confidence_value));')
    return flask.jsonify(list(map(lambda x: x[0], c.fetchall())))


from django.urls import path

from . import views

app_name = 'dashboard'
urlpatterns = [
    #If you plan on using hrefs in html to a specific url pattern you have to use the app_name and url pattern name, look at templates/accounts/passwordchangesuccess.html as an example
    path('', views.dashboard_view, name='index'),
    path('search/', views.search_view, name='search'),
    path('transcript_msg_receiver', views.trint_webhook_receiver, name='transcript_msg_receiver'),
    path('recording/update_file_title', views.update_file_title, name='update_file_title'),
    path('recording/update_prompt_selection', views.update_prompt_selection, name='update_prompt_selection'),
    path('recording/<int:pk>/<int:user_id>/trigger_summarization', views.trigger_summarization, name='trigger_summarization'),
    path('recording/<int:pk>/delete', views.delete_recording, name='delete_recording'),
    path('recording/<int:pk>/hold', views.hold_recording, name='hold_recording'),
]


# -*- coding: utf-8 -*-
"""
Created on Fri Jun  8 11:14:24 2018

@author: hp
"""

import cv2
import matplotlib.pyplot as plt

def emptyfunction():
    pass

def main():
    path="C:\\Users\\hp\\Downloads\\forsk ml project\\opencv program\\image\\standard_test_images\\"

    imgpath2=path +"peppers_color.tif"
    imgpath1=path +"mandril_color.tif"

    img1=cv2.imread(imgpath1,1)
    img2=cv2.imread(imgpath2,1)

    #add two images by weight and give blending effect to image
    output=cv2.addWeighted(img1,0.5,img2,0.5,0)
    windowName="Transition Demo"
    cv2.namedWindow(windowName)
    # it only create trackbar on window but doesnot give blending effect
    cv2.createTrackbar('Alpha',windowName,0,100,emptyfunction)

    while True:
        cv2.imshow(windowName,output)
        if cv2.waitKey(1)==27:
            break
        # blending effect provide by this 'trackbarpos'
        alpha=cv2.getTrackbarPos('Alpha',windowName)/100
        beta=1-alpha

        #finally give blending effect to two images
        output=cv2.addWeighted(img1,alpha,img2,beta,0)
        print(alpha,beta)

    cv2.destroyAllWindows()
if __name__=="__main__":
    main()

openCV3 tutorial/opencv13 mini project  trackbar blending app.py