import numpy as np
import pandas as pd
import re
from bs4 import BeautifulSoup
import gzip
import os
# import matplotlib.pyplot as plt

def read_abundance(file_path, index_col=0, return_sum=-1, return_mean=-1):
    """
    Abundance table should be tab seprated.
    args:
        file_path: file path of abundance table
        index_col: which column is the indexes (row names), e.g: -1 for last column, 0 for the first column
        return_sum: -1(return df.DataFrame of abundance table), or 1(sums of each row), or 0(sums of each column)
        return_mean: -1(return df.DataFrame of abundance table), or 1(means of each row), or 0(means of each column),
                    if not return_sum==-1, this argument will be disabled.
    """
    df = pd.read_csv(file_path, sep="\t", index_col=index_col)
    df = df.loc[:, df.dtypes != np.object]
    return df.sum(axis=return_sum) if not return_sum == -1 else (df if return_mean == -1 else df.mean(axis=return_mean))

def read_to_html_table(file_path, table_class=False, thead_class=False):
    file_type = re.search(r'[^\.]+$', file_path).group()
    if file_type == 'txt' or file_type == 'csv':
        out_df = pd.read_csv(file_path, sep='\t')
    elif file_type == "html":
        out_df = pd.read_html(file_path)[0]
    # pdb.set_trace()
    table = out_df.to_html(index=False)
    if table_class or thead_class:
        table = BeautifulSoup(table, 'lxml')
        if table_class:
            table.table['class'] = table_class
        if thead_class:
            table.thead['class'] = thead_class
        table = table.prettify()
    return table

def get_kingdom_ratio(species_abundance_table):
    df = read_abundance(species_abundance_table, index_col=-1, return_sum=1)
    kingdom = [re.search('^[^;]+', s).group().replace('k__', '')
               for s in df.index]
    # pdb.set_trace()
    df = df.groupby(kingdom).sum()
    try:
        del df['Environmentalsamples']
    except Exception:
        pass

    def series_to_str(series, value_format=':.2%'):
        return ', '.join([('{}({%s})' % (value_format)).format(k, v) for k, v in series.items()])

    # max_abundance = max(df)
    # explode = [0.1 if i == max_abundance else 0 for i in df]
    # plt.pie(df, explode=explode, labels=df.index, labeldistance=1.1,
    #         autopct='%2.2f%%', shadow=False, startangle=90, pctdistance=0.6)
    # plt.show()
    return "您样本中，检测到的物种有：{}; 物种占比情况：{}。".format(series_to_str(df, value_format=':.0f'), series_to_str(df / df.sum()))

def format_file(in_fp, out_fp, **kwargs):
    with open(in_fp, 'r') as f:
        out = f.read()
        for k, v in kwargs.items():
            out = out.replace("{{%s}}" % (k), v)
    with open(out_fp, 'w') as f:
        f.write(out)

def update_html_properties(html_fp, format_dict, out_fp, filter_function=False, use_selector=False):
    """
    in a format_dict, you should never use '' or "" in a CSS selector
    """
    with open(html_fp, "r", encoding="utf-8") as file:
        fcontent = file.read()
    sp = BeautifulSoup(fcontent, 'lxml')
    for k, v in format_dict.items():
        # pdb.set_trace()
        if use_selector:
            label_list = sp.select(k)
        else:
            tag = re.search(r'^[^\[]+', k).group()
            attrs = re.search(r'\[(.+)\]', k)
            attrs = dict([attr.split('=')
                          for attr in attrs.group(1).split('&')]) if attrs else {}
            label_list = sp.findAll(tag, attrs=attrs)
        for label in label_list:
            for p, n in v.items():
                if label.has_attr(p) and (filter_function(label[p]) if filter_function else True):
                    label[p] = n.format(value=label[p])
    with open(out_fp, "w", encoding="utf-8") as out:
        out.write(sp.prettify())

def read_file_n_lines(file, n):
    """ Read a file n lines at a time """

    line_set = []
    with (gzip.open(file, 'r') if file.endswith('.gz') else open(file, 'r')) as file_handle:
        for line in file_handle:
            if len(line_set) == n:
                yield line_set
                line_set = []
            line_set.append(line)

    # yield the last set
    if len(line_set) == n:
        yield line_set

def iter_fa(fasta, trim_line_break=False):
    with open(fasta, 'r') as file_handle:
        l1 = file_handle.readline()
        header = l1.rstrip('\n') if trim_line_break else l1
        if not header.startswith('>'):
            raise Exception("File is not a standard fasta!")
        seq_set = []
        for line in file_handle:
            li = line.rstrip('\n') if trim_line_break else line
            if line.startswith('>'):
                yield header, "".join(seq_set)
                header = li
                seq_set = []
            else:
                seq_set.append(li)

        # yield the last seq
        yield header, "".join(seq_set)

def trunc_fa(fasta, n, out_dir=False):
    dirname = out_dir or os.path.dirname(fasta)
    basename = os.path.basename(fasta)
    out_files = []
    for i in range(n):
        out_dir = os.path.join(dirname, "{basename}.trunk{i}".format(**locals()))
        if not os.path.exists(out_dir):
            os.makedirs(out_dir)
        out_files.append(open(os.path.join(out_dir, basename), 'w'))
    ln = 0
    for h, s in iter_fa(fasta, trim_line_break=False):
        out_files[ln % n].write(h + s)
        ln += 1
    for fh in out_files:
        fh.close()


'''
Author       : xujiawei
Date         : 2021-08-13 11:42:31
Description  : description
'''
import os

import yaml

# 默认配置，启动时输出到 default_config.yaml
default_config = {
    'server': {'host': '0.0.0.0', 'port': 9965},
    'picture': {'backup_days': 2, 'save_pic_dir': './data/captured', 'delay_before_capture_sec': 0.5, 'capture_count': 3, 'capture_interval_sec': 0.5},
    'video': {'save_video': False, 'backup_days': 2, 'save_video_dir': './data/video'},
    'record': {'backup_days': 180, 'save_record_dir': './data/csv'},
    'vehicle_type_recognization': {'url': 'http://0.0.0.0:7000/prediction/vehicle_type_recognization'},
    'vehicle_detection': {'enable': True, 'url': 'http://0.0.0.0:7000/prediction/vehicle_detection', 'min_vehicle_bottom_line': 0.75},
    'vehicle_class_classification': {'enable': True, 'url': 'http://0.0.0.0:7000/prediction/vehicle_class_classification'},
    'cockpit_height_classification': {'enable': True, 'url': 'http://0.0.0.0:7000/prediction/cockpit_height_classification'},
    'vehicle_type_agent_server': {'enable': True, 'url': 'http://127.0.0.1:9201', 'send_video': False},
    'lane': {'binding_gd_index': 3, 'reverse_lane': False, 'wait_for_vehplate_time_sec': 0, 'gd_delay_time_sec': 0},
    'camera': {'name': 'imx290_170', 'type': 'imx290', 'device': 0, 'rotate': 90, 'scale': 0.5, 'brightness_range': '(90, 150)', 'exposure_range': '(1, 80)', 'gain_range': '(0, 100)', 'auto_exposure_hour_range': '(0,0)', 'default_exposure': 10},
    'splicer': {'frame_width': 360, 'frame_height': 640, 'detect_vehicle_view': '(400, 620, 170, 260)', 'calc_offset_view': '(400, 620, 30, 330)', 'splice_x_pos': 170, 'min_detect_contour_size': 150, "detect_car_leave_by_video": False},
    'ui_server': {'url': 'http://127.0.0.1:5000'},
    'processes_message_queue_type': 'RedisMessageQueue'
}

config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), '../config/default_config.yaml'))
special_config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), '../config/special_config.yaml'))

def load(file_path):
    if os.path.exists(file_path):
        with open(file_path, 'r', encoding='utf8') as f:
            cfg = yaml.load(f, Loader=yaml.FullLoader)
    else:
        cfg = {}
    return cfg

def export(config, file_path):
    with open(file_path, 'w', encoding='utf8') as w:
        yaml.dump(config, w, default_flow_style=False)

def combine(older, newer):

    for k, v in newer.items():
        if not isinstance(v, dict):
            older[k] = newer[k]
        else:
            combine(older[k], newer[k])

    return older

export(default_config, config_file)
special_config = load(special_config_file)
config = combine(default_config, special_config)


from django.shortcuts import render, redirect
from .forms import RecordForm ,PostForm
from .models import *
from django.http import HttpResponse , HttpResponseRedirect
from django.views.generic import TemplateView, FormView, CreateView, ListView, DetailView ,UpdateView, DeleteView
from django.urls import reverse_lazy
from django.contrib import messages
from django.db.models import Q
from .templatetags import tag
from notifications.signals import notify

# Create your views here.
def RecordView(request):
    if request.method=="POST":
        form=RecordForm(request.POST,request.FILES)
        if form.is_valid():
            if not request.user.is_authenticated:
              return HttpResponse("Your Registraion /Login is required")
            object=form.save(commit=False) # commit =False as form still without user data

            object.user=request.user # get logged in user from session data

            # AS WE DEFINED ONE To ONE MODEL RELATION FOR ONE USER  ONLY ONE  ENTRY

            if RecordModel.objects.filter(user=request.user).exists() :
                return HttpResponse(" Your already created your Profile")
            object.save()
            subject=form.cleaned_data['subject'] # get subject objects for iteration of multiple subject entry
            for i in subject:
                object.subject.add(i)
                object.save()
            group=form.cleaned_data['group'] # get group objects for iteration of multiple group entry

            for i in group:
                object.group.add(i)
                object.save()
            messages.success(request,'Record Form has been Successfully Submitted')
            #return HttpResponse("Success")
            return redirect ('/template/')
    else:
        form=RecordForm()
        return render (request, 'education/education.html', {'form':form,'User':request.user})

def RecordDisplay(request):
    record=RecordModel.objects.all()
    record=RecordModel.items.all()
    user=request.user
    if user :
       record_user=RecordModel.objects.filter(user=user.id)

    subject=SubjectModel.objects.all()

    #subject=SubjectModel.objects.filter()

    #user=User.objects.filter(id=5)

   # user=User.objects.filter()

    group=GroupModel.objects.all()
    math=SubjectModel.objects.get(name='Mathematics')
    record_math=math.subject_set.all()
   # record_math_user=math.subject_set.get(user=request.user.id)

    science=GroupModel.objects.get(name='Science')
    record_science=science.group_set.all()
    business=GroupModel.objects.get(name='Business Studies')
    record_business=business.group_set.all()

    Record_user=record.get(id=request.user.id)
    Record_user.views.add(request.user)
    views= Record_user.views.count()

    liked=False
    recorduser=RecordModel.objects.get(id=user.id)
    if recorduser.likes.filter(id=user.id).exists():
        liked=True

    return render (request,'education/record_display.html', {'record':record,'subject':subject,'group':group
         ,'record_math':record_math,'record_science':record_science,'record_business':record_business,'record_user':record_user,
         'user':user,'like':liked,'views':views})

class templateview(TemplateView):
    template_name= 'topic/topics.html'   #'fileupload.html'

    def get_context_data(self, **kwargs):

        context=super().get_context_data(**kwargs)
        context['msg1']="Topic Page => this maeesage from TemplateView"
        context['msg2']="No topic has been added yet => this maeesage from TemplateView"

        return context

class RecordView_by_class(FormView):
      template_name='education/education.html'
      form_class=RecordForm
      success_url='/template'

      def form_valid(self, form):
           #login=self.request.user
           if not self.request.user.is_authenticated:
              return HttpResponse("Log in is required")

           if RecordModel.objects.filter(user=self.request.user).exists() :
                return HttpResponse(" Your have already created your Profile")

           form.save()
           messages.success(self.request,'Record Form Successfully Submitted')
           return super().form_valid(form)
      def form_invalid(form):

        return super().form_invalid(form)
      def get_success_url(self)-> str :
          return super().get_success_url()
          #return reverse_lazy ('')

class PostCreateView(CreateView):
    model=PostModel
    form_class=PostForm
    template_name='education/postcreate.html'
    success_url='/'

    def form_valid(self, form):

         try :
           form.instance.user=self.request.user

           form.save()
           return HttpResponse ("Post has been Successfully Created")
           #return render (self.request, 'education/postcreate.html', {'user': form.instance.user,'msg':'Post has been Successfully Created'} )

          # return super().form_valid(form)
         except :
            #if self.request.user=='':
              #return HttpResponse ("Login is required")
            #else:
                 #return render (self.request, 'education/postcreate.html' )
            return redirect ('../post_create' )     # PostCreateView.as_view()
            #return HttpResponse ("Login is required")
    def form_invalid(form):

          return super().form_invalid(form)
    def get_success_url(self)-> str :
          return super().get_success_url()
         # return reverse_lazy ('topic')

class PostListView(ListView):
    template_name='education/listview.html'
    #queryset=PostModel.objects.filter()
    queryset=RecordModel.objects.filter()

    context_object_name='post'

    def get_context_data(self, **kwargs):

         context= super().get_context_data(**kwargs)
         #context['post'] =context.get('object_list')
         context['msg'] ='This  Total  Post  List'
         context['subjects']=SubjectModel.objects.all()
         context['group']=GroupModel.objects.all()
         return context

class PostDetailView(DetailView):
   template_name='education/postdetails.html'
   model=RecordModel
  # model=PostModel

   def get_context_data(self, **kwargs):
         self.object.views.add(self.request.user)
         liked=False
         if self.object.likes.filter(id=self.request.user.id).exists():
          liked=True
         context= super().get_context_data(**kwargs)
         post=context.get('object')
         #comment=Comment.objects.all()

         comments=Comment.objects.filter(post=self.object.id ,parent_comment= None)
         replies=Comment.objects.filter(post=self.object.id ).exclude(parent_comment=None)
         DictofReply={}
         for reply in replies :
            if reply.parent_comment.id not in DictofReply.keys():
                DictofReply[reply.parent_comment.id]=[reply]
            else :
                DictofReply[reply.parent_comment.id].append(reply)
         context['post'] =context.get('object')  # =>  this line is optional
         context['msg'] ='This is Individual Post Details'

         context['like'] =liked
         context['comments'] =comments
         #context['replies'] =replies
         context['DictofReply'] = DictofReply

         #
         # context['comment'] =comment
         return context

class PostEditView(UpdateView):
    model=RecordModel
    #model=PostModel
    #form_class=PostForm
    form_class=RecordForm
    template_name='education/postcreate.html'
    #success_url='/'

    def get_success_url(self) :
          id=self.object.id
          #return super().get_success_url()
          return reverse_lazy ('PostDetailView',kwargs={'pk':id})

class PostDeleteView(DeleteView):
     model=RecordModel
     template_name='education/postdelete.html'
     success_url=reverse_lazy ('PostListView')

def search (request):
    query=request.POST.get('search',' ')
    if query :
        queryset=(Q(name__icontains=query))|(Q(address__icontains=query))|(Q(medium__icontains=query))|(Q(category__icontains=query))|(Q(subject__name__icontains=query))|(Q(group__name__icontains=query))
        results=RecordModel.objects.filter(queryset).distinct()
    else:
        results=[]
    context={'result':results}
    return render(request, 'education/search.html',context)

def filter (request):

    if request.method=="POST" :
        subject=request.POST['subject']
        group=request.POST['group']
        amount_from=request.POST['amount_from']
        amount_to=request.POST['amount_to']
        available=request.POST['available']

        if subject or group:
           queryset=(Q(subject__name__icontains=subject)) & (Q(group__name__icontains=group)) # & for both instead of | (or)
           results=RecordModel.objects.filter(queryset).distinct()
           if available :
              results=results.filter(available=True)
           if amount_from :
             results=results.filter(amount__gte=amount_from)
           if amount_to:
             results=results.filter(amount__lte=amount_to)
        else:
           results=[]
        context={'result':results}
        return render(request, 'education/search.html',context)
    #return render(request, 'education/listview.html')
def likepost(request, id):
    if request.method=="POST":

        record_user=RecordModel.objects.get(id=id)
        postId=record_user.id
        if record_user.likes.filter(id=request.user.id).exists():
           record_user.likes.remove(request.user)
        else :
           record_user.likes.add(request.user)
           if request.user != record_user.user :
            notify.send(request.user, recipient=record_user.user , verb="has liked your post" + f''' <a href="/education/post_details/{postId}/ " > Go </a>''')
    return HttpResponseRedirect(request.META.get('HTTP_REFERER'))
    #return redirect(f'../post_details/{postId}/')
def add_comment (request):
    if request.method=="POST":
        comment=request.POST['text'] # New Comment or reply of a comment
        parent_id=request.POST['parent_id'] # id of main comment (if)
        post_id=request.POST['post_id']

        #if (request.POST['parent_id']).strip():
           # ID=True
        #else :
           # ID=False
        post=RecordModel.objects.get(id=post_id)
        if len(comment)==0 :
            return HttpResponseRedirect(request.META.get('HTTP_REFERER')) #  back to comment page/original page
        if parent_id.strip():
            parent_comment=Comment.objects.get(id=parent_id)
            newcomment=Comment(text=comment, user=request.user, post=post, parent_comment=parent_comment) # comment reply
            newcomment.save()
        else :
           newcomment=Comment(text=comment, user=request.user, post=post ) # new comment
           newcomment.save()
    return HttpResponseRedirect(request.META.get('HTTP_REFERER')) #  back to comment page/original page

def notification (request):
    return render (request, 'education/notification.html')


import random

# 指定a、b、c的值
import torch

from my_model import MyModel

a = 5
b = 7
c = 9
d = 6
# z = a * x + b * y + c
data = [(random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)) for x, y, z in
        zip(range(1000), range(1000), range(1000))]
the_model = MyModel(3, 1)
the_model.load_state_dict(torch.load('best_model.pt'))
the_model.eval()
for x, y, z in data:
    w = a * x + b * y + c * z + d
    data = torch.tensor([x, y, z], dtype=torch.float32)
    pred_w = the_model(data)
    print(f'TRUE_W={w} | PRED_W={int(pred_w.item())}')

Solving equations using deep learning/make_data_and_val.py


import database.node as nd
import utils.screen_utils as su

class Recipe_Manager():

	def __init__(self, cookbook_data):
		self.cookbook_data = cookbook_data
		self.recipe_node = nd.Recipe_Node(name='')
		self._reset_()

	def _reset_(self):
		self.ingredient_name = None
		self.quantity_type = None
		self.quantity = None
		self.QI_list = []

		self.recipe_node.name = ''
		self.recipe_node.quantity_type = self.quantity_type
		self.recipe_node.quantity = self.quantity
		self.recipe_node.QI_list = self.QI_list
		self.recipe_node.update_observers()

	@property
	def recipe_name(self):
		return recipe_node.name

	@recipe_name.setter
	def recipe_name(self, value):
		self.recipe_node.name = value

	def add_QI(self):
		self.QI_list.append(nd.Quantity_Ingredient(self.quantity, self.quantity_type, self.ingredient_name))
		self.recipe_node.QI_list = self.QI_list
		self.recipe_node.update_observers()

		# reset so that we throw an error if something is not updated appropriately
		self.ingredient_name = None
		self.quantity_type = None
		self.quantity = None

	def add_recipe(self):

		if (self.recipe_node.name == None) or (self.recipe_node.name == ''):
			return

		new_node = nd.Recipe_Node(name=self.recipe_node.name)
		new_node.QI_list = self.recipe_node.QI_list.copy()

		self.cookbook_data.add(new_node)
		self.cookbook_data.save()
		self.cookbook_data.update_observers()

class Recipe_Manager_Screen_Button(su.Screen_Button):
	def __init__(self, screen_manager, screen_name, recipe_manager, **kwargs):
		super().__init__(screen_manager, screen_name, **kwargs)
		self.recipe_manager = recipe_manager

	def manager_action(self):
		raise NotImplementedError

	def on_press(self):
		self.manager_action()
		super().on_press()


"""
Custom dataset processing/generation functions should be added to this file
"""

import pathlib
from sklearn.datasets import fetch_20newsgroups

from tqdm.auto import tqdm

from .. import paths
from ..log import logger

__all__ = [
    'process_20_newsgroups'
]

def process_20_newsgroups(*, extract_dir='20_newsgroups',
                          metadata=None, unpack_dir=None,
                          opts={"subset":"all", "remove":"('headers', 'footers', 'quotes')"}):
    """
    Process 20 newsgroups into (data, target, metadata) format.

    Parameters
    ----------
    unpack_dir: path
        The interim parent directory the dataset files have been unpacked into.
    extract_dir: str
        Name of the directory of the unpacked files relative to the unpack_dir. Note that
    opts: dict default {"subset":"all", "remove"="('headers', 'footers', 'quotes')"}
        Options to pass to sklearn.datasets.fetch_20newsgroups.

    Returns
    -------
    A tuple:
        (data, target, additional_metadata)

    """
    if metadata is None:
        metadata = {}

    if unpack_dir is None:
        unpack_dir = paths['interim_data_path']
    else:
        unpack_dir = pathlib.Path(unpack_dir)
    data_dir = unpack_dir / f"{extract_dir}"

    news = fetch_20newsgroups(**opts)

    return news.data, news.target, metadata

{{ cookiecutter.repo_name }}/{{ cookiecutter.module_name }}/data/process_functions.py


from os import getcwd

from PyQt5 import QtCore, QtWidgets, QtGui
from PyQt5.QtWidgets import QDialog, QTableWidget, QTableWidgetItem, QDialogButtonBox, QMessageBox, QSpacerItem, \
    QFileDialog

from file import saveReportAsPdf
from reporter import Report

class Window:
    def showReport(self, report: Report, name):
        dialog = Dialog(700, 500, "Report")

        header = QtWidgets.QLabel()
        header.setFont(getFont(12))
        header.setText(report.decision)
        header.setWordWrap(True)

        model = QtWidgets.QLabel()
        model.setFont(getFont(10))
        model.setText("Model probability: {:.1%}".format(report.probability))
        model.setWordWrap(True)

        desc = QtWidgets.QLabel()
        desc.setFont(getFont(10))
        desc.setText(report.message)
        desc.setWordWrap(True)

        list = QtWidgets.QLabel()
        list.setFont(getFont(10))
        list.setText(createBulletList(report.significantParameters))
        list.setWordWrap(True)

        tableLabel = QtWidgets.QLabel()
        tableLabel.setFont(getFont(10))
        tableLabel.setContentsMargins(0, 20, 0, 0)
        tableLabel.setText("Table showing how each parameter affects the final result")
        tableLabel.setWordWrap(True)

        buttonBox = QDialogButtonBox(QDialogButtonBox.Save | QDialogButtonBox.Cancel)
        dataToSave = {"decision": header.text(), "probability": model.text(), "message": report.message, "name": name,
                      "param": report.significantParameters, "table": report.reasoning,
                      "description": tableLabel.text()}
        buttonBox.accepted.connect(lambda: dialog.save(dataToSave))
        buttonBox.rejected.connect(dialog.reject)

        dialog.addWidget(header)
        dialog.addWidget(model)
        dialog.addWidget(desc)
        dialog.addWidget(list)
        dialog.addWidget(tableLabel)
        dialog.addWidget(createTable(report.reasoning))
        dialog.addWidget(buttonBox, alignment=QtCore.Qt.AlignBottom)
        dialog.showDialog()

    def showWarning(self, message, description):
        msg = QMessageBox()
        msg.setIcon(QMessageBox.Warning)
        msg.setText(message)

        msg.setInformativeText(description)
        msg.setWindowTitle("Warning")

        spacer = QSpacerItem(350, 30)
        layout = msg.layout()
        layout.addItem(spacer, 1, 0, 1, 0)

        msg.exec_()

class Dialog(QDialog):
    def __init__(self, width, height, title):
        super().__init__()
        self.setWindowTitle(title)
        self.resize(width, height)
        self.setAttribute(QtCore.Qt.WA_DeleteOnClose)
        self.setWindowFlag(QtCore.Qt.WindowContextHelpButtonHint, False)

        layoutWidget = QtWidgets.QWidget(self)
        layoutWidget.setFixedWidth(width)
        layoutWidget.setFixedHeight(height)
        self.verticalLayout = QtWidgets.QVBoxLayout(layoutWidget)
        self.verticalLayout.setContentsMargins(10, 10, 10, 10)

    def addWidget(self, widget: QtWidgets, alignment=None):
        if alignment is None:
            self.verticalLayout.addWidget(widget)
        else:
            self.verticalLayout.addWidget(widget, alignment=alignment)

    def save(self, data):
        file = QFileDialog.getSaveFileName(None, 'Save file', getcwd(), "Report pdf (*.pdf)")[0]
        if file is not None and file != '':
            saveReportAsPdf(file, data)

    def showDialog(self):
        self.exec_()

def createBulletList(elements):
    bulletList = "
"
    for a in elements:
        bulletList = bulletList + "
{}
".format(a)
    bulletList = bulletList + "
"
    return bulletList

def createTable(reasoning):
    rows = 3
    columns = 13
    redBackground = QtGui.QColor(214, 92, 92)
    greenBackground = QtGui.QColor(92, 214, 92)

    tableWidget = QTableWidget()
    tableWidget.setRowCount(rows)
    tableWidget.setFixedHeight(200)
    tableWidget.setColumnCount(columns)
    for i in range(columns):
        tableWidget.setColumnWidth(i, 80)

    columnNames = []
    index = 0
    for n in reasoning:
        columnNames.append(n.name)
        tableWidget.setItem(0, index, QTableWidgetItem("{:.3}".format(n.score)))
        tableWidget.setItem(1, index, QTableWidgetItem("{:.2%}".format(n.rate)))
        tableWidget.setItem(2, index, QTableWidgetItem("{}".format(n.patient_value)))
        if n.score > 0:
            for i in range(rows):
                tableWidget.item(i, index).setBackground(redBackground)
        elif n.score < 0:
            for i in range(rows):
                tableWidget.item(i, index).setBackground(greenBackground)

        index = index + 1

    tableWidget.setHorizontalHeaderLabels(columnNames)
    tableWidget.setVerticalHeaderLabels(["score", "rate", "patient"])
    return tableWidget

def getFont(size):
    font = QtGui.QFont()
    font.setPointSize(size)
    return font


from setuptools import setup, find_packages
from codecs import open
from os import path

here = path.abspath(path.dirname(__file__))

with open(path.join(here, 'README.lst'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='Oculi',
    version='0.1.0',
    description='Picamera control and motion detect frontend',
    long_description=long_description,
    author='Jun Tan',
    author_email='jforce716@gmail.com',
    classifier=[
        'Development Status :: 3 - Alpha',
        'Topic :: PiCamera :: Application :: Motion detect',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 3.6'
    ],
    keywords='PiCamera, montion detect',
    packages=find_packages(where='./oculi'),
    install_requires=['flask', 'picamera', 'Pillow']
)


"""
Radial profile of cooling time, mass weighted
"""
import yt
import ytree
import os
import sys
import matplotlib.pyplot as plt
import numpy as np

# macros
x = "seed1-bh-only-cooling-time-"
y = sys.argv[-1]  # plot  name
star_pos0 = [0.49030654, 0.50721054, 0.49178297] # found with smartstar-find.py

# load data
root_dir = "/home/sgordon/disk14/cirrus-runs-rsync/seed1-bh-only/270msun/BF-0.0078125"

# all ds
DD = []
ds1 = yt.load(os.path.join(root_dir, sys.argv[1])) # t = 124.760, before particle formation
label1 = sys.argv[1]
DD.append(ds1)
ds2 = yt.load(os.path.join(root_dir, sys.argv[2])) # later time
label2 = sys.argv[2]
DD.append(ds2)
if str(sys.argv[3]).startswith('DD'):
    ds3 = yt.load(os.path.join(root_dir, sys.argv[3])) # latest time
    label3 = sys.argv[3]
    DD.append(ds3)

# define dark_matter_mass field + add to all ds
def _t_cool_abs(field, data):
    return np.abs(data[("gas", "cooling_time")].in_units("s"))

for ds in DD:
    ds.add_field(("gas", "cooling_time_abs"), function=_t_cool_abs, units="s", sampling_type="cell") # need to define with units here.

""" Make Sphere of Most Massive Halo """

# Load seed1 merger tree of dataset (up to DD0118 in gas run)
a = ytree.load('/home/sgordon/disk14/pop3/gas+dm-L3/rockstar_halos/out_0.list')
a1 = ytree.load('/home/sgordon/disk14/pop3/gas+dm-L3/tree_810/tree_810.h5')
r_halo = a1[0]["virial_radius"].to('pc')
r = ds1.quan(r_halo.d, "pc") # virial radius

# Make initial sphere centred on the star at the time of formation with radius = 3 * virial radius
spheres = []
for i, ds in enumerate(DD):
    sphere = ds.sphere(star_pos0, 3*r)
    spheres.append(sphere)

""" Make Profile Plots weighted by cell mass """

all_fields = ["cell_mass", "matter_mass", "cooling_time", "cooling_time_abs"]
all_units = {"radius": "pc", "cell_mass": "Msun", "matter_mass": "Msun", "cooling_time_abs": "s", "cooling_time": "s"}

# Radial profile weighted by = 'cell_mass'
radial_profiles = []
for i, sp in enumerate(spheres):
    radial_profile = yt.create_profile(
        sp,
        "radius",
        all_fields,
        units=all_units,
        weight_field="cell_mass",
        accumulation=False,
        logs={"radius": True} # Set true for more bins
    )
    radial_profiles.append(radial_profile)

""" Plot """
plt.rcParams["font.family"] = "serif"
plt.rcParams['lines.linewidth'] = 2

font = {'family': 'serif',
        'color':  'black',
        'weight': 'normal',
        'size': 16,
        }

font2 = {'family': 'serif',
        'color':  'darkred',
        'weight': 'normal',
        'size': 14,
        }

plt.loglog(radial_profiles[0].x.value,
           radial_profiles[0][("gas","cooling_time_abs")],
           color='b', linestyle='solid', label=label1)
plt.loglog(radial_profiles[1].x.value,
           radial_profiles[1]["cooling_time_abs"],
           color='r', linestyle='solid', label=label2)
if str(sys.argv[3]).startswith('DD'):
    plt.loglog(radial_profiles[2].x[radial_profiles[2].used],
               radial_profiles[2]["cooling_time"][radial_profiles[2].used],
               color='r', linestyle='solid', label=label3)

plt.xlabel(r"r (pc)", fontdict=font)
plt.ylabel(r"t (s)", fontdict=font)

plt.legend(loc="lower left", fontsize=14,  ncol=1) # upper/lower

plt.tick_params(axis="x", which='minor', length = 4, direction="in")
plt.tick_params(axis="x", which='major', labelsize = 14,  width=2, length=7, direction="in")
# plt.yaxis.tick_right()
# plt.yaxis.set_label_position("right")
# plt.tick_params(axis="y", which='major', labelsize = 14)

plot_name = 'radial-profile-plot-' + str(x) + str(y) + '.pdf'
plt.savefig('plots/' + plot_name, dpi=100)
print("created ", plot_name)


# models.py
# Author: Jacob Schreiber <jmschreiber91@gmail.com>

import time
import numpy
import torch

from tqdm import trange

from bpnetlite.losses import MNLLLoss
from bpnetlite.losses import log1pMSELoss

from bpnetlite.performance import pearson_corr
from bpnetlite.performance import calculate_performance_measures

from bpnetlite.logging import Logger

@torch.inference_mode()
def trt_predict(model, X, cell_states, read_depths, batch_size=32):
	"""Makes predictions using fixed batch sizes.

	This function takes in a model and makes predictions for an entire data
	set using fixed batch sizes. This is identical to the `.predict` method
	that comes with the methods except that it constructs a final, fixed-size
	batch for the final elements. For instance, if you are using a batch size
	of 32 and there are 40 elements, the second batch here will have size
	32 but only fill in the first 8 elements and then truncate the remaining
	out. This is necessary when using ahead-of-time compilation, such as
	torch2trt, to speed up model inference, because these methods require
	fixed tensor sizes.

	Parameters
	----------
	X: torch.tensor, shape=(-1, 4, 2114)
		A one-hot encoded sequence tensor.

	cell_states: torch.tensor, shape=(-1, n_inputs)
		A tensor representing the state of each cell that is passed into
		the model, e.g. through PCA or LSI.

	read_depths: torch.tensor, shape=(-1, 1)
		A tensor containing the read depth of each cell or aggregated
		across similar cells.

	batch_size: int, optional
		The number of elements to use in each batch. Default is 64.

	dtype: torch datatype
		The data type to c

	Returns
	-------
	y_hat: torch.tensor, shape=(-1, 1000)
		A tensor containing the predicted profiles.
	"""

	starts = numpy.arange(0, X.shape[0], batch_size)
	ends = starts + batch_size

	y = []

	for start, end in zip(starts, ends):
		if end >= X.shape[0]:
			idx = X.shape[0] - start

			_X = torch.zeros(batch_size, X.shape[1], X.shape[2], dtype=X.dtype,
				device=X.device)
			_X[:idx] = X[start:end]

			_cs = torch.zeros(batch_size, cell_states.shape[1],
				dtype=cell_states.dtype, device=cell_states.device)
			_cs[:idx] = cell_states[start:end]

			_rd = torch.zeros(batch_size, read_depths.shape[1],
				dtype=read_depths.dtype, device=read_depths.device)
			_rd[:idx] = read_depths[start:end]
		else:
			_X = X[start:end]
			_cs = cell_states[start:end]
			_rd = read_depths[start:end]

		_y = model(_X, _cs, _rd)

		if end >= X.shape[0]:
			idx = X.shape[0] - start

			_y = _y[:idx]

		y.append(_y)

	return torch.cat(y)

class DragoNNFruit(torch.nn.Module):
	"""An entire DragoNNFruit model with all components.

	DragoNNFruit is a model for analyzing single cell data sets that include
	an accessibility component. It can be used on scATAC-seq data alone or it
	can be used on multimodal data where one of the components is scATAC-seq.

	Briefly, the model is primarily a ChromBPNet model that uses an external
	factor to modify the parameters of the accessibility model. This external
	factor can be LSI representations from the same scATAC-seq data being
	predicted or it can come from the other modalities, e.g. scRNA-seq, if the
	data is multimodal.

	A DragoNNFruit model is made up of three components: (1) the bias model,
	(2) the dynamic accessibility model, and (3) the dynamic controller. These
	components are defined below, but the first two are based on ChromBPNet.

	Parameters
	----------
	bias: torch.nn.Module
		This model takes in sequence and outputs the shape one would expect in
		ATAC-seq data due to Tn5 bias alone. This is usually a BPNet model
		from the bpnet-lite repo that has been trained on GC-matched non-peak
		regions.

	accessibility: torch.nn.Module
		This model takes in sequence and outputs the accessibility one would
		expect due to the components of the sequence, but also takes in a cell
		representation which modifies the parameters of the model, hence,
		"dynamic." This model is usually a DynamicBPNet model, defined below.

	name: str
		The name to prepend when saving the file.
	"""

	def __init__(self, bias, accessibility, name, alpha=1):
		super(DragoNNFruit, self).__init__()

		for parameter in bias.parameters():
			parameter.requires_grad = False

		self.bias = bias
		self.accessibility = accessibility
		self.name = name
		self.alpha = alpha
		self.logger = Logger(["Epoch", "Iteration", "Training Time",
			"Validation Time", "Training MNLL", "Validation MNLL",
			"Validation Profile Correlation", "Validation Count Correlation",
			"Saved?"], verbose=True)

	def forward(self, X, cell_states, read_depths):
		"""A forward pass through the network.

		This function is usually accessed through calling the model, e.g.
		doing `model(x)`. The method defines how inputs are transformed into
		the outputs through interactions with each of the layers.

		Parameters
		----------
		X: torch.tensor, shape=(-1, 4, 2114)
			A one-hot encoded sequence tensor.

		cell_states: torch.tensor, shape=(-1, n_nodes)
			A tensor representing the state of each cell that is passed into
			the model, e.g. through PCA or LSI.

		read_depths: torch.tensor, shape=(-1, 1)
			A tensor containing the read depth of each cell or aggregated
			across similar cells.

		Returns
		-------
		y_profile: torch.tensor, shape=(-1, 1000)
			The predicted logit profile for each example. Note that this is not
			a normalized value.
		"""

		return read_depths + self.bias(X)[0][:, 0] + self.accessibility(X,
			cell_states)

	@torch.no_grad()
	def predict(self, X, cell_states, read_depths, batch_size=16,
		reduction=None, verbose=False):
		"""A method for making batched predictions.

		This method will take in a large number of cell states and provide
		predictions in a batched manner without storing the gradients. Useful
		for inference step.

		Parameters
		----------
		X: torch.tensor, shape=(-1, 4, 2114)
			A one-hot encoded sequence tensor.

		cell_states: torch.tensor, shape=(-1, n_inputs)
			A tensor representing the state of each cell that is passed into
			the model, e.g. through PCA or LSI.

		read_depths: torch.tensor, shape=(-1, 1)
			A tensor containing the read depth of each cell or aggregated
			across similar cells.

		batch_size: int, optional
			The number of elements to use in each batch. Default is 64.

		reduction: None, 'logsumexp', or 'sum', optional
			Whether to reduce the predictions along the final axis. Useful to
			reduce the memory overhead of making large number of predictions.
			If None, perform no reduction. Default is None.

		verbose: bool, optional
			Whether to display a progress bar across batches. Default is False.

		Returns
		-------
		y_hat: torch.tensor, shape=(-1, 1000) or (-1, 1)
			A tensor containing the predicted profiles.
		"""

		y_hat = []
		for start in trange(0, len(X), batch_size, disable=not verbose):
			X_batch = X[start:start+batch_size]
			cs_batch = cell_states[start:start+batch_size]
			rd_batch = read_depths[start:start+batch_size]

			y_hat_ = self(X_batch, cs_batch, rd_batch)
			if reduction == 'sum':
				y_hat_ = torch.sum(y_hat_, dim=-1)
			elif reduction == 'logsumexp':
				y_hat_ = torch.logsumexp(y_hat_, dim=-1)

			y_hat.append(y_hat_)

		y_hat = torch.cat(y_hat)
		return y_hat

	@torch.no_grad()
	def predict_cross(self, X, cell_states, read_depths, batch_size=64,
		reduction=None, verbose=False):
		"""A method for making batched predictions on a locus/cell cross.

		This method makes predictions for a cross between loci and cell states.
		Use this method and not `predict` when you want to make predictions
		for all loci for all cell states. Because the cross is constructed
		in batches the amount of memory required is limited and the size
		of the loci and cell states do not need to match in the first
		dimension.

		Parameters
		----------
		X: torch.tensor, shape=(n_loci, 4, 2114)
			A one-hot encoded sequence tensor.

		cell_states: torch.tensor, shape=(n_states, n_inputs)
			A tensor representing the state of each cell that is passed into
			the model, e.g. through PCA or LSI.

		read_depths: torch.tensor, shape=(n_states, 1)
			A tensor containing the read depth of each cell or aggregated
			across similar cells.

		batch_size: int, optional
			The number of elements to use in each batch. Default is 64.

		reduction: None, 'logsumexp', or 'sum', optional
			Whether to reduce the predictions along the final axis. Useful to
			reduce the memory overhead of making large number of predictions.
			If None, perform no reduction. Default is None.

		verbose: bool, optional
			Whether to display a progress bar across states. Default is False.

		Returns
		-------
		y_hat: torch.tensor, shape=(n_loci, n_states, 1000)
			A tensor containing the predicted profiles.
		"""

		n_states = cell_states.shape[0]
		y_hat = []

		for i in trange(n_states, disable=not verbose):
			_y_hat = self.predict(X, cell_states[i:i+1].expand(X.shape[0], -1),
				read_depths=read_depths[i:i+1].expand(X.shape[0], -1),
				batch_size=batch_size, reduction=reduction)
			y_hat.append(_y_hat)

		return torch.stack(y_hat)

	def _train_step(self, X, cell_states, read_depths, y, optimizer):
		optimizer.zero_grad()

		with torch.autocast(device_type='cuda', dtype=torch.bfloat16):
			y_hat = self(X, cell_states, read_depths)

		y_hat_ = torch.nn.functional.log_softmax(y_hat.flatten(), dim=-1)
		loss = MNLLLoss(y_hat_, y.flatten())

		loss_ = loss.item()
		loss = loss.backward()

		torch.nn.utils.clip_grad_norm_(self.parameters(), 1)
		optimizer.step()
		return loss_

	def fit(self, training_data, validation_data, optimizer,
		n_validation_samples=5000, max_epochs=100, batch_size=64,
		validation_iter=100, verbose=True):
		"""Fit an entire DragoNNFruit model to the data.
		"""

		X_valid, y_valid, c_valid, r_valid = zip(*[validation_data[i]
			for i in range(n_validation_samples)])
		X_valid = torch.stack(X_valid).cuda().type(torch.float32)
		y_valid = torch.stack(y_valid).cuda().type(torch.float32)
		c_valid = torch.stack(c_valid).cuda()
		r_valid = torch.stack(r_valid).cuda()

		start, best_corr = time.time(), 0
		self.logger.start()
		for epoch in range(max_epochs):
			for i, (X, y, cell_states, read_depths) in enumerate(training_data):
				X = X.cuda().type(torch.float32)
				y = y.cuda()
				cell_states = cell_states.cuda()
				read_depths = read_depths.cuda()

				train_loss = self._train_step(X, cell_states, read_depths, y,
					optimizer)

				if verbose and i % validation_iter == 0:
					train_time = time.time() - start
					tic = time.time()

					# Make predictions
					y_hat = self.predict(X_valid, c_valid, r_valid, batch_size)

					# Calculate MNLL loss
					y_hat_ = torch.nn.functional.log_softmax(y_hat.flatten(),
						dim=-1)
					valid_loss = MNLLLoss(y_hat_, y_valid.flatten()).item()

					# Calculate count correlation
					y_hat_ = torch.logsumexp(y_hat, dim=-1)
					valid_count_corr = pearson_corr(y_hat_, y_valid.sum(dim=-1))
					valid_count_corr = valid_count_corr.mean().item()

					# Calculate profile correlation
					y_hat = torch.nn.functional.log_softmax(y_hat.flatten(),
						dim=-1).reshape(*y_hat.shape)
					y_hat = torch.exp(y_hat)
					valid_profile_corr = pearson_corr(y_hat,
						y_valid).mean().item()

					valid_time = time.time() - tic

					self.logger.add([epoch, i, train_time, valid_time,
						train_loss, valid_loss, valid_profile_corr,
						valid_count_corr, valid_profile_corr > best_corr])

					self.logger.save("{}.log".format(self.name))

					if valid_profile_corr > best_corr:
						best_corr = valid_profile_corr
						torch.save(self, "{}.best.torch".format(self.name))

					start = time.time()
					torch.save(self, "{}.{}.torch".format(self.name, epoch))

class DynamicBPNet(torch.nn.Module):
	"""A BPNet model whose weights are modified by an external factor.

	This model has the same architecture as the original BPNet model except
	that parameters can be controlled by some external factor. Specifically,
	in this model, the bias terms are controlled using the external factor,
	and that external factor is the cell state.

	Because the intended application of this is single cell data, the final
	predictions are modified using the read depth.

	Parameters
	----------
	controller: torch.nn.Module
		This model takes in the cell representation and outputs a
		representation that will be fed into the dynamic accessibility model.
		This model is usually a CellStateController model, as defined below.

	n_filters: int, optional
		The number of filters to use in each convolution layer. Default is 128.

	n_layers: int, optional
		The number of residual dilated convolutions sandwiched between the
		first and the last layer. Does not include the size of the first
		and the last layer. Default is 8.

	trimming: int or None, optional
		The amount of trimming to apply to input sequences to go from the size
		of the input to the size of the output predictions. If none, will
		automatically trim to 2 ** n_layers + 37. Default is None.

	conv_bias: bool, optional
		Whether the convolutions should have learned bias terms in addition
		to those learned from the cell state repressentation. Mostly only
		useful when transferring weights from a pre-trained model.
	"""

	def __init__(self, controller, n_filters=128, n_layers=8, trimming=None,
		conv_bias=False):
		super(DynamicBPNet, self).__init__()

		self.trimming = trimming or 2 ** n_layers + 37
		self.n_filters = n_filters
		self.n_layers = n_layers

		self.iconv = torch.nn.Conv1d(4, n_filters, kernel_size=21, padding=10)
		self.irelu = torch.nn.ReLU()
		self.fconv = torch.nn.Conv1d(n_filters, 1, kernel_size=75, padding=37,
			bias=conv_bias)

		self.biases = torch.nn.ModuleList([
			torch.nn.Linear(controller.n_outputs, n_filters) for i in range(
				n_layers)
		])

		self.convs = torch.nn.ModuleList([
			torch.nn.Conv1d(n_filters, n_filters, kernel_size=3, stride=1,
				dilation=2**i, padding=2**i, bias=conv_bias) for i in range(1,
					n_layers+1)
		])

		self.relus = torch.nn.ModuleList([
			torch.nn.ReLU() for i in range(n_layers)
		])

		self.controller = controller

	def forward(self, X, cell_states):
		"""A forward pass through the network.

		This function is usually accessed through calling the model, e.g.
		doing `model(x)`. The method defines how inputs are transformed into
		the outputs through interactions with each of the layers.

		Parameters
		----------
		X: torch.tensor, shape=(-1, 4, 2114)
			A one-hot encoded sequence tensor.

		cell_states: torch.tensor, shape=(-1, n_nodes)
			A tensor representing the state of each cell that is passed into
			the model, e.g. through PCA or LSI.

		Returns
		-------
		y_profile: torch.tensor, shape=(-1, 1000)
			The predicted logit profile for each example. Note that this is not
			a normalized value.
		"""

		start, end = self.trimming, X.shape[2] - self.trimming

		cell_states = self.controller(cell_states)
		X = self.irelu(self.iconv(X))

		for i in range(self.n_layers):
			X_conv = self.convs[i](X)
			X_bias = self.biases[i](cell_states).unsqueeze(-1)

			X = X + self.relus[i](X_conv + X_bias)

		y_profile = self.fconv(X)[:, 0, start:end]
		return y_profile

	@torch.no_grad()
	def predict(self, X, cell_states, batch_size=64, reduction=None,
		verbose=False):
		"""A method for making batched predictions.

		This method makes predictions for a paired set of loci and cell states.
		Each locus and cell state are considered individually and so the number
		of loci must match the number of cell states even if many predictions
		are being made for the same cell state or locus. In that case, one
		must pass copies of the locus or cell state to ensure that the shapes
		match.

		Parameters
		----------
		X: torch.tensor, shape=(-1, 4, 2114)
			A one-hot encoded sequence tensor.

		cell_states: torch.tensor, shape=(-1, n_inputs)
			A tensor representing the state of each cell that is passed into
			the model, e.g. through PCA or LSI.

		batch_size: int, optional
			The number of elements to use in each batch. Default is 64.

		reduction: None, 'logsumexp', or 'sum', optional
			Whether to reduce the predictions along the final axis. Useful to
			reduce the memory overhead of making large number of predictions.
			If None, perform no reduction. Default is None.

		verbose: bool, optional
			Whether to display a progress bar across batches. Default is False.

		Returns
		-------
		y_hat: torch.tensor, shape=(-1, 1000)
			A tensor containing the predicted profiles.
		"""

		y_hat = []
		for start in trange(0, len(X), batch_size, disable=not verbose):
			X_batch = X[start:start+batch_size]
			cs_batch = cell_states[start:start+batch_size]

			y_hat_ = self(X_batch, cs_batch)
			if reduction == 'sum':
				y_hat_ = torch.sum(y_hat_, dim=-1)
			elif reduction == 'logsumexp':
				y_hat_ = torch.logsumexp(y_hat_, dim=-1)

			y_hat.append(y_hat_)

		y_hat = torch.cat(y_hat)
		return y_hat

	@torch.no_grad()
	def predict_cross(self, X, cell_states, batch_size=64, reduction=None,
		verbose=False):
		"""A method for making batched predictions on a locus/cell cross.

		This method makes predictions for a cross between loci and cell states.
		Use this method and not `predict` when you want to make predictions
		for all loci for all cell states. Because the cross is constructed
		in batches the amount of memory required is limited and the size
		of the loci and cell states do not need to match in the first
		dimension.

		Parameters
		----------
		X: torch.tensor, shape=(n_loci, 4, 2114)
			A one-hot encoded sequence tensor.

		cell_states: torch.tensor, shape=(n_states, n_inputs)
			A tensor representing the state of each cell that is passed into
			the model, e.g. through PCA or LSI.

		batch_size: int, optional
			The number of elements to use in each batch. Default is 64.

		reduction: None, 'logsumexp', or 'sum', optional
			Whether to reduce the predictions along the final axis. Useful to
			reduce the memory overhead of making large number of predictions.
			If None, perform no reduction. Default is None.

		verbose: bool, optional
			Whether to display a progress bar across states. Default is False.

		Returns
		-------
		y_hat: torch.tensor, shape=(n_loci, n_states, 1000)
			A tensor containing the predicted profiles.
		"""

		n_states = cell_states.shape[0]
		y_hat = []

		for i in trange(n_states, disable=not verbose):
			_y_hat = self.predict(X, cell_states[i:i+1].expand(X.shape[0], -1),
				batch_size=batch_size, reduction=reduction)
			y_hat.append(_y_hat)

		return torch.stack(y_hat)

	def load_conv_weights(self, filename, freeze=False):
		"""Load convolution weights from a pretrained ChromBPNet model.

		This function will load the convolution weights from a pre-trained
		model. Specifically, it will load the convolutions from the
		`accessibility` attribute of the model. Must have the same number
		of layers and filter sizes.

		Parameters
		----------
		filename: str
			The filename of the model to load.

		freeze: bool, optional
			Whether to freeze the layers whose weights are loaded.
		"""

		device = next(self.parameters()).device
		model = torch.load(filename, map_location='cpu').to(device)
		model = model.accessibility

		self.iconv.weight = torch.nn.Parameter(model.iconv.weight)
		self.iconv.weight.requires_grad = not freeze

		self.iconv.bias = torch.nn.Parameter(model.iconv.bias)
		self.iconv.bias.requires_grad = not freeze

		for sconv, mconv in zip(self.convs, model.rconvs):
			sconv.weight = torch.nn.Parameter(mconv.weight)
			sconv.weight.requires_grad = not freeze

			sconv.bias = torch.nn.Parameter(mconv.bias)
			sconv.bias.requires_grad = not freeze

		self.fconv.weight = torch.nn.Parameter(model.fconv.weight)
		self.fconv.weight.requires_grad = not freeze

class CellStateController(torch.nn.Module):
	"""A controller of DynamicBPNet parameters using cell state.

	This component of the DragoNNFruit model concerns taking in cell states
	and transforming them somehow. It is passed into the DynamicBPNet model
	and provides the representations that are used to control the parameters
	of the model.

	Parameters
	----------
	n_inputs: int
		The number of dimensions in the input cell representation.

	n_nodes: int, optional
		The number of nodes in the internal neural network layers. Default
		is 256.

	n_outputs: int, optional
		The number of dimensions in the output cell representation. Default
		is 64.

	n_layers: int, optional
		The total number of internal layers to apply to the representations,
		not including the first layer that transforms the representations from
		input representations to the internal representations or the last
		layer that transforms from the internal representations to the output
		representations. Default is 0.
	"""

	def __init__(self, n_inputs, n_nodes=256, n_outputs=64, n_layers=0):
		super(CellStateController, self).__init__()

		self.n_inputs = n_inputs
		self.n_nodes = n_nodes
		self.n_outputs = n_outputs
		self.n_layers = n_layers

		self.ifc = torch.nn.Linear(n_inputs, n_nodes)
		self.irelu = torch.nn.ReLU()

		self.fcs = torch.nn.ModuleList([
			torch.nn.Linear(n_nodes, n_nodes) for i in range(n_layers)
		])
		self.relus = torch.nn.ModuleList([
			torch.nn.ReLU() for i in range(n_layers)
		])

		self.ffc = torch.nn.Linear(n_nodes, n_outputs)

	def forward(self, cell_states):
		"""A forward pass through the network.

		This function is usually accessed through calling the model, e.g.
		doing `model(x)`. The method defines how inputs are transformed into
		the outputs through interactions with each of the layers.

		Parameters
		----------
		cell_states: torch.tensor, shape=(-1, n_inputs)
			A tensor representing the state of each cell that is passed into
			the model, e.g. through PCA or LSI.

		Returns
		-------
		cell_states: torch.tensor, shape=(-1, n_outputs)
			A tensor representing the transformed state of each cell.
		"""

		cell_states = self.irelu(self.ifc(cell_states))
		for i in range(self.n_layers):
			cell_states = self.relus[i](self.fcs[i](cell_states))

		cell_states = self.ffc(cell_states)
		return cell_states

	@torch.no_grad()
	def predict(self, cell_states, batch_size=64):
		"""A method for making batched predictions.

		This method will take in a large number of cell states and provide
		predictions in a batched manner without storing the gradients. Useful
		for inference step.

		Parameters
		----------
		cell_states: torch.tensor, shape=(-1, n_inputs)
			A tensor representing the state of each cell that is passed into
			the model, e.g. through PCA or LSI.

		batch_size: int, optional
			The number of elements to use in each batch. Default is 64.

		Returns
		-------
		cell_states: torch.tensor, shape=(-1, n_outputs)
			A tensor representing the transformed state of each cell.
		"""

		y_hat = []
		for start in range(0, len(cell_states), batch_size):
			cs_batch = cell_states[start:start+batch_size]

			y_hat_ = self(cs_batch)
			y_hat.append(y_hat_)

		y_hat = torch.cat(y_hat)
		return y_hat


"""
This module handles daily brief processing.
"""

import logging
import datetime
from typing import Dict, Any

import pyttsx3

from server.api.weather import fetch_weather
from server.api.news import fetch_news_headlines
from server.api.covid import fetch_covid_data

__speech_engine = pyttsx3.init()

def daily_brief(alarm_info: Dict[str, Any]):
    """
    Gives the user a brief of the current weather, the top news, and the local covid infection rate.
    """

    logging.info("Daily brief initiated on %s.", datetime.datetime.now())

    alarm_title = alarm_info["title"]

    brief_message = f"""
    Hello! This is a scheduled daily brief titled: {alarm_title}.
    {__covid_brief()}
    {__weather_brief() if alarm_info["include_weather"] else ""}
    {__news_brief() if alarm_info["include_news"] else ""}
    This is the end of your briefing. Have a nice day.
    """

    logging.info("brief message: %s", brief_message)
    __speech_engine.say(brief_message)
    __speech_engine.runAndWait()

def __covid_brief() -> str:
    """
    Generates a brief message about the latest covid19 data.
    """

    try:
        (
            is_latest_covid_data_available,
            _,
            new_cases,
            cumulative_cases,
            new_deaths,
            cumulative_deaths,
        ) = fetch_covid_data()

        return f"""
        First, some Covid-19 update.
        {'Today, ' if is_latest_covid_data_available else 'Latest data is not available, so previous data will be recapped.'}
        In England, there are {new_cases} number of new cases,
        and unfortunately {new_deaths} people lost their battle against Covid19.
        In total, there are {cumulative_cases} number of cases,
        and {cumulative_deaths} lives are lost in this pandemic.
        """
    except:
        return "Unfortunately an error occurred when getting latest covid data."

def __weather_brief() -> str:
    """
    Generates a brief message about the current weather.
    """

    weather = fetch_weather(lat=50.718410, long=-3.533899)

    return f"""
    Currently in your location, expect {weather['weather'][0]['description']}.
    The temperature is {weather['main']['temp']}.
    """

def __news_brief() -> str:
    """
    Generates a brief message of latest news headlines.
    """

    news = fetch_news_headlines(country="gb")

    if len(news) == 0:
        return "There are no top news for you right now."

    return "Also, here are some top news headlines for you.\n" + "\n".join(
        [f"{article['title']}; " for article in news]
    )


# Future developments can include getting total wallet (including tokens) or tracking multiple wallets
import requests

params = {
    'collection': 'the-wanderers',
    'limit': 1
}

r = requests.get("https://api.opensea.io/api/v1/assets", params=params)

print(r.json())
# def current_price():
#
# if __name__ == "__main__":
#     print("Hello World")
# input opensea link or api call results
# pull all NFTs from a single wallet
# create webapp to list individual prices of each NFT and total price at end
# compare last sold price and current lowest listing price and take the min of both values
# return min


import sys
from collections import defaultdict, deque
read = sys.stdin.readline

def dfs(g, v):
    stack = [v]
    way = []
    while stack:
        node = stack.pop()
        if node not in way:
            way.append(node)
            stack.extend(sorted(g[node], reverse=True))
    return way

def bfs(g, v):
    dq = deque([v])
    way = []
    while dq:
        node = dq.popleft()
        if node not in way:
            way.append(node)
            dq.extend(sorted(g[node]))
    return way

if __name__ == "__main__":
    n, m, v = map(int, read().split())
    graph = defaultdict(list)
    for _ in range(m):
        a, b = map(int, read().split())
        graph[a].append(b)
        graph[b].append(a)
    print(*dfs(graph, v))
    print(*bfs(graph, v))


import copy

import matplotlib.pyplot as plt
import matplotlib as mpl
import numpy as np
import seaborn as sns
import pandas as pd
from scipy.io import loadmat
from scipy.signal import savgol_filter
from scipy import fftpack
from scipy import signal
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from scipy.stats.kde import gaussian_kde
from matplotlib.ticker import MaxNLocator
from matplotlib import animation, rc
import hdf5storage
import pwlf
import time
from IPython.display import display, clear_output
from sklearn.preprocessing import StandardScaler
from obspy.signal import cross_correlation
import copy

def node_reduction(data: np.ndarray, n_clusters: int, max_n_clusters=20, n_components=12, sample_labels=None, plot=True):
    """ Reduces node dimension to n_clusters.

    :param data: Data of size (n_samples, n_nodes)
    :param n_clusters: Number of clusters
    :param max_n_clusters: Maximum number of clusters for evaluation
    :param n_components: Number of PCA components
    :return: df: DataFrame
    """
    n_samples = data.shape[0]
    n_nodes = data.shape[1]

    # Perform PCA for dimensionality reduction
    print('Perform PCA')
    pca = PCA(n_components=n_components)
    pca_result = pca.fit_transform(data.T)

    if plot:
        fig = plt.figure(figsize=(7,3.4))
        gs = fig.add_gridspec(nrows=2, ncols=21)
        ax = [[] for i in range(3)]
        cm = plt.cm.get_cmap('viridis')

        ax[0] = fig.add_subplot(gs[:1, :6])
        plt.scatter(pca_result[:, 0], pca_result[:, 1], c=pca_result[:, 2], cmap=cm)
        plt.clim(np.min(pca_result[:, 2]), np.max(pca_result[:, 2]))
        ax[0].set_xlabel('1st PC [a.u.]')
        ax[0].set_ylabel('2nd PC [a.u.]')
        ax[0].spines['right'].set_visible(False), ax[0].spines['top'].set_visible(False)
        cb = plt.colorbar(ax=ax[0])
        cb.set_label(label='3rd PC [a.u.]')

    # Check score of K-Means for max_n_clusters
    n_clusters_list = list(range(1, max_n_clusters + 1))
    score = []
    for i, val in enumerate(n_clusters_list):
        km = KMeans(n_clusters=val)
        clusters = km.fit(pca_result)
        score.append(clusters.inertia_ / n_nodes)

    if plot:
        ax[1] = fig.add_subplot(gs[:1, 9:14])
        plt.plot(n_clusters_list, score, c='black')
        ax[1].set_xlabel('Nr. of clusters')
        ax[1].set_ylabel('MSE [a.u.]')
        #plt.title('Distance to nearest cluster center')
        ax[1].xaxis.set_major_locator(MaxNLocator(integer=True))
        ax[1].spines['right'].set_visible(False), ax[1].spines['top'].set_visible(False)

    # Perform K-Means on data
    print('Perform KMeans')
    km = KMeans(n_clusters=n_clusters)
    clusters = km.fit(pca_result)

    df = pd.DataFrame(pca_result)
    df['Cluster'] = clusters.labels_

    if plot:
        ax[2] = fig.add_subplot(gs[:1, 16:])
        palette = sns.color_palette('muted', n_clusters)
        sns.scatterplot(x=0, y=1, data=df, s=50, hue='Cluster', style='Cluster', palette=palette)
        ax[2].set_xlabel('1st PC [a.u.]')
        ax[2].set_ylabel('2nd PC [a.u.]')
        ax[2].spines['right'].set_visible(False), ax[2].spines['top'].set_visible(False)
        ax[2].get_legend().remove()
        #plt.title('Clustering of the first two princ. components')
        fig.savefig('figures/Ch2_NodeReduction.png', dpi=300)

    # Make DataFrame
    df = pd.DataFrame()
    df['sample'] = np.tile(np.arange(0, n_samples), n_nodes)
    df['node'] = np.repeat(np.arange(0, n_nodes), n_samples)
    df['cluster'] = np.repeat(list(clusters.labels_), n_samples)
    df['value'] = data.T.flatten()
    if sample_labels is not None:
        df['sample_label'] = np.tile(sample_labels, n_nodes)

    return df

def acf(data: np.ndarray, n_lags, partial=False):
    """ Auto Correlation Function.

    :param data: Array of size (n_samples, n_nodes)
    :param n_lags: Lag size [samples]
    :param partial: Choose for partial or non-partial
    :return:
    """
    n_samples = data.shape[0]
    n_nodes = data.shape[1]

    lags = [i for i in range(n_lags)]
    acf_list = []

    if partial:
        for i in range(n_nodes):
            corr = [1. if lag == 0 else np.corrcoef(data[lag:, i], data[:-lag, i])[0][1] for lag in lags]
            acf_list.append(np.array(corr))
            acf_array = np.array(acf_list).T
    else:
        for i in range(n_nodes):
            mean = np.mean(data[:, i])
            var = np.var(data[:, i])
            data_p = data[:, i] - mean
            corr = [1. if lag == 0 else np.sum(data_p[lag:] * data_p[:-lag]) / n_samples / var for lag in lags]
            acf_list.append(np.array(corr))
            acf_array = np.array(acf_list).T

    # Confidence
    conf = (1.96/np.sqrt(n_samples), -1.96/np.sqrt(n_samples))

    return acf_array, conf

def fft(data: np.ndarray, fs: float, downscale_factor=None):
    """

    :param data:
    :param fs:
    :param downscale_factor:
    :return:
    """
    n_samples = data.shape[0]
    n_nodes = data.shape[1]

    spect = np.zeros((n_samples, n_nodes))
    for i in range(n_nodes):
        print('Compute FFT of channel: ' + str(i))
        spect[:, i] = np.abs(fftpack.fft(data[:, i]))
    freq = fftpack.fftfreq(n_samples) * fs

    # Cut negative frequencies
    spect_pos = spect[:len(freq[freq >= 0])]
    freq_pos = freq[freq >= 0]

    if downscale_factor is not None:
        spect_resampled = []
        print('Downscale FFTs')
        for i in range(n_nodes):
            spect_filtered = savgol_filter(np.abs(spect_pos[:, i]), int(n_samples/500)-2, 3)
            spect_resampled.append(signal.resample_poly(spect_filtered, up=3, down=3*downscale_factor))
        freq_resampled = signal.resample_poly(freq_pos, up=3, down=3*downscale_factor)

        return np.array(spect_resampled).T, freq_resampled

    return spect_pos, freq_pos

def distribution(data: np.ndarray, xlim: tuple=None):
    """

    :param data:
    :param xlim:
    :return:
    """
    n_nodes = data.shape[1]

    if xlim is None:
        xlim = (np.min(data), np.max(data))
    kde_x = np.linspace(xlim[0], xlim[1], 200)
    density = []
    for i in range(n_nodes):
        print('Compute KDE of channel: ' + str(i))
        kde = gaussian_kde(data[:, i])
        density.append(kde(kde_x))

    return kde_x, np.array(density).T

def plot_distribution(data: np.ndarray, xlim: tuple=None, n_clusters: int=6, qq_plot: bool=True, title: str=''):
    print('Status: Get distributions.')
    kde_x, kde = distribution(data, xlim=xlim)

    print('Status: Reduction of nodes.')
    df_kde = node_reduction(kde, n_clusters=n_clusters, sample_labels=kde_x, plot=False)

    sns.set_style('whitegrid')
    palette = sns.color_palette('muted', n_clusters)

    if qq_plot:
        normal = np.sort(np.random.normal(size=data.shape[0]))
        df_qq = node_reduction(np.sort(data, axis=0), n_clusters=n_clusters, sample_labels=normal)

        fig = plt.figure(figsize=(8, 16))
        gs = fig.add_gridspec(nrows=int(n_clusters / 2 + 1), ncols=2)
        ax = [[] for i in range(n_clusters)]
        ax[0] = fig.add_subplot(gs[:1, :])
        sns.lineplot(x='sample_label', y='value', data=df_kde, hue='cluster', palette=palette, ax=ax[0])
        ax[0].set_xlabel('Magnitude')
        ax[0].set_ylabel('Density')
        ax[0].set_title(title)

        row = np.repeat(np.arange(1, int(n_clusters / 2 + 2)), 2)
        col = np.tile(np.array([[0, 1], [1, 2]]), 3)
        for i in range(n_clusters):
            ax[i] = fig.add_subplot(gs[row[i]:row[i + 2], col[0, i]:col[1, i]])
            for k in range(df_qq.shape[0]):
                if df_qq['cluster'][k] == i:
                    node = df_qq['node'][k]
                    break
            sns.scatterplot(x='sample_label', y='value', data=df_qq.where(df_qq['node'] == node),
                            ax=ax[i], label='from cluster ' + str(i), edgecolor=None, color=palette[i])
            ax[i].set_xlabel('Theoretical quantiles')
            ax[i].set_ylabel('Sample quantiles')
            ax[i].legend()
        plt.tight_layout()

    else:
        plt.figure(figsize=(6, 4))
        sns.lineplot(x='sample_label', y='value', data=df_kde, hue='cluster', palette=palette)
        plt.xlabel('Magnitude')
        plt.ylabel('Density')
        plt.title('PCA on density distributions')

    print('Status: Save plot.')
    #plt.savefig('../code/figures/Ch2_DataDistribution' + title + '.png')

def acf_plot(data: np.ndarray, n_clusters=5, n_lags=1000):
    """

    :param data:
    :param n_clusters:
    :param n_lags:
    :return:
    """
    acf, conf = acf(data, n_lags=n_lags)
    df = node_reduction(acf, n_clusters)
    plt.figure(figsize=(12, 8))
    sns.set_style('whitegrid')
    plt.xlim(0, n_lags)
    sns.lineplot(x='sample', y='value', data=df, hue='cluster', palette='colorblind')
    plt.plot([0, n_lags], [conf[1], conf[1]], color='black', ls='--')
    plt.plot([0, n_lags], [conf[0], conf[0]], color='black', ls='--')
    plt.xlabel('Lag')
    plt.ylabel('Correlation')
    plt.title('Autocorrelation')
    plt.savefig('../doc/figures/preprocess_acf.png')

def fft_plot(data: np.ndarray, n_clusters=5):
    """

    :param data:
    :param n_clusters:
    :return:
    """
    spect, freq = pre.fft(train_set.numpy(), fs=512, downscale_factor=20)
    df = pre.node_reduction(spect, n_clusters, sample_labels=freq)
    plt.figure(figsize=(12, 8))
    sns.set_style('whitegrid')
    # for i in range(60):
    #    plt.plot(freq, spect[:, i], color='tab:blue')
    sns.lineplot(x='sample_label', y='value', data=df, hue='cluster', palette='colorblind')

    plt.plot([120, 120], [0, 800], color='black', ls='--')
    plt.xlabel('Frequency [Hz]')
    plt.ylabel('Amplitude [-]')
    title = ['High frequency spectrum', 'Low frequency spectrum']
    save_name = ['fft_high', 'fft_low']
    xlim = [150, 25]
    for i in range(2):
        plt.xlim(0, xlim[i])
        plt.ylim(0, 800)
        plt.title(title[i])
        if i == 1:
            plt.plot([.5, .5], [0, 800], color='black', ls='--')
        plt.savefig('../doc/figures/preprocess_' + save_name[i])

def anim_connectome(patient_ID: str, hour: str, sperseg: float, soverlap: float, save=False, anim=True, fps=10):
    """

    """
    # Import signal
    ID = patient_ID
    h = hour
    data = hdf5storage.loadmat('../data/ID' + ID + '_' + h + '.mat')
    info = loadmat('../data/ID' + ID + '_info.mat')
    data = data['EEG']
    fs = float(info['fs'])

    nperseg = int(sperseg * fs)
    noverlap = int(soverlap * fs)

    corrmats = np.zeros((data.shape[0], data.shape[0], int((data.shape[1] - nperseg) / (nperseg - noverlap))))
    for i in range(corrmats.shape[2]):
        corrmats[:, :, i] = np.corrcoef(data[:, int(i * (nperseg - noverlap)):int(i * (nperseg - noverlap) + nperseg)])
    corr_val = np.mean(np.mean(np.abs(corrmats), axis=0), axis=0)

    if save:
        np.save('../data/corrmat_ID' + ID + '_' + h + '.npy', corrmats)
        np.save('../data/corrmean_ID' + ID + '_' + h + '.npy', corr_val)

    if anim:
        # equivalent to rcParams['animation.html'] = 'html5'
        rc('animation', html='html5')

        def update(frame):
            level.set_array(corrmats[:, :, frame].ravel())
            dot.set_data(t[frame], corr_val[frame])
            line.set_data([t[frame], t[frame]], [0, 30])
            return level

        corr = np.ones((corrmats.shape[0], corrmats.shape[0]))
        sns.set_style('whitegrid')

        fig = plt.figure(figsize=(10, 10))
        gs = fig.add_gridspec(2, 2)

        ax0 = fig.add_subplot(gs[:1, :])
        f, t, Sxx = signal.spectrogram(data[0, :], fs, nperseg=nperseg, noverlap=noverlap)
        t = t[:-1] / 60
        ax0.pcolormesh(t, f, Sxx[:, :-1], vmin=0, vmax=20, cmap='viridis')
        ax0.set_title('ID' + ID + ' h' + h + ' Spectrogram of Channel 00')
        ax0.set_ylabel('Frequency [Hz]')
        ax0.set_ylim(0, 30)
        ax0.set_xlabel('Time [min]')
        line, = ax0.plot([], [], color='red')

        ax1 = fig.add_subplot(gs[1:, :1])
        level = ax1.pcolormesh(corr, cmap='bwr', vmin=-1, vmax=1)
        ax1.invert_yaxis()
        ax1.set_title('Correlation matrix')

        ax2 = fig.add_subplot(gs[1:, 1:])
        ax2.plot(t, corr_val)
        ax2.set_title('Mean abs. correlation')
        ax2.set_xlim(0, t[-1])
        ax2.set_ylim(0, np.max(corr_val) + 0.2)
        ax2.set_xlabel('Time [min]')
        dot, = ax2.plot([], [], 'o', color='red')

        ani = animation.FuncAnimation(fig, update, frames=corrmats.shape[2], interval=20)

        ani.save('../doc/figures/corr_ID' + ID + '_' + h + '.gif', writer='imagemagick', fps=fps)

def nonlin_corr_metric(x: np.ndarray, y: np.ndarray, t_shift: int):
    """
    :param x:
    :param y:
    :param t_shift: In samples
    :param save_name:
    :return:
    """
    break_locs = np.linspace(-5, 5, 50)
    h2_shift = []  # Array with correlation per time shift
    r2_shift = []  # Array with correlation per time shift

    for i in range(2 * t_shift):
        # Cut data to time shift
        y_data = y[2 * t_shift - i: len(y)-i]
        x_data = x[t_shift: -t_shift]
        # Normalize data
        y_norm = (y_data - np.mean(y_data)) / np.std(y_data)
        x_norm = (x_data - np.mean(x_data)) / np.std(x_data)
        # Non-linear fit
        g = pwlf.PiecewiseLinFit(x_norm, y_norm)
        g.fit_with_breaks(break_locs)
        expect = np.mean(np.abs(y_norm - g.predict(x_norm)))  # Uniform distribution
        h2_shift.append(1 - expect**2 / np.var(y_norm))
        # Linear fit
        r2_shift.append(np.corrcoef(x_norm, y_norm)[0, 1]**2)

    h2 = h2_shift[int(np.argmax(np.asarray(h2_shift)))]
    h2_dt = int(np.argmax(np.asarray(h2_shift)) - t_shift)  # dt for best correlation from x to y
    r2 = r2_shift[int(np.argmax(np.asarray(r2_shift)))]
    r2_dt = int(np.argmax(np.asarray(r2_shift)) - t_shift)  # dt for best correlation from x to y

    return r2, h2, r2_dt, h2_dt, r2_shift, h2_shift

def plot_nonlin_corr(r2, r2_dt=None, h2=None, h2_dt=None, save_name='default'):
    fs = 256

    if h2 is not None:
        sns.set_style('ticks')
        fig = plt.figure(figsize=(7.4, 3))
        gs = fig.add_gridspec(nrows=1, ncols=2)
        cmap = 'viridis'

        ax0 = fig.add_subplot(gs[:, :1])
        sns.heatmap(r2, cmap=cmap, cbar_kws={'label': '$r_2$ [-]'})
        #ax0.set_title('r2 linear correlation')
        #ax0.get_xaxis().set_visible(False)

        ax1 = fig.add_subplot(gs[:, 1:])
        sns.heatmap(h2, cmap=cmap, cbar_kws={'label': '$h_2$ [-]'})
        #ax1.set_title('h2 non-linear correlation')
        #ax1.get_xaxis().set_visible(False), ax1.get_yaxis().set_visible(False)

        #ax2 = fig.add_subplot(gs[1:, :1])
        #vlim = np.max(np.abs(r2_dt))
        #sns.heatmap(r2_dt / fs * 1000, cmap='bwr', vmin=-vlim, vmax=vlim)
        #ax2.set_title('r2 time shift [ms]')

        #ax3 = fig.add_subplot(gs[1:, 1:])
        #vlim = np.max(np.abs(h2_dt))
        #sns.heatmap(h2_dt / fs * 1000, cmap='bwr', vmin=-vlim, vmax=vlim)
        #ax3.set_title('h2 time shift [ms]')
        #ax3.get_yaxis().set_visible(False)

        ax0.set_xlabel('Channel index [-]')
        ax0.set_ylabel('Channel index [-]')
        ax1.set_xlabel('Channel index [-]')
        ax1.set_ylabel('Channel index [-]')

    else:
        fig = plt.figure(figsize=(8, 10))
        gs = fig.add_gridspec(2, 1)
        cmap = 'viridis'

        ax0 = fig.add_subplot(gs[:1, :1])
        sns.heatmap(r2, cmap=cmap)
        ax0.set_title('r2 linear correlation')
        ax0.get_xaxis().set_visible(False)

        #ax2 = fig.add_subplot(gs[1:, :1])
        #vlim = np.max(np.abs(r2_dt))
        #sns.heatmap(r2_dt / fs * 1000, cmap='bwr', vmin=-vlim, vmax=vlim)
        #ax2.set_title('r2 time shift [ms]')

    plt.tight_layout()
    plt.savefig('../doc/figures/fc_' + save_name + '.png', dpi=300)
    plt.close()

def nonlin_corr(patient_id: str, time_begin: list, duration: float,
                           t_shift=0.2, plot_name=None):
    """

    :param patient_id:
    :param time_begin: List with [hour, minute].
    :param duration: In seconds.
    :param t_shift: In seconds.
    :param plot_name:
    :return:
    """
    # Load and prepare data
    data_mat = loadmat('../data/' + patient_id + '_' + str(time_begin[0]) + 'h.mat')
    info_mat = loadmat('../data/' + patient_id + '_info.mat')
    fs = float(info_mat['fs'])
    sample_begin = int(time_begin[1] * 60 * fs)
    sample_end = sample_begin + int(duration * fs)
    data_raw = data_mat['EEG'][:20, sample_begin:sample_end].transpose()
    # Resample to 256 Hz
    data_raw = signal.resample(data_raw, num=int(data_raw.shape[0] / fs * 256), axis=0)
    fs = 256

    # Compute correlation
    h2 = np.zeros((data_raw.shape[1], data_raw.shape[1]))
    r2 = np.zeros((data_raw.shape[1], data_raw.shape[1]))
    h2_dt = np.zeros((data_raw.shape[1], data_raw.shape[1]))
    r2_dt = np.zeros((data_raw.shape[1], data_raw.shape[1]))
    #h2_shift_list = []
    #r2_shift_list = []

    start_time = time.time()

    for i in range(data_raw.shape[1]):
        for j in range(data_raw.shape[1]):
            r2[i, j], h2[i, j], r2_dt[i, j], h2_dt[i, j], _, _ = nonlin_corr_metric(
                data_raw[:, i], data_raw[:, j], t_shift=int(t_shift * fs))
            # h2_shift_list.append(h2_shift)
            # r2_shift_list.append(r2_shift)
        t_rem = (time.time() - start_time) / (i + 1) * (data_raw.shape[1] - i + 1)
        clear_output()
        print(f'Computed columns: {i}/{data_raw.shape[1]} | Time remaining [min]: {t_rem / 60:.3}')

    save_name = '../data/fc_' + patient_id + '_' + str(time_begin[0]) + 'h' + str(time_begin[1]) + 'min_final'
    np.save(save_name + '_r2.npy', r2)
    np.save(save_name + '_r2_dt.npy', r2_dt)
    np.save(save_name + '_h2.npy', h2)
    np.save(save_name + '_h2_dt.npy', h2_dt)

    if plot_name is not None:
        plot_nonlin_corr(r2=r2, h2=h2, r2_dt=r2_dt, h2_dt=h2_dt, save_name=plot_name)

def lin_corr(patient_id: str, time_begin: list, duration: float, t_lag=0.7, critical_corr=0.7):
    """

    :param patient_id:
    :param time_begin: List with [hour, minute].
    :param duration: In seconds.
    :param t_lag: In seconds.
    :param critical_corr:
    :return:
    """
    # Load and prepare data
    data_mat = loadmat('../data/' + patient_id + '_' + str(time_begin[0]) + 'h.mat')
    info_mat = loadmat('../data/' + patient_id + '_info.mat')
    fs = float(info_mat['fs'])
    sample_begin = int(time_begin[1] * 60 * fs)
    sample_end = sample_begin + int(duration * fs)
    data_raw = data_mat['EEG'][:, sample_begin:sample_end].transpose()

    # fs = 1024
    # t = np.linspace(0, 5*60, 5*60*fs)
    # f = [.5, 4, 20, 50, 120]
    # data_raw = np.zeros((len(t), len(f)))
    # for i, val in enumerate(f):
    #     data_raw[:, i] = 100 * np.sin(2 * np.pi * val * t)

    n_lag = int(t_lag * fs)
    factor = 0.99

    # Compute normalized cross correlation (NCC)
    cctl = np.zeros((data_raw.shape[1], data_raw.shape[1], (n_lag * 2) + 1))
    for from_ in range(data_raw.shape[1]):
        for to_ in range(data_raw.shape[1]):
            x = data_raw[:, to_]
            y = data_raw[n_lag:-n_lag, from_]
            cctl[from_, to_, :] = cross_correlation.correlate_template(x, y)

    # Calculate peak cross correlation (cc) and corresponding time lag (tl)
    sign = np.sign(np.max(cctl, axis=2) - np.abs(np.min(cctl, axis=2)))
    cc = np.multiply(np.max(np.abs(cctl), axis=2), sign)
    mask = np.where(np.abs(cc) > critical_corr, 1, np.nan)
    tl_n = np.argmax(np.abs(cctl), axis=2)
    tl = (tl_n - n_lag) * mask / fs * 1000  # in [ms]
    tl_no_mask = (tl_n - n_lag) / fs * 1000  # in [ms], used for plots

    # Calculate mean tau
    # Tile and stack values for future operations
    tl_n_stacked = np.dstack([tl_n] * cctl.shape[2])
    arg_tau_stacked = factor * np.dstack([cc] * cctl.shape[2])
    mask_stacked = np.dstack([np.where(np.abs(cc) > critical_corr, 1, 0)] * cctl.shape[2])
    t_indices_tiled = np.tile(np.arange(0, cctl.shape[2]), (cctl.shape[0], cctl.shape[0], 1))
    # Get indices of values close to factor of peak cross correlation
    close_indices = np.isclose(cctl, arg_tau_stacked, rtol=1e-1) * t_indices_tiled
    # Create mask to separate negative and positive tau
    higher_tau_mask = np.where(close_indices - tl_n_stacked > 0, 1, 0)
    lower_tau_mask = np.where((tl_n_stacked - close_indices > 0) & (close_indices != 0), 1, 0)
    # Eliminate possible third occurrence of np.isclose() to factor
    higher_edge_indices = np.where(np.diff(higher_tau_mask) == -1, 1, 0) * t_indices_tiled[:, :, :-1]
    higher_edge_indices = np.min(np.where(higher_edge_indices == 0, np.inf, higher_edge_indices), axis=2)
    higher_third_occ_mask = np.where(t_indices_tiled > np.dstack([higher_edge_indices] * cctl.shape[2]), 0, 1)
    lower_edge_indices = np.where(np.diff(lower_tau_mask) == 1, 1, 0) * t_indices_tiled[:, :, :-1]
    lower_edge_indices = np.max(lower_edge_indices, axis=2)
    lower_third_occ_mask = np.where(t_indices_tiled < np.dstack([lower_edge_indices] * cctl.shape[2]), 0, 1)
    # Apply masks (apply mask for critical correlation separately to get all taus for plots)
    higher_tau_masked_all = close_indices * higher_tau_mask * higher_third_occ_mask
    higher_tau_masked = higher_tau_masked_all * mask_stacked
    lower_tau_masked_all = close_indices * lower_tau_mask * lower_third_occ_mask
    lower_tau_masked = lower_tau_masked_all * mask_stacked
    # Compute median along time lag axis and ignore zero entries
    higher_tau = np.ma.median(np.ma.masked_where(higher_tau_masked == 0, higher_tau_masked), axis=2).filled(0)
    lower_tau = np.ma.median(np.ma.masked_where(lower_tau_masked == 0, lower_tau_masked), axis=2).filled(0)
    # Get taus without mask for critical correlation for plots
    higher_tau_all = np.ma.median(np.ma.masked_where(higher_tau_masked_all == 0, higher_tau_masked_all)
                                  , axis=2).filled(0)
    higher_tau_all = (higher_tau_all - n_lag) / fs * 1000  # in [ms]
    lower_tau_all = np.ma.median(np.ma.masked_where(lower_tau_masked_all == 0, lower_tau_masked_all)
                                 , axis=2).filled(0)
    lower_tau_all = (lower_tau_all - n_lag) / fs * 1000  # in [ms]
    # Calculate mean distance for tau to cc
    tau_n = (higher_tau - lower_tau) / 2
    tau = np.where(tau_n == 0, np.nan, tau_n) / fs * 1000  # in [ms]
    tau = np.abs(tau)  # this line has to be removed in the future

    # Additional masks for plots (diagonal, upper triangle, ...)
    tl_masked = tl.copy()
    cc_masked = cc.copy()
    np.fill_diagonal(tl_masked, np.nan)
    np.fill_diagonal(cc_masked, np.nan)
    cc_masked[np.triu_indices(cc_masked.shape[0], k=1)] = np.nan

    # Plot cc, tl and tau
    # General settings
    sns.set_style('ticks')
    fig = plt.figure(figsize=(7, 9))
    gs = fig.add_gridspec(3, 10)
    cmap_div = copy.copy(mpl.cm.get_cmap('seismic'))
    cmap_div.set_bad('dimgrey')
    cmap_uni = copy.copy(mpl.cm.get_cmap('viridis'))
    cmap_uni.set_bad('dimgrey')

    # Subplot: Peak cross correlation
    ax0 = fig.add_subplot(gs[:1, :5])
    sns.heatmap(cc_masked, cmap=cmap_div, vmin=-1, vmax=1, cbar_kws={'label': 'Peak cross correlation [-]'})
    ax0.set_xlabel('Channel index [-]'), ax0.set_ylabel('Channel index [-]')

    # Subplot: Histogram of peak cross correlation
    sns.set_style('white')
    ax1 = fig.add_subplot(gs[:1, 6:])
    ax1.add_patch(mpl.patches.Rectangle(xy=(-critical_corr, 0), width=2*critical_corr, height=330,
                                        facecolor='red', edgecolor='red', alpha=.1))
    sns.distplot(cc_masked, kde=False)
    ymin, ymax = ax1.get_ylim()
    xmin, xmax = ax1.get_xlim()
    #label = 'Critical corr. = +/- ' + str(critical_corr)
    #plt.plot([critical_corr, critical_corr], [ymin, ymax], linestyle='--', color='black', label=label)
    #plt.plot([-critical_corr, -critical_corr], [ymin, ymax], linestyle='--', color='black')
    ax1.set_xlim(xmin, xmax), ax1.set_ylim(ymin, 330)
    #plt.legend()
    ax1.set_xlabel('Peak cross correlation [-]'), ax1.set_ylabel('Nr. of occurrence [-]')
    ax1.spines['right'].set_visible(False), ax1.spines['top'].set_visible(False)

    # Subplot: Time lag
    sns.set_style('ticks')
    ax2 = fig.add_subplot(gs[1:2, :5])
    vlim = np.nanmax(np.abs(tl))
    sns.heatmap(tl_masked, cmap=cmap_div, vmin=-vlim, vmax=vlim, cbar_kws={'label': 'Corresponding time lag [ms]'})
    ax2.set_xlabel('Channel index [-]'), ax2.set_ylabel('Channel index [-]')

    # Subplot: Histogram of time lag
    sns.set_style('white')
    ax3 = fig.add_subplot(gs[1:2, 6:])
    sns.distplot(tl_masked, kde=False, bins=[(i - 11) * 2 - 1 for i in range(24)])
    ax3.set_xlim(-22, 22)
    ax3.set_xlabel('Time lag [ms]'), ax3.set_ylabel('Nr. of occurrence [-]')
    ax3.spines['right'].set_visible(False), ax3.spines['top'].set_visible(False)

    # Subplot: Tau
    sns.set_style('ticks')
    ax4 = fig.add_subplot(gs[2:, :5])
    sns.heatmap(tau, cmap=cmap_uni, cbar_kws={'label': 'Corresponding $\\tau$ [ms]'})
    ax4.set_xlabel('Channel index [-]'), ax4.set_ylabel('Channel index [-]')

    # Subplot: Histogram of tau
    sns.set_style('white')
    ax5 = fig.add_subplot(gs[2:, 6:])
    auto_corr = tau.copy()
    auto_corr[np.diag_indices(auto_corr.shape[0])] = np.nan
    sns.distplot(auto_corr, kde=False, label='Cross correlated')
    sns.distplot(np.diagonal(tau), kde=False, label='Auto correlated', bins=10)
    plt.legend(loc='upper left', frameon=False)
    ax5.set_xlabel('$\\tau$ [ms]'), ax5.set_ylabel('Nr. of occurrence [-]')
    ax5.spines['right'].set_visible(False), ax5.spines['top'].set_visible(False)

    plt.tight_layout()
    save_name = patient_id + '_' + str(time_begin[0]) + 'h' + str(time_begin[1]) + 'm'
    plt.savefig('../doc/figures/cc_' + save_name + '.png', dpi=300)
    plt.close()

    # t vector for plots
    sns.set_style('ticks')
    plt.figure(figsize=(5, 4))
    t = np.arange(0, cctl.shape[2])
    t = (t - n_lag) / fs * 1000
    n0 = 2  # Base node
    N = 5  # Number of line plots
    peaks_x, peaks_y, taus_x_0, taus_x_1, taus_y = [], [], [], [], []
    # Colors
    cmap = plt.get_cmap('viridis')
    colors = [cmap(i) for i in np.linspace(0, 1, N)]
    ax6 = plt.gca()
    ax6.add_patch(mpl.patches.Rectangle(xy=(-t_lag*1000, -critical_corr), width=t_lag*2000, height=2*critical_corr,
                                        facecolor='red', edgecolor='red', alpha=.1))

    for i in range(N):
        n1 = n0 + i - int(N/2)  # Reference node
        plt.plot(t, cctl[n0, n1, :], color='black', lw='1.2')
        if n1 == n0:
            plt.plot(t, cctl[n0, n1, :], color='red', lw='.9')
        else:
            plt.plot(t, cctl[n0, n1, :], color=colors[i], lw='.9')
        if cc[n0, n1] >= critical_corr or cc[n0, n1] <= -critical_corr:
            peaks_x.append(tl_no_mask[n0, n1])
            peaks_y.append(cc[n0, n1])
            taus_x_0.append(higher_tau_all[n0, n1])
            taus_x_1.append(lower_tau_all[n0, n1])
            taus_y.append(cc[n0, n1] * factor)

    #cb = plt.colorbar(ax=ax)
    #cb.set_label(label='Nodes')

    plt.scatter(peaks_x, peaks_y, color='black', marker='d', label='Peak', zorder=N + 1)
    plt.scatter(taus_x_0, taus_y, color='black', marker='<', label='Right tau', zorder=N + 1)
    plt.scatter(taus_x_1, taus_y, color='black', marker='>', label='Left tau', zorder=N + 1)
    ymin, ymax = plt.gca().get_ylim()
    plt.plot([0, 0], [ymin, ymax], color='black', lw='.3')
    #         color='black', linestyle='--', label='Critical corr.', lw=.5)
    #plt.plot([-t_lag*1000, t_lag*1000], [-critical_corr, -critical_corr], color='black', linestyle='--', lw=.5)
    plt.ylim(ymin, ymax)
    plt.xlabel('Time lag [ms]'), plt.ylabel('NCC [-]')
    #plt.title('Normalized cross correlation: examples')
    plt.legend(loc='upper right', frameon=False)
    plt.xlim(-t_lag*1000, t_lag*1000)
    ax6.spines['right'].set_visible(False), ax6.spines['top'].set_visible(False)
    save_name = patient_id + '_' + str(time_begin[0]) + 'h' + str(time_begin[1]) + 'm'
    plt.savefig('../doc/figures/cctl_' + save_name + '.png', dpi=300)
    plt.close()

    # Plot raw signal
    plt.figure(figsize=(2.5, 6))
    sns.set_style('white')
    node_idx = [i + n0 - int(N / 2) for i in range(N)]
    plot_duration = 4  # [s]
    offset_array = np.linspace(0, (N - 1) * 200, N)
    for i in range(N):
        plt.plot(data_raw[:int(plot_duration * fs), node_idx[i]] + offset_array[i], color='black', lw='1.2')
        if i == int(N/2):
            plt.plot(data_raw[:int(plot_duration * fs), node_idx[i]] + offset_array[i], color='red', lw='.8')
        else:
            plt.plot(data_raw[:int(plot_duration * fs), node_idx[i]] + offset_array[i], color=colors[i], lw='.8')
    plt.plot([(plot_duration - 1) * fs, plot_duration * fs], [-200, -200], color='black', lw='1.5')
    plt.plot([plot_duration * fs - 8, plot_duration * fs - 8], [-200, 100], color='black', lw='1.5')
    ax7 = plt.gca()
    ax7.set_yticks(offset_array)
    ax7.set_yticklabels(['Ch. ' + str(i) for i in node_idx])
    ax7.get_yticklabels()[int(N/2)].set_weight('bold')
    ax7.set_xticklabels([])
    ax7.set_xlim((-10, int(plot_duration * fs)))
    ax7.spines['right'].set_visible(False), ax7.spines['top'].set_visible(False)
    ax7.spines['left'].set_visible(False), ax7.spines['bottom'].set_visible(False)
    plt.tight_layout()
    plt.savefig('../doc/figures/ccex_' + save_name + '.png', dpi=300)
    plt.close()

def determine_sample_size(patient_id: list=None, time_begin: list=None, max_sample_size: float=None, dt: float=None,
                          save_name='default', load_name=None):
    """

    :param patient_id: List with string or multiple strings.
    :param time_begin: List with list with [hour, minute] or multiple lists.
    :param max_sample_size: In seconds.
    :param dt: In seconds.
    :param save_name: String.
    :param load_name: If not None, only plot is applied.
    :return:
    """
    if load_name is None:
        corr_dts = []
        t_size = np.arange(dt, max_sample_size, dt).tolist()
        for i, id_ in enumerate(patient_id):
            print('Computes job ' + str(i) + '/' + str(len(patient_id) - 1))
            # Load and prepare data
            data_mat = loadmat('../data/' + id_ + '_' + str(time_begin[i][0]) + 'h.mat')
            info_mat = loadmat('../data/' + id_ + '_info.mat')
            fs = float(info_mat['fs'])
            sample_begin = int(time_begin[i][1] * 60 * fs)
            sample_end = sample_begin + int(max_sample_size * fs)
            data_raw = data_mat['EEG'][:, sample_begin:sample_end].transpose()

            # Get correlation matrices
            corr = np.zeros((len(t_size) + 1, data_raw.shape[1], data_raw.shape[1]))
            corr_dt = np.zeros((len(t_size), data_raw.shape[1], data_raw.shape[1]))
            sc = StandardScaler()
            for j, t in enumerate(t_size):
                data_norm = sc.fit_transform(data_raw[:int(t * fs), :])
                corr[j + 1, :, :] = np.corrcoef(data_norm.T)
                corr_dt[j, :, :] = corr[j + 1, :, :] - corr[j, :, :]

            corr_dts.append(np.mean(np.mean(np.abs(corr_dt), axis=1), axis=1))

        # Make DataFrame
        df = pd.DataFrame()
        for i, id_ in enumerate(patient_id):
            sub_df = pd.DataFrame()
            sub_df['unique_id'] = (np.ones(len(t_size)) * i).astype(int)
            sub_df['Patient ID'] = [id_ for _ in range(len(t_size))]
            sub_df['dt'] = [dt for _ in range(len(t_size))]
            time_begin_str = id_ + ': ' + str(time_begin[i][0]) + 'h' + str(time_begin[i][1]) + 'm'
            sub_df['time_begin'] = [time_begin_str for _ in range(len(t_size))]
            sub_df['corr_dt'] = corr_dts[i]
            sub_df['t_size'] = t_size
            df = df.append(sub_df, ignore_index=True)
        df.to_pickle('../data/sample_size_det_' + save_name + '.pkl')

    else:
        df = pd.read_pickle('../data/sample_size_det_' + load_name + '.pkl')

    # Plot results
    palette = sns.color_palette('muted', 3)
    df['t_size'] = df['t_size']/60
    sns.set_style('ticks')
    plt.figure(figsize=(4, 3))
    sns.lineplot(x='t_size', y='corr_dt', data=df, hue='Patient ID', palette=palette)
    plt.legend(frameon=False)
    plt.xlabel('Duration [min]')
    plt.ylabel('$\Delta R(d)$ [-]')
    plt.xlim(df['t_size'].min(), df['t_size'].max())
    plt.ylim(0, 0.04)
    ax = plt.gca()
    ax.spines['right'].set_visible(False), ax.spines['top'].set_visible(False)
    plt.tight_layout()
    plt.savefig('../doc/figures/sample_size_det_' + save_name + '.png', dpi=300)
    plt.close()

def determine_fc_change(patient_id: list=None, time_begin: list=None, max_sample_size: float=None, t_length: float=None,
                        save_name='default', load_name=None):
    if load_name is None:
        corr_dts = []
        dt = 60
        t_steps = np.arange(dt, max_sample_size, dt).tolist()
        swa = np.load('swa.npy')
        smooth = np.load('swa_smooth.npy')
        for i, id_ in enumerate(patient_id):
            print('Computes job ' + str(i) + '/' + str(len(patient_id) - 1))
            # Load and prepare data
            data_mat = loadmat('../data/' + id_ + '_' + str(time_begin[i][0]) + 'h.mat')
            info_mat = loadmat('../data/' + id_ + '_info.mat')
            fs = float(info_mat['fs'])
            sample_begin = int(time_begin[i][1] * 60 * fs)
            sample_end = sample_begin + int((max_sample_size + t_length) * fs)
            data_raw = data_mat['EEG'][:, sample_begin:sample_end].transpose()

            # Get correlation matrices
            corr = np.zeros((len(t_steps) + 1, data_raw.shape[1], data_raw.shape[1]))
            corr_dt = np.zeros((len(t_steps), data_raw.shape[1], data_raw.shape[1]))
            sc = StandardScaler()
            data_norm = sc.fit_transform(data_raw[int(0 * fs):int(t_length * fs), :])
            corr[0, :, :] = np.corrcoef(data_norm.T)
            for j, t in enumerate(t_steps):
                data_norm = sc.fit_transform(data_raw[int(t * fs):int((t + t_length) * fs), :])
                corr[j + 1, :, :] = np.corrcoef(data_norm.T)
                corr_dt[j, :, :] = corr[j + 1, :, :] - corr[j, :, :]

            corr_dts.append(np.mean(np.mean(np.abs(corr_dt), axis=1), axis=1))

            t_steps = np.asarray(t_steps)/60 + 0 # + 10
            #plt.figure(figsize=(3, 3))  # 2, 3
            fig, ax0 = plt.subplots(figsize=(4, 3))
            #ax0.scatter(t_steps, corr_dts[0], label='1. NREM Segment of P7', color='black', s=4)
            ax0.plot(t_steps, corr_dts[0], label='1. NREM Segment of P7', color='black', lw=1)
            ax0.set_xlabel('Starting time [min]')
            ax0.set_ylabel('$\Delta R(t_0)$ [-]')

            ax1 = ax0.twinx()
            ax1.plot(np.linspace(0, 60, len(smooth)), smooth, lw=1.7, color='darkred', alpha=.4)
            ax1.set_ylabel('SW band power\ndensity [$\mu$V$^2$/Hz]', color='darkred')
            ax1.tick_params(axis='y', labelcolor='darkred')

            ax0.spines['top'].set_visible(False), ax1.spines['top'].set_visible(False)
            plt.xlim(0, 60)
            plt.tight_layout()
            plt.savefig('figures/fig_Ch2_changeFC.png', dpi=300)
            plt.close()
            break