import logging
import json
import os

from django.http import HttpResponse
from pysearpc import SearpcError
from rest_framework import status
from rest_framework.authentication import SessionAuthentication
from rest_framework.permissions import IsAuthenticated
from rest_framework.views import APIView
import seaserv
from seaserv import seafile_api

from seahub.api2.authentication import TokenAuthentication
from seahub.api2.permissions import IsRepoAccessible
from seahub.api2.throttling import UserRateThrottle
from seahub.api2.utils import api_error
from seahub.base.templatetags.seahub_tags import email2nickname
from seahub.base.accounts import User
from seahub.share.signals import share_repo_to_user_successful, \
    share_repo_to_group_successful
from seahub.utils import (is_org_context, is_valid_username,
                          send_perm_audit_msg)

logger = logging.getLogger(__name__)
json_content_type = 'application/json; charset=utf-8'

class DirSharedItemsEndpoint(APIView):
    """Support uniform interface(list, share, unshare, modify) for sharing
    library/folder to users/groups.
    """
    authentication_classes = (TokenAuthentication, SessionAuthentication)
    permission_classes = (IsAuthenticated, IsRepoAccessible)
    throttle_classes = (UserRateThrottle, )

    def list_user_shared_items(self, request, repo_id, path):
        username = request.user.username
        if path == '/':
            share_items = seafile_api.list_repo_shared_to(username, repo_id)
        else:
            share_items = seafile_api.get_shared_users_for_subdir(repo_id,
                                                                  path, username)
        ret = []
        for item in share_items:
            ret.append({
                "share_type": "user",
                "user_info": {
                    "name": item.user,
                    "nickname": email2nickname(item.user),
                },
                "permission": item.perm,
            })
        return ret

    def list_group_shared_items(self, request, repo_id, path):
        username = request.user.username
        if path == '/':
            share_items = seafile_api.list_repo_shared_group_by_user(username, repo_id)
        else:
            share_items = seafile_api.get_shared_groups_for_subdir(repo_id,
                                                                   path, username)
        ret = []
        for item in share_items:
            ret.append({
                "share_type": "group",
                "group_info": {
                    "id": item.group_id,
                    "name": seaserv.get_group(item.group_id).group_name,
                },
                "permission": item.perm,
            })
        return ret

    def handle_shared_to_args(self, request):
        share_type = request.GET.get('share_type', None)
        shared_to_user = False
        shared_to_group = False
        if share_type:
            for e in share_type.split(','):
                e = e.strip()
                if e not in ['user', 'group']:
                    continue
                if e == 'user':
                    shared_to_user = True
                if e == 'group':
                    shared_to_group = True
        else:
            shared_to_user = True
            shared_to_group = True

        return (shared_to_user, shared_to_group)

    def get_sub_repo_by_path(self, request, repo, path):
        if path == '/':
            raise Exception("Invalid path")

        # get or create sub repo
        username = request.user.username
        if is_org_context(request):
            org_id = request.user.org.org_id
            sub_repo = seaserv.seafserv_threaded_rpc.get_org_virtual_repo(
                org_id, repo.id, path, username)
        else:
            sub_repo = seafile_api.get_virtual_repo(repo.id, path, username)

        return sub_repo

    def get_or_create_sub_repo_by_path(self, request, repo, path):
        username = request.user.username
        sub_repo = self.get_sub_repo_by_path(request, repo, path)
        if not sub_repo:
            name = os.path.basename(path)
            # create a sub-lib,
            # use name as 'repo_name' & 'repo_desc' for sub_repo
            if is_org_context(request):
                org_id = request.user.org.org_id
                sub_repo_id = seaserv.seafserv_threaded_rpc.create_org_virtual_repo(
                    org_id, repo.id, path, name, name, username)
            else:
                sub_repo_id = seafile_api.create_virtual_repo(repo.id, path,
                                                              name, name, username)
            sub_repo = seafile_api.get_repo(sub_repo_id)

        return sub_repo

    def get_repo_owner(self, request, repo_id):
        if is_org_context(request):
            return seafile_api.get_org_repo_owner(repo_id)
        else:
            return seafile_api.get_repo_owner(repo_id)

    def get(self, request, repo_id, format=None):
        """List shared items(shared to users/groups) for a folder/library.
        """
        repo = seafile_api.get_repo(repo_id)
        if not repo:
            return api_error(status.HTTP_404_NOT_FOUND, 'Library %s not found.' % repo_id)

        shared_to_user, shared_to_group = self.handle_shared_to_args(request)

        path = request.GET.get('p', '/')
        if seafile_api.get_dir_id_by_path(repo.id, path) is None:
            return api_error(status.HTTP_404_NOT_FOUND, 'Folder %s not found.' % path)

        ret = []
        if shared_to_user:
            ret += self.list_user_shared_items(request, repo_id, path)

        if shared_to_group:
            ret += self.list_group_shared_items(request, repo_id, path)

        return HttpResponse(json.dumps(ret), status=200,
                            content_type=json_content_type)

    def post(self, request, repo_id, format=None):
        """Update shared item permission.
        """
        username = request.user.username
        repo = seafile_api.get_repo(repo_id)
        if not repo:
            return api_error(status.HTTP_404_NOT_FOUND, 'Library %s not found.' % repo_id)

        path = request.GET.get('p', '/')
        if seafile_api.get_dir_id_by_path(repo.id, path) is None:
            return api_error(status.HTTP_400_BAD_REQUEST, 'Directory not found.')

        if username != self.get_repo_owner(request, repo_id):
            return api_error(status.HTTP_403_FORBIDDEN, 'Permission denied.')

        shared_to_user, shared_to_group = self.handle_shared_to_args(request)

        permission = request.data.get('permission', 'r')
        if permission not in ['r', 'rw']:
            return api_error(status.HTTP_400_BAD_REQUEST, 'permission invalid.')

        path = request.GET.get('p', '/')
        if seafile_api.get_dir_id_by_path(repo.id, path) is None:
            return api_error(status.HTTP_404_NOT_FOUND, 'Folder %s not found.' % path)

        if path == '/':
            shared_repo = repo
        else:
            try:
                sub_repo = self.get_sub_repo_by_path(request, repo, path)
                if sub_repo:
                    shared_repo = sub_repo
                else:
                    # unlikely to happen
                    return api_error(status.HTTP_404_NOT_FOUND, 'Failed to get sub repo')
            except SearpcError as e:
                logger.error(e)
                return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, 'Internal Server Error')

        if shared_to_user:
            shared_to = request.GET.get('username')
            if shared_to is None or not is_valid_username(shared_to):
                return api_error(status.HTTP_400_BAD_REQUEST, 'Email %s invalid.' % shared_to)

            try:
                User.objects.get(email=shared_to)
            except User.DoesNotExist:
                return api_error(status.HTTP_400_BAD_REQUEST, 'Invalid user, should be registered')

            if is_org_context(request):
                org_id = request.user.org.org_id
                seaserv.seafserv_threaded_rpc.org_set_share_permission(
                    org_id, shared_repo.id, username, shared_to, permission)
            else:
                seafile_api.set_share_permission(shared_repo.id, username,
                                                 shared_to, permission)

            send_perm_audit_msg('modify-repo-perm', username, shared_to,
                                repo_id, path, permission)

        if shared_to_group:
            gid = request.GET.get('group_id')
            try:
                gid = int(gid)
            except ValueError:
                return api_error(status.HTTP_400_BAD_REQUEST, 'group_id %s invalid.' % gid)
            group = seaserv.get_group(gid)
            if not group:
                return api_error(status.HTTP_404_NOT_FOUND, 'Group %s not found.' % gid)

            if is_org_context(request):
                org_id = request.user.org.org_id
                seaserv.seafserv_threaded_rpc.set_org_group_repo_permission(
                    org_id, gid, shared_repo.id, permission)
            else:
                seafile_api.set_group_repo_permission(gid, shared_repo.id,
                                                      permission)

            send_perm_audit_msg('modify-repo-perm', username, gid,
                                repo_id, path, permission)

        return HttpResponse(json.dumps({'success': True}), status=200,
                            content_type=json_content_type)

    def put(self, request, repo_id, format=None):
        username = request.user.username
        repo = seafile_api.get_repo(repo_id)
        if not repo:
            return api_error(status.HTTP_404_NOT_FOUND, 'Library %s not found.' % repo_id)

        path = request.GET.get('p', '/')
        if seafile_api.get_dir_id_by_path(repo.id, path) is None:
            return api_error(status.HTTP_404_NOT_FOUND, 'Folder %s not found.' % path)

        if username != self.get_repo_owner(request, repo_id):
            return api_error(status.HTTP_403_FORBIDDEN, 'Permission denied.')

        if path != '/':
            try:
                sub_repo = self.get_or_create_sub_repo_by_path(request, repo, path)
            except SearpcError as e:
                logger.error(e)
                return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, 'Failed to get sub repo.')
        else:
            sub_repo = None

        share_type = request.data.get('share_type')
        if share_type != 'user' and share_type != 'group':
            return api_error(status.HTTP_400_BAD_REQUEST, 'share_type invalid.')

        permission = request.data.get('permission', 'r')
        if permission not in ['r', 'rw']:
            return api_error(status.HTTP_400_BAD_REQUEST, 'permission invalid.')

        shared_repo = repo if path == '/' else sub_repo
        result = {}
        result['failed'] = []
        result['success'] = []

        if share_type == 'user':
            share_to_users = request.data.getlist('username')
            for to_user in share_to_users:
                if not is_valid_username(to_user):
                    result['failed'].append({
                        'email': to_user,
                        'error_msg': 'Nutzername oder E-Mail-Adresse ungültig.'
                        })
                    continue

                try:
                    User.objects.get(email=to_user)
                except User.DoesNotExist:
                    result['failed'].append({
                        'email': to_user,
                        'error_msg': 'Es wurde kein Nutzer mit der E-Mail-Adresse %s gefunden. Benutzen Sie die Option „Download-Link erstellen” bzw. „Upload-Link erstellen”, um Elemente für nicht-registrierte Nutzer freizugeben.' % to_user
                        })
                    continue

                try:
                    if is_org_context(request):
                        org_id = request.user.org.org_id
                        seaserv.seafserv_threaded_rpc.org_add_share(
                            org_id, shared_repo.id, username, to_user,
                            permission)
                    else:
                        seafile_api.share_repo(shared_repo.id, username,
                                               to_user, permission)

                    # send a signal when sharing repo successful
                    share_repo_to_user_successful.send(sender=None,
                                                       from_user=username,
                                                       to_user=to_user,
                                                       repo=shared_repo)
                    result['success'].append({
                        "share_type": "user",
                        "user_info": {
                            "name": to_user,
                            "nickname": email2nickname(to_user),
                        },
                        "permission": permission
                    })

                    send_perm_audit_msg('add-repo-perm', username, to_user,
                                        repo_id, path, permission)
                except SearpcError as e:
                    logger.error(e)
                    result['failed'].append({
                        'email': to_user,
                        'error_msg': 'Internal Server Error'
                        })
                    continue

        if share_type == 'group':
            group_ids = request.data.getlist('group_id')
            for gid in group_ids:
                try:
                    gid = int(gid)
                except ValueError:
                    return api_error(status.HTTP_400_BAD_REQUEST, 'group_id %s invalid.' % gid)
                group = seaserv.get_group(gid)
                if not group:
                    return api_error(status.HTTP_404_NOT_FOUND, 'Group %s not found' % gid)

                try:
                    if is_org_context(request):
                        org_id = request.user.org.org_id
                        seafile_api.add_org_group_repo(shared_repo.repo_id,
                                                       org_id, gid, username,
                                                       permission)
                    else:
                        seafile_api.set_group_repo(shared_repo.repo_id, gid,
                                                   username, permission)

                    share_repo_to_group_successful.send(sender=None,
                        from_user=username, group_id=gid, repo=shared_repo)

                    result['success'].append({
                        "share_type": "group",
                        "group_info": {
                            "id": gid,
                            "name": group.group_name,
                        },
                        "permission": permission
                    })

                    send_perm_audit_msg('add-repo-perm', username, gid,
                                        repo_id, path, permission)
                except SearpcError as e:
                    logger.error(e)
                    result['failed'].append({
                        'group_name': group.group_name,
                        'error_msg': 'Internal Server Error'
                        })
                    continue

        return HttpResponse(json.dumps(result),
            status=200, content_type=json_content_type)

    def delete(self, request, repo_id, format=None):
        username = request.user.username
        repo = seafile_api.get_repo(repo_id)
        if not repo:
            return api_error(status.HTTP_404_NOT_FOUND, 'Library %s not found.' % repo_id)

        path = request.GET.get('p', '/')
        if seafile_api.get_dir_id_by_path(repo.id, path) is None:
            return api_error(status.HTTP_404_NOT_FOUND, 'Folder %s not found.' % path)

        if username != self.get_repo_owner(request, repo_id):
            return api_error(status.HTTP_403_FORBIDDEN, 'Permission denied.')

        shared_to_user, shared_to_group = self.handle_shared_to_args(request)

        if path == '/':
            shared_repo = repo
        else:
            try:
                sub_repo = self.get_sub_repo_by_path(request, repo, path)
                if sub_repo:
                    shared_repo = sub_repo
                else:
                    return api_error(status.HTTP_404_NOT_FOUND, 'Sub-library not found.')
            except SearpcError as e:
                logger.error(e)
                return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, 'Failed to get sub-library.')

        if shared_to_user:
            shared_to = request.GET.get('username')
            if shared_to is None or not is_valid_username(shared_to):
                return api_error(status.HTTP_400_BAD_REQUEST, 'Email %s invalid.' % shared_to)

            try:
                User.objects.get(email=shared_to)
            except User.DoesNotExist:
                return api_error(status.HTTP_400_BAD_REQUEST, 'Invalid user, should be registered')

            if is_org_context(request):
                org_id = request.user.org.org_id
                seaserv.seafserv_threaded_rpc.org_remove_share(
                    org_id, shared_repo.id, username, shared_to)
            else:
                seaserv.remove_share(shared_repo.id, username, shared_to)

            permission = seafile_api.check_permission_by_path(repo.id, path,
                                                              shared_to)
            send_perm_audit_msg('delete-repo-perm', username, shared_to,
                                repo_id, path, permission)

        if shared_to_group:
            group_id = request.GET.get('group_id')
            try:
                group_id = int(group_id)
            except ValueError:
                return api_error(status.HTTP_400_BAD_REQUEST, 'group_id %s invalid' % group_id)

            # hacky way to get group repo permission
            permission = ''
            for e in seafile_api.list_repo_shared_group_by_user(username, shared_repo.id):
                if e.group_id == group_id:
                    permission = e.perm
                    break

            if is_org_context(request):
                org_id = request.user.org.org_id
                seaserv.del_org_group_repo(shared_repo.id, org_id, group_id)
            else:
                seafile_api.unset_group_repo(shared_repo.id, group_id, username)

            send_perm_audit_msg('delete-repo-perm', username, group_id,
                                repo_id, path, permission)

        return HttpResponse(json.dumps({'success': True}), status=200,
                            content_type=json_content_type)


from collections import Counter

with open('input.txt', 'r') as f:
    raw_data = [line.split('\n') for line in f.read().split('\n\n')]

if __name__ == '__main__':

    group_yes = []
    common_yes = []

    for data in raw_data:
        n_answers = len(data)
        answers = ''.join(data)
        answers_count = Counter(answers)

        group_yes.append(len(answers_count))
        common_yes.append(len([answers_count[q] for q in answers_count if answers_count[q] == n_answers]))

    print(f'Union {sum(group_yes)}')
    print(f'Intersection {sum(common_yes)}')


from django.db import models
from django.contrib.auth.models import AbstractUser, UserManager
import datetime

# Create your models here.
class Institute(models.Model):
    name = models.CharField(max_length = 200, verbose_name='学院名称')
    found_time = models.DateField(blank = True, null = True, verbose_name='成立时间')
    location = models.TextField(blank = True, null = True, verbose_name='所在地址')
    teacher_num = models.PositiveIntegerField(default=0, editable=False, verbose_name='教师人数')

    def increase_teacher_num(self):
        self.teacher_num += 1
        self.save(update_fields=['teacher_num'])

    def __str__(self):
        return self.name

class Major(models.Model):
    name = models.CharField(max_length = 200, verbose_name='专业名称')
    found_time = models.DateField(blank = True, null = True, verbose_name='成立日期')
    institute = models.ForeignKey(Institute, on_delete = models.CASCADE, verbose_name='所在学院')
    # student_num = models.PositiveIntegerField(default = 0, blank=True)

    def __str__(self):
        return self.name

class SectionInfo(models.Model):
    year = models.CharField(max_length=100, default=datetime.datetime.now().year, verbose_name='入学年份')
    semester = models.CharField(max_length=100, choices=(('autumn', u'秋季学期'), ('spring', u'春季学期'), ('summer',u'夏季学期')), default='autumn', verbose_name='学期')
    institute = models.ForeignKey(Institute, on_delete=models.DO_NOTHING, verbose_name='学院')
    major = models.ForeignKey(Major, on_delete = models.DO_NOTHING, verbose_name='专业')
    student_num = models.PositiveIntegerField(default=0, blank=True, null=True, editable=False, verbose_name='学生人数')

    def increase_student_num(self):
        self.student_num += 1
        self.save(update_fields=['student_num'])

    def __str__(self):
        return str(self.institute) + ' ' + str(self.major) + '专业 ' + str(self.year) + '级'

    class Meta:
        ordering = ['-year', 'semester', 'institute', 'major']

class User(AbstractUser):
    sno = models.CharField(max_length=200, blank=True, null=True, verbose_name='学工号')
    section_info = models.ForeignKey(SectionInfo, null=True, on_delete=models.DO_NOTHING, verbose_name='部门信息')
    # enroll_year = models.ForeignKey(EachYearMajorInfo, on_delete=models.CASCADE)
    # institute = models.ForeignKey(Institute, on_delete=models.CASCADE, null = True)
    # major = models.ForeignKey(Major, on_delete=models.CASCADE, null=True, blank=True)
    # class_num = models.IntegerField(null=True, blank=True)
    sex = models.CharField(max_length=8, choices=(("male", u"男"), ("female", u"女"),("unknown", u"未知")), default="male",verbose_name="性别")
    birthday = models.DateField(null=True, verbose_name='出生日期')
    type = models.CharField(max_length=8, choices=(('student', u'学生'), ('teacher', u'教师'), ('unknown', u'未知')), default="student",verbose_name='类型')
    # level = models.ForeignKey(Level, null=True, blank=True, on_delete=models.SET_NULL)

    def save(self, *args, **kwargs):
        if not self.sno:
            sno_str = ''
            if self.section_info:
                if self.type == 'student':
                    self.section_info.increase_student_num()
                    sno_str = str(self.section_info.year) + str('%04d' % self.section_info.institute_id) + str('%04d' % self.section_info.major_id) + str('%04d' % self.section_info.student_num)
                else:
                    self.section_info.institute.increase_teacher_num()
                    sno_str = str(self.section_info.year) + str('%04d' % self.section_info.institute_id) + str('%04d' % self.section_info.institute.teacher_num)
            self.sno = sno_str
        super(User, self).save(*args, **kwargs)

    class Meta(AbstractUser.Meta):
        pass

class Course(models.Model):
    name = models.CharField(max_length = 200, verbose_name='课程名称')
    create_time = models.DateField(blank = True,null=True, verbose_name='成立时间')
    total_hours = models.PositiveIntegerField(verbose_name='总学时')
    credit = models.PositiveIntegerField(verbose_name='学分')
    teacher = models.ForeignKey(User, limit_choices_to={'type': 'teacher'}, null = True, blank=True, on_delete = models.SET_NULL, verbose_name='任教老师')
    institute = models.ForeignKey(Institute, null = True, blank=True, on_delete = models.SET_NULL, verbose_name='开办学院')
    regular_proportion1 = models.FloatField(default=0.1, verbose_name='平时成绩1占百分比')
    regular_proportion2 = models.FloatField(default=0.4, verbose_name='平时成绩2占百分比')
    final_proportion = models.FloatField(default=0.5, verbose_name='期末成绩占百分比')

    def __str__(self):
        return self.name

    class Meta:
        ordering = ['institute']

class Score(models.Model):
    course = models.ForeignKey(Course, on_delete = models.DO_NOTHING, verbose_name='课程名称')
    student = models.ForeignKey(User, on_delete = models.CASCADE, verbose_name='学生')
    regular_grade1 = models.FloatField(default = 0.0, verbose_name='平时成绩1')
    regular_grade2 = models.FloatField(default = 0.0, verbose_name='平时成绩2')
    final_grade = models.FloatField(default = 0.0, verbose_name='期末成绩')
    total_grade = models.FloatField(default = 0.0, verbose_name='总成绩')
    credit = models.PositiveIntegerField(default = 0, verbose_name='获得学分绩点')

    class Meta:
        ordering = ['course_id', 'student_id']


import os

import faiss
import numpy as np

from descriptor_lib.descriptor_group import DescriptorGroup
import config

class FaissEngine(object):

    def __init__(self,
            descriptor_group,
            filename):
        """Descriptor indexing and storage engine powered by Faiss.

        Args:
            descriptor_group: DescriptorGroup, the descriptor_group
                to be indexing or to be stored.
            filename: str, filename for store index file.

        """
        self.descriptor_group = descriptor_group
        self.X = None
        self.index = None
        self.filename = filename

    def _get_X(self):
        """Convert feature of descriptor_group to matrix."""
        if self.X is None:
            dg = self.descriptor_group
            self.X = np.zeros(
                    len(dg), dg.feature_size,
                    dtype=dg.dtype)
            for i, desc in enumerate(dg.description_table.values()):
                self.X[i, :] = desc.feature_vec
        return self.X

    def get_index(self):
        """Get index of """
        if self.index is None:
            self.index = faiss.IndexFlatL2(self._get_X())
        return self.index

    def save_index(self):
        """Save the index file."""
        index = self.get_index()
        path = os.path.join(config.db_feature_index_path, self.filename)
        faiss.write_index(index, path)

    def load_index(self):
        """Load index from file."""
        path = os.path.join(config.db_feature_index_path, self.filename)
        self.index = faiss.read_index(path)

    def k_similar_samples(self, query, k=1):
        """Query the most similar k samples."""
        index = self.get_index()
        D, I = index.search(query, k)
        return D, I


import random
from data import rooms, facilitators, time_slots

# Selection
def softmax_selection(population, normalized_scores):
    selected_schedules = []

    while len(selected_schedules) < len(population):
        selected_index = random.choices(range(len(population)), weights=normalized_scores, k=1)[0]
        selected_schedule = population[selected_index]

        # Check if any room in the selected schedule exceeds capacity
        room_usage = {room: 0 for room in rooms}

        valid_schedule = True
        for activity in selected_schedule:
            room = activity['room']
            if room_usage[room] + activity['expected_enrollment'] > rooms[room]:
                valid_schedule = False
                break
            room_usage[room] += activity['expected_enrollment']

        if valid_schedule:
            selected_schedules.append(selected_schedule)

    return selected_schedules

# Crossover
def pmx_crossover(parent1, parent2):
    # Initialize child schedules with the same structure as parents
    child1 = [None] * len(parent1)
    child2 = [None] * len(parent2)

    # Choose two random crossover points
    point1 = random.randint(0, len(parent1) - 2)
    point2 = random.randint(point1 + 1, len(parent1) - 1)

    # Copy the selected portion from parent1 to child1, and from parent2 to child2
    child1[point1:point2 + 1] = parent1[point1:point2 + 1]
    child2[point1:point2 + 1] = parent2[point1:point2 + 1]

    # Fill in the remaining slots in child1 and child2
    for i in range(len(parent1)):
        if i < point1 or i > point2:
            # Find values from parent2 that are not in child1
            while parent2[i] in child1[point1:point2 + 1]:
                i = parent2.index(parent1[i])
            child1[i] = parent2[i]

            # Find values from parent1 that are not in child2
            while parent1[i] in child2[point1:point2 + 1]:
                i = parent1.index(parent2[i])
            child2[i] = parent1[i]

    return child1, child2

# Mutation
def mutate_schedule(schedule):
    mutated_schedule = schedule.copy()

    # Select a random activity to mutate
    activity_index = random.randint(0, len(mutated_schedule) - 1)
    mutated_activity = mutated_schedule[activity_index]

    # Mutate time and facilitator (room assignment remains unchanged)
    mutation_type = random.choice(["time", "facilitator"])

    if mutation_type == "time":
        # Change the time to a random valid time slot
        mutated_schedule[activity_index]['time'] = random.choice(time_slots)
    else:
        # Change the facilitator to a random valid facilitator
        mutated_schedule[activity_index]['facilitator'] = random.choice(facilitators)

    return mutated_schedule


# -*- coding: utf-8 -*-

import sys
import torch
import torch.nn as nn
import torch.nn.functional as F
import random
import numpy as np
#from torch.autograd import Variable
from utils.utils import print_time, lens2mask

class Attention(nn.Module):

    def __init__(self, hidden_size, method, coverage, cuda):
        super(Attention, self).__init__()
        self.method = method
        self.cuda = cuda

        if method == 'dot':
            pass #no need to use a layer

        elif method == 'general':
            self.W_a = nn.Linear(hidden_size, hidden_size, bias=False)
            if cuda: self.W_a.cuda()

        elif method == 'concat':
            self.W_a_h = nn.Linear(hidden_size, hidden_size, bias=False) #linear layer for enc hidden states (enc_output) 'context'
            self.W_a_s = nn.Linear(hidden_size, hidden_size, bias=True)  #linear layer for dec_state (dec_hidden) 'query' (contains bias: b_attn)
            self.v = nn.Linear(hidden_size, 1, bias=False)
            if coverage: self.W_a_c = nn.Linear(1, hidden_size, bias=False) #lineary layer for coverage vector
            if cuda:
                self.W_a_h.cuda()
                self.W_a_s.cuda()
                self.v.cuda()
                if coverage: self.W_a_c.cuda()

        else: sys.exit("error: bad attention method {} option. Use: dot OR general OR concat\n".format(method))

    def forward(self, query, context, len_src_batch, coverage=None):
        #query [B, H] (dec_output, h_t in Luong)
        #context [S, B, H] (enc_hidden, \hat{h}_s in Luong)
        #coverage [B, S] sum over the previous attention vectors
        self.B = query.size(0)
        self.H = query.size(1)
        self.S = context.size(0)

        if self.method in ["general", "dot"]:
            if self.method == "general":
                query = self.W_a(query) #[B,H]
            query = query.view(self.B, 1, self.H)  #[B,1,H]
            context = context.permute(1,2,0) #[B,H,S]
            scores = torch.bmm(query, context).squeeze(1) # [B,1,H] x [B,H,S] --> [B,1,S] --> [B,S]

        else:
            ### context
            context = context.permute(1,0,2) #[B,S,H]
            context = context.contiguous().view(-1, self.H) #[B*S,H]
            context = self.W_a_h(context)
            ### query
            query = self.W_a_s(query) #query [B, H]
            query = query.unsqueeze(1).expand(self.B, self.S, self.H) #[B, 1, H] => [B, S, H]
            query = query.contiguous().view(-1, self.H) # [B*S,H]
            scores = context + query # [B*S,H]
            ### coverage
            if coverage is not None:
                coverage = coverage.view(-1, 1) # [B*S,1]
                coverage = self.W_a_c(coverage) # [B*S,H]
                scores = scores + coverage # [B*S,H]
            scores = torch.tanh(scores) # [B*S,H]
            scores = self.v(scores)  # [B*S,1]
            scores = scores.view(-1, self.S)  # [B,S]

        mask = lens2mask(len_src_batch, self.S) #[B,S]
        if self.cuda:
            mask = mask.cuda()
#        mask = mask.unsqueeze(1)  #[B,1,S]
        scores.masked_fill_(1-mask, -float('inf')) ### Fills elements of scores with -inf where 1-mask is one (padded words)

        #normalize scores
        align = F.softmax(scores, dim=1) #[B, S]
        return align


import functools
import rlax

from agents import td_agent
from agents.td_agent import losses
from losses import usfa as usfa_losses
from losses import cumulants
from utils import data as data_utils
from experiments.common import nets

def default_loss_kwargs(config):
  return dict(
      discount=config.discount,
      importance_sampling_exponent=config.importance_sampling_exponent,
      burn_in_length=config.burn_in_length,
      max_replay_size=config.max_replay_size,
      store_lstm_state=config.store_lstm_state,
      max_priority_weight=config.max_priority_weight,
      tx_pair=config.tx_pair,
      bootstrap_n=config.bootstrap_n,
      clip_rewards=config.clip_rewards,
      loss_coeff=config.loss_coeff,
      priority_weights_aux=config.priority_weights_aux,
      priority_use_aux=config.priority_use_aux,
      )

def q_aux_sf_loss(config):
  """Create auxilliary Q-learning loss for SF
  """
  if config.q_aux == "single":
    loss = usfa_losses.QLearningAuxLoss
  elif config.q_aux == "ensemble":
    loss = usfa_losses.QLearningEnsembleAuxLoss
  else:
    raise RuntimeError(config.q_aux)

  if config.sf_loss == 'n_step_q_learning':
    tx_pair = rlax.IDENTITY_PAIR
  elif config.sf_loss == 'transformed_n_step_q_learning':
    tx_pair = rlax.SIGNED_HYPERBOLIC_PAIR
  else:
    raise NotImplementedError(config.sf_loss)

  return loss(
          coeff=config.value_coeff,
          discount=config.discount,
          sched_end=config.q_aux_anneal,
          sched_end_val=config.q_aux_end_val,
          tx_pair=tx_pair,
          mask_loss=config.qaux_mask_loss,
          target_w=config.target_phi,
          elementwise=config.elemwise_qaux_loss,
          stop_w_grad=config.stop_w_grad)

def r2d1(
  env_spec,
  default_config,
  dataclass_configs,
  NetworkCls=None,
  NetKwargs=None,
  LossFnKwargs=None,
  **kwargs):
  """Summary

  Args:
      env_spec (TYPE): Description
      default_config (TYPE): Description
      dataclass_configs (None, optional): Description
      NetworkCls (None, optional): Description
      NetKwargs (None, optional): Description
      LossFnKwargs (None, optional): Description
      **kwargs: Description

  Deleted Parameters:
      task_input (str, optional): how to input task to network

  Returns:
      TYPE: Description
  """
  config = data_utils.merge_configs(
    dataclass_configs=dataclass_configs,
    dict_configs=default_config)

  NetworkCls = NetworkCls or nets.r2d1 # default: 2M params
  NetKwargs =  NetKwargs or dict(
    config=config,
    env_spec=env_spec,
    **kwargs,
    )
  LossFn = td_agent.R2D2Learning
  LossFnKwargs = LossFnKwargs or default_loss_kwargs(config)
  LossFnKwargs.update(
    loss=config.r2d1_loss,
    mask_loss=config.q_mask_loss)

  return config, NetworkCls, NetKwargs, LossFn, LossFnKwargs

def usfa_lstm(
  env_spec,
  default_config : dict,
  dataclass_configs,
  NetworkCls=None,
  NetKwargs=None,
  LossFnKwargs=None,
  predict_cumulants=True,
  **kwargs):
  """Summary

  Args:
      env_spec (TYPE): Description
      default_config (dict): dict used to populate config
      dataclass_configs (TYPE): Description
      NetworkCls (None, optional): Description
      NetKwargs (None, optional): Description
      LossFnKwargs (None, optional): Description
      predict_cumulants (bool, optional): Description
      **kwargs: Description

  Returns:
      TYPE: Description
  """
  config = data_utils.merge_configs(
    dataclass_configs=dataclass_configs,
    dict_configs=default_config
    )

  NetworkCls = NetworkCls or nets.usfa # default: (1.96M) params
  NetKwargs=dict(
    config=config,
    env_spec=env_spec,
    predict_cumulants=predict_cumulants,
    **kwargs)

  LossFn = td_agent.USFALearning
  LossFnKwargs = default_loss_kwargs(config)

  LossFnKwargs.update(
      loss=config.sf_loss,
      mask_loss=config.sf_mask_loss)

  if predict_cumulants:
    LossFnKwargs['aux_tasks']=[
        q_aux_sf_loss(config),
        cumulants.CumulantRewardLoss(
          shorten_data_for_cumulant=True,
          coeff=config.reward_coeff,
          mask_loss=config.phi_mask_loss,
          loss=config.reward_loss,
          l1_coeff=config.phi_l1_coeff,
          wl1_coeff=config.w_l1_coeff,
          balance=config.balance_reward,
          elementwise=config.elemwise_phi_loss,
          )
    ]
    LossFnKwargs['shorten_data_for_cumulant']=True

    LossFnKwargs['extract_cumulants'] = functools.partial(
      losses.cumulants_from_preds,
      use_target=config.target_phi,
      stop_grad=True)

  return config, NetworkCls, NetKwargs, LossFn, LossFnKwargs

def msf(
  env_spec,
  default_config : dict,
  dataclass_configs,
  NetworkCls=None,
  NetKwargs=None,
  LossFnKwargs=None,
  predict_cumulants=True,
  learn_model=False,
  **kwargs):
  """Summary

  Args:
      env_spec (TYPE): Description
      default_config (dict): dict used to populate config
      dataclass_configs (TYPE): Description
      NetworkCls (None, optional): Description
      NetKwargs (None, optional): Description
      LossFnKwargs (None, optional): Description
      predict_cumulants (bool, optional): Description
      **kwargs: Description

  Returns:
      TYPE: Description
  """
  config = data_utils.merge_configs(
    dataclass_configs=dataclass_configs,
    dict_configs=default_config
    )

  NetworkCls = NetworkCls or nets.msf
  NetKwargs=dict(
    config=config,
    env_spec=env_spec,
    predict_cumulants=predict_cumulants,
    learn_model=learn_model,
    **kwargs)

  LossFn = td_agent.USFALearning
  LossFnKwargs = default_loss_kwargs(config)

  LossFnKwargs.update(
      loss=config.sf_loss,
      mask_loss=config.sf_mask_loss)

  if predict_cumulants:
    aux_tasks=[
      q_aux_sf_loss(config),
      # msfa_stats.MsfaStats(),
      cumulants.CumulantRewardLoss(
        shorten_data_for_cumulant=True,
        coeff=config.reward_coeff,
        mask_loss=config.phi_mask_loss,
        loss=config.reward_loss,
        l1_coeff=config.phi_l1_coeff,
        wl1_coeff=config.w_l1_coeff,
        balance=config.balance_reward,
        ),
    ]
    LossFnKwargs.update(
      shorten_data_for_cumulant=True,
      aux_tasks=aux_tasks)

    LossFnKwargs['extract_cumulants'] = functools.partial(
      losses.cumulants_from_preds,
      use_target=config.target_phi,
      stop_grad=True)

  if learn_model:
    if config.contrast_module_coeff > 0:
      aux_tasks.append(
          ModuleContrastLoss(
            coeff=config.contrast_module_coeff,
            extra_negatives=config.extra_module_negatives,
            temperature=config.temperature)
          )
    if config.contrast_time_coeff > 0:
      aux_tasks.append(
          TimeContrastLoss(
            coeff=config.contrast_time_coeff,
            extra_negatives=config.extra_time_negatives,
            temperature=config.temperature,
            normalize_step=config.normalize_step)
          )

  return config, NetworkCls, NetKwargs, LossFn, LossFnKwargs

def default_agent_settings(agent, env_spec, configs, config_kwargs=None, env_kwargs=None, **kwargs):
  default_config = dict()
  default_config.update(config_kwargs or {})

  agent = agent.lower()
  if agent == "r2d1":
  # Recurrent DQN/UVFA
    config, NetworkCls, NetKwargs, LossFn, LossFnKwargs = r2d1(
      env_spec=env_spec,
      default_config=default_config,
      dataclass_configs=[configs.R2D1Config()],
      **kwargs,
      )

  elif agent == "r2d1_farm":
  # UVFA + FARM
    config, NetworkCls, NetKwargs, LossFn, LossFnKwargs = r2d1(
      env_spec=env_spec,
      default_config=default_config,
      NetworkCls = nets.r2d1_farm,
      dataclass_configs=[
        configs.R2D1Config(),
        configs.FarmConfig(),
      ],
      **kwargs,
    )

  elif agent == "r2d1_no_task":
  # UVFA + noise added to goal embedding
    config, NetworkCls, NetKwargs, LossFn, LossFnKwargs = r2d1(
      env_spec=env_spec,
      default_config=default_config,
      dataclass_configs=[
        configs.R2D1Config(),
      ],
      task_input='none',
      task_embedding='none',
      **kwargs,
      )

  elif agent in ["r2d1_noise_eval", "r2d1_noise"]:
  # UVFA + noise added to goal embedding
    config, NetworkCls, NetKwargs, LossFn, LossFnKwargs = r2d1(
      env_spec=env_spec,
      default_config=default_config,
      NetworkCls=nets.r2d1_noise,
      dataclass_configs=[
        configs.NoiseConfig(),
        configs.R2D1Config(),
      ],
      **kwargs,
      )
  elif agent == "usfa":
  # USFA + cumulants from LSTM + Q-learning
    config, NetworkCls, NetKwargs, LossFn, LossFnKwargs = usfa_lstm(
        env_spec=env_spec,
        default_config=default_config,
        dataclass_configs=[configs.USFAConfig()],
        predict_cumulants=False,
        **kwargs,
      )

  elif agent == "usfa_lstm":
  # USFA + cumulants from LSTM + Q-learning
    config, NetworkCls, NetKwargs, LossFn, LossFnKwargs = usfa_lstm(
        env_spec=env_spec,
        default_config=default_config,
        dataclass_configs=[
          configs.QAuxConfig(),
          configs.RewardConfig(),
          configs.USFAConfig(),
        ],
        **kwargs,
      )

  elif agent == "msf":
  # USFA + cumulants from FARM + Q-learning (1.9M)
    config, NetworkCls, NetKwargs, LossFn, LossFnKwargs = msf(
        env_spec=env_spec,
        default_config=default_config,
        dataclass_configs=[
          configs.QAuxConfig(),
          configs.RewardConfig(),
          configs.FarmConfig(),
          configs.ModularUSFAConfig(),
        ],
        **kwargs,
      )

  elif agent == "msf_oracle":
  # USFA + cumulants from FARM + Q-learning (1.9M)
    config, NetworkCls, NetKwargs, LossFn, LossFnKwargs = msf(
        env_spec=env_spec,
        default_config=default_config,
        predict_cumulants=False,
        dataclass_configs=[
          configs.QAuxConfig(),
          configs.RewardConfig(),
          configs.FarmConfig(),
          configs.ModularUSFAConfig(),
        ],
        **kwargs,
      )

  else:
    raise NotImplementedError(agent)

  loss_label=None
  eval_network=False
  return config, NetworkCls, NetKwargs, LossFn, LossFnKwargs, loss_label, eval_network


import sys

sys.path.append('../')
from model.BasicBert.BertEmbedding import BertEmbeddings
from model.BasicBert.Bert import BertAttention
from model.BasicBert.Bert import BertLayer
from model.BasicBert.Bert import BertEncoder
from model.BasicBert.Bert import BertModel
from model.BasicBert.BertConfig import BertConfig
import torch
from utils.log_helper import logger_init

if __name__ == '__main__':
    logger_init()
    json_file = '../bert_base_chinese/config.json'
    config = BertConfig.from_json_file(json_file)
    config.__dict__['use_torch_multi_head'] = True  # 表示使用 torch框架中的MultiHeadAttention 注意力实现方法
    config.max_position_embeddings = 518 # 测试大于512时的情况
    src = torch.tensor([[1, 3, 5, 7, 9, 2, 3], [2, 4, 6, 8, 10, 0, 0]], dtype=torch.long)
    src = src.transpose(0, 1)  # [src_len, batch_size]
    print(f"input shape [src_len,batch_size]: ", src.shape)
    token_type_ids = torch.LongTensor([[0, 0, 0, 1, 1, 1, 1], [0, 0, 1, 1, 1, 0, 0]]).transpose(0, 1)
    attention_mask = torch.tensor([[True, True, True, True, True, True, True],
                                   [True, True, True, True, True, False, False]])
    # attention_mask 实际就是Transformer中指代的key_padding_mask

    # ------ BertEmbedding -------
    bert_embedding = BertEmbeddings(config)
    bert_embedding_result = bert_embedding(src, token_type_ids=token_type_ids)
    # [src_len, batch_size, hidden_size]

    # 测试类BertAttention
    bert_attention = BertAttention(config)
    bert_attention_output = bert_attention(bert_embedding_result, attention_mask=attention_mask)
    print(f"BertAttention output shape [src_len, batch_size, hidden_size]: ", bert_attention_output.shape)

    # 测试类BertLayer
    bert_layer = BertLayer(config)
    bert_layer_output = bert_layer(bert_embedding_result, attention_mask)
    print(f"BertLayer output shape [src_len, batch_size, hidden_size]: ", bert_layer_output.shape)

    # 测试类BertEncoder
    bert_encoder = BertEncoder(config)
    bert_encoder_outputs = bert_encoder(bert_embedding_result, attention_mask)
    print(f"num of BertEncoder [config.num_hidden_layers]: ", len(bert_encoder_outputs))
    print(f"each output shape in BertEncoder [src_len, batch_size, hidden_size]: ", bert_encoder_outputs[0].shape)

    # 测试类BertModel
    position_ids = torch.arange(src.size()[0]).expand((1, -1))  # [1,src_len]
    bert_model = BertModel(config)
    bert_model_output = bert_model(input_ids=src,
                                   attention_mask=attention_mask,
                                   token_type_ids=token_type_ids,
                                   position_ids=position_ids)[0]
    print(f"BertModel's pooler output shape [batch_size, hidden_size]: ", bert_model_output.shape)
    print("\n  =======  BertMolde 参数: ========")
    for param_tensor in bert_model.state_dict():
        print(param_tensor, "\t", bert_model.state_dict()[param_tensor].size())

    print(f"\n  =======  测试BertModel载入预训练模型： ========")
    model = BertModel.from_pretrained(config, pretrained_model_dir="../bert_base_chinese")


from PyQt5.QtWidgets import QDialog
from src.UI.ui_range_slider_thresholding import UiRangeSliderThresholding
from PyQt5.QtGui import QPixmap, QImage
import cv2

FORMATS = {
    1: QImage.Format_Grayscale8
}

OPERATIONS = {
    'Thresholding': 1,
    'Adaptive Thresholding': 2,
    "Otsu thresholding": 3,
}

def threshold_single_channel(channel, value, adaptive_method):
    _, channel_thresh = cv2.threshold(channel, value, 255, adaptive_method)
    return channel_thresh

def threshold_channels(image, value, adaptive_method):
    b, g, r = cv2.split(image)

class PointOperationThresholding(QDialog, UiRangeSliderThresholding):
    def __init__(self, image_data, gray):
        super().__init__()
        self.setup_ui(self)
        self.setWindowTitle("Thresholding")
        self.pixmap = None
        self.image_data = image_data
        self.image_origin = image_data
        self.gray = gray
        self.fill_combo_boxes()
        self.po_thresholding()
        self.cB_operations.currentIndexChanged.connect(self.po_thresholding)
        self.slider_max.valueChanged.connect(self.po_thresholding)
        #self.cb_block_size.currentIndexChanged.connect(self.po_thresholding)
        self.cb_block_size.valueChanged.connect(self.po_thresholding)

    def fill_combo_boxes(self):
        for key in OPERATIONS.keys():
            self.cB_operations.addItem(key)

    def update_window(self):
        height, width = self.image_data.shape[:2]
        if len(self.image_data.shape) < 3:
            image = QImage(self.image_data, width, height, width, FORMATS[self.image_data.dtype.itemsize])
        else:
            image = QImage(self.image_data, width, height, 3 * width, QImage.Format_BGR888)
        self.pixmap = QPixmap(image)
        self.setFixedSize(self.pixmap.width() + 10, self.pixmap.height() + 30)
        self.label_image.setPixmap(self.pixmap)

    def po_thresholding(self):
        value = self.slider_max.value()
        self.label_max.setText("Threshold: {0}".format(value))
        number_operation = OPERATIONS[self.cB_operations.currentText()]
        if not self.gray:
            b, g, r = cv2.split(self.image_origin)
            if number_operation == 1:
                self.cb_block_size.setEnabled(False)
                self.image_data = cv2.merge((threshold_single_channel(b, value, cv2.THRESH_BINARY),
                                            threshold_single_channel(g, value, cv2.THRESH_BINARY),
                                            threshold_single_channel(r, value, cv2.THRESH_BINARY)))
            elif number_operation == 2:
                self.cb_block_size.setEnabled(True)
                b_size = self.cb_block_size.value()
                if b_size % 2 == 0:
                    b_size = b_size + 1
                    self.cb_block_size.setValue(b_size)
                b_thresh = cv2.adaptiveThreshold(b, value, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, b_size, 5)
                g_thresh = cv2.adaptiveThreshold(g, value, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, b_size, 5)
                r_thresh = cv2.adaptiveThreshold(r, value, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, b_size, 5)
                self.image_data = cv2.merge((b_thresh, g_thresh, r_thresh))
            elif number_operation == 3:
                self.cb_block_size.setEnabled(False)
                _, b_thresh = cv2.threshold(b, 0, value, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
                _, g_thresh = cv2.threshold(g, 0, value, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
                _, r_thresh = cv2.threshold(r, 0, value, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
                self.image_data = cv2.merge((b_thresh, g_thresh, r_thresh))
        else:
            if number_operation == 1:
                self.cb_block_size.setEnabled(False)
                _, self.image_data = cv2.threshold(self.image_origin, value, 255, cv2.THRESH_BINARY)
            elif number_operation == 2:
                self.cb_block_size.setEnabled(True)
                b_size = self.cb_block_size.value()
                if b_size % 2 == 0:
                    b_size = b_size + 1
                    self.cb_block_size.setValue(b_size)
                self.image_data = cv2.adaptiveThreshold(self.image_origin, value, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
                                                        cv2.THRESH_BINARY, b_size, 5)
            elif number_operation == 3:
                self.cb_block_size.setEnabled(False)
                _, self.image_data = cv2.threshold(self.image_origin, 0, value, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
        self.update_window()


import tensorflow as tf
import numpy as np
import random
from tensorflow.keras.preprocessing.image import img_to_array, load_img
import matplotlib.pyplot as plt
import time
import os

physical_devices = tf.config.experimental.list_physical_devices('GPU')
if len(physical_devices) > 0:
    tf.config.experimental.set_memory_growth(physical_devices[0], True)

model = tf.keras.models.load_model(
    '/home/amritpal/PycharmProjects/100-days-of-code/100_days_of_code/Skin_disease_working_version_minimal'
    '/saved_model/model_name_RGB_224__0.55__1.61__12-3_2:59')
image_directory = 'img.jpg'

model_save_path = 'model_layers/' + str(time.time())
os.mkdir(model_save_path)

def Look_into_filter(model, image_directory):
    # Let's define a new Model that will take an image as input, and will output
    # intermediate representations for all layers in the previous model after
    # the first.
    successive_outputs = [layer.output for layer in model.layers[1:]]

    # visualization_model = Model(img_input, successive_outputs)
    visualization_model = tf.keras.models.Model(inputs=model.input, outputs=successive_outputs)

    '''# Let's pick a random input image of a normal or pneumonia from the training set.
    normal_img_files = [os.path.join(train_normal_dir, f) for f in train_normal_fnames]
    pneumonia_img_files = [os.path.join(train_pneumonia_dir, f) for f in train_pneumonia_fnames]

    img_path = random.choice(normal_img_files + pneumonia_img_files)'''

    img = load_img(image_directory, target_size=(224, 224))  # this is a PIL image
    x = img_to_array(img)  # Numpy array with shape (150, 150, 3)
    x = x.reshape((1,) + x.shape)  # Numpy array with shape (1, 150, 150, 3)
    x /= 255.0  # Rescale by 1/255

    # Let's run our image through our network, thus obtaining all intermediate representations for this image.
    successive_feature_maps = visualization_model.predict(x)
    layer_names = [layer.name for layer in model.layers]  # names of the layers, to make our plot

    # Now let's display our representations
    for layer_name, feature_map in zip(layer_names, successive_feature_maps):
        if len(feature_map.shape) == 4:
            # Just do this for the conv / maxpool layers, not the fully-connected layers
            n_features = feature_map.shape[-1]  # number of features in the feature map
            size = feature_map.shape[1]  # feature map shape (1, size, size, n_features)

            # We will tile our images in this matrix
            display_grid = np.zeros((size, size * n_features))

            # Postprocess the feature to be visually palatable
            for i in range(n_features):
                x = feature_map[0, :, :, i]
                x -= x.mean()
                x /= x.std()
                x *= 64
                x += 128
                x = np.clip(x, 0, 255).astype('uint8')
                display_grid[:, i * size: (i + 1) * size] = x
            # Tile each filter into a horizontal grid

            # Display the grid
            scale = 20. / n_features
            plt.figure(figsize=(scale * n_features, scale))
            plt.title(layer_name)
            plt.grid(False)
            plt.imshow(display_grid, aspect='auto', cmap='viridis')
            plt.savefig(model_save_path + '/' + layer_name + str(time.time()) + '.png')

Look_into_filter(model, image_directory)


from itertools import izip
import os.path # exists

from pyffi.formats.nif import NifFormat
from pyffi.utils import unique_map
import pyffi.utils.tristrip
import pyffi.utils.vertex_cache
import pyffi.spells
import pyffi.spells.nif
import pyffi.spells.nif.fix
import pyffi.spells.nif.modify
from pyffi.spells.nif.optimize import *

# localization
#import gettext
#_ = gettext.translation('pyffi').ugettext
_ = lambda msg: msg # stub, for now

# set flag to overwrite files
__readonly__ = False

# example usage
__examples__ = """* Standard usage:

    python niftoaster.py optimize /path/to/copy/of/my/nifs

* Optimize, but do not merge NiMaterialProperty blocks:

    python niftoaster.py optimize --exclude=NiMaterialProperty /path/to/copy/of/my/nifs
"""

class SpellOptimizeGeometry2(pyffi.spells.nif.NifSpell):
    """Optimize all geometries:
      - remove duplicate vertices
      - triangulate
      - recalculate skin partition
      - recalculate tangent space
    """

    SPELLNAME = "opt_geometry"
    READONLY = False

    # spell parameters
    VERTEXPRECISION = 3
    NORMALPRECISION = 3
    UVPRECISION = 5
    VCOLPRECISION = 3

    def __init__(self, *args, **kwargs):
        pyffi.spells.nif.NifSpell.__init__(self, *args, **kwargs)
        # list of all optimized geometries so far
        # (to avoid optimizing the same geometry twice)
        self.optimized = []

    def datainspect(self):
        # do not optimize if an egm or tri file is detected
        filename = self.stream.name
        if (os.path.exists(filename[:-3] + "egm")
            or os.path.exists(filename[:-3] + "tri")):
            return False
        # so far, only reference lists in NiObjectNET blocks, NiAVObject
        # blocks, and NiNode blocks are checked
        return self.inspectblocktype(NifFormat.NiTriBasedGeom)

    def branchinspect(self, branch):
        # only inspect the NiAVObject branch
        return isinstance(branch, NifFormat.NiAVObject)

    def optimize_vertices(self, data):
        self.toaster.msg("removing duplicate vertices")
        # get map, deleting unused vertices
        return unique_map(
            vhash
            for i, vhash in enumerate(data.get_vertex_hash_generator(
                vertexprecision=self.VERTEXPRECISION,
                normalprecision=self.NORMALPRECISION,
                uvprecision=self.UVPRECISION,
                vcolprecision=self.VCOLPRECISION)))

    def branchentry(self, branch):
        """Optimize a NiTriStrips or NiTriShape block:
          - remove duplicate vertices
          - retriangulate for vertex cache
          - recalculate skin partition
          - recalculate tangent space

        @todo: Limit the length of strips (see operation optimization mod for
            Oblivion!)
        """
        if not isinstance(branch, NifFormat.NiTriBasedGeom):
            # keep recursing
            return True

        if branch in self.optimized:
            # already optimized
            return False

        if branch.data.additional_data:
            # occurs in fallout nv
            # not sure how to deal with additional data
            # so skipping to be on the safe side
            self.toaster.msg(
                "mesh has additional geometry data"
                " which is not well understood: not optimizing")
            return False

        # we found a geometry to optimize

        # we're going to change the data
        self.changed = True

        # cover degenerate case
        if branch.data.num_vertices < 3 or branch.data.num_triangles == 0:
            self.toaster.msg(
                "less than 3 vertices or no triangles: removing branch")
            self.data.replace_global_node(branch, None)
            return False

        self.optimized.append(branch)

        # shortcut
        data = branch.data

        v_map, v_map_inverse = self.optimize_vertices(data)

        self.toaster.msg("(num vertices was %i and is now %i)"
                         % (len(v_map), len(v_map_inverse)))

        # optimizing triangle ordering
        # first, get new triangle indices, with duplicate vertices removed
        triangles = list(pyffi.utils.vertex_cache.get_unique_triangles(
            (v_map[v0], v_map[v1], v_map[v2])
            for v0, v1, v2 in data.get_triangles()))
#        old_atvr = pyffi.utils.vertex_cache.average_transform_to_vertex_ratio(
#            triangles)
#        self.toaster.msg("optimizing triangle ordering")
#        new_triangles = pyffi.utils.vertex_cache.get_cache_optimized_triangles(
#            triangles)
#        new_atvr = pyffi.utils.vertex_cache.average_transform_to_vertex_ratio(
#            new_triangles)
#        if new_atvr < old_atvr:
#            triangles = new_triangles
#            self.toaster.msg(
#                "(ATVR reduced from %.3f to %.3f)" % (old_atvr, new_atvr))
#        else:
#            self.toaster.msg(
#                "(ATVR stable at %.3f)" % old_atvr)
        # optimize triangles to have sequentially ordered indices
        self.toaster.msg("optimizing vertex ordering")
        v_map_opt = pyffi.utils.vertex_cache.get_cache_optimized_vertex_map(
            triangles)
        triangles = [(v_map_opt[v0], v_map_opt[v1], v_map_opt[v2])
                      for v0, v1, v2 in triangles]
        # update vertex map and its inverse
        for i in xrange(data.num_vertices):
            try:
                v_map[i] = v_map_opt[v_map[i]]
            except IndexError:
                # found a trailing vertex which is not used
                v_map[i] = None
            if v_map[i] is not None:
                v_map_inverse[v_map[i]] = i
            else:
                self.toaster.logger.warn("unused vertex")
        try:
            new_numvertices = max(v for v in v_map if v is not None) + 1
        except ValueError:
            # max() arg is an empty sequence
            # this means that there are no vertices
            self.toaster.msg(
                "less than 3 vertices or no triangles: removing branch")
            self.data.replace_global_node(branch, None)
            return False
        del v_map_inverse[new_numvertices:]

        # use a triangle representation
        if not isinstance(branch, NifFormat.NiTriShape):
            self.toaster.msg("replacing branch by NiTriShape")
            newbranch = branch.get_interchangeable_tri_shape(
                triangles=triangles)
            self.data.replace_global_node(branch, newbranch)
            branch = newbranch
            data = newbranch.data
        else:
            data.set_triangles(triangles)

        # copy old data
        oldverts = [(v.x, v.y, v.z) for v in data.vertices]
        oldnorms = [(n.x, n.y, n.z) for n in data.normals]
        olduvs   = [[(uv.u, uv.v) for uv in uvset] for uvset in data.uv_sets]
        oldvcols = [(c.r, c.g, c.b, c.a) for c in data.vertex_colors]
        if branch.skin_instance: # for later
            oldweights = branch.get_vertex_weights()
        # set new data
        data.num_vertices = new_numvertices
        if data.has_vertices:
            data.vertices.update_size()
            for i, v in enumerate(data.vertices):
                old_i = v_map_inverse[i]
                v.x = oldverts[old_i][0]
                v.y = oldverts[old_i][1]
                v.z = oldverts[old_i][2]
        if data.has_normals:
            data.normals.update_size()
            for i, n in enumerate(data.normals):
                old_i = v_map_inverse[i]
                n.x = oldnorms[old_i][0]
                n.y = oldnorms[old_i][1]
                n.z = oldnorms[old_i][2]
        # XXX todo: if ...has_uv_sets...:
        data.uv_sets.update_size()
        for j, uvset in enumerate(data.uv_sets):
            for i, uv in enumerate(uvset):
                old_i = v_map_inverse[i]
                uv.u = olduvs[j][old_i][0]
                uv.v = olduvs[j][old_i][1]
        if data.has_vertex_colors:
            data.vertex_colors.update_size()
            for i, c in enumerate(data.vertex_colors):
                old_i = v_map_inverse[i]
                c.r = oldvcols[old_i][0]
                c.g = oldvcols[old_i][1]
                c.b = oldvcols[old_i][2]
                c.a = oldvcols[old_i][3]
        del oldverts
        del oldnorms
        del olduvs
        del oldvcols

        # update skin data
        if branch.skin_instance:
            self.toaster.msg("update skin data vertex mapping")
            skindata = branch.skin_instance.data
            newweights = []
            for i in xrange(new_numvertices):
                newweights.append(oldweights[v_map_inverse[i]])
            for bonenum, bonedata in enumerate(skindata.bone_list):
                w = []
                for i, weightlist in enumerate(newweights):
                    for bonenum_i, weight_i in weightlist:
                        if bonenum == bonenum_i:
                            w.append((i, weight_i))
                bonedata.num_vertices = len(w)
                bonedata.vertex_weights.update_size()
                for j, (i, weight_i) in enumerate(w):
                    bonedata.vertex_weights[j].index = i
                    bonedata.vertex_weights[j].weight = weight_i

            # update skin partition (only if branch already exists)
            if branch.get_skin_partition():
                self.toaster.msg("updating skin partition")
                if isinstance(branch.skin_instance,
                              NifFormat.BSDismemberSkinInstance):
                    # get body part indices (in the old system!)
                    triangles, trianglepartmap = (
                        branch.skin_instance.get_dismember_partitions())
                    maximize_bone_sharing = True
                    # update mapping
                    new_triangles = []
                    new_trianglepartmap = []
                    for triangle, trianglepart in izip(triangles, trianglepartmap):
                        new_triangle = tuple(v_map[i] for i in triangle)
                        # it could happen that v_map[i] is None
                        # these triangles are skipped
                        # see for instance
                        # falloutnv/meshes/armor/greatkhans/greatkhan_v3.nif
                        # falloutnv/meshes/armor/tunnelsnake01/m/outfitm.nif
                        if None not in new_triangle:
                            new_triangles.append(new_triangle)
                            new_trianglepartmap.append(trianglepart)
                    triangles = new_triangles
                    trianglepartmap = new_trianglepartmap
                else:
                    # no body parts
                    triangles = None
                    trianglepartmap = None
                    maximize_bone_sharing = False
                # use Oblivion settings
                branch.update_skin_partition(
                    maxbonesperpartition=18, maxbonespervertex=4,
                    stripify=False, verbose=0,
                    triangles=triangles, trianglepartmap=trianglepartmap,
                    maximize_bone_sharing=maximize_bone_sharing)

        # update morph data
        for morphctrl in branch.get_controllers():
            if isinstance(morphctrl, NifFormat.NiGeomMorpherController):
                morphdata = morphctrl.data
                # skip empty morph data
                if not morphdata:
                    continue
                # convert morphs
                self.toaster.msg("updating morphs")
                # check size and fix it if needed
                # (see issue #3395484 reported by rlibiez)
                # remap of morph vertices works only if
                # morph.num_vertices == len(v_map)
                if morphdata.num_vertices != len(v_map):
                    self.toaster.logger.warn(
                        "number of vertices in morph ({0}) does not match"
                        " number of vertices in shape ({1}):"
                        " resizing morph, graphical glitches might result"
                        .format(morphdata.num_vertices, len(v_map)))
                    morphdata.num_vertices = len(v_map)
                    for morph in morphdata.morphs:
                        morph.arg = morphdata.num_vertices # manual argument passing
                        morph.vectors.update_size()
                # now remap morph vertices
                for morph in morphdata.morphs:
                    # store a copy of the old vectors
                    oldmorphvectors = [(vec.x, vec.y, vec.z)
                                       for vec in morph.vectors]
                    for old_i, vec in izip(v_map_inverse, morph.vectors):
                        vec.x = oldmorphvectors[old_i][0]
                        vec.y = oldmorphvectors[old_i][1]
                        vec.z = oldmorphvectors[old_i][2]
                    del oldmorphvectors
                # resize matrices
                morphdata.num_vertices = new_numvertices
                for morph in morphdata.morphs:
                    morph.arg = morphdata.num_vertices # manual argument passing
                    morph.vectors.update_size()

        # recalculate tangent space (only if the branch already exists)
        if (branch.find(block_name='Tangent space (binormal & tangent vectors)',
                        block_type=NifFormat.NiBinaryExtraData)
            or (data.num_uv_sets & 61440)
            or (data.bs_num_uv_sets & 61440)):
            self.toaster.msg("recalculating tangent space")
            branch.update_tangent_space()

        # stop recursion
        return False


#!/usr/bin/python3
import argparse
import base64
import csv
import os
import re
import smtplib
import socket
import sys
import time
import schedule
import json
import pandas as pd
from datetime import datetime
from email import encoders
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import mechanicalsoup
from bs4 import BeautifulSoup
from selenium import webdriver
from mailjet_rest import Client
from decimal import Decimal
from slackclient import SlackClient

btcPrice = {}

class LocalBitcoinsSpider:
    __previos_data = []
    __data = {}
    __total_updated = 0

    def __init__(self, input_url, csv_file, from_address=None, to_address=None, subject=None,
                 email_body=None, api_public_key = None, api_secret_key = None, slack_token=None, btc=None):
        self.__url = input_url
        self.__output_csv = csv_file + '.csv' if not str(csv_file).endswith('.csv') else csv_file
        self.__from_address = from_address
        self.__to_address = to_address
        self.__subject = subject
        self.__email_body = email_body
        self.__api_public_key = api_public_key
        self.__api_secret_key = api_secret_key
        self.__slack_token = slack_token
        self.btc = btc
        try:
            self.__data = pd.read_csv(self.__output_csv, index_col='trader')
        except:
            self.__data = pd.DataFrame(
                             {'tx_total': 0,
                              'reputation': 0,
                              'price': 0,
                              'volume': 0,
                              'age': 0}, index=['null'])

    def __enter__(self):
        self.__build_opener()
        self.__previos_data.clear()
        if os.path.exists(self.__output_csv):
            self.__data = pd.read_csv(self.__output_csv, index_col='trader')
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.__browser:
            del self.__browser

    def __build_opener(self):
        socket.setdefaulttimeout(60)
        self.__browser = mechanicalsoup.Browser(
            soup_config={'features': 'lxml'}
        )

    def grab_data(self):
        try:
            print('=== URL: {} ==='.format(self.__url))
            cap = webdriver.DesiredCapabilities.PHANTOMJS
            cap["phantomjs.page.settings.javascriptEnabled"] = True
            cap[
                "phantomjs.page.settings.userAgent"] = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'
            driver = webdriver.PhantomJS('./phantomjs', desired_capabilities=cap)
            driver.get(self.__url)

            print('Script is going to sleep for 10 seconds...')
            time.sleep(10)

            print('Trying to get results from page...')
            driver.get(self.__url)
            data = driver.page_source
            driver.close()

            soup = BeautifulSoup(data)
            if not soup:
                print('No response data found from url: {}'.format(self.__url))
                return

            results_link = soup.find('a', {'name': 'results'})
            if not results_link:
                print('No result found!')
                return

            table = results_link.find_next_sibling('table')
            if not table:
                print('No result found!')
                return

            tr_list = table.find_all('tr')
            if tr_list and len(tr_list) > 1:
                for tr in tr_list[1:]:
                    td_list = tr.find_all('td')
                    if not td_list or len(td_list) < 4:
                        continue

                    try:
                        trader1, trader2, trader3 = '', '', ''
                        trader_content = self.__clean_data(td_list[0].text.strip())
                        trader_m = re.search(r'^([^\(]*?)\(([^\)]*?)\)', trader_content)
                        if trader_m:
                            trader_part1 = trader_m.group(1).strip()
                            trader_part2 = trader_m.group(2).strip()
                            trader_part2_split = re.split(r';', trader_part2)
                            trader = trader_part1 + ', ' + ', '.join(
                                [re.sub(r'[^\d]', '', word) for word in trader_part2_split])
                            trader1, trader2, trader3 = trader.split(', ')
                        payment_method = self.__clean_data(td_list[1].text.strip())
                        price = self.__clean_data(td_list[2].text.strip().strip("USD"))
                        limits = self.__clean_data(td_list[3].text.strip())
                        csv_data = [trader1, trader2, trader3, price, limits]
                        print(csv_data)
                        if trader1 in self.__data.index:
                            print('Already exists. Skip writing...')
                            continue
                        value = Decimal(re.sub(r'[^\d.]', '', price))
                        if value<self.btc*1.17:
                            #todo raise exception instead
                            print("Price low, skipping")
                            continue
                        if (int(trader3) < 95):
                            print('Bad Reputation Skip writing...')
                            continue
                        if (int(trader2) < 31):
                            print('Bad Reputation Skip writing...')
                            continue
                        newRow = pd.DataFrame({'tx_total': trader2,
                              'reputation': trader3,
                              'price': price,
                              'volume': limits,
                              'age': 0}, index=[trader1])
                        self.__data = self.__data.append(newRow)
                        self.__total_updated+=1
                    except Exception as ex:
                        print("Exception Found:", ex)
            self.__write_data_to_csv()
        except Exception as x:
            print('Error grabbing page: ')
            print(x)
        finally:
            print("Finally")
            if (self.__total_updated > 0):
                self.send_alarm_via_slack()
                #self.send_email_via_mailjet()

    @staticmethod
    def __clean_data(data):
        try:
            result = re.sub(r'\n+', ' ', data)
            return re.sub(r'\s+', ' ', result)
        except Exception as x:
            print('Error when cleanup data.' + str(x))
        return ''

    def __write_data_to_csv(self):
        df = self.__data
        df['age'] += 1
        df=df[df['age']<3]
        df.to_csv(self.__output_csv, index_label='trader')

    def send_alarm_via_slack(self):
        print("Sending ALARM")
        self.sc=SlackClient(self.__slack_token)
        try:
           self.sc.api_call(
            "chat.postMessage",
            channel="robot",
            text="Novos traders:" + str(self.__data.index.values)
            )
        except:
           print(sys.exc_info()[0])

    def send_email_via_mailjet(self):
        try:
            if not self.__from_address or not self.__to_address:
                return
            print('Sending email...')
            mailjet = Client(auth=(self.__api_public_key, self.__api_secret_key), version='v3.1')
            data = {
                'Messages': [
                    {
                        "From": {
                            "Email": self.__from_address,
                            "Name": ""
                         },
                        "To": [
                            {
                                "Email": self.__to_address,
                                "Name": ""
                            }
                        ],
                        "Subject": self.__subject,
                        "TextPart": self.__email_body + str(self.__data.index.values)
                    }
                ]
            }
            result = mailjet.send.create(data=data)
            print(result.status_code)
            #print(result.json())
            print('Email send successfully.')
        except Exception as x:
            print(x)

def updatedBtcPrice( btc ):
    date = datetime.now().date()
    global btcPrice
    if date in btcPrice:
        return btcPrice[date]
    else:
        try:
          btcPrice[date] = ccxt.gdax().fetch_ticker('BTC/USD')['ask']
          return btcPrice[date]
        except:
          return btc

def job():
    print("Starting a job")
    with open('crawler.conf', 'r') as f:
        args = json.load(f)

    url = args['url']
    #https://localbitcoins.com/instant-bitcoins/?action=sell&country_code=US&amount=&currency=USD&place_country=US&online_provider=ALL_ONLINE&find-offers=Search

    file = 'df'
    email = args['src_email']
    subject = 'New alarms'
    body = "The following traders are available: "
    dest_email = args['dest_email']
    api_public_key = args['api_public_key']
    api_secret_key = args['api_secret_key']
    slack_token = args['slack_token']
    btc = args['btc']
    btc = updatedBtcPrice(btc)
    with LocalBitcoinsSpider(url, file, email, dest_email, subject, body, api_public_key, api_secret_key , slack_token, btc=btc) as spider:
        spider.grab_data()

if __name__ == '__main__':
    job()
    schedule.every(19).minutes.do(job)
    while 1:
        schedule.run_pending()
        time.sleep(1)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import uuid

class Migration(migrations.Migration):

    dependencies = [
        ('message', '0004_team_uid'),
    ]

    operations = [
        migrations.AlterField(
            model_name='team',
            name='uid',
            field=models.CharField(default=uuid.uuid4, unique=True, max_length=100),
            preserve_default=True,
        ),
    ]