from time import sleep
from social_interaction_cloud.abstract_connector import AbstractSICConnector
from social_interaction_cloud.action import ActionRunner
from social_interaction_cloud.basic_connector import BasicSICConnector
from social_interaction_cloud.detection_result_pb2 import DetectionResult

class Example:
    """Example that uses speech recognition. Prerequisites are the availability of a dialogflow_key_file,
    a dialogflow_agent_id, and a running Dialogflow service. For help meeting these Prerequisites see
    https://socialrobotics.atlassian.net/wiki/spaces/CBSR/pages/260276225/The+Social+Interaction+Cloud+Manual"""

    def __init__(self, server_ip: str, dialogflow_key_file: str, dialogflow_agent_id: str):
        self.sic = BasicSICConnector(server_ip, 'en-US', dialogflow_key_file, dialogflow_agent_id)
        self.action_runner = ActionRunner(self.sic)

        self.user_model = {}
        self.recognition_manager = {'attempt_success': False, 'attempt_number': 0}

    def run(self) -> None:
        self.sic.start()
        self.action_runner.run_action('say_animated', "hahahahahahahahahhhahahahahahahahahhhahahahahahahahahhhahahahahahahahahhhahahahahahahahahhhahahahahahahahahhhahahahahahahahahhhahahahahahahahahhhahahahahahahahahhhahahahahahahahahhhahahahahahahahahh")
        sleep(2)
        self.action_runner.run_action('say_animated', 'hello')
        # if (self.introduce()):
        #     print('here')
        #     self.action_runner.run_waiting_action('set_language', 'en-US')

        #     while not self.recognition_manager['attempt_success'] and self.recognition_manager['attempt_number'] < 2:
        #         self.action_runner.run_waiting_action('say', 'Hello, world! Tell me your name.')
        #         self.action_runner.run_waiting_action('speech_recognition', 'answer_name', 0, additional_callback=self.on_intent)

        #     self.reset_recognition_management()

        #     if 'name' in self.user_model:
        #         self.action_runner.run_waiting_action('say', 'Thank you!')

        #     while not self.recognition_manager['attempt_success'] and self.recognition_manager['attempt_number'] < 2:
        #         self.action_runner.run_waiting_action('say', 'How old are you?')
        #         self.action_runner.run_waiting_action('speech_recognition', 'answer_age', 0, additional_callback=self.on_intent)

        #     if 'age' in self.user_model:
        #         my_age = 5
        #         self.action_runner.run_waiting_action('say', 'Oh, I am ' + str(my_age) + ' years old, so you are ' + str(self.user_model['age'] - 5) + ' years older than me')
        # else:
        #     self.action_runner.run_waiting_action('say', 'Bye bye!')
        #     # self.action_runner.run_waiting_action('rest')

        self.sic.stop()

    def introduce(self) -> None:
        # self.action_runner.load_waiting_action('wake_up')
        self.action_runner.run_waiting_action('set_language', 'en-US')
        # self.action_runner.run_loaded_actions()
        while not self.recognition_manager['attempt_success'] and self.recognition_manager['attempt_number'] < 2:
            self.action_runner.run_waiting_action('say', 'Hello! I am Nao! Do you want to hear a story?')
            self.action_runner.run_waiting_action('speech_recognition', 'answer_decision', 0, additional_callback=self.on_intent)

        self.reset_recognition_management()

        if 'decision' in self.user_model and self.user_model['decision'] == "Yes":
            return True

        return False

    def on_intent(self, detection_result: DetectionResult) -> None:
        if detection_result and len(detection_result.parameters) > 0:

            if detection_result.intent == 'answer_name':
                self.user_model['name'] = detection_result.parameters['name'].struct_value['name']
                self.recognition_manager['attempt_success'] = True

            elif detection_result.intent == 'answer_age':
                self.user_model['age'] = detection_result.parameters['age'].struct_value['amount']
                self.recognition_manager['attempt_success'] = True

            elif detection_result.intent == 'answer_decision':
                print(detection_result.parameters['decision'].string_value)
                self.user_model['decision'] = detection_result.parameters['decision'].string_value
                self.recognition_manager['attempt_success'] = True
            else:
                self.recognition_manager['attempt_number'] += 1

        else:
            self.recognition_manager['attempt_number'] += 1

    def reset_recognition_management(self) -> None:
        self.recognition_manager.update({'attempt_success': False, 'attempt_number': 0})

example = Example('127.0.0.1',
                  'newagent-tfdt-2585f6cb1ae5.json',
                  'newagent-tfdt')
example.run()


#*************************************************************************
#*
#*  Title:            3x3 LED Cube
#*
#*  Description:      Simple LED cube for soldering practice and demonstrate some
#*                    array handling with GPIO pins
#*                       - Pins P1 to P9 are connected to LED Cathode (-)
#*                       - Layers are connected to LED Anode (+)
#*
#*  Author:           Jacob Groves, Coventry Makerspace
#*
#*************************************************************************

import RPi.GPIO as GPIO
from time import sleep

# Assign the individual LED pins a variable name . . .
P1 = 2  # 13
P2 = 3  # 19
P3 = 4  # 26
P4 = 10  # 23
P5 = 9  # 24
P6 = 11  # 25
P7 = 25  # 16
P8 = 8  # 20
P9 = 7  # 21

# Assign the layers to a nice variable . . .
Top = 17
Middle = 27
Bottom = 22

# LED sequences represented by an array . . .
aPins = [P1, P2, P3, P4, P5, P6, P7, P8, P9]
aPinsReversed = [P9, P8, P7, P6, P5, P4, P3, P2, P1]
aPinsStar = [P1, P2, P3, P6, P9, P8, P7, P4]
aPinsStar2 = [P9, P8, P7, P4, P1, P2, P3, P6]
aPinsSpiral = [P1, P2, P3, P6, P9, P8, P7, P4, P5]
aPinsSpiralReversed = [P5, P4, P7, P8, P9, P6, P3, P2, P1]

aLayers = [Top, Middle, Bottom]

# Time delay built in between pin output . .  .
delay = 0.15

#**************************************************************************
#* setupColumns - Initialisation of the pins . . .
#**************************************************************************

def setupColumns():

    # GPIO pin addressing will use the virtual number . . .
    GPIO.setmode(GPIO.BCM)

    # Initialise the pins so we can output values to them . . .
    GPIO.setup(aPins, GPIO.OUT)
    GPIO.setup(aLayers, GPIO.OUT)

    GPIO.output(aPins, False)
    GPIO.output(aLayers, True)

#**************************************************************************
#* cyclePins - Receive array of pins & light them up one by one . . .
#**************************************************************************

def cyclePins(pPins):

    for iPin in pPins:

        GPIO.output(iPin, True)
        sleep(delay)
        GPIO.output(iPin, False)

#**************************************************************************
#* cyclePinsPair - Receive 2 arrays of pins & light them up in pairs . . .
#**************************************************************************

def cyclePinsPair(pPins, pPins2):

    for iPin in range(len(pPins)):

        GPIO.output(pPins[iPin], True)
        GPIO.output(pPins2[iPin], True)
        sleep(delay)
        GPIO.output(pPins[iPin], False)
        GPIO.output(pPins2[iPin], False)

#**************************************************************************
#* Mainline logic . . .
#**************************************************************************
try:
    print('Press CTRL+C to stop script')
    i = 0
    while 1:

        # Spinning star . . .
        GPIO.output(aLayers, True)
        GPIO.output(P5, True)
        cyclePinsPair(aPinsStar, aPinsStar2)
        cyclePinsPair(aPinsStar, aPinsStar2)
        GPIO.output(P5, False)

        # Zig-zaggy columns . . .
        GPIO.output(aLayers, True)
        cyclePins(aPins)
        cyclePins(aPinsReversed)
        GPIO.output(aLayers, False)

        # Spiral columns . . .
        GPIO.output(aLayers, True)
        cyclePins(aPinsSpiral)
        cyclePins(aPinsSpiralReversed)
        GPIO.output(aLayers, False)

        # Zig-zaggy layers . . .
        GPIO.output(Top, True)
        cyclePins(aPins)
        GPIO.output(Top, False)
        GPIO.output(Middle, True)
        cyclePins(aPinsReversed)
        GPIO.output(Middle, False)
        GPIO.output(Bottom, True)
        cyclePins(aPins)
        GPIO.output(Bottom, False)
        sleep(delay)

except KeyboardInterrupt:
    GPIO.cleanup()

finally:
    GPIO.cleanup()


#!/usr/bin/env python3
"""
sp_version.py

Outputs a list of software versions used that night. Rewritten by Dylan Gatlin
 based on spVersion by Elena Malanushenko

Changelog:
2020-06-08  DG  Ported to ObserverTools in Python 3, replaced os with sub
"""

import subprocess as sub

__version__ = '3.0.0'

def main():
    softwares, versions = [], []
    idlspec2d = sub.Popen('setup idlspec2d; idlspec2d_version', shell=True,
                          stdout=sub.PIPE)
    softwares.append('idlspec2d')
    versions.append(idlspec2d.stdout.read().decode('utf-8').strip('\n'))

    plate_mapper = sub.Popen('setup idlmapper; echo "print, idlmapper_version()'
                             '" | idl 2> /dev/null &', shell=True,
                             stdout=sub.PIPE)
    softwares.append('plate-mapper3')
    versions.append(plate_mapper.stdout.read().decode('utf-8').strip('\n'))

    petunia = sub.Popen('readlink /home/sdss4/products/Linux64/petunia/current',
                        shell=True, stdout=sub.PIPE)
    softwares.append('Petunia')
    versions.append(petunia.stdout.read().decode('utf-8').strip('\n'))

    autoscheduler = sub.Popen('readlink /home/sdss4/products/Linux64/'
                              'autoscheduler/current', shell=True,
                              stdout=sub.PIPE)
    softwares.append('Autoscheduler')
    versions.append(autoscheduler.stdout.read().decode('utf-8').strip('\n'))

    sdss_module = sub.Popen("readlink /home/sdss4/products/Linux64/"
                            "sdss_python_module/current", shell=True,
                            stdout=sub.PIPE)
    softwares.append('SDSS Python Module')
    versions.append(sdss_module.stdout.read().decode('utf-8').strip('\n'))

    mangadrp = sub.Popen("/home/manga/products/Linux64/mangadrp/trunk/bin/"
                         "mangadrp_version", shell=True, stdout=sub.PIPE)
    softwares.append('mangadrp')
    versions.append(mangadrp.stdout.read().decode('utf-8').strip('\n'))

    print('{:-^42}'.format('Other Versions'))
    for s, v in zip(softwares, versions):
        print('{:<20}: {:<20}'.format(s, v))

if __name__ == '__main__':
    main()


# 2021-04-11, lucas.mayer.almeida@ccc.ufcg.edu.br
# Exibe quais pessoas atingiram a maioridade penal

def maioridade_penal(nomes, idades) :
    resultado = []
    string = ""
    nomes = nomes.split()
    idades = idades.split()
    for i in range(len(idades)) :
        if int(idades[i]) >= 18 :
            resultado.append(nomes[i])
    for i in range(len(resultado)) :
        if i == len(resultado) - 1 :
            string += resultado[i]
        else :
            string += resultado[i] + " "
    return string


import torch
# import matplotlib
# matplotlib.use('Agg')
import matplotlib.pyplot as plt

import matplotlib.patches as mpatches
import numpy as np
import config

def save_checkpoint(state, filename):
    print('----> Saving checkpoint')
    torch.save(state, filename)

def load_checkpoint(checkpoint, model_gen, model_disc, model_decoder, optimizer_gen, optimizer_disc, optimizer_decoder,
                    gen_scheduler, disc_scheduler, decoder_scheduler):
    print("----> Loading checkpoint")
    model_gen.load_state_dict(checkpoint['gen_state_dict'])
    optimizer_gen.load_state_dict(checkpoint['gen_optimizer'])
    model_disc.load_state_dict(checkpoint['disc_state_dict'])
    optimizer_disc.load_state_dict(checkpoint['disc_optimizer'])
    model_decoder.load_state_dict(checkpoint['decoder_state_dict'])
    optimizer_decoder.load_state_dict(checkpoint['decoder_optimizer'])
    gen_scheduler.load_state_dict(checkpoint['gen_scheduler'])
    disc_scheduler.load_state_dict(checkpoint['disc_scheduler'])
    decoder_scheduler.load_state_dict(checkpoint['decoder_scheduler'])

    return checkpoint['epoch_num']  #, checkpoint['TB_step']

def get_mean_std(loader):
    # var[X] = E[X**2] - E[X]**2
    channels_sum, channels_squared_sum, num_batches = 0, 0, 0

    for data, _ in loader:
        channels_sum += torch.mean(data, dim=[0, 2, 3])
        channels_squared_sum += torch.mean(data ** 2, dim=[0, 2, 3])
        num_batches += 1

    mean = (channels_sum / num_batches)
    std = (channels_squared_sum / num_batches - mean ** 2) ** 0.5

    return mean, std

def gaus2d(x=0, y=0, mx=0, my=0, sx=1, sy=1):
    return 1. / (2. * np.pi * sx * sy) ** 0.5 * np.exp(-((x - mx)**2. / (2. * sx**2.) + (y - my)**2. / (2. * sy**2.)))

x = np.linspace(-5, 5, 1000)
y = np.linspace(-5, 5, 1000)
x, y = np.meshgrid(x, y) # get 2D variables instead of 1D
z = gaus2d(x, y, mx=1, my=0, sx=1, sy=0.2)

def get_digit_distribution(digits, num_elements):
    rad_angles = [2 * np.pi * float(digit / config.NUM_CLASSES) for digit in digits]
    Rotation_matrices = [[[np.cos(angle), - np.sin(angle)], [np.sin(angle), np.cos(angle)]] for angle in
                         rad_angles]
    shifts = [[np.cos(angle), - np.sin(angle)] for angle in rad_angles]

    # create simple non-symmetric gaussian
    mx, my = 0, 0
    sx, sy = 1.5, 0.2
    mean = [mx, my]
    cov = [[sx ** 2, 0], [0, sy ** 2]]

    samples = np.random.multivariate_normal(mean, cov, (num_elements,))
    shift = np.multiply(shifts, [3 * max(sx, sy), 3 * max(sx, sy)])

    samples_new = [shift[idx] + np.matmul(samples[idx], Rotation_matrices[idx]) for idx in range(num_elements)]

    return samples_new

def save_reconstructed_images(orig_img, reconstructed_img, epoch_num, mode):
    plt.ioff()

    fig, ax = plt.subplots()
    fig.suptitle(f"Epoch {epoch_num}")
    ax1 = fig.add_subplot(121)
    ax1.imshow(orig_img)
    ax1.title.set_text('Original')
    ax2 = fig.add_subplot(122)
    ax2.imshow(reconstructed_img)
    ax2.title.set_text('Reconstructed')
    fig.savefig("results/" + mode + "/reconstructed_image/epoch_" + str(epoch_num) + ".jpg")
    plt.close(fig)

def save_latent_distribution(loader, encoder, epoch_num, mode):
    samples_distribution = np.array([])
    labels = np.array([])

    with torch.no_grad():
        for image, label in loader:
            image = image.to(config.device)
            label = label.to(config.device)

            latent_vector = encoder(image).squeeze(3).squeeze(2).cpu().numpy()
            samples_distribution = np.vstack([samples_distribution, latent_vector]) if samples_distribution.size else latent_vector
            labels = np.hstack([labels, label.cpu().numpy()]) if labels.size else label.cpu().numpy()
        plt.ioff()
        fig, ax = plt.subplots(figsize=(5, 5))
        fig.suptitle(f"Epoch {epoch_num}")
        data_colors = [config.colors[digit] for digit in labels]
        x = samples_distribution[:, 0]
        y = samples_distribution[:, 1]

        ax.scatter(x, y, c=data_colors)

        # create legend
        legend_dict = {str(i): config.colors[i] for i in range(config.NUM_CLASSES)}
        patchList = [mpatches.Patch(color=legend_dict[key], label=key) for key in legend_dict]
        plt.legend(handles=patchList)
        plt.title('Latent Space Distribution')
        # ax.set_xlim(-4, 4)
        # ax.set_ylim(-4, 4)
        fig.savefig("results/" + mode + "/latent_distribution/epoch_" + str(epoch_num) + ".jpg")
        plt.close(fig)

def plot_latent_distribution(loader, encoder):
    samples_distribution = np.array([])
    labels = np.array([])

    with torch.no_grad():
        for image, label in loader:
            image = image.to(config.device)
            label = label.to(config.device)

            latent_vector = encoder(image).squeeze(3).squeeze(2).cpu().numpy()
            samples_distribution = np.vstack([samples_distribution, latent_vector]) if samples_distribution.size else latent_vector
            labels = np.hstack([labels, label.cpu().numpy()]) if labels.size else label.cpu().numpy()

        fig, ax = plt.subplots(figsize=(5, 5))
        data_colors = [config.colors[digit] for digit in labels]
        x = samples_distribution[:, 0]
        y = samples_distribution[:, 1]

        ax.scatter(x, y, c=data_colors)

        # create legend
        legend_dict = {str(i): config.colors[i] for i in range(config.NUM_CLASSES)}
        patchList = [mpatches.Patch(color=legend_dict[key], label=key) for key in legend_dict]
        plt.legend(handles=patchList)
        plt.title('Latent Space Distribution')
        plt.show()

def plot_gmm(num_classes=config.NUM_CLASSES):
    import matplotlib.pyplot as plt
    num_classes = 10
    rad_angles = [2 * np.pi * float(digit / num_classes) for digit in range(num_classes)]
    Rotation_matrices = [[[np.cos(angle), - np.sin(angle)], [np.sin(angle), np.cos(angle)]] for angle in rad_angles]
    shifts = [[np.cos(angle), - np.sin(angle)] for angle in rad_angles]

    mx, my = 0, 0
    sx, sy = 1.5,  0.25

    mean = [mx, my]
    cov = [[sx ** 2, 0], [0, sy ** 2]]
    x = np.random.multivariate_normal(mean, cov, (1000,))

    x_total = np.array(())

    for digit in range(num_classes):
        Rotation_matrix = Rotation_matrices[digit]
        shift = np.multiply(shifts[digit], [5*max(sx, sy), 5*max(sx, sy)])

        x_new = shift + np.matmul(x, Rotation_matrix)
        x_total = np.vstack([x_total, x_new]) if x_total.size else x_new
    # plot gmm
    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.scatter(x_total[:, 0], x_total[:, 1], color='b', marker='o')
    # ax.set_xlim(-5, 5)
    # ax.set_ylim(-5, 5)
    # plt.title(f"correlation equals {digit}")
    plt.show()

def plot_batch_latent_distribution(samples, labels):
    txt = labels.cpu().numpy()
    samples_new = np.array(samples)
    fig, ax = plt.subplots()
    x = samples_new[:, 0]
    y = samples_new[:, 1]
    ax.scatter(x, y)

    for i, lab in enumerate(txt):
        ax.annotate(lab, (x[i], y[i]))


"""
Created on March 18 2021
@author: Dongfang Xu

Part of this library is based on sentence-transformers[https://github.com/UKPLab/sentence-transformers]
"""
import math
import queue

import numpy as np
import torch
import torch.multiprocessing as mp
import transformers
from sentence_transformers import SentenceTransformer, models
from sklearn.metrics.pairwise import cosine_similarity

import read_files as read

class ConceptNormalizer():
    """
    Loads or create a concept normalizer model, that can be used to map concepts/mentions to embeddings.
    :param model_name_or_path: Filepath of pre-trained LM or fine-tuned sentence-transformers. If it is a path for fine-tuned sentence-transformer, please also set sentence_transformer True.
    :sentence_transformer: This parameter can be used to create custom SentenceTransformer models from scratch.
    :search_over_synonyms: Whether to generate concept embeddings by averaging synonyms of that concept and then search over concept.
    """
    def __init__(
        self,
        model_name_or_path: str = None,
        sentence_transformer: bool = False,
        search_over_synonyms: bool = True,
    ):

        if sentence_transformer == False:
            ######## Load pre-trained models ########
            ######## word_embedding_model = models.BERT(path_to_BERT-based-models) #####
            ######## word_embedding_model = models.RoBERTa(path_to_RoBERTa-based-models) #####
            word_embedding_model = models.BERT(model_name_or_path)

            # Apply mean pooling to get one fixed sized sentence vector
            pooling_model = models.Pooling(
                word_embedding_model.get_word_embedding_dimension(),
                pooling_mode_mean_tokens=True,
                pooling_mode_cls_token=False,
                pooling_mode_max_tokens=False)

            self.concept_normalizer = SentenceTransformer(
                modules=[word_embedding_model, pooling_model])

        else:
            #### load fine-tuned sentence-BERT models ####
            self.concept_normalizer = SentenceTransformer(model_name_or_path)

        self.search_over_synonyms = search_over_synonyms
        self.concept_mentions = {}
        self.concepts = []

    def generate_embeddings(self, ontology):

        for idx, [synonym, concept] in enumerate(ontology):
            read.add_dict(self.concept_mentions, concept, synonym)

        if len(self.concepts) == 0:
            self.concepts = self.concept_mentions.keys()

        self.synonyms = []
        self.concept_mention_idx = {}
        self.idx_to_concept = {}

        idx = 0
        for concept in self.concepts:
            concept_synonyms = list(set(self.concept_mentions[concept]))
            self.synonyms += concept_synonyms
            end = idx + len(concept_synonyms)
            for index in range(idx, end):
                self.idx_to_concept[int(index)] = concept
                self.concept_mention_idx[concept] = (idx, end)
            idx = end

        self.ontology_embedding = self.concept_normalizer.encode(self.synonyms)

        if self.search_over_synonyms == False:
            ontology_embedding_avg = []
            for concept in self.concepts:
                s, e = self.concept_mention_idx[concept]
                embedding_synonyms = self.ontology_embedding[s:e]

                ontology_embedding_avg.append(
                    np.mean(embedding_synonyms, axis=0))
            self.ontology_embedding_avg = np.asarray(ontology_embedding_avg)
            self.ontology_embedding = None

    def load_ontology(self, concept_file_path=None):
        if concept_file_path is not None:
            ontology = read.read_from_tsv(concept_file_path)
            self.generate_embeddings(ontology)
        else:
            raise ValueError("Please specify the path of ontology files")

    def add_terms(self, term_concept_pairs=[]):
        """
        term_concept_pairs is a list of 2-element tuples,
        [(syn_1, concept_1), (syn_2,concept_2),...]
        """
        ontology = [[item[0], item[1]] for item in term_concept_pairs]

        self.generate_embeddings(ontology)

    def normalize(self, mention, top_k):

        mention_embedding = self.concept_normalizer.encode(
            [mention], show_progress_bar=True)

        if self.search_over_synonyms:
            similarity_matrix = cosine_similarity(mention_embedding,
                                                  self.ontology_embedding)
        else:
            similarity_matrix = cosine_similarity(mention_embedding,
                                                  self.ontology_embedding_avg)
            # similarity_matrix = similarity_matrix.astype(np.float16)
        idx = np.argsort(similarity_matrix).astype(
            np.int32)[:, ::-1][:, :top_k]

        scores_pre = [row[idx[i]]
                      for i, row in enumerate(similarity_matrix)][0]
        concepts_pre = [[self.idx_to_concept[item] for item in row]
                        for row in idx][0]

        if self.search_over_synonyms:
            synonyms_pre = [[self.synonyms[item] for item in row]
                            for row in idx][0]
            predictions = [(cui, syn, score) for cui, syn, score in zip(
                concepts_pre, synonyms_pre, scores_pre)]
            return predictions
        else:
            predictions = [(cui, self.concept_mentions[cui][:2], score)
                           for cui, score in zip(concepts_pre, scores_pre)]
            return predictions


import FWCore.ParameterSet.Config as cms

# MonitorTrackResiduals
from DQMServices.Core.DQMEDAnalyzer import DQMEDAnalyzer
MonitorTrackResiduals = DQMEDAnalyzer("MonitorTrackResiduals",
    OutputMEsInRootFile = cms.bool(False),
    # should histogramms on module level be booked and filled?
    Mod_On = cms.bool(True),
    VertexCut = cms.untracked.bool(True),
    OutputFileName = cms.string('test_monitortracks.root'),
    genericTriggerEventPSet = cms.PSet(),
    # bining and range for absolute and normalized residual histogramms
    TH1ResModules = cms.PSet(
        xmin = cms.double(-0.05),   # native unit in CMS is [cm], so these are 500um
        Nbinx = cms.int32(100),
        xmax = cms.double(0.05)
    ),
    TH1NormResModules = cms.PSet(
        xmin = cms.double(-5.0),
        Nbinx = cms.int32(100),
        xmax = cms.double(5.0)
    ),
    # input for Tracks and Trajectories, should be TrackRefitter
    # or similar
    Tracks = cms.InputTag("TrackRefitter"),
    trajectoryInput = cms.string('TrackRefitter'),
    # should all MEs be reset after each run?
    ResetAfterRun = cms.bool(True)
)


"""get_network_timezones.py

An over-engineered script for scraping networks (now called companies) from theTVDB and formatting them into something we can use.
"""

import argparse
import logging
import mimetypes
import random
import re
import sys
import time
import traceback
from pathlib import Path
from typing import Type, Union
from urllib.parse import parse_qs, urlparse

import coloredlogs
import country_converter
import pytz
import requests
import tvmaze.api
import tvmaze.models
import verboselogs
from bs4 import BeautifulSoup
from pytz import country_timezones
from requests_cache import CacheMixin
from requests_html import HTMLSession
from tvmaze.expections import NetworkNotFound, WebChannelNotFound

logger = logging.getLogger("")
country_converter_logger = logging.getLogger("country_converter")
country_converter_logger.setLevel(logging.ERROR)

class CachedHTMLSession(CacheMixin, HTMLSession):
    """Session with features from both CachedSession and HTMLSession"""

class Scraper:
    def __init__(self):
        self.session = CachedHTMLSession(cache_name="requests_cache.sqlite")

        self.location = Path(__file__).parent

        self.verbose = False
        self.debug = False
        self.quiet = False

        self.purge_old = False
        self.start_page = 1
        self.number_of_pages = -1
        self._include_types = ["Network"]
        self.throttle = True
        self.throttle_seconds = None
        self._pages_processed = 0

        self.output_file = self.location.joinpath("network_timezones.txt")
        self.discarded_file_path = self.output_file.parent.joinpath(f"DISCARDED_{self.output_file.name}")

        self.parser = argparse.ArgumentParser(
            epilog=f"""
                Network and timezones will be stored in: {self.output_file}
                All purged networks will be stored in: {self.discarded_file_path}
            """,
            description=__doc__,
            formatter_class=argparse.RawTextHelpFormatter,
        )

        self.parser.add_argument(
            "-v",
            "--verbose",
            dest="verbose",
            action="store_true",
            help="enable verbose logging",
        )
        self.parser.add_argument(
            "-d",
            "--debug",
            dest="debug",
            action="store_true",
            help="enable debug output",
        )
        self.parser.add_argument(
            "-q",
            "--quiet",
            dest="quiet",
            action="store_true",
            help="shush log output to only warnings and errors",
        )
        self.parser.add_argument(
            "-p",
            "--purge-old",
            dest="purge_old",
            action="store_true",
            help="purge items not found on theTVDB",
        )
        self.parser.add_argument(
            "-s",
            "--start-page",
            type=int,
            dest="start_page",
            default=self.start_page,
            help="page to start from",
        )
        self.parser.add_argument(
            "-n",
            "--numer-of-pages",
            type=int,
            dest="number_of_pages",
            default=self.number_of_pages,
            help="number of pages to parse",
        )
        self.parser.add_argument(
            "-o",
            "--output-file",
            dest="output_file",
            default=self.output_file,
            help="output filename",
        )
        self.parser.add_argument(
            "-i",
            "--include-types",
            dest="include_types",
            default=",".join(self.include_types),
            help="company types to include",
        )
        self.parser.add_argument(
            "--no-throttle",
            dest="throttle",
            action="store_false",
            help="do not sleep between pages",
        )
        self.parser.add_argument(
            "-t",
            "--throttle-seconds",
            type=int,
            dest="throttle_seconds",
            default=self.throttle_seconds,
            help="do not sleep between pages",
        )

        self.parser.parse_args(namespace=self)

        if self.verbose:
            self.set_log_level(logging.VERBOSE)
        elif self.debug:
            self.set_log_level(logging.DEBUG)
        elif self.quiet:
            self.set_log_level(logging.WARNING)
        else:
            self.set_log_level(logging.INFO)

        self._next_page = self.start_page
        self._new_list = {}
        self._extra_companies = {}
        self._extra_company_types = []
        self._extra_dump_files = []

        self.country_converter = country_converter.CountryConverter(include_obsolete=True)

    def set_log_level(self, level: Type[int]):
        verboselogs.install()
        global logger
        logger = verboselogs.VerboseLogger("")
        logger.addHandler(logging.StreamHandler())
        logger.setLevel(level)

        coloredlogs.install(level=level, logger=logger)
        for handler in logger.handlers:
            handler.setLevel(level)

    @property
    def include_types(self):
        return self._include_types

    @include_types.setter
    def include_types(self, value):
        self._include_types = {name.strip().lower() for name in value.split(",")}

    def scrape_tvdb(self):

        logger.info(f"Scraping page: {self._next_page}")

        companies_url = "https://thetvdb.com/companies/"

        data = self.session.get(companies_url, params={"page": self._next_page})

        soup = BeautifulSoup(data.content, "html.parser")

        company_rows = soup.find_all(class_="row mb-3")
        for _current in company_rows:
            company_name = _current.find(class_="mt-0 mb-0").get_text(strip=True)
            company_country = _current.find(class_="ml-0").get_text(strip=True)
            company_type = _current.find(class_="ml-1").get_text(strip=True)

            if company_type.lower() in [item.lower() for item in self.include_types]:
                logger.verbose(f"Found result: {company_name}:{company_country} with type {company_type}")
                self._new_list[company_name] = company_country
            else:
                if company_type not in self._extra_companies:
                    self._extra_companies[company_type] = {}
                self._extra_companies[company_type][company_name] = company_country

                # logger.verbose(f"Skipping result: {company_name}:{company_country} with type {company_type}")

        # This must be before any return due to checks such as number of pages processed,
        # so that the loop is broken unless this is reset below to a value.
        self._next_page = None

        self._pages_processed += 1
        if self.number_of_pages > 0 and self._pages_processed >= self.number_of_pages:
            logger.info(f"Reached our maximum number of pages to scrape: {self._pages_processed}")
            return

        next_page = soup.find("a", attrs={"rel": "next"})
        if next_page and next_page.get("href"):
            parsed_url = urlparse(next_page["href"])
            query = parse_qs(parsed_url.query)
            self._next_page = query.get("page")

    def __process_tvmaze_result(self, item: Union[tvmaze.models.WebChannel, tvmaze.models.Network]):
        try:
            timezone = item.country.timezone
        except AttributeError:
            timezone = ""

        if item.name not in self._new_list:
            logger.verbose(f"Found result: {item.name}:{timezone} with type {item.__class__.__name__}")
            self._new_list[item.name] = timezone
        elif timezone:
            logger.verbose(f"Setting result: {item.name}:{timezone} with type {item.__class__.__name__}")
            self._new_list[item.name] = timezone

    def scrape_tvmaze(self):
        api = tvmaze.api.Api()

        original_throttle = self.throttle_seconds
        self.throttle_seconds = 2
        self._next_page = self.start_page
        while True:
            try:
                data = api.network.get(self._next_page)
                self.__process_tvmaze_result(data)
                self.sleep()
                self._next_page += 1
            except NetworkNotFound:
                logger.info(f"Finished processing {self._next_page - 1} Networks from tvmaze")
                break

        self._next_page = self.start_page
        while True:
            try:
                data = api.web_channel.get(self._next_page)
                self.__process_tvmaze_result(data)
                self.sleep()
                self._next_page += 1
            except WebChannelNotFound:
                logger.info(f"Finished processing {self._next_page - 1} Web Channels from tvmaze")
                break

        self.throttle_seconds = original_throttle

    def scrape_tmdb(self):
        pass

    def sleep(self):
        if self.throttle:
            seconds = self.throttle_seconds
            if seconds is None or seconds <= 0:
                seconds = random.randint(1, 3)

            if self._next_page:
                logger.debug(f"Sleeping for {(seconds)}")
                time.sleep(seconds)

    def dump_extra_company_types(self):
        logger.info("Dumping excluded types to corresponding txt files, in format {company}:{country} (not timezone!)")
        for company_type, companies in self._extra_companies.items():
            location = self.location.joinpath(f"{company_type.lower().replace(' ', '_')}_timezones.txt")
            self._extra_dump_files.append(location)

            logger.info(f"Dumping company type {company_type} to {location}")

            with open(location, mode="w", encoding="utf-8") as dump_file:
                for company, country in sorted(companies.items()):
                    new_value = self.country_name_to_timezone(company, country)
                    dump_file.write(f"{company}:{new_value}\n")

        self._extra_company_types = sorted(self._extra_companies)
        self._extra_companies.clear()

    def country_name_to_timezone(self, network: str, value: str = None) -> str:
        if value == "Unknown Country":
            return ""

        try:
            pytz.timezone(value)
            return value
        except pytz.exceptions.UnknownTimeZoneError:
            for src in ("name_official", "name_short", "regex", "ISO2", "ISO3"):
                code = self.country_converter.convert(value, to="ISO2", src=src)
                if code and code != "not found":
                    break
            else:
                logger.warning(f"Did not find timezone for {network} from value {value}, returning empty!")
                return ""

            try:
                result = country_timezones.get(code, "")[0]
            except Exception as error:
                logger.debug(traceback.format_exc())
                logger.warning(f"Error parsing timezone for {network} from value {value} with {error}, returning empty")
                result = ""

        return result

    def process_line(self, network, value):
        return network, self.country_name_to_timezone(network, value)

    def run(self):
        # Load data from TVDB network select box
        while self._next_page is not None:
            self.scrape_tvdb()
            self.sleep()

        self.scrape_tvmaze()

        logger.info(f"Processed {self._pages_processed} pages from theTVDB")

        self.dump_extra_company_types()

        old_list = []
        if self.output_file.is_file():
            with open(self.output_file, encoding="utf-8") as tz_file:
                for current in tz_file.readlines():
                    network, timezone = current.strip("\n").rsplit(":", 1)
                    old_list.append({"network": network, "time_zone": timezone})

        new_incomplete = new_complete = duplicate_count = 0
        new_data = []
        invalid_data = []

        def is_name_in_list(name, lst):
            for item in lst:
                if item.startswith(name + ":"):
                    return True
            return False

        logger.verbose("Listing valid old networks + time zones")
        logger.verbose("Action - Network Name - Time Zone")

        data_to_append = []
        while old_list:
            current = old_list.pop(0)

            line_format = f"{current['network']}:{current['time_zone']}\n"

            if is_name_in_list(current["network"], new_data + invalid_data):
                duplicate_count += 1
                logger.verbose(f"Duplicate - {current['network']} - {current['time_zone']}")
                continue

            if current["network"] not in self._new_list:
                if current["time_zone"]:
                    valid_timezone = self.country_name_to_timezone(current["network"], current["time_zone"])
                    if valid_timezone:
                        invalid_data.append(line_format)
                logger.verbose(f"Invalid: {current['network']} - {current['time_zone']}")
                continue
            else:
                if current["time_zone"]:
                    valid_timezone = self.country_name_to_timezone(current["network"], current["time_zone"])
                    if valid_timezone:
                        del self._new_list[current["network"]]
                        new_data.append(line_format)

                    logger.verbose(f"Re-adding: {current['network']} - {current['time_zone']}")

        logger.verbose("--- Adding new networks ---")
        logger.verbose("Action - Network Name - Country - Guessed Time Zone")

        for network, country in self._new_list.items():
            # try to determine time zone by country name in display name
            network, timezone_guess = self.process_line(network, country)
            if timezone_guess:
                new_complete += 1
                logger.debug(f"New network: {network} - {country} - {timezone_guess}")
                new_data.append(f"{network}:{timezone_guess}\n")
            else:
                new_incomplete += 1
                fixed_value = re.sub(re.escape(network) + r"(?:.* \((.*)\))?$", r"\1", country)
                logger.debug(f"New network: {network} - {fixed_value} - Unknown")
                data_to_append.append(f"{network}:{timezone_guess}\n")

        with open(self.output_file, mode="w", encoding="utf-8") as tz_file:
            if self.purge_old:
                if new_data:
                    new_data.sort()
                    tz_file.writelines(new_data)
            else:
                if new_data or invalid_data:
                    joined_data = new_data + invalid_data
                    joined_data.sort()
                    tz_file.writelines(joined_data)
            if data_to_append:
                data_to_append.sort()
                tz_file.writelines(data_to_append)

        if self.purge_old and invalid_data:
            with open(self.discarded_file_path, mode="w", encoding="utf-8") as tz_file:
                tz_file.writelines(invalid_data)

        logger.info("--- Done ---")
        logger.info(f"Processed {self._pages_processed} pages")
        logger.info(f"Skipped all {self._extra_company_types}")
        logger.info(f"New file created [{self.output_file.name}]")
        for extra_file in self._extra_dump_files:
            logging.info(f"New file created: {extra_file}")

        logger.info(f"Total {len(new_data)}")
        logger.info(f"New with time zone {new_complete}")
        logger.info(f"New without time zone {new_incomplete}")
        logger.info(f"Duplicates: {duplicate_count}")
        if invalid_data:
            if not self.purge_old:
                logger.info(f"Invalid networks added {len(invalid_data)}")
            else:
                logger.info(f"New file created [{self.discarded_file_path.name}]")
                logger.info(f"Total discarded {len(invalid_data)}")

        self.missing_images(new_data)

    def missing_images(self, network_list):
        def is_image(name):
            return mimetypes.guess_type(name)[0] == "image"

        contents_url: str = "https://api.github.com/repos/SickChill/SickChill/contents/sickchill/gui/slick/images/network"
        contents_json: dict = requests.get(contents_url).json()

        # Split off the timezones
        network_list: list = [network.rsplit(":", 1)[0] for network in network_list]

        existing_images: dict = {image["name"].rsplit(".", 1)[0].lower(): image["name"] for image in contents_json if is_image(image["name"])}

        # Lowercase to make for easier comparison
        network_list_lower: list = [network.lower() for network in network_list]

        extra_images: list = [filename for image, filename in existing_images.items() if image not in network_list_lower]
        missing_images: list = [network for network in network_list if network.lower() not in existing_images.keys()]

        missing_location: "Path" = self.location.joinpath("missing_images.txt")
        with open(missing_location, "w", encoding="utf-8") as missing:
            missing.write("\n".join(missing_images))

        extra_location = self.location.joinpath("extra_images.txt")
        with open(extra_location, "w", encoding="utf-8") as extra:
            extra.write("\n".join(extra_images))

        logger.info(f"Missing images count: {len(missing_images)}, Extra images count: {len(extra_images)}")

if __name__ == "__main__":

    scraper = Scraper()
    if sys.getdefaultencoding() != "utf-8" or sys.getfilesystemencoding() != "utf-8":
        logger.error("Default encoding and file system encoding must me utf-8 for use this script")
        logger.info(f"File system encoding is: {sys.getfilesystemencoding()}")
        logger.info(f"Default system encoding is: {sys.getdefaultencoding()}")
        sys.exit(1)
    scraper.run()


'''
Authors: Henry Thompson, Bharat Ram Ambati
Date: 2014-10-01
Copyright: This work is licensed under a Creative Commons
Attribution-NonCommercial 4.0 International License
(http://creativecommons.org/licenses/by-nc/4.0/): You may re-use,
redistribute, or modify this work for non-commercial purposes provided
you retain attribution to any previous author(s).
'''
# fix buggy NLTK 3 :-(
# different fixes for different versions :-((((
import re, sys
import nltk
from nltk.grammar import _ARROW_RE, _PROBABILITY_RE, _DISJUNCTION_RE, Production
from nltk.draw import CFGEditor
from nltk.probability import ImmutableProbabilisticMixIn
ARROW = u'\u2192'
TOKEN = u'([\\w ]|\\\\((x[0-9a-f][0-9a-f])|(u[0-9a-f][0-9a-f][0-9a-f][0-9a-f])))+'
CFGEditor.ARROW = ARROW
CFGEditor._TOKEN_RE=re.compile(u"->|u?'"+TOKEN+u"'|u?\""+TOKEN+u"\"|\\w+|("+ARROW+u")")
CFGEditor._PRODUCTION_RE=re.compile(u"(^\s*\w+\s*)" +
                                    u"(->|("+ARROW+"))\s*" +
                                    u"((u?'"+TOKEN+"'|u?\""+TOKEN+"\"|''|\"\"|\w+|\|)\s*)*$")
nltk.grammar._TERMINAL_RE = re.compile(u'( u?"[^"]+" | u?\'[^\']+\' ) \s*', re.VERBOSE)
nltk.grammar._ARROR_RE = re.compile(u'\s* (->|'+ARROW+') \s*', re.VERBOSE)

from nltk.grammar import _TERMINAL_RE

if sys.version_info[0]>2 or sys.version_info[1]>6:
    from nltk.grammar import PCFG, CFG, ProbabilisticProduction as FixPP
    parse_grammar=CFG.fromstring
    parse_pgrammar=PCFG.fromstring
    from nltk import InsideChartParser
    def nbest_parse(self,tokens,n=None):
        parses=self.parse(tokens)
        if n is None:
            return [parse for parse in parses]
        else:
            return [parses.next() for i in range(n)]
    InsideChartParser.nbest_parse=nbest_parse
else:
    from nltk.grammar import WeightedGrammar as PCFG, WeightedProduction as FixPP
    from nltk import parse_cfg, parse_pcfg
    parse_grammar=parse_cfg
    parse_pgrammar=parse_pcfg

def fix_parse_production(line, nonterm_parser, probabilistic=False):
    """
    Parse a grammar rule, given as a string, and return
    a list of productions.
    """
    pos = 0

    # Parse the left-hand side.
    lhs, pos = nonterm_parser(line, pos)

    # Skip over the arrow.
    m = _ARROW_RE.match(line, pos)
    if not m: raise ValueError('Expected an arrow')
    pos = m.end()

    # Parse the right hand side.
    probabilities = [0.0]
    rhsides = [[]]
    while pos < len(line):
        # Probability.
        m = _PROBABILITY_RE.match(line, pos)
        if probabilistic and m:
            pos = m.end()
            probabilities[-1] = float(m.group(1)[1:-1])
            if probabilities[-1] > 1.0:
                raise ValueError('Production probability %f, '
                                 'should not be greater than 1.0' %
                                 (probabilities[-1],))

        # String -- add terminal.
        elif (line[pos] in "\'\"" or line[pos:pos+2] in ('u"',"u'")):
            m = _TERMINAL_RE.match(line, pos)
            if not m: raise ValueError('Unterminated string')
            rhsides[-1].append(eval(m.group(1)))
            pos = m.end()

        # Vertical bar -- start new rhside.
        elif line[pos] == '|':
            m = _DISJUNCTION_RE.match(line, pos)
            probabilities.append(0.0)
            rhsides.append([])
            pos = m.end()

        # Anything else -- nonterminal.
        else:
            nonterm, pos = nonterm_parser(line, pos)
            rhsides[-1].append(nonterm)

    if probabilistic:
        return [FixPP(lhs, rhs, prob=probability)
                for (rhs, probability) in zip(rhsides, probabilities)]
    else:
        return [Production(lhs, rhs) for rhs in rhsides]

if sys.version_info[0]>2 or sys.version_info[1]>6:
    nltk.grammar._read_production=fix_parse_production
else:
    nltk.grammar.parse_production=fix_parse_production

class CostedProduction(FixPP):
    """
    A probabilistic context free grammar production using costs.
    A PCFG ``ProbabilisticProduction`` is essentially just a ``Production`` that
    has an associated probability, which represents how likely it is that
    this production will be used.  In particular, the probability of a
    ``ProbabilisticProduction`` records the likelihood that its right-hand side is
    the correct instantiation for any given occurrence of its left-hand side.

    :see: ``Production``
    """
    def __init__(self, lhs, rhs, cost):
        """
        Construct a new ``ProbabilisticProduction``.

        :param lhs: The left-hand side of the new ``ProbabilisticProduction``.
        :type lhs: Nonterminal
        :param rhs: The right-hand side of the new ``ProbabilisticProduction``.
        :type rhs: sequence(Nonterminal and terminal)
        :param prob: Probability parameters of the new ``ProbabilisticProduction``.
        """
        ImmutableProbabilisticMixIn.__init__(self, logprob=-cost)
        Production.__init__(self, lhs, rhs)

    def __str__(self):
        return Production.__unicode__(self) + \
            (' [0.0]' if (self.logprob() == 0.0) else ' [%g]' % -self.logprob())

    def __repr__(self):
        return '%s'%str(self)

    def cost(self):
        return 0.0 if self.logprob() == 0.0 else -self.logprob()


n,p,q=int(input()),[],{}
for i in range(n):
    a,b=(input().split())
    q[a]=q.get(a,0)+int(b)
    p.append([a,q[a]])
print(p)
m=max(q.values())
for i,j in p:
    print(i,j)
    if q[i]==m and int(j)>=m:
	    print(i)
	    break


import numpy as np
import cv2, math, os, sys
from scipy import ndimage as ndi
from scipy import ndimage as ndi
from skimage.morphology import watershed
from skimage import color
from skimage.feature import peak_local_max

def edge_mask(x):
    mask = np.ones(x.shape) * 255
    mask[x.ndim * (slice(1, -1),)] = 0
    return mask
def find_min_intensity(input_image):
    frequency_array = np.zeros(256)
    # Now find size of image
    (height, width) = input_image.shape
    for y in range(0, height):
        for x in range(0, width):
            frequency_array[input_image[y][x]] = frequency_array[input_image[y][x]] + 1
    for i in range(1, 255):  # We ignore 0 - Black
        if frequency_array[i] != 0:
            return i
# Define File Path
dataset_dir = "DIC-C2DH-HeLa"
dataset_path = "Segmented/" + dataset_dir + "(Raw)"
# Find Direcotry in Dataset
seq_dir_list = []
for _ in [x[0] for x in os.walk(dataset_path)][1:]:
    if "Masks" not in _:
        seq_dir_list.append(_.split("\\")[1])

for seq in seq_dir_list:
    file_path = dataset_path + "/" + seq + "/"
    file_list = [f for f in os.listdir(file_path) if os.path.isfile(os.path.join(file_path, f))]
    print(f"Begining processing {seq}...")

    for file in file_list:
        print(f"Processing {file}")
        # READ IMAGE-----------------------------------------------------------------------------------
        img_num = file.split(".")[0]
        # Original - define output image
        path_img = file_path + file
        img_ori_gry = cv2.imread(path_img, cv2.IMREAD_GRAYSCALE)
        # Raw Image Number Capturing
        raw_num = int(file.split(".")[0])-1
        if raw_num < 10:
            raw_img_num = "t00" + str(raw_num) + ".tif"
        elif raw_num >= 10 and raw_num < 100:
            raw_img_num = "t0" + str(raw_num) + ".tif"
        else:
            raw_img_num = "t" + str(raw_num) + ".tif"

        # Raw Image reading
        img_ori_path = file_path.replace("Segmented", "RawData").replace("(Raw)", "") + raw_img_num
        img_ori = cv2.imread(img_ori_path)
        # Distance File Path
        dis_img_path = path_img.replace("(Raw)", "(DisT)")
        img_dis_gry = cv2.imread(dis_img_path, cv2.IMREAD_GRAYSCALE)

        #Use Big and small Kernel for different purposes
        kernel_1 = np.ones((5, 5), np.uint8)
        kernel_2 = np.ones((3, 3), np.uint8)

        marker = ndi.label(peak_local_max(img_dis_gry, indices=False, labels=img_dis_gry))[0]
        ws_labels_2 = watershed(-img_dis_gry, marker, mask=img_dis_gry)
        if seq == "Sequence 4":
            eroded = cv2.morphologyEx(ws_labels_2.astype(np.uint8), cv2.MORPH_ERODE, kernel_1, iterations=7)
        elif seq == "Sequence 3" or seq == "Sequence 2" or seq == "Sequence 1":
            eroded = cv2.morphologyEx(ws_labels_2.astype(np.uint8), cv2.MORPH_ERODE, kernel_1, iterations=10)
        r_0 = eroded.copy()
        # Now find the large cells:
        while True:
            r_i = cv2.bitwise_and(cv2.dilate(r_0, kernel_2), img_ori_gry)
            if cv2.countNonZero(r_0) == cv2.countNonZero(r_i):
                reconstructed = r_0
                break
            else:
                r_0 = r_i.copy()
        kernel_3 = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
        kernel_4 = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
        if seq == "Sequence 4":
            closed_holes = cv2.morphologyEx(reconstructed, cv2.MORPH_CLOSE, kernel_1, iterations=1)
            close_eroded = cv2.morphologyEx(closed_holes, cv2.MORPH_ERODE, kernel_1, iterations=8)
            out = cv2.morphologyEx(close_eroded, cv2.MORPH_DILATE, kernel_2, iterations=15)
        elif seq == "Sequence 3":
            closed_holes = cv2.morphologyEx(reconstructed, cv2.MORPH_CLOSE, kernel_1, iterations=1)
            close_eroded = cv2.morphologyEx(closed_holes, cv2.MORPH_ERODE, kernel_1, iterations=7)
            out = cv2.morphologyEx(close_eroded, cv2.MORPH_DILATE, kernel_4, iterations=15)
        elif seq == "Sequence 2":
            closed_holes = cv2.morphologyEx(reconstructed, cv2.MORPH_CLOSE, kernel_1, iterations=1)
            close_eroded = cv2.morphologyEx(closed_holes, cv2.MORPH_ERODE, kernel_2, iterations=10)
            out = cv2.morphologyEx(close_eroded, cv2.MORPH_DILATE, kernel_4, iterations=7)
        elif seq == "Sequence 1":
            closed_holes = cv2.morphologyEx(reconstructed, cv2.MORPH_CLOSE, kernel_1, iterations=1)
            close_eroded = cv2.morphologyEx(closed_holes, cv2.MORPH_ERODE, kernel_3, iterations=10)
            out = cv2.morphologyEx(close_eroded, cv2.MORPH_DILATE, kernel_4, iterations=10)
        # Segmented Image Saving (Gray-Scale)
        seg_dir = "Segmented/" + dataset_dir + "/" + seq
        path_seg = seg_dir + '/' + file
        cv2.imwrite(path_seg, out)


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from bndl.compute.accumulate import Accumulator
from bndl.compute.broadcast import broadcast
from bndl.compute.collections import DistributedCollection
from bndl.compute.files import files
from bndl.compute.ranges import RangeDataset
from bndl.execute.context import ExecutionContext

class ComputeContext(ExecutionContext):
    '''
    :class:`ComputeContext` is the main 'handle' into a cluster of BNDL workers from the 'driver' node.

    It provides a means to create partitioned distributed data sets (which can be then transformed and
    combined), broadcast data, create accumulators to collect data into, etc.
    '''
    @property
    def default_pcount(self):
        '''
        The default number of partitions for data sets. This either the `bndl.compute.pcount`
        configuration value or the number of workers (`ctx.worker_count`) times the
        `bndl.execute.concurrency` configuration value (which defaults to 1).
        '''
        pcount = self.conf['bndl.compute.pcount']
        if pcount:
            return pcount
        if self.worker_count == 0:
            self.await_workers()
        return self.worker_count * self.conf['bndl.execute.concurrency']

    def collection(self, collection, pcount=None, psize=None):
        '''
        Create a data set from a python iterable (e.g. a list, dict or iterator).

        Note that iterators are consumed immediately since data sets need to be deterministic in
        their output.

        By default pcount (or `ctx.default_pcount`) partitions are created. pcount must *not* be
        set when psize is set.

        Args:

            collection (iterable): The iterable to partition and distribute.
            pcount (int or None): The number of partition to build.
            psize (int or None): The maximum number of records per partition.

        Example::

            >>> ctx.collection(list('abcd')).nlargest(2)
            ['d', 'c']
        '''
        if isinstance(collection, range):
            return self.range(collection.start, collection.stop, collection.step, pcount=pcount)
        else:
            return DistributedCollection(self, collection, pcount, psize)

    def range(self, start, stop=None, step=1, pcount=None):
        '''
        Create a distributed partitioned data set of a range of numbers.

        Args:
            start (int): The start or stop value (if no stop value is given).
            stop (int): The stop value or None if start is the stop value.
            step (int): The step between each value in the range.
            pcount (int): The number of partitions to partition the range into.

        Examples::

            >>> ctx.range(10).collect()
            [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
            >>> ctx.range(10).glom().collect()
            [range(0, 2), range(2, 5), range(5, 7), range(7, 10)]
            >>> ctx.range(5, 20).glom().map(len).collect()
            [3, 4, 4, 4]
            >> ctx.range(5, 10).stats()
            <Stats count=5, mean=7.0, min=5.0, max=9.0, var=2.0, stdev=1.4142135623730951, skew=0.0, kurt=-1.3>

        '''
        return RangeDataset(self, start, stop, step, pcount)

    def accumulator(self, initial):
        '''
        Create an :class:`Accumulator <bndl.compute.accumulate.Accumulator>` with an initial value.

        Args:
            initial: The initial value of the accumulator.

        Example::

            >>> from bndl.compute import ctx
            >>> accum = ctx.accumulator(0)
            >>> def mapper(i):
            ...     global accum
            ...     accum += i
            ...     return i
            ...
            >>> ctx.range(10).sum()
            45
            >>> accum.value
            0
            >>> ctx.range(10).map(mapper).sum()
            45
            >>> accum.value
            45
            >>> ctx.range(10).map(mapper).sum()
            45
            >>> accum.value
            90
        '''
        accumulator = Accumulator(self, self.node.name, initial)
        self.node.service('accumulate').register(accumulator)
        return accumulator

    broadcast = broadcast
    files = files


import requests
from helper import get_legis_body, save_file    # helper functions for getting and saving legislation
from bs4 import BeautifulSoup

# disable warnings for HTTPS certificate verification
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

url = "https://www.mlis.gov.mm"
s = requests.Session()
s.verify = False        # disable HTTPS certificate verification (not ideal)

# Get set of all laws with parallel translations
list_url = url + '/mLsScList.do'

payload = {"pageSize":150,"upperLawordKndCode":"0100","queryType":"07","ordrType":"01","query":"","selFont":"PYI"}
lawordSn_set = set()

pageNo = 1
not_empty = True
while not_empty:
  payload["pageIndex"] = pageNo
  r = s.post(list_url, json=payload)
  if len(r.json()['list']) == 0:
    not_empty = False
  else:
    for item in r.json()['list']:
      lawordSn_set.add(item["lawordSn"])
  pageNo += 1

# generate final list of legislation objects, each with a pointer to EN and MY versions
legis_list = []
for item in lawordSn_set:
  legis_list.append({"EN": item})

# Obtain Burmese translations for each piece of legislation
view_legis_url = "https://www.mlis.gov.mm/mLsView.do"

for legis in legis_list:
  payload = {"lawordSn": legis["EN"]}
  r = s.get(view_legis_url, params = payload)
  soup = BeautifulSoup(r.text, "lxml")

  try:
    translation_link = soup.find('a', class_='btn-convert').get('href')
    translation_lawordSn = translation_link.split("lawordSn=")[1]
    legis["MY"] = translation_lawordSn
  except:
    print(f"translation link unavailable for {legis['EN']}")
    legis["MY"] = None

# Obtain English and Burmese versions of the law in parallel and save to similarly named files (legislation title followed by _MY or _EN)
for legis in legis_list:
  en = legis["EN"]
  my = legis["MY"]

  title_en, paras_en = get_legis_body(en, s)
  save_file(title_en + "_EN", paras_en)

  if my != None:
    title_my, paras_my = get_legis_body(my, s)
    save_file(title_en + "_MY", paras_my)


import os

def add_cwd_nginx_addon_segment(powerline):
    cwd = powerline.cwd or os.getenv('PWD')

    if cwd[0] == '~':
        long_cwd = os.environ['HOME'] + cwd[1:]
    else:
        long_cwd = cwd

    for i in range(len(NGINX_ROOT_LIST)):
        server_root = NGINX_ROOT_LIST[i]
        server_root_len = len(server_root)
        if long_cwd[:server_root_len] == server_root:
            powerline.cwd = (NGINX_SERVER_ICON + ' ' + NGINX_SERVER_LIST[i] + long_cwd[server_root_len:]).encode('utf-8')
            return
    return


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""

EGE DOĞAN DURSUN
051700000006
EGE ÜNİVERSİTESİ
MÜHENDİSLİK FAKÜLTESİ
BİLGİSAYAR MÜHENDİSLİĞİ BÖLÜMÜ
MNIST VERİ SETİ EĞİTİM YARIŞMASI PROJESİ
TARİH: 26 NİSAN 2020

ULAŞILAN TEST ACCURACY DEĞERİ : "%99.62"

"""

#Import the necessary libraries
import keras
from keras.datasets import mnist
import matplotlib.pyplot as plt
from keras import backend as K
from keras.models import load_model
import os
import tensorflow as tf
import seaborn as sns
import numpy as np

#height and width of the images
rows, cols = 28, 28
num_classes = 10

#Get the dataset
(_, _), (x_test, y_test) = mnist.load_data()

#Format the data for the channels
if K.image_data_format() == 'channels_first':
    x_test = x_test.reshape(x_test.shape[0], 1, rows, cols)
    input_shape = (1, rows, cols)

else:
    x_test = x_test.reshape(x_test.shape[0], rows, cols, 1)
    input_shape = (rows, cols, 1)

#Preprocess and normalize the data
x_test = x_test.astype('float32')

x_test = x_test / 255

y_test = keras.utils.to_categorical(y_test, num_classes)

#Load the model
cwd = os.getcwd()
path = os.path.join(cwd, 'model.hdf5')
model = load_model(path)

#Evaluate the model
score = model.evaluate(x_test, y_test, verbose=1)

#Show model performance
print("_________________")
print("Model Loss : ", score[0])
print("Model Accuracy : %", score[1]*100)
print("__________________")

#Create predictions
predictions = model.predict(x_test)

y_trues =[]
for i in range(0, len(y_test)):
    y_trues.append(np.argmax(y_test[i]))

y_preds = []
for j in range(0, len(predictions)):
    y_preds.append(np.argmax(predictions[j]))

#Create the confusion matrix
con_mat = tf.math.confusion_matrix(
        labels = y_trues,
        predictions = y_preds,
        ).numpy()

#Print the confusion matrix
print("\n HATA MATRİSİ : \n")
figure = plt.figure(figsize=(8, 8))
sns.heatmap(con_mat, annot=True,cmap=plt.cm.Blues, fmt='g')
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()


import pyautogui as pygui
from time import sleep
a = pygui.getWindowsWithTitle("Devolução Cliente")[0]
a.activate()
sleep(2)
pygui.write("c01".upper())

pygui.hotkey("tab")
pygui.write("comercial".upper())

# input('ESSE TAB TA ERRADO POIS VARIA DE ACORDO COM A QUANTIDADE')
# [pygui.hotkey('tab') for i in range(5)]

# pygui.hotkey('tab', 'enter', interval=1)
pygui.keyDown("shift")
[pygui.hotkey('tab') for i in range(16)]
pygui.keyUp("shift")

pygui.write("PA")
sleep(.5)
pygui.hotkey('tab', 'enter', interval=1)

# input('teste')

pygui.hotkey("f8")

pygui.getWindowsWithTitle("devolucao")[0].activate()
pygui.hotkey("up")
a.activate()


from twilio.rest import TwilioRestClient

sid = "Your ID"
token = "Your Token"

phone_number = " "
twilio_number = " "
message = '''
the-message
'''

Client=TwilioRestClient(sid, token)
Client.messages.create(from_=twilio_number, to=phone_number, body=message)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# Imports
import os
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
import numpy as np
import tensorflow as tf

import kanji_prepper as prep

from tensorflow.contrib import learn
from tensorflow.contrib.learn.python.learn.estimators import model_fn as model_fn_lib

tf.logging.set_verbosity(tf.logging.INFO)

FLAGS = None

# Our application logic will be added here
def cnn_model_fn(features, labels, mode):
    classes = 1721
    """Model function for CNN."""
    # Input layer
    input_layer = tf.reshape(features, [-1, 32, 32, 1])

    # Convolutional Layer #1
    conv1 = tf.layers.conv2d(
        inputs=input_layer,
        filters=32,
        kernel_size=[3, 3],
        padding="same",
        activation=tf.nn.relu)

    # Convolutional Layer #2
    conv2 = tf.layers.conv2d(
        inputs=conv1,
        filters=32,
        kernel_size=[3, 3],
        padding="same",
        activation=tf.nn.relu)

    # Pooling Layer #1
    pool1 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)

    # Convolutional Layer #3
    conv3 = tf.layers.conv2d(
        inputs = pool1,
        filters = 64,
        kernel_size = [3, 3],
        padding = "same",
        activation = tf.nn.relu)

    # Convolutional Layer #4
    conv4 = tf.layers.conv2d(
        inputs = conv3,
        filters = 64,
        kernel_size = [3, 3],
        padding = "same",
        activation = tf.nn.relu)

    # Pooling Layer #2
    pool2 = tf.layers.max_pooling2d(inputs = conv4, pool_size = [2, 2], strides = 2)

    # Convolutional Layer #4
    conv5 = tf.layers.conv2d(
        inputs = pool2,
        filters = 128,
        kernel_size = [3, 3],
        padding = "same",
        activation = tf.nn.relu)

    # Pooling Layer #3
    pool3 = tf.layers.max_pooling2d(inputs = conv5, pool_size = [2, 2], strides = 2)

    # Dense Layer
    pool3_flat = tf.reshape(pool3, [-1, 4 * 4 * 128])
    dense = tf.layers.dense(inputs = pool3_flat, units = 1024, activation = tf.nn.relu)
    dropout = tf.layers.dropout(
        inputs = dense, rate = 0.5, training=mode == learn.ModeKeys.TRAIN)

    # Logits Layer
    logits = tf.layers.dense(inputs = dropout, units = classes)

    loss = None
    train_op = None

    # Calculate Loss (for both TRAIN and EVAL modes)
    if mode != learn.ModeKeys.INFER:
        onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=classes)
        loss = tf.losses.softmax_cross_entropy(
            onehot_labels=onehot_labels, logits=logits)

    # Configure the Training Op (for TRAIN mode)
    if mode == learn.ModeKeys.TRAIN:
        train_op = tf.contrib.layers.optimize_loss(
            loss=loss,
            global_step=tf.contrib.framework.get_global_step(),
            learning_rate=0.001,
            optimizer="SGD")

    # Generate Predictions
    predictions = {
      "classes": tf.argmax(
          input=logits, axis=1),
      "probabilities": tf.nn.softmax(
          logits, name="softmax_tensor")
    }

    # Return a ModelFnOps object
    return model_fn_lib.ModelFnOps(
        mode=mode, predictions=predictions, loss=loss, train_op=train_op)

def main(unused_argv):
    classes = 1721
    # Load training and eval data
    training, t_labels, validation, v_labels = prep.data_from_base('training_data')
    # t_labels = onehot_labels(t_labels, classes)
    # v_labels = onehot_labels(v_labels, classes)

    # Create the Estimator
    hiragana_classifier = learn.Estimator(
        model_fn=cnn_model_fn, model_dir="/tmp/kanji_cnn_test2")
    # Set up logging for predictions
    tensors_to_log = {"probabilities": "softmax_tensor"}
    logging_hook = tf.train.LoggingTensorHook(
        tensors=tensors_to_log, every_n_iter=100)
    # Train the model
    hiragana_classifier.fit(
        x=training,
        y=t_labels,
        batch_size=50,
        steps=1000,
        monitors=[logging_hook])
    # Configure the accuracy metric for evaluation
    metrics = {
        "accuracy":
            learn.MetricSpec(
                metric_fn=tf.metrics.accuracy, prediction_key="classes"),
    }
    # Evaluate the model and print results
    eval_results = hiragana_classifier.evaluate(
        x=validation, y=v_labels, metrics=metrics)
    print(eval_results)

if __name__ == "__main__":
    tf.app.run()


import math
import codecs
class Decryption():
    #ct = '101000011110000011000000000001001100101001101101110010000001111000101000001000010011010100010111101001001100000110011011011011011101110100010110010010011010001111101001001011011010000000101101100001011001010100110011001111001011100011010001010010011100110100010010010001010011000000110001111110110011100100010000000001010011001110110110101111010001000101110001101100000001011110111000001110000001000010111011101110100111110111101100111010110011010101011000'
    init = None
    #key = 'password'
    #s = ''
    IV = None

    def __init__(self, finbin_text, passwd, initial_vector):
        self.key = passwd
        self.IV = initial_vector
        self.xor(finbin_text, self.key)

    def xor(self, finbin_text, key):
        self.init = bin(int.from_bytes(key.encode(), 'big')) [2:]
        print('Key in Binary:', self.init)

        lenct = len(finbin_text)
        lenkey = len(self.init)

        final_key = ""

        if(isinstance(lenct/lenkey, float)==True):
            div_num = int(lenct/lenkey)
            increment_num = div_num+1

        else:
            increment_num = int(lenct/lenkey)

        for i in range (0,increment_num):
            final_key+=self.init

        self.init = final_key[:lenct]

        print('New Key:' +self.init+' Lenght: '+str(len(self.init)))

        self.ux = '{1:0{0}b}'.format(len(finbin_text), int(finbin_text, 2) ^ int(self.init, 2))
        print('Text in binary:', self.ux)
        self.split(self.ux)

    def split(self, ux):
        a = len(self.ux)
        self.b = int(a/2)
        print(self.b)

        self.chunk1 = self.ux[:self.b]
        self.chunk2 = self.ux[self.b:]

        print(self.ux)
        print('Chunk 1:',self.chunk1)
        print('chunk 2:',self.chunk2)

        self.keychunk1Xor(self.chunk1, self.b)
        self.keychunk2Xor(self.chunk2)

    def keychunk1Xor(self, chunk1, b):
        self.init = bin(int.from_bytes(self.key.encode(), 'big'))[2:]
        print('Key in Binary:', self.init)
        lenct = len(chunk1)
        lenkey = len(self.init)

        final_key = ""

        if(isinstance(lenct/lenkey, float)==True):
            div_num = int(lenct/lenkey)
            increment_num = div_num+1

        else:
            increment_num = int(lenct/lenkey)

        for i in range (0,increment_num):
            final_key+=self.init
        self.reverse(self.chunk1, self.b)

    def reverse(self, chunk1, b):
        self.f=chunk1[b-1::-1]
        print('Reversed Chunk:',self.f)

    def keychunk2Xor(self, chunk2):
        self.init = bin(int.from_bytes(self.key.encode(), 'big'))[2:]
        print('Key in Binary:', self.init)
        lenct = len(chunk2)
        lenkey = len(self.init)

        final_key = ""

        if(isinstance(lenct/lenkey, float)==True):
            div_num = int(lenct/lenkey)
            increment_num = div_num+1

        else:
            increment_num = int(lenct/lenkey)

        for i in range (0,increment_num):
            final_key+=self.init
        self.shift(self.chunk2)

    def shift(self,chunk2):
        self.g=(self.chunk2[-7:]+chunk2[:-7])
        print('Shifted Chunk2:',self.g)
        self.concatinate(self.f,self.g)

    def concatinate(self,f,g):
        self.h = self.f + self.g
        print('Concatinated:{} Length:{}'.format(self.h, len(self.h)))

        self.plaintext = self.h[1:]
        print("plaintext: {} length: {} ".format(self.plaintext, len(self.plaintext)))
        self.plaintext_Xor_IV(self.plaintext, self.IV)

    def plaintext_Xor_IV(self, plaintext, IV):
        self.IVbin = bin(int.from_bytes(IV.encode(), 'big'))[2:]
        IVbin_length = len(self.IVbin)
        cipher_length = len(self.plaintext)
        x = math.ceil(cipher_length/IVbin_length)
        self.IVbin *= x
        self.IVbin = self.IVbin[:len(self.plaintext)]
        print("padded cipher_IV_xor: ", self.IVbin)
        print(f"the length of ciphertext: {len(self.plaintext)}")
        print(f"the length of the key: {len(self.IVbin)}")
        self.cIV_xor = '{1:0{0}b}'.format(len(self.plaintext), int(self.plaintext, 2) ^ int(self.IVbin, 2))
        print(f"XORed cipher and IV: {self.cIV_xor}")
        print(f"length: {len(self.cIV_xor)}")
        self.bintostring(self.cIV_xor)

    def bintostring(self, cIV_xor):
        self.final = ""
        for i in range(0, len(self.cIV_xor), 8):
            bytes = self.cIV_xor[i:i+7]
            decstring = int(bytes, 2)
            self.final += chr(decstring)

        print(self.final)
        self.rot13(self.final)

    # def rot13(self, final):
    #     pt13 = codecs.decode(final, 'rot_13')
    #     print(f"rot13 plaintext: {pt13}")
    #     self.pt13 = pt13

    def rot13(self, final):

        alphabet2 = [i for i in '123456789,.?!@#$ %^&*()_+=-:/[]{}|']
        alphabet = [i for i in 'abcdefghijklmnopqrstuvwxyz']
        rotated_plain_text = []
        #print(plaintext)
        row1 = alphabet[:13]
        row1_symb = alphabet2[:17]
        row2 = alphabet[13:]
        row2_symb = alphabet2[17:]
        print(f'row1 sym: {row1_symb}')
        print(f'row2 sym: {row2_symb}')
        for i in final :
            if i in row1:
                rotated_plain_text.append(row2[row1.index(i)])
            if i in row2:
                rotated_plain_text.append(row1[row2.index(i)])
            if i in row1_symb:
                rotated_plain_text.append(row2_symb[row1_symb.index(i)])
            if i in row2_symb:
                rotated_plain_text.append(row1_symb[row2_symb.index(i)])
            # if i in alphabet2:
            #     rotated_plain_text.append(i)
        print('check out for this: ', rotated_plain_text)
        pt13 = ''.join(rotated_plain_text)
        print(f'rotated-plain_text1xx: {pt13}')
        self.pt13 = pt13

    def get_text(self):
        return self.pt13

if __name__ == "__main__":
    window = Decryption()


# execute block of code multiple time
i = 1
while i<=4:
    print(i)
    i += 1
print("Finish")
count = 0
check = True
rep = "Guy"
ques = ""
while ques != rep and check:
    ques = input("Entrer la rep : ")
    count += 1
    if count == 3:
        check = False
        print("You loose")


# compare the security groups rules extracted from aws envirnoment with the corresponding baseline from Excel spreadsheet

with open('/usr/local/src/security_scripts/tmp/aws_out.txt', 'r') as t1, open('/usr/local/src/security_scripts/tmp/baseline_out.txt', 'r') as t2:
    file1 = t1.readlines()
    file2 = t2.readlines()

with open('/usr/local/src/security_scripts/outputs/result.txt', 'w+') as outFile:
    outFile.write("-----------------------------------------\n")
    outFile.write("Rules that are in BASELINE but not in AWS\n")
    for line in file2:
        if line not in file1:
            outFile.write(line)
    outFile.write("\n-----------------------------------------\n")
    outFile.write("Rules that are in AWS but not in BASELINE\n")
    for line in file1:
        if line not in file2:
            outFile.write(line)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# In[2]:

path = "hw1_data\\"
X_train = pd.read_csv(path + "X_train.csv", header=None).values
X_test = pd.read_csv(path + "X_test.csv", header=None).values
y_train = pd.read_csv(path + "y_train.csv", header=None).values
y_test = pd.read_csv(path + "y_test.csv", header=None).values

# In[3]:

Wrr = []
df_lambdas = []
lambda_values = range(5001)

# In[4]:

for lambda_value in lambda_values:
    XTX = np.matmul(X_train.transpose(),X_train)
    lambdaI = lambda_value * np.identity(XTX.shape[0])
    XTy = np.matmul(X_train.transpose(),y_train)
    XTX_lambdaI_inv = np.linalg.inv(lambdaI + XTX)
    W = np.matmul(XTX_lambdaI_inv, XTy)
    df_lambda = np.trace(np.matmul(np.matmul(X_train,XTX_lambdaI_inv),X_train.transpose()))
    df_lambdas.append(df_lambda)
    Wrr.append([i[0] for i in W.tolist()])

# In[5]:

w = pd.DataFrame(Wrr)
w.columns= ["cylinders", "displacement", "horsepower", "weight", "acceleration", "year made", "bias"]
w["df_lambda"] = df_lambdas

# In[6]:

plt.figure(figsize=(7,7))
for i in w.columns[:-1]:
    plt.plot(list(w["df_lambda"]),list(w[i]), label = i, marker = "o", markersize = 4,  linewidth=2)
plt.legend()
plt.xlabel("df(lambda)")
plt.show()

# # Prediction

# In[7]:

y=[]
Wrr=[]
lambda_values = range(51)
rmse_list = []
for lambda_value in lambda_values:
    XTX = np.matmul(X_train.transpose(),X_train)
    lambdaI = lambda_value * np.identity(XTX.shape[0])
    XTy = np.matmul(X_train.transpose(),y_train)
    XTX_lambdaI_inv = np.linalg.inv(lambdaI + XTX)
    W = np.matmul(XTX_lambdaI_inv, XTy)
    Wrr.append([i[0] for i in W.tolist()])
    y_predicted = np.matmul(X_test,W)
    y.append([i[0] for i in y_predicted.tolist()])
    rmse = np.sqrt(np.mean((y_predicted - y_test)**2))
    rmse_list.append(rmse)

# In[8]:

plt.figure(figsize=(5,5))
plt.plot(lambda_values,rmse_list)
plt.xlabel("Lambda")
plt.ylabel("RMSE")
plt.title("Effect of Lambda on RMSE(Test Data)")
plt.show()

# # pth-order polynomial regression

# In[12]:

p_values = [1,2,3]

# In[13]:

lambda_values = range(101)
rmses = pd.DataFrame()
for p in p_values:
    rmse_list = []
    X_temp_train = X_train
    X_temp_test = X_test
    if(p!=1):
        for j in range(2,p+1):
            for k in range(6):
                mean = np.mean(np.array(X_temp_train[:,k])**j)
                std = np.sqrt(np.var(np.array(X_temp_train[:,k])**j))
                X_temp_train = np.append(X_temp_train, np.array(np.array(np.array(X_temp_train[:,k]**j) - mean)/std)
                                         .reshape(np.array((np.array(X_temp_train[:,k]**2) - mean)).shape[0], 1),1)
                X_temp_test = np.append(X_temp_test, np.array(np.array(np.array(X_temp_test[:,k]**j) - mean)/std)
                                        .reshape(np.array((np.array(X_temp_test[:,k]**2) - mean)).shape[0], 1),1)

#     print(X_temp_test.shape)
#     pd.DataFrame(X_temp_train).to_csv("training data" + str(p))
    for lambda_value in lambda_values:
        XTX = np.matmul(X_temp_train.transpose(),X_temp_train)
        lambdaI = lambda_value * np.identity(XTX.shape[0])
        XTy = np.matmul(X_temp_train.transpose(),y_train)
        XTX_lambdaI_inv = np.linalg.inv(lambdaI + XTX)
        W = np.matmul(XTX_lambdaI_inv, XTy)

        y_predicted = np.matmul(X_temp_test,W)
        rmse = np.sqrt(np.mean((y_predicted - y_test)**2))
        rmse_list.append(rmse)
    rmses[p] = rmse_list

# In[11]:

plt.figure(figsize=(5,5))
for i in rmses.columns:
    plt.plot(lambda_values,rmses[i], label = ("p = " + str(i)))
plt.legend()
plt.xlabel("Lambda")
plt.ylabel("RMSE")
plt.title("Effect of Lambda and order of Polynomial(p) on RMSE(Test Data)")
plt.show()

# In[ ]:

Regression and Maximum Likelihood Estimation/HW1_pds2136.py


import bisect
import sys
n = int(input())
m = int(input())
k_arr = list(map(int, input().split()))
k_arr = sorted(k_arr)

kk_arr = [k_arr[i]+k_arr[j] for i in range(n) for j in range(n)]
kk_arr = list(set(kk_arr))

for i in range(n):
    for j in range(n):
        x = m - (k_arr[i]+k_arr[j])
        if bisect.bisect_left(kk_arr, x)!=bisect.bisect_right(kk_arr, x):
            print("Yes")
            sys.exit()

print("No")


import json

class Configuration:
    def __init__(self, file_path = None):
        # get file path
        self.file_path = file_path

        # if no file path
        if self.file_path is None:
            # initialize empty data
            self.game = None
            self.campaign = None
            self.faction_group = None
            self.faction = None
            self.difficulty = None
            self.imperium = None
            self.influence = None
            self.taxes = None
            self.edicts = list()
            self.regions = list()
            self.technologies = list()
        else:
            # load data from file
            self.load(self.file_path)

    def load(self, file_path):
        # open file for reading
        with open(file_path, 'r') as file:
            # load json as data object
            data = json.load(file)

            # initialize configuration class data
            self.game = data.get('game')
            self.game = None
            self.campaign = None
            self.faction_group = None
            self.faction = None
            self.difficulty = None
            self.imperium = None
            self.influence = None
            self.taxes = None
            self.edicts = list()
            self.regions = list()
            self.technologies = list()


import _functools

def adder(a,*b):

    sum = 0 + a

    for i in b:

       sum = sum + i

    return sum

def adder1(a,*b):

    for i in b:

       a = a + i

    return a

def adder3(**arg):

   mysum = []

   for i in arg.keys():
       mysum.append(arg[i])

   return _functools.reduce(lambda x, y: x + y ,mysum)

def adder4(**arg):

   mysum = []

   for i in arg.keys():
       mysum.append(arg[i])

   return _functools.reduce(lambda x, y: x + y ,mysum)

def copydict(**arg):

    return {x : arg[x] for x in arg.keys()}

dict1 = {'b':1, 'c':8}
dict2 = {'e':13,'r':67, 'b':100}

def adddict(dict1, dict2):

    g = zip(dict1, dict1.values(), dict2, dict2.values())

    return  {}

print(adder(1,2,3,4,5))
print(type(adder1(1,2)))
print(adder3(ugly = "s", bad = 'e', fun = 'r', fine = 'g', Excellent = 'e', cool = 'i'))
print(adder4(ugly = 1, bad = 2, fun = 3, fine = 4, Excellent = 5, cool = 6))
print(copydict(ugly = 1, bad = 2, fun = 3, fine = 4, Excellent = 5, cool = 6))
print(adddict(dict1, dict2))


# Data.py
# --------------
# Licensing Information:  You are free to use or extend this project for
# educational purposes provided that (1) you do not distribute or publish
# solutions, (2) you retain this notice, and (3) you provide clear
# attribution to The Ohio State University, including a link to http://aritter.github.io/courses/5523_fall18.html
#
# Attribution Information: This assignment was developed at The Ohio State University
# by Alan Ritter (ritter.1492@osu.edu).

from collections import namedtuple
import sys

FeatureVal = namedtuple("FeatureVal", "feature, value")

class MushroomData:
    def __init__(self, fileName):
        self.data = []
        self.features = set()
        for line in open(fileName):
            line = line.strip()
            attributes = line.split(',')
            for i in range(1,len(attributes)):
                self.features.add(FeatureVal(i, attributes[i]))
            self.data.append(attributes)


import math
import random
import cv2

#各层神经元数
layer1_count = 625
layer2_count = 25
layer3_count = 2
#各层的输入输出变量组
layer1_in = [x for x in range(layer1_count)]      #输入层的输入
layer1_out = [x for x in range(layer1_count)]     #输入层输出
layer2_in = [x for x in range(layer2_count)]      #隐藏层输入
layer2_out = [x for x in range(layer2_count)]     #隐藏层输出
layer3_in = [x for x in range(layer3_count)]      #输出层输入
layer3_out = [x for x in range(layer3_count)]     #输出层输出
#各层的权值
layer12_w = [[x for x in range(layer1_count)] for x in range(layer2_count)]      #12层权值
layer23_w = [[x for x in range(layer2_count)] for x in range(layer3_count)]      #23层权值
layer12_aw = [[x for x in range(layer1_count)] for x in range(layer2_count)]    #12层权值的调整量adjust
layer23_aw = [[x for x in range(layer2_count)] for x in range(layer3_count)]    #23层权值的调整量
#各层的阈值
layer2_limit = [x for x in range(layer2_count)]  #第2层阈值
layer3_limit = [x for x in range(layer3_count)]  #第3层阈值
layer2_alimit = [x for x in range(layer2_count)]  #第2层阈值调整量
layer3_alimit = [x for x in range(layer3_count)]  #第3层阈值调整量
#各层误差
layer2_error = [x for x in range(layer2_count)]   #隐含层误差
layer3_error = [x for x in range(layer3_count)]   #输出层误差
#输出层目标值
target = [x for x in range(layer3_count)]

#数据读取
#img:图像数组 isface:是否是人脸图像
def DataInput(img, isface):
    size = img.shape
    count = 0
    for i in range(25):
        for j in range(25):
            if len(size) == 3:
                if img[i][j][0] > 255:
                    img[i][j][0] = 255
                layer1_in[count] = img[i][j][0]/255.0
            else:
                if img[i][j] > 255:
                    img[i][j] = 255
                layer1_in[count] = img[i][j]/255.0
            count += 1
    if isface == True:
        target[0] = 0.9
        target[1] = 0.1
    else:
        target[0] = 0.1
        target[1] = 0.9

#各个变量的初始化
def ParaseInit():
    for i in range(layer1_count):
        layer1_in[i] = 0.0
        layer1_out[i] = 0.0
    for i in range(layer2_count):
        layer2_alimit[i] = 0
        layer2_error[i] = 0
        layer2_in[i] = 0
        layer2_out[i] = 0
        layer2_limit[i] = random.uniform(-0.04, 0.04)
        for j in range(layer1_count):   #第一二层的权值初始化
            layer12_w[i][j] = random.uniform(-0.04, 0.04)
            layer12_aw[i][j] = 0
    for i in range(layer3_count):
        layer3_alimit[i] = 0
        layer3_error[i] = 0
        layer3_in[i] = 0
        layer3_out[i] = 0
        target[i] = 0
        layer3_limit[i] = random.uniform(-0.04, 0.04)
        for j in range(layer2_count):
            layer23_w[i][j] = random.uniform(-0.04, 0.04)
            layer23_aw[i][j] = 0

#计算各层输出
def CaculateLayerOutput():
    for i in range(layer1_count):       #计算第一层神经元输出（第一层为线性，直接等）
        layer1_out[i] = layer1_in[i]
    for i in range(layer2_count):       #计算第二层每个神经元的输出
        temp = 0.0
        for j in range(layer1_count):   #输入层的神经元个数
            temp += layer12_w[i][j] * layer1_out[j]
        temp += layer2_limit[i]        #加上隐含层每个神经元的阈值
        layer2_out[i] = 1 / (1 + math.exp(-temp))   #根据S函数算出输出
    for i in range(layer3_count):       #计算第三层每个神经元的输出
        temp = 0.0
        for j in range(layer2_count):   #隐含层的神经元个数
            temp += layer23_w[i][j] * layer2_out[j]
        temp += layer3_limit[i]        #加上输出层每个神经元的阈值
        layer3_out[i] = 1 / (1 + math.exp(-temp))   #根据S函数算出输出

#计算输出层和隐含层的误差项
def CaculateLayerError():
    for i in range(layer3_count):       #计算输出层误差
        layer3_error[i] = (target[i] - layer3_out[i]) * layer3_out[i] * (1 - layer3_out[i])
    for i in range(layer2_count):       #计算隐含层误差
        temp = 0.0
        for j in range(layer3_count):   #对输出层误差的累加
            temp += layer3_error[j] * layer23_w[j][i]
        layer2_error[i] = layer2_out[i] * (1 - layer2_out[i]) * temp

#计算权值和阈值的调整量
def CaculateAdjust():
    for i in range(layer3_count):       #计算第三层的阈值、权值调整量
        layer3_alimit[i] = (alpha / (1.0 + layer2_count)) * (layer3_alimit[i] + 1) * layer3_error[i]     #阈值调整量
        for j in range(layer2_count):
            layer23_aw[i][j] = (alpha / (1.0 + layer2_count)) * (layer23_aw[i][j] + 1) * layer3_error[i] * layer2_out[j]
    for i in range(layer2_count):       #计算第2层的阈值、权值调整量
        layer2_alimit[i] = (alpha / (1.0 + layer1_count)) * (layer2_alimit[i] + 1) * layer2_error[i]     #阈值调整量
        for j in range(layer1_count):
            layer12_aw[i][j] = (alpha / (1.0 + layer1_count)) * (layer12_aw[i][j] + 1) * layer2_error[i] * layer1_out[j]

#计算各层权值和阈值调整后的值
def CaculateAdjusted():
    for i in range(layer3_count):       #第三层
        layer3_limit[i] += layer3_alimit[i]     #第三层阈值
        for j in range(layer2_count):
            layer23_w[i][j] += layer23_aw[i][j]
    for i in range(layer2_count):       #第2层
        layer2_limit[i] += layer2_alimit[i]     #第2层阈值
        for j in range(layer1_count):
            layer12_w[i][j] += layer12_aw[i][j]

#每算完一张图片，计算总误差是否满足要求，满足返回true，否则返回false
def CaculateTotalError():
    total_error = 0.0
    for i in range(layer3_count):
        total_error += math.pow(target[i] - layer3_out[i], 2)
    total_error *= 0.5
    if total_error <= epsilon:
        return True
    else:
        return False

#BP的一个过程
def BPNN(img, isface):
    DataInput(img, isface)
    CaculateLayerOutput()
    CaculateLayerError()
    if CaculateTotalError():
        return True
    CaculateAdjust()
    CaculateAdjusted()
    return False

#把训练好的权值和阈值保存到文件中
def SaveParase():
    save_file = open("parase.txt", 'w')
    for i in range(layer2_count):
        for j in range(layer1_count):
            save_file.write(str(layer12_w[i][j]) + "\n")
    for i in range(layer3_count):
        for j in range(layer2_count):
            save_file.write(str(layer23_w[i][j]) + "\n")
    for i in range(layer2_count):
        save_file.write(str(layer2_limit[i]) + "\n")
    for i in range(layer3_count):
        save_file.write(str(layer3_limit[i]) + "\n")
    save_file.close()

#将训练好的权值和阈值读取出来
def ReadParase():
    read_file = open("parase.txt", 'r')
    for i in range(layer2_count):
        for j in range(layer1_count):
            temp_str = read_file.readline()
            layer12_w[i][j] = float(temp_str)
    for i in range(layer3_count):
        for j in range(layer2_count):
            temp_str = read_file.readline()
            layer23_w[i][j] = float(temp_str)
    for i in range(layer2_count):
        temp_str = read_file.readline()
        layer2_limit[i] = float(temp_str)
    for i in range(layer3_count):
        temp_str = read_file.readline()
        layer3_limit[i] = float(temp_str)
    read_file.close()

#人脸检测函数
def FaceDetect(img):
    # ParaseInit()
    # ReadParase()
    DataInput(img, True)
    CaculateLayerOutput()
    if layer3_out[0] > layer3_out[1]:
        return True
    else:
        return False

#精度控制参数、学习率
alpha = 0.4         #学习率
epsilon = 0.1      #精度控制参数