# -*- coding: utf-8 -*-

"""tests for app.api.decorators"""
import collections
from mock import Mock, MagicMock, patch, call

from tests.unit import UnitTestCase
from app.api import (anonymous_required, token_auth_required, http_auth_required,
                     session_auth_required, auth_required, roles_required, roles_accepted, one_of,
                     permissions_required, permissions_accepted)

from app.exceptions import BadRequestException, UnauthorizedException

class DecoratorsTestCase(UnitTestCase):

    @patch('app.api.decorators.authenticated')
    def test_anonymous_required(self, mock_authenticated):
        mock_authenticated.return_value = False

        @anonymous_required
        def test():
            return 'anonymous_required'

        self.assertEqual(test(), 'anonymous_required',
                         'test() should return {}'.format('anonymous_required'))

        mock_authenticated.return_value = True
        self.assertRaises(BadRequestException, test)

    @patch('app.api.decorators.current_user')
    @patch('app.api.decorators.verify_jwt')
    def test_token_auth_required_unauthorized(self, mock_verify_jwt, mock_current_user):

        mock_current_user.is_authenticated.return_value = False

        @token_auth_required('realm')
        def test():
            return 'token_auth_required'

        with self.assertRaises(UnauthorizedException) as uae:
            test()
        ex = uae.exception
        self.assertIsInstance(ex, UnauthorizedException)
        self.assertTrue(ex.message, 'Invalid Token')
        mock_verify_jwt.assert_called_once_with('realm')
        mock_current_user.is_authenticated.assert_called_once_with()

    @patch('app.api.decorators.identity_changed')
    @patch('app.api.decorators._request_ctx_stack')
    @patch('app.api.decorators.current_app')
    @patch('app.api.decorators.current_user')
    @patch('app.api.decorators.verify_jwt')
    def test_token_auth_required_authorized(self, mock_verify_jwt, mock_current_user,
                                            mock_current_app, mock_request_ctx_stack,
                                            mock_identity_changed):

        mock_current_user.is_authenticated.return_value = True

        @token_auth_required('realm')
        def test():
            return 'token_auth_required'

        self.assertEqual(test(), 'token_auth_required',
                         'test() should return {}'.format('token_auth_required'))

        mock_verify_jwt.assert_called_once_with('realm')
        mock_current_user.is_authenticated.assert_called_once_with()
        mock_current_app._get_current_object.assert_called_once_with()
        self.assertEqual(mock_request_ctx_stack.top.user, mock_current_user)
        self.assertEqual(mock_identity_changed.send.call_count, 1)

    @patch('app.api.decorators.http_authenticated')
    def test_http_auth_required(self, mock_http_authenticated):
        mock_http_authenticated.return_value = True

        @http_auth_required
        def test():
            return 'http_auth_required'

        self.assertEqual(test(), 'http_auth_required',
                         'test() should return {}'.format('http_auth_required'))

        mock_http_authenticated.return_value = False
        self.assertRaises(UnauthorizedException, test)

    @patch('app.api.decorators.session_authenticated')
    def test_session_auth_required(self, mock_session_authenticated):
        mock_session_authenticated.return_value = True

        @session_auth_required
        def test():
            return 'session_auth_required'

        self.assertEqual(test(), 'session_auth_required',
                         'test() should return {}'.format(session_auth_required))

        mock_session_authenticated.return_value = False
        self.assertRaises(UnauthorizedException, test)

    @patch('app.api.decorators.token_authenticated')
    @patch('app.api.decorators.http_authenticated')
    @patch('app.api.decorators.session_authenticated')
    def test_auth_required(self, mock_token_authenticated, mock_http_authenticated,
                           mock_session_authenticated):

        mock_token_authenticated.return_value = False
        mock_http_authenticated.return_value = False
        mock_session_authenticated.return_value = False

        @auth_required('token', 'basic', 'session')
        def test():
            return 'authenticated'

        self.assertRaises(UnauthorizedException, test)

        mock_token_authenticated.return_value = True

        self.assertEqual(test(), 'authenticated', 'test() should return {}'.format('authenticated'))

        mock_token_authenticated.return_value = False
        mock_http_authenticated.return_value = True
        self.assertEqual(test(), 'authenticated', 'test() should return {}'.format('authenticated'))

        mock_http_authenticated.return_value = False
        mock_session_authenticated.return_value = True
        self.assertEqual(test(), 'authenticated', 'test() should return {}'.format('authenticated'))

    @patch('app.api.decorators.auth_permissions_required')
    def test_permissions_required(self, mock_permissions_required):
        permissions = [Mock(), Mock()]

        permissions_required(*permissions)

        self.assertEqual(mock_permissions_required.call_count, 1)
        # see: http://stackoverflow.com/questions/9623114/check-if-two-unordered-lists-are-equal
        self.assertEqual(collections.Counter(mock_permissions_required.call_args[0]),
                         collections.Counter(permissions))
        exception_handler = mock_permissions_required.call_args[1].get('exception_handler')
        self.assertTrue(callable(exception_handler))
        self.assertRaises(UnauthorizedException, exception_handler, permissions[1])

    @patch('app.api.decorators.auth_permissions_accepted')
    def test_permissions_accepted(self, mock_auth_permission_accepted):
        perm1 = MagicMock()
        perm1.__str__.return_value = 'perm1'
        perm2 = MagicMock()
        perm2.__str__.return_value = 'perm2'
        permissions = [perm1, perm2]

        permissions_accepted(*permissions)

        self.assertEqual(mock_auth_permission_accepted.call_count, 1)
        self.assertEqual(len(mock_auth_permission_accepted.call_args), 2)
        # see: http://stackoverflow.com/questions/9623114/check-if-two-unordered-lists-are-equal
        self.assertEqual(collections.Counter(mock_auth_permission_accepted.call_args[0]),
                         collections.Counter(permissions))
        exception_handler = mock_auth_permission_accepted.call_args[1].get('exception_handler')
        self.assertTrue(callable(exception_handler))
        with self.assertRaises(UnauthorizedException) as uae:
            exception_handler(permissions)
            ex = uae.exception
            self.assertIsInstance(ex, UnauthorizedException)
            self.assertEqual(ex.message, 'Invalid Permission')
            self.assertEqual(ex.description,
                             'required perm1 or perm2 permission was invalid or not provided')

    @patch('app.api.decorators.auth_roles_required')
    def test_roles_required(self, mock_roles_required):
        roles = ['admin', 'editor']
        roles_required(*roles)

        self.assertEqual(mock_roles_required.call_count, 1)
        self.assertEqual(len(mock_roles_required.call_args), 2)
        # see: http://stackoverflow.com/questions/9623114/check-if-two-unordered-lists-are-equal
        self.assertEqual(collections.Counter(mock_roles_required.call_args[0]),
                         collections.Counter(roles))
        exception_handler = mock_roles_required.call_args[1].get('exception_handler')
        self.assertTrue(callable(exception_handler))
        self.assertRaises(UnauthorizedException, exception_handler, roles)

    @patch('app.api.decorators.auth_roles_accepted')
    def test_roles_accepted(self, mock_roles_accepted):
        roles = ['admin', 'editor']
        roles_accepted(*roles)

        self.assertEqual(mock_roles_accepted.call_count, 1)
        self.assertEqual(len(mock_roles_accepted.call_args), 2)
        # see: http://stackoverflow.com/questions/9623114/check-if-two-unordered-lists-are-equal
        self.assertEqual(collections.Counter(mock_roles_accepted.call_args[0]),
                         collections.Counter(roles))
        exception_handler = mock_roles_accepted.call_args[1].get('exception_handler')
        self.assertTrue(callable(exception_handler))
        self.assertRaises(UnauthorizedException, exception_handler, roles)

    def test_one_of(self):

        from app.exceptions import ApplicationException

        def result():
            return 'one_of'

        mock_decorator_1 = Mock(return_value=result)
        mock_decorator_2 = Mock(return_value=result)
        mock_decorator_3 = Mock(side_effect=Exception('exception'))
        mock_decorator_4 = Mock(
            side_effect=ApplicationException('app exception', description='description exception'))

        @one_of(mock_decorator_1, mock_decorator_2)
        def test1():
            return result

        self.assertEqual(test1(), 'one_of', 'test1() should return {}'.format('one_of'))

        @one_of(mock_decorator_3, mock_decorator_2)
        def test2():
            return result

        self.assertEqual(test2(), 'one_of', 'test2() should return {}'.format('one_of'))

        @one_of(mock_decorator_3, mock_decorator_4)
        def test3():
            return result

        with self.assertRaises(ApplicationException) as ex:
            test3()
        ex_3 = ex.exception
        self.assertIsInstance(ex_3, ApplicationException)
        self.assertEqual(ex_3.message, 'app exception or exception')
        self.assertEqual(ex_3.description, 'description exception')

        mock_decorator_3.side_effect = Exception('exception')
        mock_decorator_4.side_effect = ApplicationException('app exception',
                                                            description='description exception')

        @one_of(mock_decorator_4, mock_decorator_3)
        def test_4():
            return result

        with self.assertRaises(ApplicationException) as ex:
            test_4()

        ex_4 = ex.exception
        self.assertIsInstance(ex_4, ApplicationException)
        self.assertEqual(ex_4.message, 'exception or app exception')
        self.assertEqual(ex_4.description, 'description exception')

        @one_of()
        def test5():
            return result()
        self.assertEqual(test5(), 'one_of', 'test5() should return {}'.format('one_of'))

    @patch('app.api.decorators.OffsetPagination')
    def test_paginated_one(self, mock_offset_pagination):

        from app.api.decorators import paginated

        mock_query = MagicMock()
        mock_args = MagicMock()

        mock_query.one.return_value = '1'

        mock_args.get.side_effect = [
            True
        ]

        @paginated
        def test():
            return mock_query, mock_args

        result = test()

        expected_result = {
            'data': ['1'],
            'paging': {
                'count': 1,
                'offset': 0,
                'limit': 1,
                'previous': None,
                'next': None
            }
        }

        self.assertEqual(result, expected_result)
        mock_args.get.assert_called_once_with('one', False)
        mock_query.one.assert_called_once_with()
        self.assertFalse(mock_offset_pagination.called)

    @patch('app.api.decorators.OffsetPagination')
    def test_paginated_many(self, mock_offset_pagination):

        from app.api.decorators import paginated

        pagination = MagicMock()
        mock_offset_pagination.return_value = pagination

        pagination.data = ['hi', 'there']
        pagination.count = 7
        pagination.offset = 0
        pagination.limit = 2
        pagination.prev_url = None
        pagination.next_url = 'next_url'

        mock_query = MagicMock()
        mock_args = MagicMock()

        mock_args.get.side_effect = [
            None,
            0,
            2
        ]

        @paginated
        def test():
            return mock_query, mock_args

        result = test()

        expected_result = {
            'data': ['hi', 'there'],
            'paging': {
                'count': 7,
                'offset': 0,
                'limit': 2,
                'previous': None,
                'next': 'next_url'
            }
        }

        mock_offset_pagination.assert_called_once_with(mock_query, offset=0, limit=2)
        mock_args.get.assert_has_calls([call('one', False),
                                        call('offset', None),
                                        call('limit', None)])
        self.assertEqual(result, expected_result)


# Wrap a gamestream session around a launcher program with configurable background tasks and resolution switching
import win32api
import win32.lib.win32con as win32con
import win32gui
import pywintypes
import pyautogui
import configparser
import subprocess
import psutil
import os
import sys
from time import sleep

# Window enumeration handler function per https://www.blog.pythonlibrary.org/2014/10/20/pywin32-how-to-bring-a-window-to-front/
def windowEnumerationHandler(hwnd, top_windows):
    top_windows.append((hwnd, win32gui.GetWindowText(hwnd)))

def set_resolution(gamestream_width, gamestream_height):
    print("Switching resolution to {0}x{1}".format(gamestream_width, gamestream_height))
    devmode = pywintypes.DEVMODEType()
    devmode.PelsWidth = int(gamestream_width)
    devmode.PelsHeight = int(gamestream_height)
    devmode.Fields = win32con.DM_PELSWIDTH | win32con.DM_PELSHEIGHT
    win32api.ChangeDisplaySettings(devmode, 0)

def get_process_name(p):
    # If there are permission errors reading a process name, it's probably not the one we want, so skip it.
    try:
        p_name = p.name()
    except (PermissionError, psutil.AccessDenied):
        p_name = ""
        pass
    return p_name

def reset_launcher_resolution(gamestream_width, gamestream_height, launcher_window_name):
    # Check to ensure desired GSLP resolution is still set whenever the launcher is in focus in case it didn't reset when exiting a game
    focused_window = win32gui.GetWindowText(win32gui.GetForegroundWindow()).lstrip()
    #print("Trying to match", launcher_window_name, focused_window)
    if focused_window.startswith(launcher_window_name):
        #print("Matched")
        current_width = str(win32api.GetSystemMetrics(0))
        current_height = str(win32api.GetSystemMetrics(1))
        if current_width != gamestream_width and current_height != gamestream_height:
            print("Resolutions don't match, changing from", current_width, current_height, "to", gamestream_width, gamestream_height)
            set_resolution(gamestream_width, gamestream_height)

# Define a default config file to write if we're missing one
config_filename = 'gamestream_playnite.ini'
default_config = """[LAUNCHER]
# The path to your Playnite.FullscreenApp.exe
launcher_path = %%LOCALAPPDATA%%\Playnite\Playnite.FullscreenApp.exe
# Name of the window to watch to close the session when it's gone
launcher_window_name = Playnite

[BACKGROUND]
# List as many exe's or bat's as you want here. They will run at the start of the GameStream session and be killed at the end.
# background_exe_1 = C:\Program Files (x86)\JoyToKey\JoyToKey.exe
# background_exe_2 = C:\WINDOWS\system32\mspaint.exe

[SETTINGS]
# Set debug = 1 to leave a window running after gamestream to see error messages from GSLP
debug = 0
# Set sleep_on_exit to 1 to put the computer to sleep after the session
sleep_on_exit = 0
# Set close_watch_method to "process" to wait for the launcher process to totally die to exit (can be problematic if it closes to tray), or "window" to just wait the the window to close
close_watch_method = window
"""

# Write the default config
if not os.path.exists(config_filename):
    with open(config_filename, 'w') as out_file:
        out_file.write(default_config)

# Target resolution for gamestream environment
try:
    gamestream_width = sys.argv[1]
    gamestream_height = sys.argv[2]
    # If there's a 3rd argument after the .py/.exe, use it as a custom launcher path
    if len(sys.argv) == 4:
        config_filename = sys.argv[3]
except IndexError:
    print("Error parsing arguments. Did you mean to run one of the .bat launcher scripts?")
    print("Usage: gamestream_launchpad.exe 1920 1080 [config.ini]")
    input("Press Enter to exit.")
    sys.exit(1)

# Parse the config file and assume defaults otherwise
config = configparser.ConfigParser()
config.read(config_filename)
cfg_launcher_path = config['LAUNCHER'].get('launcher_path', r'%LOCALAPPDATA%\Playnite\Playnite.FullscreenApp.exe')
cfg_launcher_window_name = config['LAUNCHER'].get('launcher_window_name', 'Playnite')
cfg_bg_paths = config['BACKGROUND']
debug = config['SETTINGS'].get('debug', '0')
sleep_on_exit = config['SETTINGS'].get('sleep_on_exit', '0')
close_watch_method = config['SETTINGS'].get('close_watch_method', 'window')

launcher_exec_name = os.path.basename(cfg_launcher_path)

# Set resolution to target
set_resolution(gamestream_width, gamestream_height)

# Start background programs, if they're available
for path in cfg_bg_paths:
    expanded_path = os.path.expandvars(cfg_bg_paths[path])
    if os.path.exists(expanded_path):
        print("Launching", expanded_path)
        exec_name = os.path.basename(expanded_path)
        # Kill the process first if it's already running
        if exec_name in (get_process_name(p) for p in psutil.process_iter()):
            os.system('taskkill /f /im ' + exec_name)
        # Start the process
        subprocess.Popen(expanded_path)

# A launcher value of false will create a wait inside of the console instead watching a program
if cfg_launcher_path.lower() == "false":
    input('Press enter to end the GameStream session.')
else:
    # Minimize all windows
    print("Minimizing windows")
    pyautogui.hotkey('winleft', 'd')

    # Playnite has to be killed before it will start in fullscreen mode
    if "Playnite" in launcher_exec_name:
        if "Playnite.FullscreenApp.exe" in (get_process_name(p) for p in psutil.process_iter()):
            os.system('taskkill /f /im ' + "Playnite.FullscreenApp.exe")
        if "Playnite.DesktopApp.exe" in (get_process_name(p) for p in psutil.process_iter()):
            os.system('taskkill /f /im ' + "Playnite.DesktopApp.exe")

        # Move mouse cursor into the lower-right corner to pseudo-hide it because sticks out in playnite fullscreen
        pyautogui.FAILSAFE = False
        pyautogui.moveTo(9999, 9999, duration = 0)

    # Start game launcher
    print("Starting game launcher")
    launcher_exe = os.path.expandvars(cfg_launcher_path)
    subprocess.Popen(launcher_exe)

    # Focus launcher in the foreground and maximize
    results = []
    top_windows = []
    launcher_focused = False
    while not launcher_focused:
        win32gui.EnumWindows(windowEnumerationHandler, top_windows)
        for i in top_windows:
            if cfg_launcher_window_name in i[1]:
                if not 'Fullscreen' in cfg_launcher_path:
                    print("Maximizing", cfg_launcher_window_name)
                    win32gui.ShowWindow(i[0], 3)
                print("Focusing", cfg_launcher_window_name)
                win32gui.SetForegroundWindow(i[0])
                launcher_focused = True
                launcher_window_handle = i[0]
                break
        sleep(1)

    # Watch for closing the launcher window to return to the system's original configuration
    if close_watch_method == "window":
        print("Watching for launcher window to close")
        while win32gui.IsWindowVisible(launcher_window_handle):
            #print("Visible:", launcher_window_handle)
            sleep(2)
            reset_launcher_resolution(gamestream_width, gamestream_height, cfg_launcher_window_name)
    # Alternative method that waits for the process to die (can be problematic if it minimizes to system tray)
    elif close_watch_method == "process":
        print("Watching for launcher process to die")
        while True:
            if launcher_exec_name in (get_process_name(p) for p in psutil.process_iter()):
                sleep(2)
                reset_launcher_resolution(gamestream_width, gamestream_height, cfg_launcher_window_name)
            else:
                break
    else:
        print("No valid close_watch_method in the config. Press Enter when you're done.")
        input()

# Terminate background programs, if they're available
for path in cfg_bg_paths:
    expanded_path = os.path.expandvars(cfg_bg_paths[path])
    if os.path.exists(expanded_path):
        exec_name = os.path.basename(expanded_path)
        print("Terminating", exec_name)
        if exec_name in (get_process_name(p) for p in psutil.process_iter()):
            os.system('taskkill /f /im ' + exec_name)

# Kill gamestream
print("Terminating GameStream session.")
if "nvstreamer.exe" in (get_process_name(p) for p in psutil.process_iter()):
    os.system('taskkill /f /im nvstreamer.exe')

# Restore original resolution
print('Restoring original resolution.')
win32api.ChangeDisplaySettings(None, 0)

if sleep_on_exit == '1':
# Put computer to sleep
    print("Going to sleep")
    os.system("rundll32.exe powrprof.dll,SetSuspendState 0,1,0")

if debug == '1':
    # Leave window open for debugging
    input("Paused for debug review. Press Enter key to close.")


#!/usr/bin/env python

import pytest

import netmiko
from DEVICE_CREDS import *

def setup_module(module):

    module.EXPECTED_RESPONSES = {
        'enable_prompt' : 'RP/0/0/CPU0:XRv-1#',
        'base_prompt'   : 'RP/0/0/CPU0:XRv-1',
        'interface_ip'  : '169.254.254.181',
        'config_mode'   : '(config)'
    }

    show_ver_command = 'show version'
    commit_history_cmd = 'show configuration commit list'
    module.basic_command = 'show ipv4 int brief'

    SSHClass = netmiko.ssh_dispatcher(cisco_xr['device_type'])
    net_connect = SSHClass(**cisco_xr)

    module.show_version = net_connect.send_command(show_ver_command)
    module.show_ip = net_connect.send_command(module.basic_command)

    net_connect.enable()
    module.enable_prompt = net_connect.find_prompt()

    current_commit_history = net_connect.send_command(commit_history_cmd)

    # get the current 10 digit commit Label ID
    module.current_commit = current_commit_history.split('\n')[4].split()[1]
    module.config_mode = net_connect.config_mode()
    config_commands = ['logging monitor warning', 'logging buffered 30720100', 'logging console errors']
    net_connect.send_config_set(config_commands, commit=True)

    new_commit_history = net_connect.send_command(commit_history_cmd)

    # get the new 10 digit commit Label ID
    module.new_commit = new_commit_history.split('\n')[4].split()[1]

    module.config_commands_output = net_connect.send_command("show run | inc logging")

    net_connect.disconnect()

def test_enable_mode():
    assert enable_prompt == EXPECTED_RESPONSES['enable_prompt']

def test_config_mode():
    assert EXPECTED_RESPONSES['config_mode'] in config_mode

def test_command_set():
    assert 'logging buffered 30720100' in config_commands_output
    assert 'logging console errors' in config_commands_output

def test_commit():
    assert int(current_commit) < int(new_commit)


# -*- coding: utf-8 -*-
import scrapy
import pdb
from transfer_crawl.items import idMapEnglish
from scrapy_redis.spiders import RedisSpider
import traceback
import time, datetime
from compute_algorithm.chinese_2_english import Chinese_2_english

c_2_e = Chinese_2_english()
# scrapy crawl china2code
# class China2codeSpider(scrapy.Spider):
class China2codeSpider(RedisSpider):
    name = 'china2code'
    allowed_domains = ['http://liansai.500.com']
    start_urls = []
    redis_key = 'china2code:start_urls'

    def start_requests(self):
        for url in self.start_urls:
            yield scrapy.Request(url)

    def parse(self, response):
        trs = response.xpath('//table[@class="lstable1 ljifen_top_list_s jTrHover"]/tr')
        for tr in trs:
            if len(tr.xpath('td')) < 7:
                continue
            tds = tr.xpath('td')
            team_id = tds[1].xpath('a')[0].xpath('@href').extract()[0].split('/')[2]
            team_title = tds[1].xpath('a')[0].xpath('@title').extract()[0]
            english_name = c_2_e.get(team_title)
            single_item = idMapEnglish()
            single_item['team_id'] = team_id
            single_item['english_name'] = english_name
            yield single_item


from flask_restful import Resource
from flask import request
from models import db, User
import random
import string

class DeleteAccount(Resource):
    def post(self):
        print(request.data)
        json_data = request.get_json(force=True)

        header = request.headers["Authorization"]

        if not header:
            return {"status": 'Failed'}, 403
        else:
            print(header)
            if header == "HPU9mKF2A8J4MwyXqQmiuvdwXaxnFT2ASnOvWiIOrZ9X3qWkrp":
                user = db.session.query(User).filter_by(api_key = json_data["api_key"]).first()
                if user:
                    db.session.delete(user)
                    db.session.commit()
                else:
                    return {"status": 'Failed because no User was found'}, 401

            else:
                return {"status": 'Failed because theres the wrong authentication'}, 402

        return {"status": 'success'}, 201


import os
import warnings

import numpy as np

def softmax(logits):
    """Transforms predictions into probability values.

    Parameters
    ----------
    logits : array_like
        The logits predicted by the model.

    Returns
    -------
    `numpy.ndarray`
        Probability values corresponding to the logits.
    """

    assert logits.ndim == 1

    # for numerical reasons we subtract the max logit
    # (mathematically it doesn't matter!)
    # otherwise exp(logits) might become too large or too small
    logits = logits - np.max(logits)
    e = np.exp(logits)
    return e / np.sum(e)

def crossentropy(label, logits):
    """Calculates the cross-entropy.

    Parameters
    ----------
    logits : array_like
        The logits predicted by the model.
    label : int
        The label describing the target distribution.

    Returns
    -------
    float
        The cross-entropy between softmax(logits) and onehot(label).

    """

    assert logits.ndim == 1

    # for numerical reasons we subtract the max logit
    # (mathematically it doesn't matter!)
    # otherwise exp(logits) might become too large or too small
    logits = logits - np.max(logits)
    e = np.exp(logits)
    s = np.sum(e)
    ce = np.log(s) - logits[label]
    return ce

def batch_crossentropy(label, logits):
    """Calculates the cross-entropy for a batch of logits.

    Parameters
    ----------
    logits : array_like
        The logits predicted by the model for a batch of inputs.
    label : int
        The label describing the target distribution.

    Returns
    -------
    np.ndarray
        The cross-entropy between softmax(logits[i]) and onehot(label)
        for all i.

    """

    assert logits.ndim == 2

    # for numerical reasons we subtract the max logit
    # (mathematically it doesn't matter!)
    # otherwise exp(logits) might become too large or too small
    logits = logits - np.max(logits, axis=1, keepdims=True)
    e = np.exp(logits)
    s = np.sum(e, axis=1)
    ces = np.log(s) - logits[:, label]
    return ces

def binarize(x, values, threshold=None, included_in="upper"):
    """Binarizes the values of x.

    Parameters
    ----------
    values : tuple of two floats
        The lower and upper value to which the inputs are mapped.
    threshold : float
        The threshold; defaults to (values[0] + values[1]) / 2 if None.
    included_in : str
        Whether the threshold value itself belongs to the lower or
        upper interval.

    """
    lower, upper = values

    if threshold is None:
        threshold = (lower + upper) / 2.0

    x = x.copy()
    if included_in == "lower":
        x[x <= threshold] = lower
        x[x > threshold] = upper
    elif included_in == "upper":
        x[x < threshold] = lower
        x[x >= threshold] = upper
    else:
        raise ValueError('included_in must be "lower" or "upper"')
    return x

def imagenet_example(shape=(224, 224), data_format="channels_last", bounds=(0, 255)):
    """ Returns an example image and its imagenet class label.

    Parameters
    ----------
    shape : list of integers
        The shape of the returned image.
    data_format : str
        "channels_first" or "channels_last"
    bounds : tuple
        smallest and largest allowed pixel value

    Returns
    -------
    image : array_like
        The example image.

    label : int
        The imagenet label associated with the image.

    NOTE: This function is deprecated and will be removed in the future.
    """
    assert len(shape) == 2
    assert data_format in ["channels_first", "channels_last"]

    from PIL import Image

    path = os.path.join(os.path.dirname(__file__), "example.png")
    image = Image.open(path)
    image = image.resize(shape)
    image = np.asarray(image, dtype=np.float32)
    image = image[:, :, :3]
    assert image.shape == shape + (3,)
    if data_format == "channels_first":
        image = np.transpose(image, (2, 0, 1))
    if bounds != (0, 255):
        image = image / 255 * (bounds[1] - bounds[0]) + bounds[0]
    return image, 282

def samples(
    dataset="imagenet",
    index=0,
    batchsize=1,
    shape=(224, 224),
    data_format="channels_last",
    bounds=(0, 255),
):
    """ Returns a batch of example images and the corresponding labels

    Parameters
    ----------
    dataset : string
        The data set to load (options: imagenet, mnist, cifar10,
        cifar100, fashionMNIST)
    index : int
        For each data set 20 example images exist. The returned batch
        contains the images with index [index, index + 1, index + 2, ...]
    batchsize : int
        Size of batch.
    shape : list of integers
        The shape of the returned image (only relevant for Imagenet).
    data_format : str
        "channels_first" or "channels_last"
    bounds : tuple
        smallest and largest allowed pixel value

    Returns
    -------
    images : array_like
        The batch of example images

    labels : array of int
        The labels associated with the images.

    """
    from PIL import Image

    images, labels = [], []
    basepath = os.path.dirname(__file__)
    samplepath = os.path.join(basepath, "data")
    files = os.listdir(samplepath)

    if batchsize > 20:
        warnings.warn(
            "foolbox.utils.samples() has only 20 samples and repeats itself if batchsize > 20"
        )

    for idx in range(index, index + batchsize):
        i = idx % 20

        # get filename and label
        file = [n for n in files if "{}_{:02d}_".format(dataset, i) in n][0]
        label = int(file.split(".")[0].split("_")[-1])

        # open file
        path = os.path.join(samplepath, file)
        image = Image.open(path)

        if dataset == "imagenet":
            image = image.resize(shape)

        image = np.asarray(image, dtype=np.float32)

        if dataset != "mnist" and data_format == "channels_first":
            image = np.transpose(image, (2, 0, 1))

        images.append(image)
        labels.append(label)

    images = np.stack(images)
    labels = np.array(labels)

    if bounds != (0, 255):
        images = images / 255 * (bounds[1] - bounds[0]) + bounds[0]
    return images, labels

def onehot_like(a, index, value=1):
    """Creates an array like a, with all values
    set to 0 except one.

    Parameters
    ----------
    a : array_like
        The returned one-hot array will have the same shape
        and dtype as this array
    index : int
        The index that should be set to `value`
    value : single value compatible with a.dtype
        The value to set at the given index

    Returns
    -------
    `numpy.ndarray`
        One-hot array with the given value at the given
        location and zeros everywhere else.

    """

    x = np.zeros_like(a)
    x[index] = value
    return x

def flatten(x):
    shape = (x.shape[0], -1)
    return x.reshape(shape)

def atleast_kd(x, k):
    shape = x.shape + (1,) * (k - x.ndim)
    return x.reshape(shape)

def accuracy(fmodel, inputs, labels):
    logits = fmodel.forward(inputs)
    predictions = logits.argmax(axis=-1)
    accuracy = (predictions == labels).mean()
    return accuracy


import cv2
import numpy as np
import matplotlib.pyplot as plt

def HW4(bonus=False):
    """

    :return:
    """
    MIN_MATCH_COUNT = 10
    # part 1
    # 1.Load the "stones" image and convert it to grayscale.
    img1 = cv2.imread('stones.png')
    gray = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
    # 2.Detect SIFT features and compute descriptors for them using sift.detectAndCompute().
    sift = cv2.xfeatures2d.SIFT_create()
    kp1, des_stone = sift.detectAndCompute(gray, None)
    img1 = cv2.drawKeypoints(img1,kp1,img1)
    # 3.Show the SIFT features on the image.
    # cv2.imshow('sift', draw_keypts)
    # 4.Iterate through the frames of the video and detect SIFT features & descriptors on each frame
    cap = cv2.VideoCapture('input.mov')
    # Part 2, 1.Create a "brute force" matcher (BFMatcher) object.
    bf = cv2.BFMatcher()
    i = 0
    inlier_ = outlier_ = 0
    frame_size = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
    filename = 'overlay_Chan.avi' if bonus else 'boarder_Chan.avi'
    video = cv2.VideoWriter(filename=filename, fourcc=cv2.VideoWriter_fourcc('M', 'P', 'E', 'G'), fps=30,
                            frameSize=frame_size)

    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        sift = cv2.xfeatures2d.SIFT_create()
        kp2, des_frame = sift.detectAndCompute(gray, None)
        # frames.append(cv2.drawKeypoints(frame, kp2, frame))
        # part 2
        # 2.At each video frame, compute the first two nearest-neighbor matches for each keypoint in the video frame
        # (matching to the target "stones" image).
        matches = bf.knnMatch(des_stone, des_frame, k=2)
        # 3.Apply the "ratio test" to filter out unreliable matches, using a threshold of 0.7
        # store all the good matches as per Lowe's ratio test.
        good = []
        for m, n in matches:
            if m.distance < 0.7 * n.distance:
                good.append(m)
        # 4. Draw the matches between the video frame and the target image using cv.drawMatches()
        if len(good) > MIN_MATCH_COUNT:
            src_pts = np.float32([kp1[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
            dst_pts = np.float32([kp2[m.trainIdx].pt for m in good]).reshape(-1, 1, 2)
            # part 3
            # 1.Fit a homography to the matches using findHomography() with the RANSAC option.
            # Note that you might need to invert the resulting homography matrix using np.linalg.inv().
            H, inlier = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0)
            matchesMask = inlier.ravel().tolist()
            print(matchesMask)
            h, w, d = img1.shape
            pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2)
            dst = cv2.perspectiveTransform(pts, H)
            # Draw polygon around the tracking target in the video frame using cv.polylines()
            frame = cv2.polylines(frame, [np.int32(dst)], True, 255, 3, cv2.LINE_AA)
        else:
            print("Not enough matches are found - {}/{}".format(len(good), MIN_MATCH_COUNT))
            matchesMask = None
        # 4.Draw the inlier matches between the video frame and the target image using cv.drawMatches()

        draw_params = dict(matchColor=(0, 255, 0),  # draw matches in green color
                           singlePointColor=None,
                           matchesMask=matchesMask,  # draw only inliers
                           flags=2)
        inlier_matches = cv2.drawMatches(img1, kp1, frame, kp2, good, None, **draw_params)
        # cv2.imwrite('inlier_matches'+str(i)+'.png', inlier_matches)
        if not i:
            inlier_h, inlier_w, inlier_d = inlier_matches.shape
            video2 = cv2.VideoWriter(filename='matches.avi', fourcc=cv2.VideoWriter_fourcc('M', 'P', 'E', 'G'),
                                     fps=15,
                                     frameSize=(inlier_w, inlier_h))
        video2.write(inlier_matches)
        if bonus:
            # bonus
            # Use the calculate homography to warp the overlay image (see cv.perspectiveWarp() ).
            overlay = cv2.imread('overlay.png')
            # To put the overlay on the image, you need to create a mask which has ones over the warped overlay and zeros
            # outside of it.  To do this, create an image of all ones and
            # warp it using perspectiveWarp().
            mask = np.ones(overlay.shape)
            mask = cv2.warpPerspective(mask, H, frame_size)
            overlay = cv2.warpPerspective(overlay, H, frame_size)
            # Calculate the composite image as (1-mask) * frame + mask * overlay.
            comp_img = np.multiply(1-mask, frame) + np.multiply(mask, overlay)
            frame = np.uint8(comp_img)
        video.write(frame)
        i += 1
    # part 3  5. Calculate the average percentage of inliers.
    for i in range(len(good)):
        if matchesMask[i]:
            inlier_ += 1
        else:
            outlier_ += 1
    percent_inlier = inlier_ / (inlier_ + outlier_) * 100
    percent_outlier = 100-percent_inlier
    print("percent inlier = %.6f" % percent_inlier)
    print("percent outlier = %.6f" % percent_outlier)
    video.release()
    cv2.destroyAllWindows()

if __name__ == '__main__':
    # True to generate output with overlay
    # False to generate output with border
    HW4(True)


from django.urls import include, path, re_path

def build_integration_urlpatterns(list_view_cls, config_form_view_cls,
                                  namespace=None):
    """Build URL patterns for integration pages.

    This will produce a set of URL patterns for the integration administration
    pages that take advantage of the specialized views provided. These can
    be used in a :file:`urls.py`, either returned as a ``urlpatterns``
    variable or included using :py:func:`~django.conf.urls.include`.

    Args:
        list_view_cls (type, optional):
            The view responsible for listing available integrations. This
            should be a subclass of
            :py:class:`~djblets.integrations.views.BaseIntegrationListView`.

            If ``None``, the integration list URL will not be provided in the
            URL patterns. Consumers should define their own
            ``integration-list`` URL with the appropriate namespace in order to
            handle any URL reversing that may occur.

        config_form_view_cls (type):
            The view responsible for configuring an integration. This should
            be a subclass of
            :py:class:`~djblets.integrations.views.
            BaseIntegrationConfigFormView`.

            If ``None``, the integration configuration URLs will not be
            provided in the URL patterns. This *will* break the integrations
            list URL unless consumers provide their own
            ``integration-add-config`` and ``integration-change-config`` URL
            registrations.

        namespace (unicode, optional):
            The namespace to use for all URL names.

            For legacy reasons, this is an empty string. If embedding into
            other parts of the app, this should be changed. For instance,
            you may want the admin version to use ``admin``.
    """
    urlpatterns = []

    if list_view_cls is not None:
        urlpatterns.append(
            path('', list_view_cls.as_view(), name='integration-list'))

    if config_form_view_cls is not None:
        urlpatterns.append(
            re_path(r'^(?P<integration_id>[A-Za-z0-9_\.]+)/configs/', include([
                path('add/',
                     config_form_view_cls.as_view(),
                     name='integration-add-config'),
                path('<int:config_id>/',
                     config_form_view_cls.as_view(),
                     name='integration-change-config'),
            ])))

    return [
        path('', include(urlpatterns, namespace=namespace)),
    ]


#!/usr/bin/env python

import subprocess as sp
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import vplot as vpl
import os
import sys
import matplotlib.patches as mpatches

#Variable List
dest = "KDwarfStatic"
num = 100
L_sun = 3.846e26
a_earth = 1
listf = "list_KDwarfStatic"

# if the list file exists, extract data for plotting

lum0, obliq0, semi0, inst, snowball, northCapL, northCapS, southCapL, southCapS, icebeltL, icebeltS, iceFree, tGlobal = np.loadtxt(
    listf, unpack=True)

PolarCaps = np.zeros(num*num)
MoistGreen = np.zeros(num*num)
for i in np.arange(num*num):

    if (
    northCapL[i] == 1 and southCapL[i] == 1 and icebeltS[i] == 0 and icebeltL[i] == 0 or
    northCapS[i] == 1 and northCapL[i] == 1 and southCapS[i] == 1 and southCapL[i] == 1 and icebeltS[i] == 0 and icebeltL[i] == 0 or
    northCapS[i] == 1 and southCapS[i] == 1 and icebeltS[i] == 0 and icebeltL[i] == 0 or
    northCapL[i] == 1 and southCapS[i] == 1 and icebeltS[i] == 0 and icebeltL[i] == 0 or
    northCapS[i] == 1 and southCapL[i] == 1 and icebeltS[i] == 0 and icebeltL[i] == 0
    ):
        PolarCaps[i] = 1

    if tGlobal[i] >= 70 and iceFree[i] == 1:
        MoistGreen[i] = 1

lum0 = np.reshape(lum0, (num, num))
obliq0 = np.reshape(obliq0, (num, num)) * 180 / np.pi
semi0 = np.reshape(semi0, (num, num)) / 1.49598e11
inst = np.reshape(inst, (num, num)) / 1350
snowball = np.reshape(snowball, (num, num))
northCapL = np.reshape(northCapL, (num, num))
northCapS = np.reshape(northCapS, (num, num))
southCapL = np.reshape(southCapL, (num, num))
southCapS = np.reshape(southCapS, (num, num))
icebeltL = np.reshape(icebeltL, (num, num))
icebeltS = np.reshape(icebeltS, (num, num))
iceFree = np.reshape(iceFree, (num, num))
tGlobal = np.reshape(tGlobal, (num, num))

PolarCaps = np.reshape(PolarCaps, (num, num))
MoistGreen = np.reshape(MoistGreen, (num, num))

plt.figure(figsize=(9,6.5))
plt.ylabel("Instellation [Earth]", fontsize=16)
plt.xlabel("Obliquity [$^\circ$]", fontsize=16)
plt.ylim(0.867,1.211)
plt.xlim(0,90)

iFF = plt.contourf(obliq0, inst, iceFree, [0, 1], colors = vpl.colors.dark_blue)
sNF = plt.contourf(obliq0, inst, snowball, [0.5, 1], colors = '#efefef')
PcF = plt.contourf(obliq0,inst,PolarCaps, [0.5, 1], colors = vpl.colors.purple)
icF = plt.contourf(obliq0,inst,icebeltL, [0.5, 1], colors = vpl.colors.pale_blue)

h1, _ = iFF.legend_elements()
h2, _ = icF.legend_elements()
h3, _ = sNF.legend_elements()
h4, _ = PcF.legend_elements()

plt.legend([h1[0], h2[0], h3[0], h4[0]], [ 'Ice Free', 'Ice Belt', 'Snowball','Polar Ice Caps'],loc = 'upper left', bbox_to_anchor=(0, 1.02, 1, 0.102),ncol=4, mode="expand", borderaxespad=0)

if (sys.argv[1] == 'pdf'):
    plt.savefig(dest + '.pdf', dpi=300)
if (sys.argv[1] == 'png'):
    plt.savefig(dest + '.png', dpi=300)

plt.show()
plt.close()


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

from sklearn.preprocessing import PolynomialFeatures
import statsmodels.formula.api as sm

def get_best_features(X, y):

    poly_reg = PolynomialFeatures(degree = 2)
    X = poly_reg.fit_transform(X)

    last_rsquared_adj = 0
    X = np.append(arr = np.ones((X.shape[0], 1)).astype(int), values = X, axis = 1)

    X_columns = [i for i in range(X.shape[1])]
    X_opt = np.float64(X[:, X_columns])
    regressor_OLS = sm.OLS(endog = y, exog = X_opt).fit()

    drop_list = [np.argmax(regressor_OLS.pvalues)]

    while not (not drop_list):
#        print ("Old rsquared : ", regressor_OLS.rsquared_adj)
        X_opt = np.delete(X_opt,drop_list , axis=1)
        last_rsquared_adj = regressor_OLS.rsquared_adj
        regressor_OLS = sm.OLS(endog = y, exog = X_opt).fit()
        drop_list = []
#        print ("New rsquared : ", regressor_OLS.rsquared_adj)

        if regressor_OLS.rsquared_adj >= last_rsquared_adj \
        and (max(regressor_OLS.pvalues) - 0.05) > 0.02:
                drop_list.append(np.argmax(regressor_OLS.pvalues))

    return (X_opt)


from markupsafe import Markup
from fanboi2.helpers.formatters import format_markdown
from fanboi2.models import DBSession, Page
from fanboi2.cache import cache_region as cache_region_

def _get_internal_page(slug, cache_region=cache_region_):
    """Returns a content of internal page.

    :param slug: An internal page slug.
    :type slug: String
    :rtype: String or None
    """
    def _creator():
        page = DBSession.query(Page).filter_by(
                namespace='internal',
                slug=slug).\
            first()
        if page:
            return page.body
    return cache_region.get_or_create(
        'partial:%s' % (slug,),
        _creator,
        expiration_time=43200)

def global_css(context, request, cache_region=cache_region_):
    """Returns a string of inline global custom CSS for site-wide CSS override.
    This custom CSS is the content of ``internal:global/css`` page.

    :param context: A :class:`mako.runtime.Context` object.
    :param request: A :class:`pyramid.request.Request` object.
    :param cache_region: Optional cache region to cache this partial.

    :type context: mako.runtime.Context or None
    :type request: pyramid.request.Request
    :type cache_region: dogpile.cache.region.CacheRegion
    :rtype: Markup or None
    """
    page = _get_internal_page('global/css', cache_region)
    if page:
        return Markup(page)

def global_appendix(context, request, cache_region=cache_region_):
    """Returns a HTML of global appendix content. This appendix content is the
    content of ``internal:global/appendix`` page.

    :param context: A :class:`mako.runtime.Context` object.
    :param request: A :class:`pyramid.request.Request` object.
    :param cache_region: Optional cache region to cache this partial.

    :type context: mako.runtime.Context or None
    :type request: pyramid.request.Request
    :type cache_region: dogpile.cache.region.CacheRegion
    :rtype: Markup or None
    """
    page = _get_internal_page('global/appendix', cache_region)
    if page:
        return format_markdown(context, request, page)

def global_footer(context, request, cache_region=cache_region_):
    """Returns a HTML of global footer content. This footer content is the
    content of ``internal:global/footer`` page.

    :param context: A :class:`mako.runtime.Context` object.
    :param request: A :class:`pyramid.request.Request` object.
    :param cache_region: Optional cache region to cache this partial.

    :type context: mako.runtime.Context or None
    :type request: pyramid.request.Request
    :type cache_region: dogpile.cache.region.CacheRegion
    :rtype: Markup or None
    """
    page = _get_internal_page('global/footer', cache_region)
    if page:
        return Markup(page)


#!/usr/bin/env python3
#--- coding: utf-8 ---

import os
import argparse
import pickle
import math
import numpy as np
import copy

def distance(list1, list2):
    assert len(list1) == len(list2)
    dis_sqr = 0
    for d1, d2 in zip(list1, list2):
        dis_sqr += (d1 - d2) * (d1 - d2)
    return math.sqrt(dis_sqr)

def extract_app_feature(op_list, op_centers, max_locality):
    total_time = 0.0
    each_center_time = []

    for i in range(len(op_centers)):
        each_center_time.append(0.0)

    op_perf_dict = {}
    op_center_dict = {}
    for each_op in op_list:
        if (not each_op.is_aid_op) and (each_op.comp_instrs != 0):

            locality = (each_op.mem_trans / each_op.comp_instrs)
            norm_loc = locality / max_locality

            dis_array = [distance([each_op.parallelism, norm_loc], c)
                         for c in op_centers]
            min_i = np.argmin(dis_array)

            op_key_list = copy.deepcopy(sorted(each_op.input_tensor_name))
            op_key_list.extend(sorted(each_op.output_tensor_name))
            op_key = ','.join(op_key_list)
            elapsed_time = float(each_op.elapsed_time)

            if (op_key not in op_perf_dict) or (elapsed_time < op_perf_dict[op_key]):
                op_perf_dict[op_key] = elapsed_time
                op_center_dict[op_key] = min_i

    for op_key in op_perf_dict.keys():
        total_time += op_perf_dict[op_key]
        each_center_time[op_center_dict[op_key]] += op_perf_dict[op_key]

    app_feature = [(t / total_time) for t in each_center_time]
    return app_feature

def merge_two_clusters(label_a, label_b, app_label):
    assert label_a != label_b
    for app_name in app_label.keys():
        if app_label[app_name] == label_a:
            app_label[app_name] = label_b
    return

def update_label_center(app_dict, app_label):
    label_center = {}
    num_points = {}

    for app_name in app_label.keys():
        ndims = len(app_dict[app_name])
        label = app_label[app_name]
        label_center[label] = np.zeros(ndims).tolist()
        num_points[label] = 0

    for app_name in app_label.keys():
        label = app_label[app_name]
        ndims = len(app_dict[app_name])
        for i in range(ndims):
            label_center[label][i] += app_dict[app_name][i]
        num_points[label] += 1

    for label in label_center.keys():
        ndims = len(label_center[label])
        for i in range(ndims):
            label_center[label][i] /= num_points[label]

    return label_center

def comp_cohesion(app_dict, app_label, label_center):
    dis_array = [distance(app_dict[x], label_center[app_label[x]])
                 for x in app_dict.keys()]
    return np.max(dis_array)

def main():

    parser = argparse.ArgumentParser(
        description="cluster applications according to operator cluster results",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('input_dir', help="specify the directory under which "
                        "profiling results are saved")
    parser.add_argument('--op_centers', default="op_centers.txt",
                        help="specify the file of op centers")
    parser.add_argument('--suffix', default=".pickle", help="specify the suffix of "
                        "result files")
    parser.add_argument('--max_locality', default="max_locality.txt",
                        help="specify the value of max locality for normalization")
    parser.add_argument('--output_dir', default=None,
                        help="specify the directory where output results are saved")
    parser.add_argument('--output_file', default="app_cluster.pickle",
                        help="specify the filename of application cluster outputs")

    args = parser.parse_args()

    print('Loading Operator Centers...')
    op_centers = []
    with open(args.op_centers, 'r') as f:
        lines = f.readlines()
        for line in lines:
            center = []
            locs = line.split(',')
            for each_loc in locs:
                each_loc.strip('\n ')
                center.append(float(each_loc))
            op_centers.append(center)

    print('Loading Max Locality...')
    max_locality = 25.0
    with open(args.max_locality, 'r') as f:
        line = f.readline()
        line.strip('\n ')
        max_locality = float(line)

    print('Loading and Extracting Operator Features...')
    app_dict = {}
    for dirpath, dirnames, filenames in os.walk(args.input_dir):
        for input_file in filenames:
            if args.suffix in input_file:
                with open(os.path.join(dirpath, input_file), 'rb') as f:
                    op_list = pickle.load(f)
                index = input_file.find(args.suffix)
                app_name = input_file[:index]
                app_dict[app_name] = extract_app_feature(op_list, op_centers,
                                                         max_locality)

    for app_name in app_dict.keys():
        print('App: {}, feature: {}'.format(app_name, app_dict[app_name]))

    print('Starting App Clustering...')
    app_label = {}
    label_center = {}
    for app_name in app_dict.keys():
        label = len(app_label) + 1
        app_label[app_name] = label
        label_center[label] = app_dict[app_name]

    result_list = []
    iter_num = 0
    tmp_app_label = copy.deepcopy(app_label)
    result_list.append([iter_num, 0.0, tmp_app_label])
    while len(label_center) > 1:
        iter_num += 1

        dis_dict = {}
        for i in label_center.keys():
            for j in label_center.keys():
                if not i == j:
                    dis_dict[(i, j)] = distance(label_center[i], label_center[j])

        label_a, label_b = min(dis_dict.items(), key=lambda x: x[1])[0]

        merge_two_clusters(label_a, label_b, app_label)
        label_center = update_label_center(app_dict, app_label)

        cohesion = comp_cohesion(app_dict, app_label, label_center)
        tmp_app_label = copy.deepcopy(app_label)
        result_list.append([iter_num, cohesion, tmp_app_label])
        print('Iteration: {}, Cohesion: {}'.format(iter_num, cohesion))

    if args.output_dir is None:
        args.output_dir = os.path.join(os.getenv('LOG_OUTPUT_DIR'), 'outputs')

    with open(os.path.join(args.output_dir, args.output_file), 'wb') as f:
        output_tuple = (app_dict, result_list)
        pickle.dump(output_tuple, f)

if __name__ == '__main__':
    main()


# -*-coding:utf-8-*-
from multiprocessing import Process
import time

NUM = 99

def task(name):
    print(f'{name} is running')
    time.sleep(1)
    global NUM
    NUM += 1
    print(f'{name} is done, {NUM}')

if __name__ == '__main__':
    p = Process(target=task, args=('zh',)) # create subprocess

    p.start() # send a message to os and prepare subprocess resources
    # p.join()
    print(f'main process {NUM}')


import datetime
from typing import Any, Dict, List, Type, TypeVar, Union, cast

import attr
from dateutil.parser import isoparse

from ..models.creation_text_classification_record_explanation import (
    CreationTextClassificationRecordExplanation,
)
from ..models.creation_text_classification_record_inputs import (
    CreationTextClassificationRecordInputs,
)
from ..models.creation_text_classification_record_metadata import (
    CreationTextClassificationRecordMetadata,
)
from ..models.task_status import TaskStatus
from ..models.text_classification_annotation import TextClassificationAnnotation
from ..types import UNSET, Unset

T = TypeVar("T", bound="CreationTextClassificationRecord")

@attr.s(auto_attribs=True)
class CreationTextClassificationRecord:
    """Text classification record

    Attributes:
    -----------

    inputs: Dict[str, Any]
        The input data text

    multi_label: bool
        Enable text classification with multiple predicted/annotated labels.
        Default=False

    explanation: Dict[str, List[TokenAttributions]]
        Token attribution list explaining predicted classes per token input.
        The dictionary key must be aligned with provided record text. Optional"""

    inputs: CreationTextClassificationRecordInputs
    id: Union[Unset, int, str] = UNSET
    metadata: Union[CreationTextClassificationRecordMetadata, Unset] = UNSET
    event_timestamp: Union[Unset, datetime.datetime] = UNSET
    status: Union[Unset, TaskStatus] = UNSET
    prediction: Union[TextClassificationAnnotation, Unset] = UNSET
    annotation: Union[TextClassificationAnnotation, Unset] = UNSET
    multi_label: Union[Unset, bool] = False
    explanation: Union[CreationTextClassificationRecordExplanation, Unset] = UNSET
    additional_properties: Dict[str, Any] = attr.ib(init=False, factory=dict)

    def to_dict(self) -> Dict[str, Any]:
        inputs = self.inputs.to_dict()

        id: Union[Unset, int, str]
        if isinstance(self.id, Unset):
            id = UNSET
        else:
            id = self.id

        metadata: Union[Unset, Dict[str, Any]] = UNSET
        if not isinstance(self.metadata, Unset):
            metadata = self.metadata.to_dict()

        event_timestamp: Union[Unset, str] = UNSET
        if not isinstance(self.event_timestamp, Unset):
            event_timestamp = self.event_timestamp.isoformat()

        status: Union[Unset, TaskStatus] = UNSET
        if not isinstance(self.status, Unset):
            status = self.status

        prediction: Union[Unset, Dict[str, Any]] = UNSET
        if not isinstance(self.prediction, Unset):
            prediction = self.prediction.to_dict()

        annotation: Union[Unset, Dict[str, Any]] = UNSET
        if not isinstance(self.annotation, Unset):
            annotation = self.annotation.to_dict()

        multi_label = self.multi_label
        explanation: Union[Unset, Dict[str, Any]] = UNSET
        if not isinstance(self.explanation, Unset):
            explanation = self.explanation.to_dict()

        field_dict: Dict[str, Any] = {}
        field_dict.update(self.additional_properties)
        field_dict.update(
            {
                "inputs": inputs,
            }
        )
        if id is not UNSET:
            field_dict["id"] = id
        if metadata is not UNSET:
            field_dict["metadata"] = metadata
        if event_timestamp is not UNSET:
            field_dict["event_timestamp"] = event_timestamp
        if status is not UNSET:
            field_dict["status"] = status
        if prediction is not UNSET:
            field_dict["prediction"] = prediction
        if annotation is not UNSET:
            field_dict["annotation"] = annotation
        if multi_label is not UNSET:
            field_dict["multi_label"] = multi_label
        if explanation is not UNSET:
            field_dict["explanation"] = explanation

        return field_dict

    @classmethod
    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
        d = src_dict.copy()
        inputs = CreationTextClassificationRecordInputs.from_dict(d.pop("inputs"))

        def _parse_id(data: Any) -> Union[Unset, int, str]:
            data = None if isinstance(data, Unset) else data
            id: Union[Unset, int, str]
            return cast(Union[Unset, int, str], data)

        id = _parse_id(d.pop("id", UNSET))

        metadata: Union[CreationTextClassificationRecordMetadata, Unset] = UNSET
        _metadata = d.pop("metadata", UNSET)
        if not isinstance(_metadata, Unset):
            metadata = CreationTextClassificationRecordMetadata.from_dict(_metadata)

        event_timestamp: Union[Unset, datetime.datetime] = UNSET
        _event_timestamp = d.pop("event_timestamp", UNSET)
        if not isinstance(_event_timestamp, Unset):
            event_timestamp = isoparse(_event_timestamp)

        status: Union[Unset, TaskStatus] = UNSET
        _status = d.pop("status", UNSET)
        if not isinstance(_status, Unset):
            status = TaskStatus(_status)

        prediction: Union[TextClassificationAnnotation, Unset] = UNSET
        _prediction = d.pop("prediction", UNSET)
        if not isinstance(_prediction, Unset):
            prediction = TextClassificationAnnotation.from_dict(_prediction)

        annotation: Union[TextClassificationAnnotation, Unset] = UNSET
        _annotation = d.pop("annotation", UNSET)
        if not isinstance(_annotation, Unset):
            annotation = TextClassificationAnnotation.from_dict(_annotation)

        multi_label = d.pop("multi_label", UNSET)

        explanation: Union[CreationTextClassificationRecordExplanation, Unset] = UNSET
        _explanation = d.pop("explanation", UNSET)
        if not isinstance(_explanation, Unset):
            explanation = CreationTextClassificationRecordExplanation.from_dict(
                _explanation
            )

        creation_text_classification_record = cls(
            inputs=inputs,
            id=id,
            metadata=metadata,
            event_timestamp=event_timestamp,
            status=status,
            prediction=prediction,
            annotation=annotation,
            multi_label=multi_label,
            explanation=explanation,
        )

        creation_text_classification_record.additional_properties = d
        return creation_text_classification_record

    @property
    def additional_keys(self) -> List[str]:
        return list(self.additional_properties.keys())

    def __getitem__(self, key: str) -> Any:
        return self.additional_properties[key]

    def __setitem__(self, key: str, value: Any) -> None:
        self.additional_properties[key] = value

    def __delitem__(self, key: str) -> None:
        del self.additional_properties[key]

    def __contains__(self, key: str) -> bool:
        return key in self.additional_properties


import matplotlib.pyplot as plt
import numpy as np
from matplotlib.pyplot import cm
from scipy.spatial import Voronoi

from points_distributions import points as data_set
from SOM.functions import voronoi_finite_polygons_2d
from SOM.K_means import K_means as KM
from SOM.Kohonen_network import Kohonen_network as KN
from SOM.Neuron_gas import Neuron_gas as NG

def plot(black, redPointsInTime, out, title):
    colors = iter(cm.rainbow(np.linspace(0, 1, len(redPointsInTime))))
    # Set up plot
    fig = plt.figure(figsize=(10, 10))
    ax = fig.add_subplot(111)
    l, = plt.plot([], [], 'r-')
    plt.grid()
    # split black points into x and y
    black_x, black_y = zip(*black)
    ax.scatter(black_x, black_y, color='black', s=7)
    plt.xlim([-10, 10])
    plt.ylim([-10, 10])
    plt.title(title)

    i1 = iter(redPointsInTime)
    i2 = iter(redPointsInTime)
    next(i2)
    # print("redPointsInTime", redPointsInTime)
    for nr, (xy_cur, xy_next) in enumerate(zip(i1, i2)):
        xs_cur, ys_cur = zip(*xy_cur)
        xs_next, ys_next = zip(*xy_next)
        color = next(colors)
        plt.plot([], [], color=color, label="iteracja {}-{}".format(nr, nr + 1))
        plt.scatter(xs_cur, ys_cur, color="red", s=20)
        plt.scatter(xs_next, ys_next, color="red", s=20)
        for x_cur, y_cur, x_next, y_next in zip(xs_cur, ys_cur, xs_next, ys_next):
            dx = x_next - x_cur
            dy = y_next - y_cur
            if(dx != 0 or dy != 0):
                plt.arrow(x_cur, y_cur, dx, dy,
                          color=color, head_width=0.15, head_length=0.25,
                          length_includes_head=True)
    lgd = ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05),
                    fancybox=True, shadow=True, ncol=5)
    plt.savefig(out + ".png", bbox_extra_artists=(lgd,), bbox_inches='tight')

def plotVoronoi(black, redPointsInTime, out, title, koh_line_swt):

    red_at_last = list(redPointsInTime[-1])
    vor = Voronoi(red_at_last)
    # plot
    fig = plt.figure(figsize=(7, 7))
    # https://stackoverflow.com/questions/20515554/colorize-voronoi-diagram
    regions, vertices = voronoi_finite_polygons_2d(vor)
    # colorize
    for region in regions:
        polygon = vertices[region]
        plt.fill(*zip(*polygon), alpha=0.4)
    # plot lines beteeen kononen network
    if koh_line_swt:
        print("koh plt")
        red_iter = iter(red_at_last)
        red_iter_next = iter(red_at_last)
        next(red_iter_next)
        for point, point_next in zip(red_iter, red_iter_next):
            plt.plot([point[0], point_next[0]], [
                     point[1], point_next[1]], 'r--')

    for p in red_at_last:
        plt.scatter(p[0], p[1], color='red', s=10)

    plt.grid()
    # split black points into x and y
    black_x, black_y = zip(*black)
    plt.title(title)
    plt.scatter(black_x, black_y, color='black', s=1)
    plt.xlim([-10, 10])
    plt.ylim([-10, 10])
    plt.savefig(out + ".png")

Neuron_number = 10
som = {"Gas_Neuronowy": NG,
       "Siec_kohonena": KN,
       "k-Srednie": KM}
iter_number = 10
for som_name, SOM_type in som.items():
    for points_name, points in data_set.items():
        map = SOM_type(Neuron_number, points)
        redInTime = []
        for _ in range(iter_number):
            neuron_pos = list(map.getNeurons())
            redInTime.append(neuron_pos)
            map.iter_once()
        koh = False
        if som_name == "Siec_kohonena":
            koh = True
            print("kon " + som_name)
        plotVoronoi(points, redInTime, "./network_evo/" +
                    som_name + "_" + points_name + "_vor", som_name, koh)
        plot(points, redInTime, "./network_evo/" +
             som_name + "_" + points_name + "_in_time", som_name)


from matplotlib import pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from util import *

### fix the random seed to make our experiment repeatable
np.random.seed(10)

### get the data
X_train, y_train, X_test, y_test = get_data()

### linear regression
lr = LinearRegression()
lr.fit(X_train, y_train)
compute_train_test_error(X_train, y_train, X_test, y_test, lr.coef_, lr.intercept_)
plot_fit(X_train, y_train, X_test, y_test, lr.coef_, lr.intercept_, 'red', "Ordinary")

### ridge regression
rr = Ridge(alpha=10)
rr.fit(X_train, y_train)
compute_train_test_error(X_train, y_train, X_test, y_test, rr.coef_, rr.intercept_)
plot_fit(X_train, y_train, X_test, y_test, rr.coef_, rr.intercept_, 'blue', "Ridge")
plt.show()


# -*- coding: utf-8 -*-
#上一行为了中文注释

#由于在缺陷中提到的那个乱码问题，我在\scrapy_country_and_label文件夹下顺便爬了国家和标签
import scrapy
import json
import time
import re

# def deal_luanma():
#     f = open('Americas-.txt','w+', encoding='utf-8')
#     res=(f.read().replace('聽', ' '))
#     f.write(res)
#     f.close()

class CountrySpider(scrapy.Spider):
    name = 'countryspider'
    def start_requests(self):
        with open(r'wiki_country/querys/input.json', 'r') as f:
            querys = json.load(f)#把国家load进来
        #print("###"+querys)
        path = 'http://en.wikipedia.org/wiki/'
        for query in querys['query']:
            url = path + '_'.join(query.split(' '))#对于比如United States去空格为_
            print(url)
            yield scrapy.Request(url=url ,callback=self.parse)#以parse方式发出request
#meta={'country_name':query}

    def parse(self, response):
        fields = response.css('.infobox').xpath('.//th[@scope="row"]')#infobox中获取标签如Area Population
        res = {}
        for field in fields:
            key = field.xpath('.//text()').extract_first()#对应一个个标签比如Area
            #' ' 是一种奇怪的空格？？？？？？？？？？？？？？？/怎么办
            value = field.xpath('string(../td)').extract_first().replace('\xa0', ' ')
            #string可以解决有很多值 把所有值都提取
            #获取标签对应值，比如Area是4254900m平方米 如果有空格符就换成一般空格
            value = re.sub(r'\[(\d{1,2})\]','',value)
            #维基会有类似论文中的上标表示出处，将其删去
            res[key] = value#res生成关于infobox标签的键值对'
        f = open(r'wiki_country/querys/fields.txt',encoding='utf-8')
        #ff='Area,Population,GDP (nominal),HDI,Demonym,Countries,Languages,Time zones'
        field_set = f.read().split(',')#读入标签并分割成集合
        #field_set = ff.split(',')
        country_name = response.css('.firstHeading::text').extract_first()#大标题 也就是国家名字
        #timestamp= str(int(time.time()))#获取系统时间
        res_path=r'wiki_country/results'
        f_name=res_path+'\\'+country_name+'.txt'#要导入结果的文件名
        fw = open(f_name, 'w',encoding='utf-8')#覆盖写
        fw.write('CountryName')
        fw.write('#' + '#'.join(field_set) + '\n')#把标签献给文件
        fw.write(country_name)
        for field in field_set[:]:#准备写值
            fw.write('#')
            if field in res:#有值就写没值就过
                fw.write(res[field].replace('\n',''))#多行的可能会有\n
        fw.close()
        f.close()