import http.client, urllib
import json
import os
import collections
import time, datetime

pushover_api_token = os.environ.get('PUSHOVER_API_TOKEN', "nonesuch")
pushover_user_key = os.environ.get('PUSHOVER_USER_KEY', "nonesuch")

def send_notification(payload):
    conn = http.client.HTTPSConnection("api.pushover.net:443")
    conn.request("POST", "/1/messages.json",
    urllib.parse.urlencode({
        "token": pushover_api_token,
        "user": pushover_user_key,
        "message": payload,
    }), { "Content-type": "application/x-www-form-urlencoded" })
    conn.getresponse()

conn = http.client.HTTPSConnection("www.recreation.gov")

old_happy_isles_payload = ""
old_lyell_payload = ""

while True:
    print(f"Trying in 10 seconds (currently {datetime.datetime.now()})")
    time.sleep(10)
    payload = ""
    available = collections.defaultdict(list)

    for date in ["2023-07-01", "2023-08-01"]:
        conn.request("GET", f"/api/permitinyo/445859/availability?start_date={date}&commercial_acct=false", payload)
        res = conn.getresponse()
        data = res.read()
        data_json = json.loads(data.decode("utf-8"))["payload"]

        for date, divisions in data_json.items():
            for division_code, division in divisions.items():
                if division_code in ["44585918", "44585922"] and division["remaining"] > 0 and not division["is_walkup"] and division["remaining"] == division["total"]:
                    available[division_code].append((date, division["remaining"]))

    happy_isles_dates = ", ".join([f"{ent[1]} spots on {ent[0]}" for ent in available["44585918"]])
    happy_isles_payload = f"Happy Isles Permits Available: {happy_isles_dates}"

    lyell_dates = ", ".join([f"{ent[1]} spots on {ent[0]}" for ent in available["44585922"]])
    lyell_payload = f"Lyell Canyon Permits Available: {lyell_dates}"

    if happy_isles_payload==old_happy_isles_payload:
        if lyell_payload==old_lyell_payload:
            continue

    old_lyell_payload = lyell_payload
    old_happy_isles_payload=happy_isles_payload

    for dates, payload in [(available["44585918"], happy_isles_payload), (available["44585922"], lyell_payload)]:
        if dates:
            print(f"Sending notifications... {payload}")
            send_notification(payload=payload)


import numpy as np
import cv2 as cv
# 鼠标回调函数

def draw_circle(event, x, y, flags, param):
    if event == cv.EVENT_LBUTTONDBLCLK:
        cv.circle(img1, (x, y), 100, (255, 0, 0), 3)

# 创建一个黑色图像，一个窗口，然后和回调绑定
img1 = cv.imread("001.jpg")
cv.namedWindow('image', cv.WINDOW_AUTOSIZE)
cv.setMouseCallback('image', draw_circle)
while(1):
    cv.imshow('image', img1)
    if cv.waitKey(20) & 0xFF == 27:
        break
cv.destroyAllWindows()


import telegram, requests, json, utils, subprocess, facebook
from creadcials import *
import tweepy

# First command
def hello(bot, update):
    chat_id = update.message.chat_id
    bot.send_chat_action(chat_id=chat_id, action=telegram.ChatAction.TYPING)
    update.message.reply_text(
        'Hello {}'.format(update.message.from_user.first_name))

# Get random hans
def hans(bot, update):
    chat_id = update.message.chat_id
    bot.send_chat_action(chat_id=chat_id, action=telegram.ChatAction.TYPING)
    text = utils.randomCN()
    update.message.reply_text(text)

# Get random zen
def zen(bot, update):
    chat_id = update.message.chat_id
    bot.send_chat_action(chat_id=chat_id, action=telegram.ChatAction.TYPING)
    zen = requests.get('https://api.github.com/zen')
    update.message.reply_text(zen.text)

# Get random jokes
def laugh(bot, update):
    chat_id = update.message.chat_id
    bot.send_chat_action(chat_id=chat_id, action=telegram.ChatAction.TYPING)
    headers = {'Accept': 'text/plain'}
    # url = 'https://icanhazdadjoke.com/'
    url = 'http://is.snssdk.com/neihan/stream/mix/v1/?mpic=1&webp=1&essence=1&content_type=-102&message_cursor=-1&am_longitude=110&am_latitude=120&am_city=%E5%8C%97%E4%BA%AC%E5%B8%82&am_loc_time=1489226058493&count=1&min_time=1489205901&screen_width=1450&do00le_col_mode=0&iid=3216590132&device_id=32613520945&ac=wifi&channel=360&aid=7&app_name=joke_essay&version_code=612&version_name=6.1.2&device_platform=android&ssmix=a&device_type=sansung&device_brand=xiaomi&os_api=28&os_version=6.10.1&uuid=326135942187625&openudid=3dg6s95rhg2a3dg5&manifest_version_code=612&resolution=1450*2800&dpi=620&update_version_code=6120'
    joke_json = requests.get(url)
    joke = json.loads(joke_json.text)
    text = joke['data']['data'][0]['group']['content']
    update.message.reply_text(text)

# Get news from sources
def news(bot, update):
    chat_id = update.message.chat_id
    bot.send_chat_action(chat_id=chat_id, action=telegram.ChatAction.TYPING)

    url_source = 'https://newsapi.org/v1/sources?language=en'
    source_json = requests.get(url_source)
    source_object = json.loads(source_json.text)

    name_list = ['BBC News', 'Buzzfeed', 'CNN', 'Engadget', 'Google News', 'Hacker News', 'National Geographic', 'The Economist', 'The Telegraph', 'The Verge', 'The Washington Post', 'Time', 'USA Today']
    source_list = []
    for item in source_object['sources']:
        if(item['name'] in name_list):
            source_list.append(telegram.InlineKeyboardButton(item['name'], callback_data='news/' + item['id']))

    reply_markup = telegram.InlineKeyboardMarkup(utils.build_menu(source_list, n_cols=2))
    bot.send_message(chat_id=chat_id, text="Please choose a news source", reply_markup=reply_markup).message_id

def send_news(bot, update, name):
    chat_id = update.callback_query.from_user.id
    bot.answer_callback_query(callback_query_id=update.callback_query.id, text='Here you go!')
    bot.send_chat_action(chat_id=chat_id, action=telegram.ChatAction.TYPING)
    source = name
    url = 'https://newsapi.org/v1/articles?source={source}&apiKey={apiKey}'.format(source=source, apiKey=news_api)
    news_json = requests.get(url)
    news_object = json.loads(news_json.text)
    for item in news_object['articles']:
        text = ('*{title}*\n[Link]({link})'.format(title=item['title'], link=item['url']))
        bot.send_message(chat_id=chat_id, text=text, parse_mode=telegram.ParseMode.MARKDOWN, disable_web_page_preview=False)

def echo(bot, update):
    bot.send_message(chat_id=update.message.chat_id, text=update.message.text)

def post(bot, update):
    chat_id = update.message.chat_id
    bot.send_chat_action(chat_id=chat_id, action=telegram.ChatAction.TYPING)
    source_list = [
        telegram.InlineKeyboardButton('Yes!', callback_data='post/1'),
        telegram.InlineKeyboardButton('No', callback_data='post/0')
    ]

    reply_markup = telegram.InlineKeyboardMarkup(utils.build_menu(source_list, n_cols=2))
    bot.send_message(chat_id=chat_id, text="Would you like to post it?", reply_markup=reply_markup, reply_to_message_id=update.message.message_id)

    file_object = open('message_meta.py', 'w')
    try:
        file_object.write(str(update.message))
    finally:
        file_object.close()

def dopost(bot, update):
    # Post
    # Channel
    chat_id = update.callback_query['message']['chat']['id']
    bot.send_chat_action(chat_id=chat_id, action=telegram.ChatAction.TYPING)
    graph = facebook.GraphAPI(access_token=facebook_auth_token, version="2.1")
    chat_id = TyteKaChannel

    file_object = open('message_meta.py', 'r')
    try:
        msg_str = file_object.read()
    finally:
          file_object.close()
    msg = eval(msg_str)

    if (not msg['photo']):
        # On channel
        bot.send_message(chat_id=chat_id, text=msg['text'])
        # On facebook
        graph.put_object(
            parent_object="me",
            connection_name="feed",
            message=msg['text'])
        api = auth_twitter()
        api.update_status(status = msg['text'])

    else:
        file = bot.getFile(msg['photo'][-1]['file_id'])
        path = file['file_path']
        # Channel
        bot.send_photo(chat_id=chat_id, photo=msg['photo'][-1]['file_id'], caption=msg['caption'] if 'caption' in msg.keys() else None)
        # print('message_files/pics/' + path.split('/')[-1])
        # Facebook
        graph.put_photo(image=open('message_files/pics/' + path.split('/')[-1], 'rb'), message=msg['caption'] if 'caption' in msg.keys() else 'No caption.')
        # Twitter
        api = auth_twitter()
        api.update_with_media('message_files/pics/' + path.split('/')[-1], status = msg['caption'] if 'caption' in msg.keys() else 'No caption.')

    subprocess.call(['rm', 'message_files', '-rf'])
    subprocess.call(['mkdir', 'message_files'])

def auth_twitter():
    auth = tweepy.OAuthHandler(t_api_key, t_api_sec)
    auth.set_access_token(t_token, t_sec)
    return tweepy.API(auth)


from django.test import TestCase
from Players.models import DimPlayers, FactPlayerstats, DimPositions, DimPositioncategory
from ProjectFifa.Players.models import DimClubs
from django.db.models import Q, Avg

from ProjectFifa.ProjectFifa.settings import MIDDLEWARE

# Create your tests here.

class correctplayerandyear(TestCase):

    def setUp(self):
        DimPlayers.objects.create(sofifa_id=123, short_name= 'Cristiano Ronaldo', long_name = 'fifjskjfks', dob='1990-10-10')
        FactPlayerstats.objects.create(sofifa_id=123, year= 2016)

    def test_player_is_ronaldo_and_year_is_2016(self):
        #Checks if the correct player and correct year is filtered
        query = 'Cristiano Ronaldo'
        request_year = 2015
        player = FactPlayerstats.objects.filter(sofifa_id__short_name__icontains=query, year=request_year)
        ronaldo = player[0].sofifa_id.short_name
        year = player[0].year
        self.assertEqual(ronaldo, 'Cristiano Ronaldo')
        self.assertEqual(year, 2015)

class correctclub(TestCase):
    #Checks to see if the correct club is selected
    def setUp(self):
        DimClubs.objects.create(clubid=1, club_name='London United', leagueid=1)

    def test_club_is_london_year(self):
        query = 'London United'
        club = DimClubs.objects.get(clubid=1)

class correctclubstats(TestCase):
    #Checks to see if the correct stats are calculated
    def setUp(self):
        DimClubs.objects.create(clubid=1, club_name="London United", leagueid=1)
        DimPositions.objects.create(positionid=1, position="ST", position_categoryid=1)
        DimPositions.objects.create(positionid=2, position="LW", position_categoryid=1)
        DimPositions.objects.create(positionid=3, position="CM", position_categoryid=2)
        DimPositions.objects.create(positionid=4, position="DM", position_categoryid=2)
        DimPositions.objects.create(positionid=5, position="CB", position_categoryid=3)
        DimPositions.objects.create(positionid=6, position="GK", position_categoryid=3)
        DimPositions.objects.create(positionid=7, position="SUB", position_categoryid=4)
        DimPositions.objects.create(positionid=8, position="RES", position_categoryid=4)
        DimPositioncategory.objects.create(position_category="ATT", position_categoryid=1)
        DimPositioncategory.objects.create(position_category="MID", position_categoryid=2)
        DimPositioncategory.objects.create(position_category="DEF", position_categoryid=3)
        DimPositioncategory.objects.create(position_category="NONE", position_categoryid=4)
        FactPlayerstats.objects.create(sofifa_id=1, club_name='London United', positionid=1, overall= 50, year=2021)
        FactPlayerstats.objects.create(sofifa_id=2, club_name='London United', positionid=2, overall= 20, year=2021)
        FactPlayerstats.objects.create(sofifa_id=3, club_name='London United', positionid=3, overall= 30, year=2021)
        FactPlayerstats.objects.create(sofifa_id=4, club_name='London United', positionid=4, overall= 70, year=2021)
        FactPlayerstats.objects.create(sofifa_id=5, club_name='London United', positionid=5, overall= 90, year=2021)
        FactPlayerstats.objects.create(sofifa_id=6, club_name='London United', positionid=6, overall= 25, year=2021)

    def test_if_clubstats_is_correct(self):
        clubname = "London United"
        request_year = 2021
        #Check if the stats of ATT MID DEF is calculated correctly
        from django.db.models import Func
        class Round_club_avg(Func):
            function = "ROUND"
            template = "%(function)s(%(expressions)s::numeric, 0)"
        aggregate_scores = DimPositioncategory.objects.annotate(num_players = Round_club_avg(Avg('dimpositions__factplayerstats__overall', filter=Q(dimpositions__factplayerstats__club_name= clubname)&Q(dimpositions__factplayerstats__year=request_year)))).filter(~Q(position_categoryid = 3))
        ATT = aggregate_scores[0].position_category
        MID = aggregate_scores[1].position_category
        DEF = aggregate_scores[2].position_category
        self.assertEqual(ATT, 35)
        self.assertEqual(MID, 50)
        #true value is 57.5 but rounds up to 60
        self.assertEqual(DEF, 60)


from flask import Flask, render_template, redirect
from config import ConfigDict
import os
import pyqrcode
import json

conf_dict = ConfigDict().conf_dict
config_path = 'config.py'
app = Flask(__name__, instance_relative_config=False)
app.config.from_pyfile(config_path)

@app.route('/', methods=['GET'])
def main_reroute():
    return redirect('index.html')

@app.route('/index.html', methods=['GET'])
def index_html():
        with open('tap_list.json', mode='r') as file_stream:
            beers_dict = json.loads(file_stream.read())
        return render_template('index.html', beers_dict=beers_dict)

@app.route('/qr_code.html', methods=['GET'])
def qr_code():
        return render_template('qr_code.html')

@app.route('/setup_keg.html', methods=['GET'])
def setup_keg():
    return render_template('setup_keg.html')

def main():
    HOST = os.getenv('SERVER_HOST', '0.0.0.0')
    PORT = int(conf_dict['server_port'])
    THREADED = app.config['THREADED']
    PROCESSES = app.config['PROCESSES']
    FLASK_DEBUG = app.config['FLASK_DEBUG']
    if app.config['USE_SSL']:
        app.run(debug=FLASK_DEBUG, host=HOST, port=PORT,
                threaded=THREADED,
                processes=PROCESSES,
                ssl_context=('server.crt', 'server.key'))
    else:
        app.run(debug=FLASK_DEBUG,
                host=HOST,
                port=PORT,
                threaded=THREADED,
                processes=PROCESSES)

if '__main__' == __name__:
    url = pyqrcode.create(conf_dict['site_url'])
    url.svg('./static/img/qr_code.svg', scale=8)
    main()


# reading FS sheet and calculate revenue
import csv
import pandas as pd
import matplotlib.pyplot as plt
import os
import re

# plot config
plt.style.use('ggplot')
plt.rcParams['figure.figsize'] = (15, 5)

filename = 'FSsheet2020.csv'

if os.path.exists(filename):
    df = pd.read_csv(filename, sep=',', error_bad_lines = False)
    df.drop(df.index[[0, 1, 2, 3, 4]], inplace = True)
    df.rename(columns = {'Unnamed: 0':'Date', 'Unnamed: 1':'Job Number'}, inplace=True)
    new_df = df[['Date', 'Job Number', 'Company - Name', 'Due date', 'Date out', 'Engineer', 'Description of samples', 'Hours']]
    new_df['Hours'].fillna(value = 1, inplace=True)
    new_df['Company - Name'].dropna()
    #pd.set_option('display.max_columns', 10)
    print(new_df[:5])

new_df['Engineer'].value_counts()
#df_wrong = df[df['Company - Name'].str.contains('-')]
#pd.set_option('display.max_rows', 1000)
#print(df['Company - Name'])
#new_df[['Company', 'Customer']] = df['Company - Name'].str.split("-", expand = True)
#print(new_df[:5])
#company_rank = new_df['Company'].value_counts().index.tolist()
#top_five = []
#for i in range(5):
#    top_five.append(comapy_rank[i])
#print(*[name for name in top_five], sep = '\n')
#
#
#new_df['Date'] = pd.to_datetime(new_df['Date'])
#new_df = new_df.set_index(new_df['Date'])
#performance = new_df.loc['1/01/2021':'2/01/2021']
#performance


import numpy as np
import matplotlib.pyplot as plt
from random import randint
from  matplotlib.animation import FuncAnimation

from os import system
cls = lambda: system("clear")

rndGrid = lambda r, c: np.array([[randint(0,1) for n in range(c)] for m in range(r)])

# cw = rndGrid(8, 10)

ax = [(r,c) for r in [-1, 0, 1] for c in [-1, 0, 1] if (r!=0) or (c!=0)]

cwSum = lambda cw: np.sum([np.roll(cw, a, (0, 1)) for a in ax], 0)

# np.logical_or(np.logical_and(cw == 1, cnt == 2), cnt == 3)*1

def newGen(cw):
    cnt = cwSum(cw)
    return np.logical_or(np.logical_and(cw == 1, cnt == 2), cnt == 3)*1

plot = plt.matshow(cw)

def init():
    plot.set_data(cw)
    return plot

def update(j):
    plot.set_data(cw)
    return [plot]

# anim = FuncAnimation(fig, update, init_func = init, frames=n_frames, interval = 30, blit=True)

plt.show()

def testcw(r, c=0):
    if c == 0:
        c = r
    cw = rndGrid(r, c)
    cnt = cwSum(cw)
    for w, n in zip(cw, cnt):
        print('\t', w, '\t', n)
    cw = newGen(cw)
    print(cw, '\n')

    cw = newGen(cw)
    print(cw, '\n')

    cw = newGen(cw)
    print(cw, '\n')

    cw = newGen(cw)
    print(cw, '\n')

    plt.matshow(cw)
    plt.show()


import datetime as dt
import math
import random
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Dict, List, Union

import botocore.exceptions
import pandas as pd

from sm_serverless_benchmarking.analysis import (compute_cost_savings,
                                                 summarize_concurrency_results,
                                                 summarize_stability_results)
from sm_serverless_benchmarking.endpoint import ServerlessEndpoint
from sm_serverless_benchmarking.report import generate_html_report
from sm_serverless_benchmarking.utils import read_example_args_file

def create_endpoint(
    model_name: str, memory_size: int = 1024, max_concurrency: int = 1
) -> ServerlessEndpoint:
    ep = ServerlessEndpoint(
        model_name=model_name, memory_size=memory_size, max_concurrency=max_concurrency
    )
    ep.create_endpoint()
    return ep

def timed_invocation(endpoint: ServerlessEndpoint, invoke_args: Dict[str, str]):
    time.sleep(random.random())
    t1 = time.perf_counter()
    response_size = 0
    try:
        response = endpoint.invoke_endpoint(invoke_args)
        response_size = int(
            response["ResponseMetadata"]["HTTPHeaders"]["content-length"]
        )
        # body = response["Body"].read()
        # response_size = sys.getsizeof(
        #     body
        # )  # calculate response size to estimate per inference cost

    except botocore.exceptions.ClientError as error:
        if error.response["Error"]["Code"] == "ThrottlingException":
            return {
                "invocation_latency": -1,
                "throttle_exception": 1,
                "start_time": t1,
                "end_time": time.perf_counter(),
            }

        elif error.response["Error"]["Code"] == "ModelError":
            if "insufficient memory" in error.response["Error"]["Message"]:
                return {
                    "invocation_latency": -1,
                    "throttle_exception": 0,
                    "insufficient_memory_error": 1,
                    "other_model_error": 0,
                    "start_time": t1,
                    "end_time": time.perf_counter(),
                }
            else:
                print(
                    f"error in endpoint {endpoint.endpoint_name} ",
                    error.response["Error"]["Message"],
                )
                return {
                    "invocation_latency": -1,
                    "throttle_exception": 0,
                    "insufficient_memory_error": 0,
                    "other_model_error": 1,
                    "start_time": t1,
                    "end_time": time.perf_counter(),
                }

    t2 = time.perf_counter()

    return {
        "invocation_latency": (t2 - t1) * 1000,
        "response_size": response_size,
        "throttle_exception": 0,
        "insufficient_memory_error": 0,
        "other_model_error": 0,
        "start_time": t1,
        "end_time": t2,
    }

def create_endpoint_configs(
    memory_size: Union[int, List[int]], max_concurrency: Union[int, List[int]]
):

    if type(memory_size) == int:
        memory_size = [memory_size]

    if type(max_concurrency) == int:
        max_concurrency = [max_concurrency]

    endpoint_configs = []
    for mem_size in memory_size:
        for max_conc in max_concurrency:
            endpoint_configs.append(
                {"memory_size": mem_size, "max_concurrency": max_conc}
            )

    return endpoint_configs

def setup_endpoints(
    model_name: str,
    memory_size: Union[int, List[int]],
    max_concurrency: Union[int, List[int]],
    sleep: int = 0,
):

    update_configs = create_endpoint_configs(memory_size, max_concurrency)

    endpoint_futures = []

    with ThreadPoolExecutor(max_workers=len(update_configs)) as executor:

        for config_kwargs in update_configs:

            future = executor.submit(create_endpoint, model_name, **config_kwargs)
            endpoint_futures.append(future)
            time.sleep(2)

        endpoints = [future.result() for future in as_completed(endpoint_futures)]

    endpoints = [endpoint for endpoint in endpoints if endpoint._created]
    time.sleep(sleep)  # sleep to increase chance of cold start

    return endpoints

def stability_benchmark(
    endpoint: ServerlessEndpoint, invoke_args_list, num_invocations=1000, error_thresh=3
):

    results = []
    errors = 0

    for _ in range(num_invocations):
        invoke_arg_idx = random.randint(0, len(invoke_args_list)-1)
        invoke_args = invoke_args_list[invoke_arg_idx]
        result = timed_invocation(endpoint, invoke_args)
        result["invoke_arg_index"] = invoke_arg_idx

        if result["invocation_latency"] == -1:
            errors += 1

        if errors >= error_thresh:
            print(
                f"Terminating benchmark for {endpoint.endpoint_name} due to excessive endpoint errors"
            )

            return results

        results.append(result)

    return results

def run_stability_benchmark(
    endpoints: List[ServerlessEndpoint],
    invoke_args_list: List[Dict[str, str]],
    num_invocations: int = 1000,
    error_thresh: int = 3,
    result_save_path: str = ".",
):

    endpoint_benchmark_futures = {}
    benchmark_results = []
    all_endpoint_metrics = []

    save_path = Path(result_save_path) / "stability_benchmark_raw_results"
    save_path.mkdir(exist_ok=True, parents=True)

    benchmark_start_time = dt.datetime.utcnow()

    with ThreadPoolExecutor(max_workers=len(endpoints)) as executor:

        for endpoint in endpoints:
            future = executor.submit(
                stability_benchmark,
                endpoint,
                invoke_args_list,
                num_invocations,
                error_thresh,
            )
            endpoint_benchmark_futures[future] = endpoint

        for future in as_completed(endpoint_benchmark_futures):
            endpoint = endpoint_benchmark_futures[future]

            result = future.result()
            df_result = pd.DataFrame(result)
            df_result["memory_size"] = endpoint.memory_size
            benchmark_results.append(df_result)

            benchmark_end_time = dt.datetime.utcnow()
            lookback_window = (
                benchmark_end_time - benchmark_start_time
            ) + dt.timedelta(hours=1)

            endpoint_metrics = endpoint.get_endpoint_metrics(lookback_window)
            df_endpoint_metrics = pd.DataFrame(endpoint_metrics)
            df_endpoint_metrics["memory_size"] = endpoint.memory_size
            all_endpoint_metrics.append(df_endpoint_metrics)

    df_benchmark_results = pd.concat(benchmark_results).reset_index(drop=True)
    df_endpoint_metrics = pd.concat(all_endpoint_metrics).reset_index(drop=True)

    df_benchmark_results.to_csv(
        save_path / "invocation_benchmark_results.csv", index=False
    )
    df_endpoint_metrics.to_csv(save_path / "endpoint_metrics.csv", index=False)

    return df_benchmark_results, df_endpoint_metrics

def concurrency_benchmark(
    endpoint: ServerlessEndpoint, invoke_args_list, num_invocations=50, num_clients=1
):

    futures = []

    with ThreadPoolExecutor(max_workers=num_clients) as executor:

        for _ in range(num_invocations):
            invoke_args = random.choice(invoke_args_list)

            futures.append(executor.submit(timed_invocation, endpoint, invoke_args))

        results = [future.result() for future in futures]

    return results

def run_concurrency_benchmark(
    endpoints: List[ServerlessEndpoint],
    invoke_args_list: List[Dict[str, str]],
    num_invocations: int = 1000,
    num_clients_multipliers: List[float] = [1, 1.5, 1.75, 2],
    result_save_path: str = ".",
):

    endpoint_benchmark_futures = {}
    benchmark_results = []
    all_endpoint_metrics = []

    save_path = Path(result_save_path) / "concurrency_benchmark_raw_results"
    save_path.mkdir(exist_ok=True, parents=True)

    benchmark_start_time = dt.datetime.now()

    with ThreadPoolExecutor(max_workers=len(endpoints)) as executor:
        seen_conc_clients = set()

        for multiplier in num_clients_multipliers:
            futures = []
            for endpoint in endpoints:
                num_clients = math.ceil(endpoint.max_concurrency * multiplier)
                max_conc_num_clients = (endpoint.max_concurrency, num_clients)

                if max_conc_num_clients in seen_conc_clients:
                    continue
                seen_conc_clients.add(max_conc_num_clients)

                future = executor.submit(
                    concurrency_benchmark,
                    endpoint,
                    invoke_args_list,
                    num_invocations,
                    num_clients,
                )
                endpoint_benchmark_futures[future] = (endpoint, num_clients)
                futures.append(future)

            while any([future.running() for future in futures]):
                time.sleep(5)

        for future in as_completed(endpoint_benchmark_futures):
            endpoint = endpoint_benchmark_futures[future][0]

            result = future.result()
            df_result = pd.DataFrame(result)
            df_result["memory_size"] = endpoint.memory_size
            df_result["max_concurrency"] = endpoint.max_concurrency
            df_result["num_clients"] = endpoint_benchmark_futures[future][1]
            benchmark_results.append(df_result)

            benchmark_end_time = dt.datetime.now()
            lookback_window = (
                benchmark_end_time - benchmark_start_time
            ) + dt.timedelta(hours=1)
            endpoint_metrics = endpoint.get_endpoint_metrics(lookback_window)
            df_endpoint_metrics = pd.DataFrame(endpoint_metrics)
            df_endpoint_metrics["memory_size"] = endpoint.memory_size
            df_endpoint_metrics["max_concurrency"] = endpoint.max_concurrency
            df_endpoint_metrics["num_clients"] = endpoint_benchmark_futures[future][1]
            all_endpoint_metrics.append(df_endpoint_metrics)

    df_benchmark_results = pd.concat(benchmark_results).reset_index(drop=True)
    df_endpoint_metrics = pd.concat(all_endpoint_metrics).reset_index(drop=True)

    df_benchmark_results.to_csv(
        save_path / "invocation_benchmark_results.csv", index=False
    )
    df_endpoint_metrics.to_csv(save_path / "endpoint_metrics.csv", index=False)

    return df_benchmark_results, df_endpoint_metrics

def tear_down_endpoints(endpoints: List[ServerlessEndpoint]):
    for endpoint in endpoints:
        try:
            endpoint.clean_up()
        except:
            pass

def run_serverless_benchmarks(
    model_name: str,
    invoke_args_examples_file: Path,
    cold_start_delay: int = 0,
    memory_sizes: List[int] = [1024, 2048, 3072, 4096, 5120, 6144],
    stability_benchmark_invocations: int = 1000,
    stability_benchmark_error_thresh: int = 3,
    include_concurrency_benchmark: bool = True,
    concurrency_benchmark_max_conc: List[int] = [2, 4, 8],
    concurrency_benchmark_invocations: int = 1000,
    concurrency_num_clients_multiplier: List[float] = [1, 1.5, 1.75, 2],
    result_save_path: str = ".",
)->str:
    """Runs a suite of SageMaker Serverless Benchmarks on the specified model_name.
    Will automatically deploy endpoints for the specified model_name and perform a tear down
    upon completion of the benchmark or an error.

    There are two types of benchmarks that are supported and both are executed by defaults
    - Stability Benchmark: Deploys an endpoint for each of the specified memory configurations and
    max concurrency of 1. Invokes the endpoint the specified number of times and determines the stable
    and most cost effective configuration

    - Concurrency Benchmark: Deploys endpoints with different max_concurrency configurations and performs
    a load test with a simulated number of concurrent clients

    Args:
        model_name (str): Name of the SageMaker Model resource
        invoke_args_examples_file (Path: Path to the jsonl file containing the example invocation arcguments
        cold_start_delay (int, optional): Number of seconds to sleep before starting the benchmark. Helps to induce a cold start on initial invocation. Defaults to 0.
        memory_sizes (List[int], optional): List of memory configurations to benchmark Defaults to [1024, 2048, 3072, 4096, 5120, 6144].
        stability_benchmark_invocations (int, optional): Total number of invocations for the stability benchmark. Defaults to 1000.
        stability_benchmark_error_thresh (int, optional): The allowed number of endpoint invocation errors before the benchmark is terminated for a configuration. Defaults to 3.
        include_concurrency_benchmark (bool, optional): Set True to run the concurrency benchmark with the optimal configuration from the stability benchmark. Defaults to True.
        concurrency_benchmark_max_conc (List[int], optional): A list of max_concurency settings to benchmark. Defaults to [2, 4, 8].
        concurrency_benchmark_invocations (int, optional): Total number of invocations for the concurency benchmark. Defaults to 1000.
        concurrency_num_clients_multiplier (List[int], optional): List of multipliers to specify the number of simulated clients which is determined by max_concurency * multiplier. Defaults to [1, 1.5, 1.75, 2].
        result_save_path (str, optional): The location to which the output artifacts will be saved. Defaults to ".".

    Returns:
        str: HTML for the generated benchmarking report
    """

    function_args = locals()
    benchmark_config = pd.Series(function_args).to_frame()

    invoke_args_examples = read_example_args_file(invoke_args_examples_file)

    stability_endpoints = setup_endpoints(
        model_name, memory_size=memory_sizes, max_concurrency=1, sleep=cold_start_delay
    )

    try:
        (
            df_stability_benchmark_results,
            df_stability_endpoint_metrics,
        ) = run_stability_benchmark(
            stability_endpoints,
            invoke_args_list=invoke_args_examples,
            num_invocations=stability_benchmark_invocations,
            error_thresh=stability_benchmark_error_thresh,
            result_save_path=result_save_path,
        )
        (
            df_stability_results,
            df_stability_summary,
            df_stability_metric_summary,
            stability_latency_distribution_fig,
        ) = summarize_stability_results(
            df_stability_benchmark_results,
            df_stability_endpoint_metrics,
            result_save_path=result_save_path,
        )

        avg_response_size = df_stability_results["response_size"].mean()
        (
            df_cost_savings,
            minimal_successful_config,
            comparable_sagemaker_instance,
            cost_vs_performance,
        ) = compute_cost_savings(
            df_stability_metric_summary,
            invoke_args_examples,
            average_response_size=avg_response_size,
            result_save_path=result_save_path
        )

    except Exception as e:
        print(f"Could not complete benchmark due to Exception: {e}")
        raise e

    finally:
        tear_down_endpoints(stability_endpoints)

    if include_concurrency_benchmark:
        concurrency_endpoints = setup_endpoints(
            model_name,
            memory_size=minimal_successful_config,
            max_concurrency=concurrency_benchmark_max_conc,
        )
        try:
            (
                df_conc_benchmark_results,
                df_conc_endpoint_metrics,
            ) = run_concurrency_benchmark(
                concurrency_endpoints,
                invoke_args_examples,
                num_invocations=concurrency_benchmark_invocations,
                num_clients_multipliers=concurrency_num_clients_multiplier,
                result_save_path=result_save_path,
            )
            (
                df_concurrency_metrics,
                df_concurrency_metric_summary,
                concurrency_latency_distribution_fig,
            ) = summarize_concurrency_results(
                df_conc_benchmark_results,
                df_conc_endpoint_metrics,
                result_save_path=result_save_path,
            )

        except Exception as e:
            print(f"Could not complete benchmark due to Exception: {e}")

        finally:
            tear_down_endpoints(concurrency_endpoints)

        report = generate_html_report(
            benchmark_config=benchmark_config,
            df_stability_summary=df_stability_summary,
            df_stability_metric_summary=df_stability_metric_summary,
            stability_latency_distribution=stability_latency_distribution_fig,
            df_cost_savings=df_cost_savings,
            cost_vs_performance=cost_vs_performance,
            optimal_memory_config=minimal_successful_config,
            comparable_instance=comparable_sagemaker_instance,
            df_concurrency_metrics=df_concurrency_metrics,
            df_concurrency_metric_summary=df_concurrency_metric_summary,
            concurrency_latency_distribution=concurrency_latency_distribution_fig,
            result_save_path=result_save_path,
        )
        return report

    else:
        report = generate_html_report(
            benchmark_config=benchmark_config,
            df_stability_summary=df_stability_summary,
            df_stability_metric_summary=df_stability_metric_summary,
            stability_latency_distribution=stability_latency_distribution_fig,
            df_cost_savings=df_cost_savings,
            cost_vs_performance=cost_vs_performance,
            optimal_memory_config=minimal_successful_config,
            comparable_instance=comparable_sagemaker_instance,
            result_save_path=result_save_path,
        )
        return report


import numpy as np
import tensorflow as tf
import tensorflow_hub as hub
import PIL

#######################################
# parameters to play with
#######################################

# just take arbitrary models, thtat were trained on large image datasets
uri_vgg = 'https://tfhub.dev/google/magenta/arbitrary-image-stylization-v1-256/2'

# img dimensions
content_size = 512
style_size = 512

# downsampling parameters
kernel_size = 3
strides = 1

# paths
p_style = "imgs/style/ships.jpg"
p_content = "imgs/content/rotwein.png"
p_result = "result.png"

def load_img_tensor(path_to_img: str, max_dim: int):
    img = tf.io.read_file(path_to_img)
    img = tf.image.decode_image(img, channels=3)
    img = tf.image.convert_image_dtype(img, tf.float32)

    shape = tf.cast(tf.shape(img)[:-1], tf.float32)
    long_dim = max(shape)
    scale = max_dim / long_dim

    new_shape = tf.cast(shape * scale, tf.int32)

    img = tf.image.resize(img, new_shape)
    img = img[tf.newaxis, :]
    return img

def tensor_to_image(tensor: tf.Tensor):
    tensor = tensor * 255
    tensor = np.array(tensor, dtype=np.uint8)
    if np.ndim(tensor) > 3:
        assert tensor.shape[0] == 1
        tensor = tensor[0]
    return PIL.Image.fromarray(tensor)

def get_hub_model(uri: str):
    return hub.load(uri)

class Predictor:
    def __init__(self, uri_model: str, kernel_size: int=3, strides: int=1):
        self.model = get_hub_model(uri_model)
        self.kernel_size = kernel_size
        self.strides = strides

    def predict(self, img_content: tf.Tensor, img_style: tf.Tensor):
        img_style = tf.nn.avg_pool(
            img_style,
            ksize=[self.kernel_size, self.kernel_size],
            strides=[self.strides, self.strides],
            padding='SAME')
        outputs = self.model(tf.constant(img_content), tf.constant(img_style))
        return outputs[0]

predictor = Predictor(uri_vgg, kernel_size, strides)
img_content = load_img_tensor(p_content, content_size)
img_style = load_img_tensor(p_style, style_size)
img_prediction = predictor.predict(img_content, img_style)
result_img = tensor_to_image(img_prediction)
result_img.save(p_result)


import sqlite3
from flask_restful import Resource, reqparse
from flask import request
from flask_jwt import jwt_required

class Item(Resource):
    parser = reqparse.RequestParser()
    parser.add_argument('price',
                        type=float,
                        required=True,
                        help="this field cannot be blank")

    @jwt_required()
    def get(self, name):
        connection = sqlite3.connect('data.db')
        cursor = connection.cursor()
        get_item = "SELECT * FROM items WHERE name = ?"
        rows = cursor.execute(get_item, (name,))
        row = rows.fetchone()
        connection.close()
        if row:
            return {"item": {"name": row[0], "price": row[1]}}
        return {"message": "item not found"}, 404

    def post(self, name):

        connection = sqlite3.connect("data.db")
        cursor = connection.cursor()
        insert_query = "INSERT INTO items VALUES(?, ?)"
        data = Item.parser.parse_args()
        cursor.execute(insert_query, (name, data['price']))
        connection.commit()
        connection.close()
        return {'item': {'name': name, 'price': data['price']}}, 201

    def delete(self, name):
        connection = sqlite3.connect("data.db")
        cursor = connection.cursor()
        deleteitem = "DELETE FROM items WHERE name=?"
        cursor.execute(deleteitem, (name,))
        connection.commit()
        connection.close()
        return {"message": "item deleted successfully"}

    @classmethod
    def getbyitem(cls, name):
        connection = sqlite3.connect("data.db")
        cursor = connection.cursor()
        getitem = "SELECT * FROM items WHERE name=?"
        row = cursor.execute(getitem, (name,))
        connection.commit()
        connection.close()
        return row

    def put(self, name):
        connection = sqlite3.connect("data.db")
        cursor = connection.cursor()
        item = Item.getbyitem(name)
        data = Item.parser.parse_args()
        if item:

            query = "UPDATE items SET price=? WHERE name=? "
            values = (data['price'], name)

        else:

            query = "INSERT INTO items VALUES(?, ?) "
            values = (name, data['price'])
            print("fresh value")

        cursor.execute(query, values)
        connection.commit()
        connection.close()
        return {"item": name, "price": data['price']}

class Items(Resource):
    def get(self):
        items = []
        connection = sqlite3.connect("data.db")
        cursor = connection.cursor()
        itemlist = "SELECT * FROM items"
        rows = cursor.execute(itemlist)
        for row in rows:
            items.append({"name": row[0], "price": row[1]})
        return {"items": items}


import os
import json
import jsonlines
from tqdm import tqdm
import random
import numpy as np

from transformers.tokenization_bert import BasicTokenizer

PAD_LABEL_ID = -100

def basic_tokenize(string):
    return BasicTokenizer().tokenize(string)

def get_vocab(paths):
    if os.path.isfile(paths['vocab_path']):
        vocab_path = paths['vocab_path']
        g = open(vocab_path, 'r', encoding='utf-8')
        voc = [s.strip() for s in g.readlines()]
    else:
        vocab = {}
        g = open(paths['vocab_path'], 'w', encoding='utf-8')
        with open(paths['gold_pair_train_path'], 'r', encoding='utf-8') as f:
            json_data = json.load(f)
            # json_data = jsonlines.Reader(f)
            train_data = []
            for item in json_data['train']:
                train_data.append(item)
            for story_index, story in enumerate(tqdm(train_data)):
                history = basic_tokenize(story['beforegold'])
                persona = basic_tokenize(story['persona'])
                gold = basic_tokenize(story['gold'])
                ori_beforegold = basic_tokenize(story['ori_beforegold'])
                ori_persona = basic_tokenize(story['ori_persona'])
                # ori_gold = basic_tokenize(story['ori_gold'])

                all = history + persona + gold + ori_beforegold + ori_persona
                for word in all:
                    if word in vocab:
                        vocab[word] += 1
                    else:
                        vocab[word] = 1

            voc = [v[0] for v in sorted(vocab.items(), key=lambda item: item[1], reverse=True)]

        for x in voc:
            g.write(x + "\n")

    return voc

def get_exp_vocab(paths):
    if os.path.isfile(paths['vocab_path']):
        g = open(paths['vocab_path'], 'r', encoding='utf-8')
        voc = [s.strip() for s in g.readlines()]
    else:
        vocab = {}
        g = open(paths['vocab_path'], 'w', encoding='utf-8')
        with open(paths['gold_pair_train_path'], 'r', encoding='utf-8') as f:
            json_data = json.load(f)
            # json_data = jsonlines.Reader(f)
            train_data = []
            for item in json_data['train']:
                train_data.append(item)
            for story_index, story in enumerate(tqdm(train_data)):
                history = basic_tokenize(story['beforegold'])
                persona = basic_tokenize(story['persona'])
                gold = basic_tokenize(story['gold'])

                temp_persona = []
                temp_persona.append(story['exp_personality']['xattr'])
                temp_persona.append(story['exp_personality']['xeffect'])
                temp_persona.append(story['exp_personality']['xintent'])
                temp_persona.append(story['exp_personality']['xneed'])
                temp_persona.append(story['exp_personality']['xreact'])
                temp_persona.append(story['exp_personality']['xwant'])
                temp_persona = list(filter('<none>'.__ne__, temp_persona))

                exp_persona = basic_tokenize(" ".join(temp_persona))
                all = history + persona + gold + exp_persona

                for word in all:
                    if word in vocab:
                        vocab[word] += 1
                    else:
                        vocab[word] = 1

            voc = [v[0] for v in sorted(vocab.items(), key=lambda item: item[1], reverse=True)]

        for x in voc:
            g.write(x + "\n")

    return voc

def get_aug_skeletons(s, mask_rate, replace_rate, vocab):
    """ This function is used for generating augmented skeletons

    Args:
        s: list. The raw skeleton
        mask_rate: The rate of background words to be masked
        replace_rate: The rate of background words to be replaced with random words.
        vocab: The vocab for getting random words.
    """
    # a for mask, b for replace, c for shuffle
    # ending string을 3가지 카테고리(랜덤 마스킹, 랜덤 단어 대치, 랜덤 순서 변경)로 augment 시켜서 리스트를 리턴
    a = s.copy()
    b = s.copy()
    c = s.copy()

    # 랜덤으로 토큰 __ 처리
    for i in range(len(s)):
        if s[i] != "__":
            rand = random.random()
            if rand < mask_rate:
                a[i] = "__"

    # 랜덤으로 토큰에 다른 토큰으로 대채함(보카 사전에 있는 단어 중에서 골라옴)
    for i in range(len(s)):
        if s[i] != "__":
            rand = random.random()
            if rand < replace_rate:
                rand_word_id = np.random.randint(0, len(vocab)-1)
                b[i] = vocab[rand_word_id]

    # 블랭크 뺀 토큰들만 다시 담아서 셔플해버림
    no_blank_word_id_in_sen = []
    no_blank_word_in_sen = []
    for i in range(len(s)):
        if s[i] != "__":
            no_blank_word_id_in_sen.append(i)
            no_blank_word_in_sen.append(s[i])
    random.shuffle(no_blank_word_in_sen)
    j = 0
    for i in range(len(s)):
        if s[i] != "__":
            c[i] = no_blank_word_in_sen[j]
            j += 1

    str_mask = " ".join([s for s in a]).strip()
    str_replace = " ".join([s for s in b]).strip()
    str_shuffle = " ".join([s for s in c]).strip()

    return [str_mask, str_replace, str_shuffle]

def bottom_up_dp_lcs(str_a, str_b, do_merge, mask_rate, replace_rate, vocab, mode):
    """Get LCS skeletons using the bottom up DP algorithm

    Args:
        str_a, : String
        do_merge: bool. Whether merge the consecutive blanks into one blank
        mask_rate: float. The rate of background words to be masked
        replace_rate: float. The rate of background words to be replaced with the random word
        vocab: list. The vocab for getting random words.
        mode: string. "train", "dev" or "test"
    """

    """ LCS 로직 수정
    <기존> : str_a와 str_b. 두 개의 스트링의 토큰들을 각각 끝 단어부터 서로 비교하면서
    공통된 단어(background word)는 살리고, 서로 다른 단어(Unique word)는 블랭크 처리

    example.
    str_a ( persona ) : i have played since i was 4 years old .
    str_b ( gold )    : never . i have been playing since i was four. it pays more than lifiting weights .

    lcs_a ( persona after process) : i have __ since i was __ __ __ .
    lcs_b ( gold after process)    : __ __ i have __ __ since i was __ __ __ __ __ __ __ .

    * merge = True --> 이어지는 / consecutive blank는 하나의 blank로 합침.
      데이터 증강 전에 이 작업을 해주고, 증강을 시킴

    lcs_a ( persona after process) : i have __ since i was __ .
    lcs_b ( gold after process)    : __ i have __ since i was __ .

    <수정> : str_a와 str_b. 두 개의 스트링의 토큰들을 각각 끝 단어부터 서로 비교하면서
    서로 다른 단어(Unique word)는 살리고, 서로 같은 단어(background words)는 personal words로 취급하여 블랭크 처리

    lcs_a ( persona after process ) :  __ __ played __ __ __ 4 years old __
    lcs_b ( gold after process )    : never . __ __ been playing __ __ __ four . it pays more than lifting weights __

    * merge = True
    lcs_a ( persona after process ) :  __ played __ 4 years old __
    lcs_b ( gold after process )    : never . __ been playing __ four . it pays more than lifting weights __

    * Aug

    lcs_a : [__ played __ 4 years old __],
            [__ played __ 4 __ old __ ] : 랜덤으로 blank 추가된 데이터
            [__ played __ 4 months old __] : 단어 사전에서 뽑아온 신규 단어를 랜덤으로 교체
            [__ 4 __ played years old __] : 랜덤으로 단어 순서를 바꿈

    이렇게 기존 데이터에 3가지 타입의 데이터를 더 붙여서 어그맨팅함
    우리는 lcs_b만을 훈련 과정에 사용한다
    """
    str_a = basic_tokenize(str_a)
    str_b = basic_tokenize(str_b)
    str_a.insert(0, "_str_")
    str_b.insert(0, "_str_")
    if len(str_a) == 0 or len(str_b) == 0:
        return 0
    # Persona sentence 길이만큼 0으로 리스트를 채우고, 그 리스트를 Gold sentence 길이만큼 반복해서 복사한 리스트를 채운다
    dp = [[0 for _ in range(len(str_b) + 1)] for _ in range(len(str_a) + 1)]
    for i in range(1, len(str_a) + 1):
        for j in range(1, len(str_b) + 1):
            if str_a[i - 1] == str_b[j - 1]:
                dp[i][j] = dp[i - 1][j - 1] + 1
            else:
                dp[i][j] = max([dp[i - 1][j], dp[i][j - 1]])

    i, j = len(str_a), len(str_b)
    LCS_a = ""
    LCS_b = ""
    a_unique = []
    b_unique = []

    # str_a와 str_b의 문장들의 가장 끝부분부터 차례대로 검사하면서, 공통 단어를 체크한다. 공통 단어가 아닌 경우를 unique 리스트에 추가한다
    while i > 0 and j > 0:
        # str_a와 str_b의 마지막 토큰들을 서로 비교하여 같을 경우에 아래 조건문 진입
        if str_a[i - 1] == str_b[j - 1] and dp[i][j] == dp[i - 1][j - 1] + 1:
            # LCS_a = str_a[i - 1] + " " + LCS_a
            # LCS_b = str_a[i - 1] + " " + LCS_b
            # merge가 True일 경우 __이 맨 앞에 있을 경우는 중복 블랭크를 없애야 하기 때문에 if 문으로 구별
            if LCS_a.startswith(" __ ") and do_merge:
                LCS_a = LCS_a
            else:
                LCS_a = " __ " + LCS_a
            if LCS_b.startswith(" __ ") and do_merge:
                LCS_b = LCS_b
            else:
                LCS_b = " __ " + LCS_b

            # LCS_a = " __ " + LCS_a
            # LCS_b = " __ " + LCS_b
            i, j = i - 1, j - 1
            continue
        # str_a와 str_b의 마지막 토큰들을 서로 비교하여 다를 경우에 아래 조건문 진입
        if dp[i][j] == dp[i - 1][j]:  # lcs_a를 채워나가는 조건문
            i, j = i - 1, j
            if LCS_a.startswith(" __ ") and do_merge:
                # LCS_a = LCS_a
                LCS_a = str_a[i] + " " + LCS_a
            else:
                # LCS_a = " __ " + LCS_a
                LCS_a = str_a[i] + " " + LCS_a
            a_unique.append((i, str_a[i]))
            continue
        if dp[i][j] == dp[i][j - 1]:  # lcs_b를 채워나가는 조건문
            i, j = i, j - 1
            if LCS_b.startswith(" __ ") and do_merge:
                # LCS_b = LCS_b
                LCS_b = str_b[j] + " " + LCS_b
            else:
                # LCS_b = " __ " + LCS_b
                LCS_b = str_b[j] + " " + LCS_b
            b_unique.append((j, str_b[j]))
            continue
    a_unique = a_unique[::-1]
    b_unique = b_unique[::-1]

    LCS_a = LCS_a[4:]  # _str_ 이후의 글자들
    LCS_b = LCS_b[4:]

    if not do_merge:
        lcsas = [LCS_a]
        lcsbs = [LCS_b]
    else:
        if mode == "train_aug":
            split_LCS_a = LCS_a.split()
            split_LCS_b = LCS_b.split()
            lcsas = [LCS_a] + get_aug_skeletons(split_LCS_a, mask_rate, replace_rate, vocab)
            lcsbs = [LCS_b] + get_aug_skeletons(split_LCS_b, mask_rate, replace_rate, vocab)
        else:
            lcsas = [LCS_a]
            lcsbs = [LCS_b]

    return lcsas, lcsbs, a_unique, b_unique, str_a[1:], str_b[1:]

def get_train_skeletons(paths, mode, do_merge, mask_rate, replace_rate, exp):

    if exp == True:
        vocab = get_exp_vocab(paths)
    else:
        vocab = get_vocab(paths)

    with open(paths['gold_pair_train_path'], 'r', encoding='utf-8') as f:
        json_data = json.load(f)
        train_data = []
        for item in json_data['train']:
            train_data.append(item)

    skeletons = []

    with open(paths['train_skeletons_path'], 'w', encoding="utf-8") as f:
        for story_index, story in enumerate(tqdm(train_data)):
            history = story['ori_beforegold']
            upg_persona = story['upg_persona']
            gold = story['gold']
            if exp:
                # atomic의 9가지 관계 중 others 관계 3가지를 제외한 6가지 관계만 관리
                # 6가지 관계 중 NLI 필터를 거쳐서 남은 1등 페르소나(빔서치 내림차순)를 사용
                # None은 NLI 필터를 거쳐서 살아남은 녀석이 없는 것임.
                temp_persona = []
                temp_persona.append(story['exp_personality']['xattr'])
                temp_persona.append(story['exp_personality']['xeffect'])
                temp_persona.append(story['exp_personality']['xintent'])
                temp_persona.append(story['exp_personality']['xneed'])
                temp_persona.append(story['exp_personality']['xreact'])
                temp_persona.append(story['exp_personality']['xwant'])

                exp_persona = list(filter('<none>'.__ne__, temp_persona))

                #"xattr": "<none>", "xeffect": "<none>", "xintent": "<none>", "xneed": "i need to drive to the diner . .", "xreact": "<none>", "xwant": "<none>"

            skeleton_persona, skeleton_gold, \
            persona_unique_word, gold_unique_word, \
            basic_toked_persona, basic_toked_gold = bottom_up_dp_lcs(upg_persona, gold, do_merge, mask_rate, replace_rate,
                                                                     vocab, mode)

            skeleton = {}
            skeleton['history'] = story['ori_beforegold']
            skeleton['gold'] = gold
            skeleton['ori_gold'] = story['ori_gold']
            skeleton['persona'] = upg_persona
            skeleton['ori_persona'] = story['ori_persona']
            skeleton['exp_personality'] = story['exp_personality']
            skeleton['used_exp'] = story['used_exp']
            skeleton['raw_skeleton_gold'] = skeleton_gold
            # skeleton['raw_skeleton_gold_unique'] = gold_unique_word
            # skeleton['raw_skeleton_persona_unique'] = persona_unique_word
            basic_toked_history = basic_tokenize(history)

            if exp:
                if not exp_persona:
                    skeleton['exp_persona'] = '[PAD]'
                    basic_toked_exp_persona = basic_tokenize(exp_persona)
                else:
                    skeleton['exp_persona'] = exp_persona
                    basic_toked_exp_persona = basic_tokenize(" ".join(exp_persona))

                sen_pre_1 = ["<premise>"] + basic_toked_history + ["<raw>"] + basic_toked_persona + [
                    "<cf>"] + basic_toked_exp_persona
                sen_after_1 = ["<raw>"] + basic_toked_gold

            else:
                sen_pre_1 = ["<premise>"] + basic_toked_history + ["<raw>"] + basic_toked_persona
                sen_after_1 = ["<raw>"] + basic_toked_gold

            # sen_pre_1은 sketch 레이블에는 관여되지 않는 컨택스트들이라서 -100으로 패딩 처리
            # gold 자리의 레이블은 토큰 길이만큼 0으로 우선 채우고, 정답이 아닌 위치의 레이블은 1로 변경
            len_pre = len(sen_pre_1)
            label_pre = [PAD_LABEL_ID for i in range(len(sen_pre_1))]
            label_gold = [0 for i in range(len(basic_toked_gold))]

            # 우리가 맞춰야 할 자리는 페르소나 문장과 골드 문장의 공통된 단어 토큰
            # 그래서 unique_words의 레이블은 1로 변환. 정답 위치의 레이블은 0으로 두고
            # for x in persona_unique_word:
            #     label_gold[x[0]] = 1
            for x in gold_unique_word:
                label_gold[x[0] - 1] = 1

            label_gold = [PAD_LABEL_ID] + label_gold
            label_raw = [label_pre, label_gold]
            ex_raw = [sen_pre_1, sen_after_1]

            assert len(label_raw[1]) == len(ex_raw[1])

            # skeleton['label_raw'] = label_raw
            # skeleton['ex_raw'] = ex_raw
            skeletons.append(skeleton)

        json.dump(skeletons, f)

def get_train_skeletons_edit(paths):
    '''
    triple_5_ForLCS 데이터를 가지고 스켈레톤을 우선 만든다 ( skeleton = LCS(전처리반영된페르소나, 골드)
    '''
    skeleton_path = paths['train_skeletons_path']
    triple_path = paths['gold_pair_train_path']
    result_path = paths['train_skeletons_edit_path']

    with open(skeleton_path, 'r', encoding='utf-8') as f:
        skeleton_item = json.load(f)
    with open(triple_path, 'r', encoding='utf-8') as t:
        triple_item = json.load(t)

    with open(result_path, 'w', encoding='utf-8') as w:
        result = []
        # triple_persona = []
        # for i, data in enumerate(triple_item):
        #     triple_persona.append(data['originP'])

        for i, data in enumerate(skeleton_item):
            edit_triple = {}
            edit_triple['history'] = data['history']
            edit_triple['persona'] = data['ori_persona']
            edit_triple['gold'] = data['gold']
            edit_triple['raw_skeleton_gold'] = data['raw_skeleton_gold']
            result.append(edit_triple)
            #print("check")
        json.dump(result, w, separators=(',', ': '))

def get_dev_skeletons(paths, mode, do_merge, exp):
    if mode == "dev":
        pair_path = paths['gold_pair_dev_path']
        skeletons_path = paths['dev_skeletons_path']
    elif mode == "test":
        pair_path = paths['gold_pair_test_path']
        skeletons_path = paths['test_skeletons_path']

    with open(pair_path, 'r', encoding='utf-8') as f:
        json_data = json.load(f)
        train_data = []
        for item in json_data['valid']:
            train_data.append(item)

    skeletons = []

    with open(skeletons_path, 'w', encoding="utf-8") as f:
        for story_index, story in enumerate(tqdm(train_data)):
            history = story['beforegold']
            upg_persona = story['upg_persona']
            gold = story['gold']
            if exp:
                # atomic의 9가지 관계 중 others 관계 3가지를 제외한 6가지 관계만 관리
                # 6가지 관계 중 NLI 필터를 거쳐서 남은 1등 페르소나(빔서치 내림차순)를 사용
                # None은 NLI 필터를 거쳐서 살아남은 녀석이 없는 것임.
                temp_persona = []
                temp_persona.append(story['exp_personality']['xattr'])
                temp_persona.append(story['exp_personality']['xeffect'])
                temp_persona.append(story['exp_personality']['xintent'])
                temp_persona.append(story['exp_personality']['xneed'])
                temp_persona.append(story['exp_personality']['xreact'])
                temp_persona.append(story['exp_personality']['xwant'])

                exp_persona = list(filter('<none>'.__ne__, temp_persona))

            skeleton_persona, skeleton_gold, \
            persona_unique_word, gold_unique_word, \
            basic_toked_persona, basic_toked_gold = bottom_up_dp_lcs(upg_persona, gold, do_merge, 0, 0, [], mode)

            skeleton = {}
            skeleton['history'] = story['ori_beforegold']
            skeleton['gold'] = gold
            skeleton['ori_gold'] = story['ori_gold']
            skeleton['persona'] = upg_persona
            skeleton['ori_persona'] = story['ori_persona']
            skeleton['exp_personality'] = story['exp_personality']
            skeleton['used_exp'] = story['used_exp']
            skeleton['raw_skeleton_gold'] = skeleton_gold
            # skeleton['raw_skeleton_gold_unique'] = gold_unique_word
            # skeleton['raw_skeleton_persona_unique'] = persona_unique_word
            basic_toked_history = basic_tokenize(history)

            if exp:
                if not exp_persona:
                    skeleton['exp_persona'] = '[PAD]'
                    basic_toked_exp_persona = basic_tokenize(exp_persona)
                else:
                    skeleton['exp_persona'] = exp_persona
                    basic_toked_exp_persona = basic_tokenize(" ".join(exp_persona))

                sen_pre_1 = ["<premise>"] + basic_toked_history + ["<raw>"] + basic_toked_persona + [
                    "<cf>"] + basic_toked_exp_persona
                sen_after_1 = ["<raw>"] + basic_toked_gold

            else:
                sen_pre_1 = ["<premise>"] + basic_toked_history + ["<raw>"] + basic_toked_persona
                sen_after_1 = ["<raw>"] + basic_toked_gold

            #
            #
            # if exp:
            #     skeleton['exp_persona'] = exp_persona
            #     basic_toked_exp_persona = basic_tokenize(exp_persona)
            #
            # skeleton['raw_skeleton_gold'] = skeleton_gold
            # skeleton['raw_skeleton_gold_unique'] = gold_unique_word
            # # skeleton['raw_skeleton_persona_unique'] = persona_unique_word
            #
            # basic_toked_history = basic_tokenize(history)
            #
            # if exp:
            #     sen_pre_1 = ["<premise>"] + basic_toked_history + ["<raw>"] + basic_toked_persona + [
            #         "<cf>"] + basic_toked_exp_persona
            #     sen_after_1 = ["<raw>"] + basic_toked_gold
            # else:
            #     sen_pre_1 = ["<premise>"] + basic_toked_history + ["<raw>"] + basic_toked_persona
            #     sen_after_1 = ["<raw>"] + basic_toked_gold

            # sen_pre_1은 sketch 레이블에는 관여되지 않는 컨택스트들이라서 -100으로 패딩 처리
            # gold 자리의 레이블은 토큰 길이만큼 0으로 우선 채우고, 정답이 아닌 위치의 레이블은 1로 변경
            len_pre = len(sen_pre_1)
            label_pre = [PAD_LABEL_ID for i in range(len(sen_pre_1))]
            label_gold = [0 for i in range(len(basic_toked_gold))]

            # 우리가 맞춰야 할 자리는 페르소나 문장과 골드 문장의 공통된 단어 토큰
            # 그래서 unique_words의 레이블은 1로 변환. 정답 위치의 레이블은 0으로 두고
            # for x in persona_unique_word:
            #     label_gold[x[0]] = 1
            for x in gold_unique_word:
                label_gold[x[0] - 1] = 1

            label_gold = [PAD_LABEL_ID] + label_gold
            label_raw = [label_pre, label_gold]
            ex_raw = [sen_pre_1, sen_after_1]

            assert len(label_raw[1]) == len(ex_raw[1])

            # skeleton['label_raw'] = label_raw
            # skeleton['ex_raw'] = ex_raw
            skeletons.append(skeleton)

        json.dump(skeletons, f)

if __name__ == '__main__':
    paths = {}
    # 최종 데이터 #

    do_merge = True
    exp = False
    paths['vocab_path'] = "dataForFinal/vocab.txt"
    paths['gold_pair_train_path'] = "trimdata/edited_triple_train.json"
    paths['train_skeletons_path'] = "dataForFinal/train_skeletons_aug.json"
    paths['train_skeletons_edit_path'] = "dataForFinal/train_skeletons_aug_edit.json"
    get_train_skeletons(paths, "train_aug", do_merge, 0.2, 0.2, exp)
    # get_train_skeletons_edit(paths)

    #dev
    do_merge = False
    exp = False
    paths['gold_pair_dev_path'] = "trimdata/edited_triple_train.json"
    paths['dev_skeletons_path'] = "dataForFinal/dev_skeletons.json"
    get_dev_skeletons(paths, "dev", do_merge, exp)

    # # --- 원규 데이터  ----#

    # train
    # do_merge = True
    # exp = True
    # paths['vocab_path'] = "dataForWG/vocab_exp.txt"
    # paths['gold_pair_train_path'] = "dataForWG/gold_pair_train_exp_atomic_filter.json"  # read
    # paths['train_skeletons_path'] = "dataForWG/train_skeletons_atomic_filter_aug.json"  # write
    # get_train_skeletons(paths, "train_aug", do_merge, 0.2, 0.2, exp)

    #dev
    # do_merge = False
    # exp = True
    # paths['gold_pair_dev_path'] = "dataForWG/gold_pair_dev_exp_atomic_filter.json"
    # paths['dev_skeletons_path'] = "dataForWG/dev_skeletons_atomic_filter_TTTT.json"
    # get_dev_skeletons(paths, "dev", do_merge, exp)
    # ----- 원규 데이터 용도 끝 -------

    # do_merge = False
    # exp = True
    # paths['vocab_path'] = "data/vocab_exp_none.txt"
    # paths['gold_pair_dev_path'] = "dataForWG/gold_pair_dev.json"
    # paths['dev_skeletons_path'] = "dataForWG/dev_skeletons.json"
    # get_dev_skeletons(paths, "dev", do_merge, exp)

    # do_merge = False
    # exp = True
    # paths['vocab_path'] = "data/vocab_exp_none.txt"
    # paths['gold_pair_dev_path'] = "data/gold_pair_dev_exp_none.json"
    # paths['dev_skeletons_path'] = "data/dev_skeletons_exp_none.json"
    # get_dev_skeletons(paths, "dev", do_merge, exp)

    # #test_exp
    # do_merge = False
    # exp = True
    # paths['gold_pair_test_path'] = "data/gold_pair_test_exp_none.json"
    # paths['test_skeletons_path'] = "data/test_skeletons_exp_none.json"
    # get_dev_skeletons(paths, "test", do_merge, exp)

    # test
    # do_merge = False
    # exp = False
    # paths['gold_pair_test_path'] = "data/gold_pair_test_wo_exp2.json"
    # paths['test_skeletons_path'] = "data/test_skeletons_wo_exp2.json"
    #
    # get_dev_skeletons(paths, "test", do_merge, exp)

    # # --- YES merge  YES Aug  TRAINING ----#
    # do_merge = True
    # paths['vocab_path'] = "data/vocab.txt"
    # paths['gold_pair_train_path'] = "data/gold_pair_train.json"
    # paths['train_skeletons_path'] = "data/train_skeletons.json"
    # #
    # get_train_skeletons(paths, "train", do_merge, 0.2, 0.2)


import csv

import networkx as nx
import pandas as pd

import nltk
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from langdetect import detect, DetectorFactory
import yake

# data are scraped from https://www.dimensions.ai/products/all-products/dimensions-free-version/
# Load the dataset
data = pd.read_csv("publications.csv", index_col="title")
data.index.name = None
data["title"] = data.index
print("size of the data:/n", data.shape)

# Preprocess the abstracts
nltk.download('stopwords')
nltk.download('wordnet')
nltk.download('punkt')
nltk.download('omw-1.4')

# Add domain-specific stopwords
additional_stopwords = ['however', 'important', 'also', 'understanding', 'one', 'many', 'several', 'within', 'among','study','area','new','tp','tibetan','plateau','sensitive','region','mountain','used']
stop_words = set(stopwords.words('english'))
stop_words.update(additional_stopwords)

# remove the abstracts that are empty
data = data[data['abstract'].notna()]

# Set the seed to ensure consistent language detection results
DetectorFactory.seed = 0

def is_english(text):
    try:
        return detect(text) == 'en'
    except:
        return False

# Filter out non-English abstracts
data = data[data['abstract'].apply(is_english)]

# Set a similarity threshold to determine which keywords are connected to which publications
similarity_threshold = 0.3

def yake_keywords(text, n_values=(1, 2, 3), top=5):
    all_keywords = []

    for n in n_values:
        kw_extractor = yake.KeywordExtractor(lan="en", n=n, top=top, features=None)
        ignored = stop_words
        keywords = kw_extractor.extract_keywords(text)
        unique_keywords = list(set(kw for kw, score in keywords if kw.lower() not in ignored))
        all_keywords.extend(unique_keywords)

    return all_keywords[:top]

# Create an empty graph
G = nx.Graph()

# Add keyword nodes to the graph
for abstract in data['abstract']:
    keywords = yake_keywords(abstract)
    for keyword in keywords:
        G.add_node(keyword)

# Create a TF-IDF vectorizer
vectorizer = TfidfVectorizer(stop_words=stop_words)
term_document_matrix = vectorizer.fit_transform(data['abstract'])

cosine_sim_matrix = cosine_similarity(term_document_matrix)

# Add publication nodes to the graph
for i in range(cosine_sim_matrix.shape[0]):
    G.add_node(i)

# Connect keyword nodes to publication nodes based on the similarity threshold
for i, abstract in enumerate(data['abstract']):
    keywords = yake_keywords(abstract)
    document_vector = term_document_matrix[i]

    for keyword in keywords[:5]:  # Extract top 5 keywords
        keyword_vector = vectorizer.transform([keyword])
        cosine_similarity_value = cosine_similarity(document_vector, keyword_vector)

        if cosine_similarity_value >= similarity_threshold:
            G.add_edge(i, keyword, weight=cosine_similarity_value)

# Compute the degree centrality for each keyword
degree_centrality = nx.degree_centrality(G)

keyword_impact = {}
for keyword in G.nodes():
    if isinstance(keyword, str):
        impact = 0
        for pub in G.neighbors(keyword):
            if isinstance(pub, int):
                impact += G.edges[keyword, pub]['weight']
        keyword_impact[keyword] = impact

# Extract the top 10 keywords
sorted_keywords = sorted(keyword_impact.items(), key=lambda x: -x[1])
top_10_keywords = [(keyword, len(list(G.neighbors(keyword))), impact) for keyword, impact in sorted_keywords[:10]]

# Print the top 10 keywords
print("Top 10 keywords:")
for keyword, impact in top_10_keywords:
    print(f"{keyword}: impact: {impact}")


# coding=utf-8
from Tkinter import *
import Sublimator
import matplotlib
import numpy as np
import datetime

matplotlib.use('TkAgg')
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib import pyplot as plt
MAX_NUM_PLOTDATA = 100

class CalibrationDialog:

    '''
        Dialog for Calibrating the lightbarrier for the current light conditions.
    '''

    def __init__(self, parent,sublimator):
        self.sublimator = sublimator
        top = self.top = Toplevel(parent)
        self.parent = parent
        Label(
            top, text="Calibration in progress.\n Lightsensor will not work if canceled.").pack()
        self.timelabel = Label(top, text="Time: 0:0")
        self.timelabel.pack()
        self.startTime = datetime.datetime.utcnow()
        self.calibrationRunning = True
        self.finishbutton = Button(top, text="Cancel Calibration", command=self.finish,foreground="red")
        self.finishbutton.pack(pady=5)

        top.wait_visibility()
        top.protocol('WM_DELETE_WINDOW', self.finish)
        # start calibration thread
        self.sublimator.calib_start()
        self.updateTime()

    def finish(self):
        # stopping calibr. thread
        self.sublimator.calib_stop()
        self.calibrationRunning = False
        self.top.destroy()

    def updateTime(self):
        if self.calibrationRunning:
            self.passedTime = datetime.datetime.utcnow() - self.startTime
            self.timelabel.config(
                text="Calibration-Time: {}:{}".format(self.passedTime.seconds / 60, self.passedTime.seconds % 60))
            if self.sublimator.hardware.activeConfiguration():
                self.parent.after(1000, self.updateTime)
            else:
                self.sublimator.lightcalibrationFinished = True
                self.finishbutton.config(text="Finish", foreground="black")
                self.timelabel.config(
                    text="Calibration finished!")

class Gui(Frame):

    '''
    Main Window for controlling the underlying program.
    '''

    def __init__(self, sublimator, master=None):
        self.sublimator = sublimator
        self.coolinglist = []
        self.heatinglist = []
        self.timelist = []
        self.seplist = []
        self.log = []

        self.progend = True
        self.sequences = self.sublimator.sequences
        self.testsequence = [x.name for x in self.sequences]
        self.variable = StringVar(master)
        self.variable.set(self.sequences[0].name)
        self.runner = 0
        self.oldlen = 0

        Frame.__init__(self, master)

        self.grid()
        self.buttoncontainer = Frame(master=self)
        self.buttoncontainer.grid(column=0, row=0)

        self.infoContainer()
        self.showText()
        self.showTextline(event=None)
        self.createDropdown()
        self.createStartButton()
        self.createCalibrationButton()
        self.showDiagram()
        self.saveDiagram()

    def infoContainer(self):
        '''
            Erstellt Container fuer die Programminformationen und setzt eine Scrollbar ein.
            '''
        self.containercan = Canvas(self, borderwidth=0, width=30)
        self.infocontainer = Frame(self.containercan)
        self.vsb = Scrollbar(self, orient="vertical", command=self.containercan.yview)
        self.containercan.configure(yscrollcommand=self.vsb.set)
        self.containercan.grid(column=0, row=2, sticky=N + S + E)
        self.vsb.grid(column=1, row=2, sticky=N + S)
        interior_id = self.containercan.create_window(
            (0, 0), window=self.infocontainer, anchor="nw")

        def _configure_infocontainer(event):
            # update the scrollbars to match the size of the inner frame
            size = (self.infocontainer.winfo_reqwidth(), self.infocontainer.winfo_reqheight())
            self.containercan.config(scrollregion="0 0 %s %s" % size)
            if self.infocontainer.winfo_reqwidth() != self.containercan.winfo_width():
                # update the self.containercan's width to fit the inner frame
                self.containercan.config(width=self.infocontainer.winfo_reqwidth())

        def _configure_containercan(event):
            if self.infocontainer.winfo_reqwidth() != self.containercan.winfo_width():
                # update the inner frame's width to fill the self.containercan
                self.containercan.itemconfigure(interior_id, width=self.containercan.winfo_width())

        self.infocontainer.bind('<Configure>', _configure_infocontainer)
        self.containercan.bind('<Configure>', _configure_containercan)

    def showText(self):
        '''
            Erstellt das Textfenster fuer die Log informationen.
            '''
        self.scrollbar = Scrollbar(self)
        self.textField = Text(self, height=10, width=90)
        self.scrollbar.grid(column=3, row=0, sticky=N + S)
        self.textField.grid(column=2, row=0)
        self.scrollbar.config(command=self.textField.yview)
        self.textField.config(
            yscrollcommand=self.scrollbar.set, state=DISABLED)

    def createDropdown(self):
        '''
        Erstellt das Dropdownmenue fuer die Auswahl der Programme.
        '''
        self.dropdown = apply(
            OptionMenu, (self.buttoncontainer, self.variable) + tuple(self.testsequence))
        self.variable.trace("w", self.showTextline)
        self.dropdown.grid(column=0, row=0, sticky=E + W + N + S)

    def createStartButton(self):
        '''
            Erstellt den Startbutton
            '''
        self.startButton = Button(self.buttoncontainer)
        self.startButton["text"] = "Start"
        self.startButton.bind("<Button-1>", self.startButton_Click)
        self.startButton.grid(column=1, row=0, sticky=E + W + N + S)

    def createCalibrationButton(self):
        '''
        Erstellt den Kalibrierungsbutton
        '''
        self.calibrationButton = Button(self.buttoncontainer)
        self.calibrationButton["text"] = "Lichtschranke kalibrieren"
        self.calibrationButton.bind("<Button-1>", self.calibrateButton_Click)
        self.calibrationButton.grid(column=0, row=2, sticky=E + W + N + S)

    def saveDiagram(self):
        self.checkvariable = IntVar()
        self.saveCheckbox = Checkbutton(
            self.buttoncontainer, text="save Diagram", variable=self.checkvariable)
        self.saveCheckbox.grid(column=0, row=1, sticky=E + W + N + S, columnspan=2)

    def startButton_Click(self, event):
        if not self.sublimator.running:
            self.sublimator.start(self.sequences[self.runner])
            self.plotData = [(0, 0, 0, 0)] * MAX_NUM_PLOTDATA
            self.startButton.config(text="Stop")
            self.saveCheckbox.configure(state="disabled")
        else:
            self.sublimator.stop()
            self.startButton.config(text="Start")
            self.saveCheckbox.configure(state="normal")

    def calibrateButton_Click(self, event):
        dialog = CalibrationDialog(root,self.sublimator)
        dialog.top.transient(self.master)
        dialog.top.grab_set()
        self.wait_window(dialog.top)
        self.calibTime = dialog.passedTime
        dialog.top.grab_release()
        self.sublimator.logger.info("Light-Calibration was running for {} minutes and {} seconds.".format(
            self.calibTime.seconds / 60, self.calibTime.seconds % 60))

    def showTextline(self, event, *args):
        '''
            sorgt fuer das fuellen des Informationscontainers bei Auswahl von Programm.
            '''
        self.runner = self.testsequence.index(self.variable.get())
        phases = Entry(self.infocontainer)
        phases.insert(END, "Phases of Program")
        phases.grid(column=0, row=0, sticky=N)
        phases.config(state=DISABLED)
        t = 0
        for heat in self.heatinglist:
            heat.destroy()
        self.heatinglist[:] = []
        for time_entry in self.timelist:
            time_entry.destroy()
        self.timelist[:] = []
        for cool in self.coolinglist:
            cool.destroy()
        self.coolinglist[:] = []
        for sep in self.seplist:
            sep.destroy()
        self.seplist[:] = []

        for i in range(len(self.sequences[self.runner].programs)):
            tlineheat = Entry(self.infocontainer)
            tlinecool = Entry(self.infocontainer)
            tlinetime = Entry(self.infocontainer)
            tlinefree = Entry(self.infocontainer)

            tlinefree.insert(END, "Phase " + str(i + 1))
            tlinefree.grid(column=0, row=t, sticky=N)
            tlinefree.config(state=(DISABLED), disabledbackground="white")
            self.seplist.append(tlinefree)

            tlineheat.config(state=NORMAL)
            tlineheat.insert(
                END, "Heating: " + str(self.sequences[self.runner].programs[i].targetHeatingTemp) + " Celsius")
            tlineheat.grid(column=0, row=t + 1, sticky=N)
            tlineheat.config(state=DISABLED)
            self.heatinglist.append(tlineheat)

            tlinecool.config(state=NORMAL)
            tlinecool.insert(
                END, "Cooling: " + str(self.sequences[self.runner].programs[i].targetCoolingTemp) + " Celsius")
            tlinecool.grid(column=0, row=t + 2, sticky=N)
            tlinecool.config(state=DISABLED)
            self.coolinglist.append(tlinecool)

            tlinetime.config(state=NORMAL)
            tlinetime.insert(
                END, "time: " + str(self.sequences[self.runner].programs[i].time) + " Sekunden")
            tlinetime.grid(column=0, row=t + 3, sticky=N)
            tlinetime.config(state=DISABLED)
            self.timelist.append(tlinetime)

            t += 4

    def button02_Click(self, event):

        if not self.sublimator.running:
            self.sublimator.start(self.sequences[self.runner - 1])
            self.plotData = [(0, 0, 0, 0)] * MAX_NUM_PLOTDATA
            self.button02.config(text="Stop")

            if self.scrollbar.get()[1] == 1.0:
                self.textField.yview(END)

        else:
            self.sublimator.stop()
            self.button02.config(text="Start")

    def showDiagram(self):
        self.plotData = [(0, 0, 0, 0, 0)] * MAX_NUM_PLOTDATA
        self.fig, self.ax = plt.subplots()
        self.ax2 = self.ax.twinx()

        # Lightbarrier Axis
        self.lightax = self.ax.twinx()
        self.fig.subplots_adjust(right=0.75)
        # Move lightbarrier axis to the right
        self.lightax.spines['right'].set_position(('axes', 1.2))
        self.lightax.set_frame_on(True)
        self.lightax.patch.set_visible(False)

        # Initalize plot lines with dummy data
        self.line1, = self.ax.plot(
            [x[0] for x in self.plotData], 'r--')  # Target Heat
        self.line2, = self.ax.plot(
            [x[1] for x in self.plotData], 'r-')  # Heat
        self.line3, = self.ax2.plot(
            [x[2] for x in self.plotData], 'b--')  # Target Cooling
        self.line4, = self.ax2.plot(
            [x[3] for x in self.plotData], 'b-')  # Cooling
        self.line5, = self.lightax.plot(
            [x[4] for x in self.plotData], 'k:')  # Light

        # Set parameter for the look of the figure
        self.ax.set_xlabel("Time")
        self.ax.set_ylim([0, 180])
        self.ax.set_ylabel(u"Heating-element in °C")
        self.ax.yaxis.label.set_color("red")
        self.ax.tick_params(axis="y", colors="red")
        self.ax2.set_ylim([0, 30])
        self.ax2.set_ylabel(u"Cooling-element in °C")
        self.ax2.yaxis.label.set_color("blue")
        self.ax2.tick_params(axis="y", colors="blue")
        self.lightax.set_ylim([0, 1])
        self.lightax.set_ylabel(u"LED-Brightness")
        self.lightax.yaxis.label.set_color("black")
        self.lightax.tick_params(axis="y", colors="black")

        # Show figure in tkinter widget
        self.canvas = FigureCanvasTkAgg(self.fig, self)
        self.canvas.show()
        self.canvas.get_tk_widget().grid()
        self.canvas._tkcanvas.grid(column=2, row=1, rowspan=100, sticky=W + S)

    def updatePlot(self):
        if self.sublimator.running:
            self.progend = False
            self.ax.set_title("")
            data = self.sublimator.datalog[:]
            if self.sublimator.progindex == 0:

                self.seplist[self.sublimator.progindex].configure(disabledbackground="cyan")

            else:
                self.seplist[self.sublimator.progindex].configure(disabledbackground="cyan")
                self.seplist[self.sublimator.progindex - 1].configure(disabledbackground="white")

            self.plotData = data
            if len(data) > MAX_NUM_PLOTDATA:
                self.plotData = data
                del self.plotData[0: len(data) - MAX_NUM_PLOTDATA]

            self.line1.set_xdata(np.arange(len(self.plotData)))
            self.line1.set_ydata([x[0] for x in self.plotData])

            self.line2.set_xdata(np.arange(len(self.plotData)))
            self.line2.set_ydata([x[1] for x in self.plotData])

            self.line3.set_xdata(np.arange(len(self.plotData)))
            self.line3.set_ydata([x[2] for x in self.plotData])

            self.line4.set_xdata(np.arange(len(self.plotData)))
            self.line4.set_ydata([x[3] for x in self.plotData])

            self.line5.set_xdata(np.arange(len(self.plotData)))
            self.line5.set_ydata([x[4] for x in self.plotData])

            # self.ax.plot()
            # self.ax2.plot()
            self.fig.canvas.draw()

        self.after(1000, self.updatePlot)

        if not self.sublimator.running:
            if self.progend is False and self.checkvariable.get() is 1:
                # Kompletter Plot wird am Ende gezeichnet wenn Checkbox aktiv
                self.saveEndPlot()

                self.progend = True

            for sep in self.seplist:
                sep.configure(disabledbackground="white")

            self.startButton.config(text="Start")
            self.saveCheckbox.configure(state="normal")

    def saveEndPlot(self):
        name = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M") + "_" + self.sequences[
            self.runner].name
        figurefile = "./figs/" + name + ".png"
        self.ax.lines.pop(0)
        self.ax2.lines.pop(0)
        self.lightax.pop(0)

        self.line1, = self.ax.plot(
            [x[0] for x in self.sublimator.datalog], 'r--')  # Target Heat
        self.line2, = self.ax.plot(
            [x[1] for x in self.sublimator.datalog], 'r-')  # Heat
        self.line3, = self.ax2.plot(
            [x[2] for x in self.sublimator.datalog], 'b--')  # Target Cooling
        self.line4, = self.ax2.plot(
            [x[3] for x in self.sublimator.datalog], 'b-')  # Cooling
        self.line5, = self.lightax.plot(
            [x[4] for x in self.sublimator.datalog], 'k:')  # Light

        # self.line1.set_xdata(np.arange(len(self.sublimator.datalog)))
        # self.line1.set_ydata([x[0] for x in self.sublimator.datalog])
        # self.line2.set_xdata(np.arange(len(self.sublimator.datalog)))
        # self.line2.set_ydata([x[1] for x in self.sublimator.datalog])
        # self.line3.set_xdata(np.arange(len(self.sublimator.datalog)))
        # self.line3.set_ydata([x[2] for x in self.sublimator.datalog])
        # self.line4.set_xdata(np.arange(len(self.sublimator.datalog)))
        # self.line4.set_ydata([x[3] for x in self.sublimator.datalog])
        # self.line5.set_xdata(np.arange(len(self.sublimator.datalog)))
        # self.line5.set_ydata([x[4] for x in self.sublimator.datalog])

        self.ax.set_title(name)
        self.fig.savefig(figurefile)
        self.sublimator.logger.info(
            "Figure with collected data was created: {}".format(figurefile))
        self.showDiagram()

    def updateConsole(self):
            '''
            Updatet die Konsolenausgabe.
            Verhindert Probleme mit Multithrading und Tkinter...
            '''

            self.newlen = len(self.sublimator.log_capture_string.getvalue())
            # Update der Konsole wenn neue Daten vorliegen
            if 0 < self.newlen != self.oldlen:
                self.oldlen = self.newlen
                # Inhalt komplett loeschen
                self.textField.configure(state=NORMAL)
                self.textField.delete(1.0, END)
                self.log = self.sublimator.log_capture_string.getvalue()
                # Neuen Inhalt einfuegen
                self.textField.insert(END, self.log)
                self.textField.configure(state=DISABLED)
                self.textField.yview(END)  # Setzt Scrollbar ans Ende
            # Update nach einer Sekunde, ruft Methode neu auf
            self.after(1000, self.updateConsole)

def _quit():
    '''
    Sorgt dafür das alle Threads in Tkinter und matplotlib richtig geschlossen werden
    :return:
    '''
    root.quit()
    root.destroy()

if __name__ == '__main__':
    sublimator = Sublimator.Sublimator()
    root = Tk()
    root.protocol("WM_DELETE_WINDOW", _quit)
    myapp = Gui(sublimator, master=root)

    myapp.master.title("Sublimator")
    myapp.master.minsize(860, 560)
    myapp.after(300, myapp.updatePlot)
    myapp.after(1000, myapp.updateConsole)
    myapp.mainloop()


from ast import alias
import os, json

dir = "/home/xelpmoc/Documents/Mailer/whatsapp"

files = os.listdir(dir)

def addAlias():
    for filename in files:
        if filename.endswith("json"):
            f = open(filename)
            print(filename)

            data = json.load(f)

            fields = data["data"]

            i = 1
            for field in fields:
                _alias =  str(field["page"]) + "_" +   str(i)
                field["alias"] = _alias

                i += 1
            f.close()

            f = open(filename, "w")
            f.write(str({"data" : fields}))
            f.close()

def testjson():
    for filename in files:
        if filename.endswith("json"):
            try:
                f = open(filename)
                data = json.load(f)
                f.close()
                if data != None:
                    print("OK:", filename)
            except:
                print("error:"  ,filename )

    print("all files tested")

testjson()


import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

class Model(nn.Module):
    """
    Normalization-Linear
    """
    def __init__(self, configs):
        super(Model, self).__init__()
        self.seq_len = configs.seq_len
        self.pred_len = configs.pred_len

        # Use this line if you want to visualize the weights
        # self.Linear.weight = nn.Parameter((1/self.seq_len)*torch.ones([self.pred_len,self.seq_len]))
        self.channels = configs.enc_in
        self.individual = configs.individual

        self.activations = {'relu': nn.ReLU(),
                    'softplus': nn.Softplus(),
                    'tanh': nn.Tanh(),
                    'selu': nn.SELU(),
                    'lrelu': nn.LeakyReLU(),
                    'prelu': nn.PReLU(),
                    'sigmoid': nn.Sigmoid()}

        if self.individual:
            self.mu = nn.ModuleList()
            for i in range(self.channels):
                self.mu.append(nn.Linear(self.seq_len,self.seq_len))
        else:
            self.mu=nn.Linear(self.seq_len,self.seq_len)

        if self.individual:
            self.var = nn.ModuleList()
            for i in range(self.channels):
                self.var.append(nn.Linear(self.seq_len,self.seq_len))
        else:
            hidden_layers = []
            hidden_layers.append(nn.Linear(self.seq_len, self.seq_len))
            hidden_layers.append(self.activations['selu'])
            hidden_layers.append(nn.InstanceNorm1d(num_features=self.seq_len))
            layers = hidden_layers
            self.var= nn.Sequential(*layers)

        if self.individual:
            self.Linear = nn.ModuleList()
            for i in range(self.channels):
                self.Linear.append(nn.Linear(self.seq_len,self.pred_len))
        else:

            self.Linear = nn.Linear(self.seq_len, self.pred_len)

        if self.individual:
            self.back = nn.ModuleList()
            for i in range(self.channels):
                self.Linear.append(nn.Linear(self.seq_len,self.seq_len))
        else:
            self.back = nn.Linear(self.seq_len, self.seq_len)

    def sample(self, mu, log_var):
        std = torch.exp(0.5*log_var)
        eps = torch.randn_like(std)
        return mu + eps*std

    def vae_loss(self,mu, logvar):
        KL_loss = -0.5 * (1 + logvar - mu.pow(2) - logvar.exp()).mean()
        return KL_loss

    def forward(self, x):
        # x: [Batch, Input length, Channel]
        seq_last = x[:,-1:,:].detach()
        x = x - seq_last

        if self.individual:
            mu = torch.zeros([x.size(0),self.seq_len,x.size(2)],dtype=x.dtype).to(x.device)
            log_var = torch.zeros([x.size(0),self.seq_len,x.size(2)],dtype=x.dtype).to(x.device)
            for i in range(self.channels):
                mu[:,:,i] = self.mu(x[:,:,i])
                log_var[:,:,i] = self.var(x[:,:,i])
            back = self.sample(mu, log_var)
            kl_divergence=self.vae_loss(mu,log_var)

        else:
            mu = self.mu(x.permute(0,2,1)).permute(0,2,1)
            log_var = self.var(x.permute(0,2,1)).permute(0,2,1)
            back = self.sample(mu, log_var)
            kl_divergence=self.vae_loss(mu,log_var)

            reproduce=self.back(back.permute(0,2,1)).permute(0,2,1)

        backward= reproduce+seq_last

        if self.individual:
            output = torch.zeros([x.size(0),self.pred_len,x.size(2)],dtype=x.dtype).to(x.device)
            for i in range(self.channels):
                output[:,:,i] = self.Linear[i](back[:,:,i])
        else:
            output = self.Linear(back.permute(0,2,1)).permute(0,2,1)
        output = output + seq_last

        return output,backward,kl_divergence # [Batch, Output length, Channel]


from nltk.stem import PorterStemmer
from nltk.tokenize import word_tokenize

ps=PorterStemmer()

words='swim swimmer swimming swims swimly'

words=word_tokenize(words)

for w in words:
 print(ps.stem(w))


import poseidon.poseidon
from requests.models import PreparedRequest

def DeleteLogicalAsset(accessKey, secretKey, orgId, url, assetId):
    accessURL = url + '/asset-service/v2.1/logical-assets'
    params = {"action": "delete", "orgId": orgId}
    req = PreparedRequest()
    req.prepare_url(accessURL, params)
    print(req.url)

    body = {"assetId": assetId}
    print(body)

    response = poseidon.poseidon.urlopen(accessKey, secretKey, req.url, body)
    print(response)


import http.client

# Mengatur host dan port server
HOST = 'localhost'
PORT = 8080

# Membuat koneksi dengan server
conn = http.client.HTTPConnection(HOST, PORT)

# Meminta input pengguna
file_path = input("Masukkan path file yang ingin Anda minta: ")

# Mengirim permintaan ke server
conn.request('GET', file_path)

# Menerima respon dari server
response = conn.getresponse()

# Membaca data respon sebagai string
data = response.read().decode('utf-8')

# Menampilkan respon di terminal client
print(data)

# Menutup koneksi dengan server
conn.close()


import pandas as pd
import matplotlib
matplotlib.use('TkAgg')
from matplotlib import pyplot as plt
import sklearn.datasets
from matplotlib import pyplot as plt
import sklearn
from sklearn.metrics import roc_curve, auc
from sklearn.cross_validation import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB

def get_iris_df():
    ds = sklearn.datasets.load_iris()
    df = pd.DataFrame(ds['data'], columns=ds['feature_names'])
    code_species_map = dict(zip(range(3), ds['target_names']))
    df['species'] = [code_species_map[c] for c in ds['target']]
    return df

df = get_iris_df()

# 분류기 이름 -> (선 스타일, 분류기 인스턴스)
CLASS_MAP = {
    '로지스틱 회귀':
        ('-', LogisticRegression()),
    '나이브 베이즈':
        ('--', GaussianNB()),
    '결정 트리':
        ('.-', DecisionTreeClassifier(max_depth=5)),
    '렌덤 포레스트':
        (':', RandomForestClassifier(
                max_depth=5, n_estimators=10,
                max_features=1)),
}
print(df)
# 데이터프레임의 행을 학습/시험셋으로 나눈다.

X, Y = df[df.columns[:3]], (df['species']=='virginica')
X_train, X_test, Y_train, Y_test = \
    train_test_split(X, Y, test_size=.8)

for name, (line_fmt, model) in CLASS_MAP.items():
    model.fit(X_train, Y_train)
    # 열마다 True/False 예측값을 저장한다.
    preds = model.predict_proba(X_test) # shape: (120, 2)
    pred = pd.Series(preds[:,1])
    fpr, tpr, thresholds = roc_curve(Y_test, pred)
    auc_score = auc(fpr, tpr)
    label = '%s: auc=%f' % (name, auc_score)
    plt.plot(fpr, tpr, line_fmt, linewidth=5, label=label)

plt.legend(loc="lower right")
plt.title('분류기 성능 비교')

plt.plot([0,1], [0, 1], 'k--') # 비교를 위해 x=y 라인 추가
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('거짓 양성비')
plt.ylabel('참 양성비')
plt.show()


#定义函数，用cKDTree匹配点与点
import numpy as np
import pandas as pd
from CoordinatesConverter import gcj02tobd09,bd09togcj02,wgs84togcj02,gcj02towgs84,wgs84tobd09,bd09towgs84,getdistance
from scipy.spatial import cKDTree
import itertools
from operator import itemgetter
#定义KDTree的函数
def ckdnearest(dfA_origin,dfB_origin,Aname = ['lon','lat'],Bname = ['lon','lat']):
    '''
    输入两个DataFrame，分别指定经纬度列名，为表A匹配表B中最近点，并计算距离
    '''
    gdA = dfA_origin.copy()
    gdB = dfB_origin.copy()
    from scipy.spatial import cKDTree
    #为gdB表的点建立KDTree
    btree = cKDTree(gdB[Bname].values)
    #在gdB的KDTree中查询gdA的点,dist为距离,idx为gdB中离gdA最近的坐标点
    dist,idx = btree.query(gdA[Aname].values,k = 1)
    #构建匹配的结果
    gdA['index'] = idx
    gdB['index'] = range(len(gdB))
    gdf = pd.merge(gdA,gdB,on = 'index')
    #计算
    if (Aname[0] == Bname[0])&(Aname[1] == Bname[1]):
        gdf['dist'] = getdistance(gdf[Aname[0]+'_x'],gdf[Aname[1]+'_y'],gdf[Bname[0]+'_x'],gdf[Bname[1]+'_y'])
    else:
        gdf['dist'] = getdistance(gdf[Aname[0]],gdf[Aname[1]],gdf[Bname[0]],gdf[Bname[1]])
    return gdf
def ckdnearest_point(gdA, gdB):
    '''
    输入两个geodataframe，gdfA、gdfB均为点，该方法会为gdfA表连接上gdfB中最近的点，并添加距离字段dsit
    '''
    #提取gdA中的所有点要素
    nA = np.array(list(gdA.geometry.apply(lambda x: (x.x, x.y))))
    #提取gdB中的所有点要素
    nB = np.array(list(gdB.geometry.apply(lambda x: (x.x, x.y))))
    #为gdB表的点建立KDTree
    btree = cKDTree(nB)
    #在gdB的KDTree中查询gdA的点,dist为距离,idx为gdB中离gdA最近的坐标点
    dist, idx = btree.query(nA, k=1)
    #构建匹配的结果
    gdA['dist'] = dist
    gdA['index'] = idx
    gdB['index'] = range(len(gdB))
    gdf = pd.merge(gdA,gdB,on = 'index')
    return gdf
def ckdnearest_line(gdfA, gdfB):
    '''
    输入两个geodataframe，其中gdfA为点，gdfB为线，该方法会为gdfA表连接上gdfB中最近的线，并添加距离字段dsit
    '''
    #提取gdA中的所有点要素
    A = np.concatenate(
        [np.array(geom.coords) for geom in gdfA.geometry.to_list()])
    #把gdfB的几何坐标提取到B，此时B为一个大list中包含多个小list，每个小list代表一个几何图形，小list中为坐标
    #B=[[[要素1坐标1],[要素1坐标2],...],[[要素2坐标1],[要素2坐标2],...]]
    B = [np.array(geom.coords) for geom in gdfB.geometry.to_list()]
    #B_ix代表B中的每个坐标点分别属于B中的哪个几何图形
    B_ix = tuple(itertools.chain.from_iterable(
        [itertools.repeat(i, x) for i, x in enumerate(list(map(len, B)))]))
    #把B表展开，B=[[要素1坐标1],[要素1坐标2],...,[要素2坐标2],[要素2坐标2],...]
    B = np.concatenate(B)
    #为B表建立KDTree
    ckd_tree = cKDTree(B)
    #在B的KDTree中查询A的点,dist为距离,idx为B中离A最近的坐标点
    dist, idx = ckd_tree.query(A, k=1)
    #由坐标点对应到几何要素
    idx = itemgetter(*idx)(B_ix)
    #构建匹配的结果
    gdfA['dist'] = dist
    gdfA['index'] = idx
    gdfB['index'] = range(len(gdfB))
    gdf = pd.merge(gdfA,gdfB,on = 'index')
    return gdf