import os
import sys
import csv
import re
import requests
import getopt
import json
import operator
import numpy as np
import pandas as pd
import seaborn as sb
import matplotlib.pyplot as plt
from collections import Counter
from pandas import DataFrame as df
from matplotlib import rcParams
from matplotlib.lines import Line2D

# function retrieve GOnumber ('GO:0034219') from GOterm ('transmembrane transport')
# parameters: go_term (str)
# return go_number (str)
def get_GOterm_to_GOnumber(go_term):
    quickgo_url="https://www.ebi.ac.uk/QuickGO/services/ontology/go/search?query="\
                +go_term+\
                "&limit=25&page=1"
    req = requests.get(quickgo_url, headers={ "Accept" : "application/json"})
    try:
        page1 = req
    except requests.exceptions.ConnectionError:
        r.status_code = "Connection refused"

    if not req.ok:
        req.raise_for_status()
        sys.exit()
    json_data = json.loads(req.text)
    for jdata in json_data["results"]:
        go_number = str(jdata["id"])
    return go_number

# function: get a list of ancestors GO (['GO:1902494', 'GO:1902495',..., 'GO:0071944'])
#           given a GOnumber ('GO:0043190') using EMBL-EBI's QuickGO browser
# parameter: go_number (str)[a valid Gene Ontology ID, e.g. GO:0048527]
# return: all ancestors (list)
def get_ancestors(go_number):
	number=go_number[3:] # to remove "GO:" and use only the number
	quickgo_url='https://www.ebi.ac.uk/QuickGO/services/ontology/go/terms/GO%3A'\
	            +number+\
	            '/ancestors?relations=is_a%2Cpart_of%2Coccurs_in%2Cregulates'
	req = requests.get(quickgo_url, headers={ "Accept" : "application/json"})
	if not req.ok:
		req.raise_for_status()
		sys.exit()
	json_data = json.loads(req.text)

	go_ancestors = []
	for k_jdata,v_jdata in json_data.items():
		if k_jdata == "results":
			for v_jd in v_jdata:
				for k_dic,v_dic in v_jd.items():
					if k_dic == "ancestors":
						for go in v_dic:
							go_ancestors.append(go)
	return go_ancestors

# function: find the GOs in raw data (output of the program GO)
# parameter: raw string that contain GOs, example: 'F:GO:0003924; C:GO:0005840'
# return: list of GOnumbers
def findall_GOs(string):
    raw_string_gos = re.findall('GO:(\d+)',string)
    list_go_numbers =[]
    for go in raw_string_gos:
        go_find = 'GO:'+go
        list_go_numbers.append(go_find)
    return list_go_numbers

# function: find the term GOs in raw data (output of the program GO)
# input : string that contein GO terms. example.: 'P:transmembrane transport; P:deoxyribonucleoside monophosphate biosynthetic process; P:transport'
def findall_terms(string):
	raw_string_terms = string.split(";")
	list_terms =[]
	for i in raw_string_terms:
		term = re.findall(':\s*([^.]+|\S+)',i)
		for t in term:
			list_terms.append(t)
	return list_terms

#############################################################################################

txt_input = sys.argv[1]
if os.path.isfile(txt_input):
	if txt_input.endswith('txt'):
		print("--> working in ",txt_input)
		csv_output= txt_input[:-4]+'_out.csv'
		png_output= txt_input[:-4]+'_out.png'
	else:
		print('This file (',txt_input,') is not txt file! Review your file')
		sys.exit()
else:
	print('The file ',txt_input,'not exists')
	sys.exit()

#step_0 : Finding the columns with GOnumbers or GOterms
print('### FINDING THE COLUMNS WITH GOnumbers and/or GOterms ###')
# to identify the headers that contain the GOnumbers we will use regular expressions
# and they will be store in:
headers_GOs = []
# to indetify the header that contain the GOterm we will use this list
# and tehy will be store in:
posibles_headers = ['GO Names list','GO Names','Annotation GO Term','InterPro GO Term','InterPro GO Names']
headers_terms = []
with open(txt_input,'r') as my_data:
	#making dictionary with all raw data
	readrows = csv.DictReader(my_data,delimiter='\t',skipinitialspace=True)
	data = {name:[] for name in readrows.fieldnames}
	for row in readrows:
		for header in readrows.fieldnames:
			data[header].append(row[header])

	for head, rows in data.items():
		if len(head) != 0 and len(rows) != 0:
			for r in rows:
				regex_GO = re.findall('.*GO:.*',r)
				if regex_GO:
					headers_GOs.append(head)
				else:
					if head in posibles_headers:
						headers_terms.append(head)

headers_GOnumber_GO_term = [list(set(headers_GOs)),list(set(headers_terms))]
headers_GOnumber_GO_term = [l for l in headers_GOnumber_GO_term if l != []] # Removing the empty lists

number_col = []
if len(headers_GOnumber_GO_term) == 0:
	print("It was not possible to find any column with GOnumber or GOterms")
	sys.exit()
else:
	if len(headers_GOnumber_GO_term) == 2:
		columns_selected = headers_GOnumber_GO_term[0] #chose the GOnumber colums
		print('This file contain GO codes in:')
		key_list = list(data)
		keys_of_interest = headers_GOnumber_GO_term[0]
		for key in keys_of_interest:
			number_col.append(key_list.index(key))
		number_col.insert(0,'GO codes')
		for n in sorted(number_col[1:]):
			print("Column ",n)
	if len(headers_GOnumber_GO_term) == 1:
		columns_selected = headers_GOnumber_GO_term[0]
		print('This file contain GO terms in:')
		key_list = list(data)
		keys_of_interest = headers_GOnumber_GO_term[0]
		for key in keys_of_interest:
			number_col.append(key_list.index(key))
		number_col.insert(0,'GO terms')
		for n in  sorted(number_col[1:]):
			print("Column ",n)

key_list = list(data)
for n,i in enumerate(key_list):
	if i == 'SeqName':
		SeqName_column = n

#step_1: geting all GOs_IDs
#        or getting GO_IDs from GO_terms
print("#### MAKING A LIST WITH UNIQUES GOs OF THE ROW DATA ###")

list_GO = []
no_retrieve = []
no_InterPro = []
with open(txt_input,'r') as my_data:
    readrows = csv.reader(my_data,delimiter='\t')
    for rw in readrows:
    	if number_col[0] == 'GO codes':
    		column = sorted(number_col[1:])
	    	regex = re.findall('GO:',rw[column[0]])
	    	if regex:
	    		GOs_retrieve = findall_GOs(rw[column[0]])
	    		for i in GOs_retrieve:
	    			list_GO.append(i)
	    	else:
	    		if len(column) == 2:
		    		no_retrieve.append(rw[SeqName_column])
		    		GOs_retrieve = findall_GOs(rw[column[1]])
		    		if len(GOs_retrieve)!=0:
		    			for i in GOs_retrieve:
		    				list_GO.append(i)
		    		else:
		    			no_InterPro.append(rw[SeqName_column])

with open(txt_input,'r') as my_data:
    readrows = csv.reader(my_data,delimiter='\t')
    for rw in readrows:
    	if number_col[0] == 'GO terms':
    		column = sorted(number_col[1:])
    		list_term  = []
    		rows_string = str(rw[column[0]])
    		term_retrieve = findall_terms(rows_string)
    		if len(term_retrieve) != 0:
    			for tr in term_retrieve:
    				list_term.append(tr)
    		else:
    			if len(column) == 2:
    				no_retrieve.append(rw[SeqName_column])
    				rows_string = str(rw[column[1]])
    				term_retrieve = findall_terms(rows_string)
    				if len(term_retrieve) != 0:
    					for tr in term_retrieve:
    						list_term.append(tr)
    				else:
    					no_InterPro.append(rw[SeqName_column])

    		for my_term in list_term:
    			my_request = get_GOterm_to_GOnumber(my_term)
    			list_GO.append(my_request)

print("SeqName without principal GO IDs found:")
if len(no_retrieve[1:]) != 0:
	for n in no_retrieve[1:]:
		print(n)
else:
	print("0")
print("\nTrying to find in InterPro ...")
print("SeqName without InterPro GO IDs found:")

if len(no_InterPro[1:]) != 0:
	for n in no_InterPro[1:]:
		print(n)
else:
	print("0")

lista_GO_uniques_pre = set(list_GO)

#step_2: building a dic and buildind a list of GO ids
print("#### BUILDING A DICTIONARY 2ND LEVEL ###")
dict_2nd_GO_term = {}
data_from_dic = csv.reader(open('dictionary_2_level.csv'),delimiter=",")
for row in data_from_dic:
	if row[4] == "is_a":
		k,v = row[2],row[3]
		dict_2nd_GO_term[k]=v

my_dic_2nd_nivel = dict_2nd_GO_term#building_dic("dictionary_2_level.csv")
list_GOs_dic = []
for k,v in my_dic_2nd_nivel.items():
	list_GOs_dic.append(k)

print("#### GETING ANCESTOR TO EACH GO ###")
lista_GO_uniques_no_retrieve_ancestral = []
lista_GO_uniques =[]
for _GO in lista_GO_uniques_pre:
	if _GO in list_GOs_dic:
		lista_GO_uniques_no_retrieve_ancestral.append(_GO)
	else:
		lista_GO_uniques.append(_GO)

lista_term_uniques_no_retrieve_ancestral = []
for i in lista_GO_uniques_no_retrieve_ancestral:
	for k,v in my_dic_2nd_nivel.items():
		if i == k:
			lista_term_uniques_no_retrieve_ancestral.append(v)

print("GOs with existing 2nd-level ancestors:")
for n in lista_GO_uniques_no_retrieve_ancestral:
	print(n)

#step_3: geting ancestors and filtering with the before lis of the dic
my_GO_and_term_ancest_selected = {}
for my_GOs in lista_GO_uniques:
	list_all_ancestors = get_ancestors(my_GOs)
	list_selected_GO_ancestors = []
	for GO_ancest in list_all_ancestors:
		if GO_ancest in list_GOs_dic:
			list_selected_GO_ancestors.append(GO_ancest)

	#step 4: retrieving the term from the selected GO id
	list_selected_GOterm_ancestors = []
	for selected_GO_ancestor in list_selected_GO_ancestors:
		for k,v in my_dic_2nd_nivel.items():
			if selected_GO_ancestor == k:
				list_selected_GOterm_ancestors.append(v)
	my_GO_and_term_ancest_selected[my_GOs]=list_selected_GOterm_ancestors

#step_3: geting a list with all terms
all_terms_pre = []
for k,v in my_GO_and_term_ancest_selected.items():
    for i in v:
        all_terms_pre.append(i)
all_terms = all_terms_pre + lista_term_uniques_no_retrieve_ancestral

#step_4: preparing data calculate frecueny and percentage
data_from_dic = csv.reader(open('dictionary_2_level.csv'),delimiter=",")
dict_categ_2ndterm = {}
for row in data_from_dic:
    k,v = (row[0]+'-'+row[1]),row[3]
    if not k in dict_categ_2ndterm:
        dict_categ_2ndterm[k] = [v]
    else:
        dict_categ_2ndterm[k].append(v)

all_terms_freq = []
for terms,freq in Counter(all_terms).items():
    terms_freq = [terms,int(freq)]
    all_terms_freq.append(terms_freq)

total = []
for i in all_terms_freq:
    total.append(i[1])

list_results = []
for i in all_terms_freq:
    suma = sum(total)
    perc = round(i[1]*100/suma,1)
    for k,v in dict_categ_2ndterm.items():
        if i[0] in v:
            list_results.append([k,i[0],i[1],perc])
list_results = sorted(list_results, key=operator.itemgetter(0))

#step_5: writing in a tabulate file
print("#### WRITING THE RESULTS ###")
final_results = []
for r in list_results:
    col1 = r[0][2:].replace('_', ' ')
    col2 = r[1].replace('_', ' ')
    col3= int(r[2])
    col4= float(r[3])
    final_results.append([col1,col2,col3,col4])

with open(csv_output,"w") as my_out:
    writerow = csv.writer(my_out,delimiter='\t')
    writerow.writerows(final_results)

#step_6: making a graph
print("#### MAKING THE GRAPH ###")
category = []
term_2nd = []
frequenc = []
percenta = []
for finalr in final_results:
    category.append(finalr[0])
    term_2nd.append(finalr[1])
    frequenc.append(finalr[2])
    percenta.append(finalr[3])

sb.set_style('darkgrid')
plt.figure(figsize=(10,.2*len(term_2nd)))

pos = np.arange(len(term_2nd))+.5
color_categories = []
for n in category:
    if n == 'cellular component':
        color_categories.append("#3367e1")
    if n == 'molecular function':
        color_categories.append("#4e904a")
    if n == 'biological process':
        color_categories.append("#e95050")

plt.barh(pos,percenta,color=color_categories, height=0.7,align='center',alpha=0.8)#edgecolor='black'
plt.yticks(pos, term_2nd)
plt.rc('xtick',labelsize=10)
plt.rc('ytick',labelsize=8)
#plt.title("GO funtional analysis")
plt.xlabel('GO distribution by level (2) in percentage')
legend_elements = [Line2D([0], [0], color='#3367e1', lw=4, label='cellular component'),
                   Line2D([0], [0], color='#4e904a', lw=4, label='molecular function'),
                   Line2D([0], [0], color='#e95050', lw=4, label='biological process')]
plt.legend(handles=legend_elements, loc='best')
plt.savefig(png_output, dpi=100,bbox_inches = "tight")

print("#### FINALIZED ###","\nCheck the files: ",csv_output,' and ',png_output)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import absl
import absl.flags as flags
import tensorflow as tf
from tensorflow.python.keras.losses import LossFunctionWrapper

import reader
from lstm import RNNLMModel, RNNLMModelTrainer

# flags.DEFINE_integer("hidden_size", 200, "hidden dim of RNN")
#
# flags.DEFINE_string("data_path", None,
#                     "Where the training/test data is stored.")
# flags.DEFINE_string("vocab_path", None,
#                     "Where the wordlist file is stored.")
# flags.DEFINE_string("save_path", "export",
#                     "Model output directory.")
# flags.DEFINE_bool("use_fp16", False,
#                   "Train using 16-bit floats instead of 32bit floats")

FLAGS = flags.FLAGS

class Config(object):
  """Small config."""
  init_scale = 0.1
  learning_rate = 1
  max_grad_norm = 5
  num_layers = 2
  num_steps = 20
  hidden_size = 200
  max_epoch = 4
  max_max_epoch = 13
  keep_prob = 1.0
  lr_decay = 0.8
  batch_size = 64

def data_type():
  return tf.float16 if FLAGS.use_fp16 else tf.float32

# this new "softmax" function we show can train a "self-normalized" RNNLM where
# the sum of the output is automatically (close to) 1.0
# which saves a lot of computation for lattice-rescoring
def new_softmax(labels, logits):
  flatten_labels = tf.reshape(labels, [-1])
  n_samples = tf.shape(flatten_labels)[0]
  flatten_logits = tf.reshape(logits, shape=[n_samples, -1])
  f_logits = tf.exp(flatten_logits)
  row_sums = tf.reduce_sum(f_logits, -1) # this is the negative part of the objf

  t2 = tf.expand_dims(flatten_labels, 1)
  range = tf.expand_dims(tf.range(n_samples), 1)
  ind = tf.concat([range, t2], 1)
  res = tf.gather_nd(flatten_logits, ind)

  return -res + row_sums - 1

class MyFastLossFunction(LossFunctionWrapper):
  def __init__(self):
    super().__init__(new_softmax)

class FastRNNLMModel(RNNLMModel):
  def __init__(self, config):
    super().__init__(config, tf.constant_initializer(-9))

  def get_loss(self, word_ids, labels, is_training=False):
    logits = self.get_logits(word_ids, is_training)
    loss_obj = MyFastLossFunction()
    return loss_obj(labels, logits)

  def get_score(self, logits):
    # In this implementation, logits can be used as dist output
    return logits

def get_config():
  return Config()

def main(_):
  # Turn this on to try the model code with this source file itself!
  __TESTING = False

  if __TESTING:
    (train_data, valid_data), word_map = reader.rnnlm_gen_data(__file__, reader.__file__)
  else:
    if not FLAGS.data_path:
      raise ValueError("Must set --data_path to RNNLM data directory")

    raw_data = reader.rnnlm_raw_data(FLAGS.data_path, FLAGS.vocab_path)
    train_data, valid_data, _, word_map = raw_data

  config = get_config()
  config.hidden_size = FLAGS.hidden_size
  config.vocab_size = len(word_map)

  if __TESTING:
    # use a much smaller scale on our tiny test data
    config.num_steps = 8
    config.batch_size = 4

  model = FastRNNLMModel(config)
  train_producer = reader.RNNLMProducer(train_data, config.batch_size, config.num_steps)
  trainer = RNNLMModelTrainer(model, config)

  valid_producer = reader.RNNLMProducer(valid_data, config.batch_size, config.num_steps)

  # Save variables to disk if you want to prevent crash...
  # Data producer can also be saved to preverse feeding progress.
  checkpoint = tf.train.Checkpoint(trainer=trainer, data_feeder=train_producer)
  manager = tf.train.CheckpointManager(checkpoint, "checkpoints/", 5)

  for i in range(config.max_max_epoch):
    lr_decay = config.lr_decay ** max(i + 1 - config.max_epoch, 0.0)
    lr = config.learning_rate * lr_decay
    trainer.train_one_epoch(train_producer, lr)
    manager.save()

    eval_loss = trainer.evaluate(valid_producer)
    print("validating: loss={}".format(eval_loss))

  # Export
  print("Saving model to %s." % FLAGS.save_path)
  spec = [tf.TensorSpec(shape=[config.num_layers, 2, 1, config.hidden_size], dtype=data_type(), name="context"),
          tf.TensorSpec(shape=[1, 1], dtype=tf.int32, name="word_id")]
  cfunc = model.single_step.get_concrete_function(*spec)
  cfunc2 = model.get_initial_state.get_concrete_function()
  tf.saved_model.save(model, FLAGS.save_path, signatures={"single_step": cfunc, "get_initial_state": cfunc2})

if __name__ == "__main__":
  absl.app.run(main)


from pathlib import Path
import subprocess
import matplotlib.pyplot as plt
from tqdm import tqdm
import cv2
from .detect import detect_in_frames
from .detection import load_detections
from .trajectories import TrajectoryBuilder, save_trajectories

def plot_trajectory_helper(trajectory, frames_path, output_file):
    start, _ = trajectory[0]
    middle, _ = trajectory[len(trajectory) // 2]
    end, _ = trajectory[-1]

    plt.figure(figsize=(15, 10))
    for ti, fi in enumerate([start, middle, end]):
        plt.subplot(1, 3, ti + 1)
        img = cv2.imread(str(frames_path / f"f{fi + 1:07d}.png"))
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        plt.imshow(img)
        trajectory.plot(max_idx=fi)
        plt.xticks([], [])
        plt.yticks([], [])
        plt.xlabel(f"Frame {fi}", fontsize=16)
    plt.savefig(output_file, bbox_inches="tight")
    plt.close()

def create_clip(project_path, clip_name, start, duration, num_workers=5):
    print("Extract clip")
    output_path = project_path / clip_name
    output_path.mkdir()
    cmd = [
        "ffmpeg",
        "-ss",
        start,
        "-i",
        str(project_path / "match.mp4"),
        "-t",
        f"00:00:{duration:02d}",
        "-c",
        "copy",
        str(output_path / "clip.mp4"),
    ]
    subprocess.check_call(cmd)
    print("Extract frames")
    frames_path = output_path / "frames"
    detections_path = output_path / "detections"
    frames_path.mkdir()
    detections_path.mkdir()
    cmd = [
        "ffmpeg",
        "-i",
        str(output_path / "clip.mp4"),
        str(frames_path / "f%07d.png"),
    ]
    subprocess.check_call(cmd)
    print("Detect players")
    num_frames = detect_in_frames(
        frames_path,
        detections_path,
        num_workers=num_workers,
    )
    print("Find player trajectories")
    trajectories_path = output_path / "trajectories"
    trajectories_path.mkdir()
    builder = TrajectoryBuilder()
    for fi in tqdm(range(num_frames)):
        detections = load_detections(detections_path / f"{fi + 1:07d}.det")
        builder.add_detections(fi, detections)
    print(f"Saving {len(builder.trajectories)} trajectories to {trajectories_path}")
    save_trajectories(builder.trajectories, trajectories_path / "trajectories")
    with (trajectories_path / "players.txt").open("w") as f:
        for ti, trajectory in enumerate(tqdm(builder.trajectories)):
            f.write(f"{ti}: player_name\n")
            plot_trajectory_helper(
                trajectory, frames_path, trajectories_path / f"{ti}.png"
            )
    print(f"Edit {trajectories_path}/players.txt and run the next step")

if __name__ == "__main__":
    from argparse import ArgumentParser

    parser = ArgumentParser(description="Create a clip for a project")
    parser.add_argument(
        "-s",
        "--start",
        type=str,
        default="00:00:00",
        help="Start time (format hh:mm:ss)",
    )
    parser.add_argument(
        "-d",
        "--duration",
        type=int,
        default=10,
        help="Duration in seconds (max 60; default: 10)",
    )
    parser.add_argument(
        "-W",
        "--num-workers",
        type=int,
        default=5,
        help="Number of workers for player detection",
    )
    parser.add_argument("-p", "--project", type=str, required=True, help="Project name")
    parser.add_argument("clip_name")
    args = parser.parse_args()
    project_path = Path(args.project)
    create_clip(
        project_path,
        args.clip_name,
        args.start,
        args.duration,
        num_workers=args.num_workers,
    )


#-*- coding: UTF-8 -*-
import unittest
import xlrd
import requests
import json
import Readini
from BeautifulReport import BeautifulReport
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# 解决verify= False报错
# 在Run中选择运行，防止pycharm使用单元测试框架运行脚本，导致数据错误

print_log= 'Y'
# 是否打印请求参数

class PatientInfoSave(Readini.Config):
    '''
    1.保存患者信息
    2.form_data_post返回请求结果
    '''
    def __init__(self):
        super(PatientInfoSave,self).__init__()

    @staticmethod
    def checks():
        if print_log== 'Y':
            return True
        elif print_log== 'N':
            return False
        else:
            return False

    def form_data_post(self,params):
        super().cfg_general()
        self.headers= {
            "Content-Type":"application/json",
            "Authorization":"",
            "clientType":"X",
            "firmId":"dev",
            "hosId":"",
            "subjectType":"",
            "version":""
        }
        self.result= requests.post(url= self.BaseUrl + "/changmugu-basic/patient/patientInfoSave",
                                    data= json.dumps(params),
                                    headers= self.headers,
                                    verify= False)
        if self.checks():
            print("请求参数:")
            print(params)
            print("请求地址:")
            print(self.BaseUrl + "/changmugu-basic/patient/patientInfoSave")
            print()
        return self.result

class BeginPISTest(Readini.ParametrizedTestCase,Readini.Config):
    '''使用unittest进行PatientInfoSave接口测试'''
    def setUp(self):
        super().cfg_general()
        self.t= PatientInfoSave()
        self.r= self.t.form_data_post(
            Readini.create_parameter_dict(
                docId= self.param[1],
                docPhone= self.param[2],
                idNo= self.param[3],
                openId= self.param[4],
                patientName= self.param[5],
                telNo= self.param[6],
            ))
        self.formR= json.loads(self.r.text)
        print("返回参数:")
        print(self.formR)
        print()

        if self.param[0]== '0':
            with open("Temporary/001.txt", "a+") as f:
                f.write(str(self.formR) + "\n")
        else:
            pass
        # 将返回参数写入文件，便于之后接口调用

    def testX(self):
        self.doc= self.param[-1]  # 重写了源文件get_testcase_property，来读取我用例文档的title列
        if self.param[0]== '0':
            self.assertEqual(self.formR["code"],20000)
        elif self.param[0]== '1':
            self.assertNotEqual(self.formR["code"],20000)
        else:
            raise Exception

def runners():
    '''
    读取测试用例文件，加载成list用于循环
    :return:
    '''
    suite= unittest.TestSuite()
    list_open= []
    file= "Cases_File/Case001_PatientInfoSave.xls"
    sheet1= xlrd.open_workbook(filename= file).sheet_by_index(0)
    for i in range(1,sheet1.nrows):
        rows= sheet1.row_values(i)
        list_open.append(rows)
    for i in list_open:
        suite.addTest(Readini.ParametrizedTestCase.parametrize(BeginPISTest, param= i))
    result= BeautifulReport(suite)
    result.report(filename= "接口测试报告_Case001_PatientInfoSave",
                  description= "PatientInfoSave接口测试报告",
                  report_dir= "report",
                  theme=  "theme_memories")

if __name__== '__main__':
    Readini.delete_temp("001.txt")
    runners()


from .common import VLABEL, ELABEL
from scipy.io import loadmat
import networkx as nx

MUTAG_MAT_FILE_NAME = './data/MUTAG.mat'

def load_mutag():
    data = loadmat(MUTAG_MAT_FILE_NAME)
    # print(data.keys())
    y = data['lmutag'].reshape((-1,))
    # print('y', y.shape)
    N, = y.shape

    for i in range(N):
        graph = nx.Graph()
        molecule = data['MUTAG'][0, i]
        node_labels = list(molecule['nl'][0,0][0].reshape((-1,)))
        for node_id, node_label in enumerate(node_labels):
            graph.add_node(node_id, **{VLABEL:int(node_label)})

        edge_list = list(molecule['el'][0,0][0])

        for v, w, elabel in edge_list:
            graph.add_edge(v-1, w-1, **{ELABEL:int(elabel)})
        # print('y[i]', i, y[i])
        yield y[i], graph


import logging
import os

import yaml

from page.app import App
from page.logit import logit

class TestVideoNews:
    """
    测试Video news tab页
    """
    def setup_class(self):
        logging.info("TestVideoNews:setup_class")
        # 获取测试数据文件名
        testcase_file = os.path.dirname(__file__).split("testcase")[0] + r"data/testcase_data/video/video_news.yaml"
        self.video_news_data = yaml.safe_load(open(testcase_file, "r", encoding="utf-8"))
        self.video_news_page = App.start_logged_in().to_video_home_page().click_news_tab()

    def setup_method(self, method):
        logging.info("TestVideoNews:setup_method")
        # 记录类名和方法名
        logging.info(type(self).__name__+":"+method.__name__)

    def teardown_class(self):
        logging.info("TestVideoNews:teardown_class")
        App.quit()

    @logit()
    def test_channels_card_title(self):
        """
        测试channels 卡片标题
        :return:
        """
        data_list = self.video_news_data["test_channels_card_title"]
        title = data_list["title"]
        actual_title = self.video_news_page.get_channels_card_title()
        assert actual_title == title

    @logit()
    def test_click_channels_more(self):
        """
        测试点击channels 查看更多
        :return:
        """
        data_list = self.video_news_data["test_click_channels_more"]
        title = data_list["title"]
        more_page = self.video_news_page.click_channels_more()
        actual_title = more_page.get_tool_bar_title()
        assert actual_title == title


from django.core.management.base import BaseCommand
from hotel.models import Hotel

# command delete_hotel
class Command(BaseCommand):
    help = 'Deletes hotels based on criteria'

    def add_arguments(self, parser):
        parser.add_argument('hotel_name', nargs='+', type=str, help='Name of the hotel to delete')

    def handle(self, *args, **kwargs):
        hotel_names = kwargs['hotel_name']
        for name in hotel_names:
            try:
                hotel = Hotel.objects.get(name=name)
                # Delete associated rooms before deleting the hotel
                rooms = hotel.rooms.all()
                for room in rooms:
                    room.delete()
                hotel.delete()
                self.stdout.write(self.style.SUCCESS(f'Successfully deleted {name} and its rooms'))
            except Hotel.DoesNotExist:
                self.stdout.write(self.style.WARNING(f'Hotel with name {name} does not exist'))


"""Utility for creating and inserting user data into the database"""
import logging

from faker import Faker

from db import database_connection

fake = Faker()

def insert_user_data_to_db():
    """Insert initial user data into the database"""
    insert_query = 'INSERT INTO public."User"(first_name, last_name, email) VALUES '
    query_bind_values = ""
    for user_count in range(5):
        first_name = fake.unique.first_name()
        last_name = fake.unique.last_name()
        email = f"{(first_name + last_name).lower()}@example.com"
        if user_count < 1:
            query_bind_values = f"('{first_name}','{last_name}','{email}')"
        else:
            query_bind_values = "".join(
                [query_bind_values, "," f"('{first_name}','{last_name}','{email}')"]
            )
    insert_query = "".join([insert_query, query_bind_values])
    conn = database_connection()
    conn.set_session(autocommit=True)
    cur = conn.cursor()
    try:
        cur.execute(insert_query)
        cur.close()
    except Exception as e:
        logging.error("Error executing query", e)
        conn.close()

if __name__ == "__main__":
    insert_user_data_to_db()


"""
    v1.0.0

    Author: Rick Nunes
    Email: sanunes.ricardo@gmail.com

    Tag: Quick and dirty

    Features:
        1. Parsing tweets
        2. Generalizing for up to 9 groups of files

    Assumptions:
        1. node_id's are exclusive to each node_file
        2. Words are exclusive to each terms file
        3. No duplicate tweets

"""
from collections import defaultdict
from datetime import datetime
import argparse
import jsonlines as jsonl
import gzip
from functools import wraps, partial
import multiprocessing as mp
from os import (
    stat,
    environ,
    popen,
    path,
    makedirs,
    listdir,
    remove
)
import threading
from sys import exit
from .utils import (
    get_node_ids,
    get_terms,
    compress_jsonl,
    get_temp_jsonl_files,
    archive,
    logging,
    log_process,
)

from .config import config

logger = logging.getLogger('MAIN')

class CategoriesDontMatchError(Exception):
    """ Raised when node and terms categories don't match """
    pass

class TweetMatcher(object):
    """
        Matches node ids in different cathegories to terms in equivalent
        cathegories, parsing tweets and matching n-grams found in terms
    """
    temp_dir = config['TEMP_DIR']
    matches = set()
    tweets = dict()

    def __init__(
        self,
        nodes_dir,
        terms_dir,
        tweets_file,
        output_dir,
        partition_to_jsonl_by=config['PARTITION_BY'],
        is_concurrent=config['IS_CONCURRENT'],
    ):
        self.nodes_dir = nodes_dir
        self.terms_dir= terms_dir
        self.is_concurrent = is_concurrent
        self.partition_to_jsonl_by = partition_to_jsonl_by
        self.tweets_file = tweets_file
        self.output_dir = output_dir

    def get_nodes_and_terms(self):
        self.nodes = get_node_ids(self.nodes_dir)
        self.terms, self.max_gram_length = get_terms(self.terms_dir)
        if len(self.nodes) != len(self.terms):
            raise CategoriesDontMatchError
        self.category_count = len(self.nodes)
        for c in range(self.category_count):
            #tweets_in_category = dict()
            self.tweets[c] = dict()

    @log_process
    def run(self):
        if not self.is_concurrent or stat(self.tweets_file).st_size < config['MAX_SIZE']:
            self.process()
        else:
            self.process_concurrent()
        self.write_matches()

    @log_process
    def process_concurrent(self):
        """
            Handles compressed tweets file in parallel after partitioning
        """
        logger.info(f"Processing {self.tweets_file}")
        lines = []
        total_partitions = 0
        partitioner = self.partition_to_jsonl_by
        with gzip.open(self.tweets_file, 'rb') as f:
            for index, line in enumerate(f):
                lines.append(line)
                if index > partitioner:
                    total_partitions += 1
                    partition_file = f"{self.temp_dir}/tweets_up_to_{partitioner}.jsonl.gz"
                    compress_jsonl(partition_file, lines)
                    partitioner += self.partition_to_jsonl_by
                    lines = []

        if lines:
            total_partitions += 1
            partitioner = partitioner - (self.partition_to_jsonl_by + len(lines)) if total_partitions > 1 else len(lines)
            last_partition_file = f"{self.temp_dir}/tweets_up_to_{partitioner}.jsonl.gz"
            compress_jsonl(last_partition_file, lines)

        logger.info(f"Partitions created: {total_partitions}")
        self.jsonl_files = get_temp_jsonl_files(self.temp_dir)
        self.get_matches_concurrent()
        self.clean_up()

    @log_process
    def process(self):
        """
            Reads compressed file sequentially and gets matches on the go
        """
        logger.info(f"Processing {self.tweets_file}")
        count_lines = 0
        count_filtered = 0
        with gzip.open(self.tweets_file, 'rb') as f:
            reader = jsonl.Reader(f)
            for i, tweet in enumerate(reader.iter(type=dict, skip_invalid=True)):
                count_lines += 1
                pp_tweet = self.pre_process_tweet(tweet['text'])
                if len(pp_tweet) < 2: continue
                count_filtered += 1
                self.get_match(pp_tweet, tweet['node_id'])
        logger.info(f"# TWEETS | PROCESSED {count_lines} | FILTERED {count_filtered} ")

    @log_process
    def read_to_memory(self):
        with gzip.open(self.tweets_file, 'rb') as f:
            reader = jsonl.Reader(f)
            for tweet in reader.iter(type=dict, skip_invalid=True):
                for i, node_set in enumerate(self.nodes):
                    if int(tweet['node_id']) in node_set:
                        pp_tweet = self.pre_process_tweet(tweet['text'])
                        # Ignore empty preprocessed tweets and single characters
                        if len(pp_tweet) < 2 and not tweet['message_id']: continue
                        try:
                            self.tweets[i][tweet['node_id']].add((tweet['message_id'], tweet['text']))
                        except KeyError:
                            self.tweets[i][tweet['node_id']] = set([(tweet['message_id'], tweet['text']),])

    @staticmethod
    def pre_process_tweet(tweet_text):
        """
            Removes some special characters, makes lower case and
            ignores embedded links
        """
        return [word.lower().strip().replace("‘", "").replace("’", "")\
                .replace(",","").replace(":","").replace("!","")\
                .replace("#","").replace("...","").replace('"','')
                for word in tweet_text.split(' ')
                if "http" not in word and len(word) > 1]

    def get_n_grams_in_text(self, pp_tweet):
        """
            Iterates over each word in tweet text and
            generates sets of n-grams starting with each
        """
        n_grams_in_text = defaultdict(set)
        for a in range(len(pp_tweet)):
            """ Creating n-gram sets """
            for step in range(1, self.max_gram_length+1):
                n_gram = " ".join(pp_tweet[a:a+step])
                try:
                    n_grams_in_text[step-1].add(n_gram)
                except IndexError as e:
                    logger.warning(e, step, self.max_gram_length)
        return n_grams_in_text

    def match_text_n_grams_with_terms(
        self,
        node_id,
        node_c,
        n_grams_in_text,
        matches_set
    ):
        """
            Matches generated n-grams from tweet text with stored terms
            in given category
        """
        for n, n_grams in n_grams_in_text.items():
            for gram in n_grams:
                try:
                    if gram in self.terms[node_c][n]:
                        matches_set.add((node_id, gram))
                except IndexError as e:
                    logger.warning(f"{e} in terms {i}, {j}")

    def get_match(self, pp_tweet, node_id, matches_set=None):
        """
            Processes match considering node id category
        """
        matches_set = self.matches if not matches_set else matches_set
        for node_category, id_set in enumerate(self.nodes):
            if int(node_id) in id_set:
                n_grams_in_text = self.get_n_grams_in_text(pp_tweet)
                self.match_text_n_grams_with_terms(
                    node_id,
                    node_category,
                    n_grams_in_text,
                    matches_set
                )

    @log_process
    def get_matches_concurrent(self):
        for file in self.jsonl_files:
            t = threading.Thread(target=self.get_matches_in_jsonl, args=(file, self.matches))
            t.start()

        main_thread = threading.currentThread()
        for t in threading.enumerate():
            if t is not main_thread:
                t.join()

    def get_matches_in_jsonl(self, file, matches_set):
        logger.info(f"Started: {file}")
        with gzip.open(file) as f:
            reader = jsonl.Reader(f)
            for tweet in reader.iter(type=dict, skip_invalid=True):
                pp_tweet = self.pre_process_tweet(tweet['text'])
                if len(pp_tweet) < 2: continue
                self.get_match(pp_tweet, tweet['node_id'],  matches_set)
        logger.info(f"Finished: {file}")

    @log_process
    def write_matches(self):
        if len(self.matches):
            output_file = self.output_dir + "result_"+datetime.today().strftime("%Y_%m_%d")+".txt"
            if not path.exists(self.output_dir): makedirs(self.output_dir[:-1])
            with open(output_file, 'w+') as w:
                for (node_id, term) in self.matches:
                    w.write(f"{node_id} {term}\n")
        else:
            logger.warning("No matches found")

    @log_process
    def clean_up(self):
        for file in self.jsonl_files:
            remove(file)
        logger.info("Removed temporary jsonl")
        logger.info("Job completed successfully")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-p",
        help="Partition gz file by p tweets, default is 100000",
        type=int,
        default=100000
    )
    parser.add_argument(
        "-rpt",
        help="Read from partition",
        type=bool,
        default=False
    )
    parser.add_argument("-pgz", help="Parse from gz", type=bool, default=True)
    parser.add_argument("-nd", help="Path to nodes directory")
    parser.add_argument("-td", help="Path to terms directory")
    parser.add_argument("-tw", help="Path to tweets file")
    parser.add_argument("-od", help="Path to output directory")
    parser.add_argument("-ard", help="Path to archive")
    parser.add_argument("-cc", help="Make matching concurrent", default=False)
    args = parser.parse_args()

    tm = TweetMatcher(
        nodes_dir=args.nd,
        terms_dir=args.td,
        tweets_file=args.tw,
        output_dir=args.od,
        partition_to_jsonl_by=args.p,
        is_concurrent=args.cc
    )
    try:
        tm.get_nodes_and_terms()
    except (
        IsADirectoryError,
        FileNotFoundError,
        CategoriesDontMatchError
    ) as e:
        # TODO: Implement email alert
        logger.warning(e)
        logger.warning("Exiting program,  directories are not properly set")
        exit()
    tm.run()


#! /usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse
import os

import numpy as np
import matplotlib.pyplot as plt

def _build_arg_parser():
    p = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter,
        description=__doc__)

    p.add_argument('in_binary_matrix')
    p.add_argument('in_ground_truth')
    p.add_argument('out_png')

    return p

def main():
    parser = _build_arg_parser()
    args = parser.parse_args()

    in_matrix = np.load(args.in_binary_matrix)
    gt = np.load(args.in_ground_truth)
    confusion_matrix = gt
    tp = 0
    fp = 0
    tn = 0
    fn = 0

    [i, j] = in_matrix.shape
    for x in range(0, i):
        for y in range(0, j):
            if in_matrix[x][y] == 1:
                if gt[x][y] == 1:
                    tp += 1
                    confusion_matrix[x][y] = 1
                if gt[x][y] == 0:
                    fp += 1
                    confusion_matrix[x][y] = 2
            if in_matrix[x][y] == 0:
                if gt[x][y] == 1:
                    fn += 1
                    confusion_matrix[x][y] = 4
                if gt[x][y] == 0:
                    tn += 1
                    confusion_matrix[x][y] = 3
    total = tp + fp + tn + fn

    print("True positives : " + str(100 * tp/total) + "%")
    print("False positives : " + str(100 * fp/total) + "%")
    print("True negatives : " + str(100 * tn/total) + "%")
    print("False negatives : " + str(100 * fn/total) + "%")

    fig, ax = plt.subplots()
    im = ax.imshow(confusion_matrix.T,
                   interpolation='nearest',
                   cmap='coolwarm')
    cbar = fig.colorbar(im, ticks=[1, 2, 3, 4])
    cbar.ax.set_yticklabels(['True Positives', 'False Positives',
                             'True Negatives', 'False Negatives'])

    plt.savefig(args.out_png, dpi=300, bbox_inches='tight')

if __name__ == "__main__":
    main()


import tkinter as tk
from tkinter import messagebox
from pyVmomi import vim
from GUI.scrollable_frame import scrollable_frame
# from core_functions.add_nic_to_vm import add_nic

def wait_for_task(task):
    """ wait for a vCenter task to finish """
    task_done = False
    while not task_done:
        if task.info.state == 'success':
            return task.info.result

        if task.info.state == 'error':
            print("there was an error")
            task_done = True

def get_obj(content, vimtype, name):
    """
    Return an object by name, if name is None the
    first found object is returned
    """
    obj = None
    container = content.viewManager.CreateContainerView(
        content.rootFolder, vimtype, True)
    for c in container.view:
        if name:
            if c.name == name:
                obj = c
                break
        else:
            obj = c
            break

    return obj

def _clone_vm(
        content, template, vm_name, si,
        datacenter_name, vm_folder, datastore_name,
        cluster_name, resource_pool, power_on, datastorecluster_name):
    """
    Clone a VM from a template/VM, datacenter_name, vm_folder, datastore_name
    cluster_name, resource_pool, and power_on are all optional.
    """

    # if none git the first one
    datacenter = get_obj(content, [vim.Datacenter], datacenter_name)

    if vm_folder:
        destfolder = get_obj(content, [vim.Folder], vm_folder)
    else:
        destfolder = datacenter.vmFolder

    if datastore_name:
        datastore = get_obj(content, [vim.Datastore], datastore_name)
    else:
        datastore = get_obj(
            content, [vim.Datastore], template.datastore[0].info.name)

    # if None, get the first one
    cluster = get_obj(content, [vim.ClusterComputeResource], cluster_name)

    if resource_pool:
        resource_pool = get_obj(content, [vim.ResourcePool], resource_pool)
    else:
        resource_pool = cluster.resourcePool

    vmconf = vim.vm.ConfigSpec()

    if datastorecluster_name:
        podsel = vim.storageDrs.PodSelectionSpec()
        pod = get_obj(content, [vim.StoragePod], datastorecluster_name)
        podsel.storagePod = pod

        storagespec = vim.storageDrs.StoragePlacementSpec()
        storagespec.podSelectionSpec = podsel
        storagespec.type = 'create'
        storagespec.folder = destfolder
        storagespec.resourcePool = resource_pool
        storagespec.configSpec = vmconf

        try:
            rec = content.storageResourceManager.RecommendDatastores(
                storageSpec=storagespec)
            rec_action = rec.recommendations[0].action[0]
            real_datastore_name = rec_action.destination.name
        except:
            real_datastore_name = template.datastore[0].info.name

        datastore = get_obj(content, [vim.Datastore], real_datastore_name)

    # set relospec
    relospec = vim.vm.RelocateSpec()
    relospec.datastore = datastore
    relospec.pool = resource_pool

    clonespec = vim.vm.CloneSpec()
    clonespec.location = relospec
    clonespec.powerOn = power_on

    print("cloning VM...")
    task = template.Clone(folder=destfolder, name=vm_name, spec=clonespec)
    wait_for_task(task)

def clone_vm(window):
    content = window.si.RetrieveContent()
    template = window.template_entry.get()

    template = get_obj(content, [vim.VirtualMachine], template)

    if template:
        _clone_vm(
            content, template, window.vm_name_entry.get(), window.si,
            window.datacenter_name_entry.get(), window.folder_entry.get(),
            # args.datastore_name,
            None,
            # args.cluster_name,
            None,
            # args.resource_pool,
            None,
            # args.power_on,
            None,
            # args.datastorecluster_name
            None
            )
        # if window.opaque_network_entry.get():
        #     vm = get_obj(content, [vim.VirtualMachine], args.vm_name)
        #     add_nic(window.si, vm, args.opaque_network)
    else:
        messagebox.showinfo("Error", "Please enter in a valid template")

def render_clone_vm(window):
    # clear the screen of anything in it before
    for child in window.central_frame.winfo_children():
        child.destroy()

    # render scrollable frame
    window.update_idletasks()
    scroll_frame = scrollable_frame(window.central_frame, window)
    scroll_frame.pack(expand=tk.YES, fill=tk.BOTH, side=tk.LEFT)
    # render template
    frame1 = tk.Frame(scroll_frame.scrollFrame.viewPort, width=565,
                      height=80, bg="#f442e8")
    frame1.pack(side=tk.TOP, fill=tk.BOTH, expand=tk.YES)
    frame1_1 = tk.Frame(frame1, width=200,
                        height=80, bg="#ffffff")
    frame1_1.pack(side=tk.LEFT, expand=tk.NO, fill=tk.Y)
    frame1_2 = tk.Frame(frame1, width=200,
                        height=80, bg="#000000")
    frame1_2.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    frame1_label = tk.Label(frame1_1, text="Enter template name:",
                            font=("Helvetica", 14), anchor="e", width=20)
    # add a resize event that increases font size
    frame1_label.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    window.template_entry = tk.Entry(frame1_2, font=("Helvetica", 14))
    window.template_entry.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    # render new VM name
    frame2 = tk.Frame(scroll_frame.scrollFrame.viewPort, width=565,
                      height=80, bg="#f442e8")
    frame2.pack(side=tk.TOP, fill=tk.BOTH, expand=tk.YES)
    frame2_1 = tk.Frame(frame2, width=200,
                        height=80, bg="#ffffff")
    frame2_1.pack(side=tk.LEFT, expand=tk.NO, fill=tk.Y)
    frame2_2 = tk.Frame(frame2, width=200,
                        height=80, bg="#000000")
    frame2_2.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    frame2_label = tk.Label(frame2_1, text="Enter new VM name:",
                            font=("Helvetica", 14), anchor="e", width=20)
    # add a resize event that increases font size
    frame2_label.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    window.vm_name_entry = tk.Entry(frame2_2, font=("Helvetica", 14))
    window.vm_name_entry.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    # optional name of datacenter
    frame3 = tk.Frame(scroll_frame.scrollFrame.viewPort, width=565,
                      height=80, bg="#f442e8")
    frame3.pack(side=tk.TOP, fill=tk.BOTH, expand=tk.YES)
    frame3_1 = tk.Frame(frame3, width=200,
                        height=80, bg="#ffffff")
    frame3_1.pack(side=tk.LEFT, expand=tk.NO, fill=tk.Y)
    frame3_2 = tk.Frame(frame3, width=200,
                        height=80, bg="#000000")
    frame3_2.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    frame3_label = tk.Label(frame3_1, text="Enter datacenter name:",
                            font=("Helvetica", 14), anchor="e", width=20)
    # add a resize event that increases font size
    frame3_label.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    window.datacenter_name_entry = tk.Entry(frame3_2, font=("Helvetica", 14))
    window.datacenter_name_entry.pack(side=tk.LEFT,
                                      expand=tk.YES, fill=tk.BOTH)
    # optional vm folder
    frame4 = tk.Frame(scroll_frame.scrollFrame.viewPort, width=565,
                      height=80, bg="#f442e8")
    frame4.pack(side=tk.TOP, fill=tk.BOTH, expand=tk.YES)
    frame4_1 = tk.Frame(frame4, width=200,
                        height=80, bg="#ffffff")
    frame4_1.pack(side=tk.LEFT, expand=tk.NO, fill=tk.Y)
    frame4_2 = tk.Frame(frame4, width=200,
                        height=80, bg="#000000")
    frame4_2.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    frame4_label = tk.Label(frame4_1, text="Enter VM folder:",
                            font=("Helvetica", 14), anchor="e", width=20)
    # add a resize event that increases font size
    frame4_label.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    window.folder_entry = tk.Entry(frame4_2, font=("Helvetica", 14))
    window.folder_entry.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    # optional datastore
    # optional cluster name
    # optional resourcePool
    # power on tick box and send it button
    frame5 = tk.Frame(scroll_frame.scrollFrame.viewPort, width=565,
                      height=80, bg="#f442e8")
    frame5.pack(side=tk.TOP, fill=tk.BOTH, expand=tk.YES)
    frame5_1 = tk.Frame(frame5, width=200,
                        height=80, bg="#ffffff")
    frame5_1.pack(side=tk.LEFT, expand=tk.NO, fill=tk.Y)
    frame5_2 = tk.Frame(frame5, width=200,
                        height=80, bg="#000000")
    frame5_2.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    # TODO add power on tick box
    # frame5_label = tk.Label(frame5_1, text="Enter VM folder:",
    #                         font=("Helvetica", 14), anchor="e", width=20)
    # frame5_label.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
    tk.Button(frame5_2, font=("Helvetica", 14),
              text="Clone VM",
              command=lambda: clone_vm(window)).pack(
              side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)


'''
Created on 20.06.2011

@author: michi
'''

from sqlalchemy.orm import object_mapper
from sqlalchemy.orm.attributes import InstrumentedAttribute
from sqlalchemy.exc import InvalidRequestError
from sqlalchemy.ext.hybrid import hybrid_property

def decorator(objOrClassOrProperty):
    '''OrmDecorator decorator(ems.model.sa.orm.base_object.OrmBaseObject)'''
    if isinstance(objOrClassOrProperty, InstrumentedAttribute):
        if hasattr(objOrClassOrProperty.class_,'__ormDecorator__'):
            return objOrClassOrProperty.class_.__ormDecorator__()
    elif hasattr(objOrClassOrProperty, '__ormDecorator__'):
        return objOrClassOrProperty.__ormDecorator__()

def friendly_name(classProperty):
    if isinstance(classProperty, InstrumentedAttribute):
        if hasattr(classProperty.class_,'__ormDecorator__'):
            deco = classProperty.class_.__ormDecorator__()
            return deco.getPropertyFriendlyName(classProperty.key)
    elif hasattr(classProperty, '__ormDecorator__'):
        return classProperty.__ormDecorator__().\
            getReprasentiveString(classProperty)

class OrmDecorator(object):
    def __init__(self, cls):
        self._class = cls
        self._shownProperties = []
        self._objectMapper = None
        self._valueFormatOptions = {}
        self._rProperties = {}
        self._colInfos = {}

    def getObjectMapper(self):
        if self._objectMapper is None:
            self._objectMapper = object_mapper(self._class())
        return self._objectMapper

    objectMapper = property(getObjectMapper)

    def getReprasentiveString(self, obj, view='default'):
        if hasattr(self._class,'__reprasentive_column__'):
            col = self._class.__reprasentive_column__
            return self.valueToFormattedString(col,
                                               obj.__getattribute__(col))
        return repr(obj)

    def getFriendlyName(self, obj=None, view='default'):
        if hasattr(self._class, "__friendlyName__"):
            return self._class.__friendlyName__
        return self._class.__name__

    def getPropertyFriendlyName(self, propertyName):
        if hasattr(self._class, "__propertyFriendlyNames__"):
            notDottedProperty = str(propertyName).split('.')[-1:][0]
            if self._class.__propertyFriendlyNames__.has_key(notDottedProperty):
                return self._class.__propertyFriendlyNames__[notDottedProperty]

        return propertyName

    def getShownProperties(self):
        if not len(self._shownProperties):
            if hasattr(self._class,'__shownProperties__'):
                self._shownProperties = self._class.__shownProperties__
            else:
                for prop in self.objectMapper.iterate_properties:
                    self._shownProperties.append(prop.key)

        return self._shownProperties

    def getDefaultOrderByProperty(self, obj):
        if hasattr(self._class,'__reprasentive_column__'):
            return self._class.__reprasentive_column__

    def getDefaultAllQuery(self, session):
        return session.query(self._class).order_by(self.getDefaultOrderByProperty(None))

    def getAll(self, session):
        return self.getDefaultAllQuery(session).all()

    def getFullTextQuery(self, session, text):
        return None

    def valueToFormattedString(self, propertyName, value):
        if hasattr(value, '__ormDecorator__'):
            dec = value.__ormDecorator__()
            return dec.getReprasentiveString(value)
        formatOption = self.getValueFormatOption(propertyName)

        newString = "%s%s%s" % (unicode(formatOption['prefix']),
                                unicode(formatOption['textFormat']).format((value)),
                                unicode(formatOption['suffix'])
                                )
        return newString

    def getValueFormatOption(self, propertyName):
        prefix = u''
        suffix = u''
        textFormat = u'{0}'

        try:
            colInfo = self.getColInfo(propertyName)
            if colInfo.has_key('unit'):
                suffix = " " + unicode(colInfo['unit'])
            if colInfo.has_key('textFormat'):
                textFormat = u"{0:" + colInfo['textFormat'] + '}'
        except AttributeError:
            pass
        except KeyError:
            pass

        return {
                'prefix':prefix,
                'suffix':suffix,
                'textFormat':textFormat
                }

    def getColInfo(self, propertyName):
        if not self._colInfos.has_key(propertyName):
            cols = self.getRProperty(propertyName).columns
            if len(cols) == 1:
                col = cols[0]
            self._colInfos[propertyName] = col.info
        return self._colInfos[propertyName]

    def getRProperty(self, propertyName):
        if not self._rProperties.has_key(propertyName):
            try:
                self._rProperties[propertyName] = self.objectMapper.\
                    get_property(propertyName)
            except InvalidRequestError:
                clsProp = self._class.__getattribute__(self._class, propertyName)
                if isinstance(clsProp, hybrid_property):
                    functionName = clsProp.fget.__name__
                    self._rProperties[propertyName] = clsProp

        return self._rProperties[propertyName]


import typing

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from scipy.interpolate import interpn

def density_scatter(ax, x, y, bins, **kwargs) -> None:
    """Create a scatter plot colored by 2d histogram density.

    Parameters
    ----------
    ax: matplotlib.axes.Axes
    x: np.ndarray
    y: np.ndarray
    bins: int
    kwargs
        any kwargs passed to 'ax.scatter'

    Returns
    -------

    References
    ----------
    Adapted from https://stackoverflow.com/a/53865762/10504481

    """
    # convert e.g. DataFrame to numpy array values
    x = np.array(x)
    y = np.array(y)

    if "cmap" not in kwargs:
        kwargs["cmap"] = "viridis"

    data, x_e, y_e = np.histogram2d(x, y, bins=bins, density=True)
    points = (0.5 * (x_e[1:] + x_e[:-1]), 0.5 * (y_e[1:] + y_e[:-1]))
    xi = np.vstack([x, y]).T
    z = interpn(points, data, xi, method="splinef2d", bounds_error=False)
    # To be sure to plot all data
    z[np.where(np.isnan(z))] = 0.0
    # Sort the points by density, so that the densest points are plotted last
    idx = z.argsort()
    x, y, z = x[idx], y[idx], z[idx]
    ax.scatter(x, y, c=z, **kwargs)

def get_figure(
    true, prediction, datalabel: str, xlabel: str, ylabel: str, figsize: tuple = (10, 7)
) -> plt.Figure:
    """Create a correlation plot for true, prediction values.

    Parameters
    ----------
    true: the true values
    prediction: the predicted values
    datalabel: str, the label for the prediction, e.g. 'MAE: 0.123 eV'
    xlabel: str, the xlabel
    ylabel: str, the xlabel
    figsize: tuple, size of the figure

    Returns
    -------
    plt.Figure

    """
    sns.set()
    fig, ax = plt.subplots(figsize=figsize)
    ax.plot(true, true, color="grey", zorder=0)  # plot the diagonal in the background
    bins = 500 if (len(true) / 10) > 500 else int(len(true) * 0.1)
    if bins < 20:
        # don't use density for very small datasets
        ax.scatter(true, prediction, marker="x", s=20.0, label=datalabel)
    else:
        density_scatter(
            ax, true, prediction, bins=bins, marker="x", s=20.0, label=datalabel
        )
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.legend()
    return fig

def get_hist(data, label, xlabel, ylabel) -> typing.Tuple[plt.Figure, plt.Axes]:
    fig, ax = plt.subplots()

    sns.histplot(
        data,
        ax=ax,
        stat="percent",
        label=label,
    )
    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.legend()

    return fig, ax

def get_histogram_figure(
    bin_edges,
    counts,
    datalabel: str,
    xlabel: str,
    ylabel: str,
    x_lim: typing.Tuple[float, float] = None,
    y_lim: typing.Tuple[float, float] = None,
    logy_scale=True,
    figsize: tuple = (10, 7),
) -> plt.Figure:
    """Creates a Matplotlib figure based on precomputed bin edges and counts.

    Parameters
    ----------
    bin_edges: np.array
        Edges of the histogram bins.
    counts: np.array
        Number of occurrences in each bin.
    datalabel: str
        Labels for the figure legend.
    xlabel: str
        X-axis label.
    ylabel: str
        Y-axis label.
    x_lim: tuple
        X-axis limits.
    y_lim: tuple
        Y-axis limits.
    figsize: tuple
        Size of the Matplotlib figure
    """
    sns.set()
    fig, ax = plt.subplots(figsize=figsize)

    ax.stairs(counts, bin_edges, label=datalabel, fill=True)

    ax.set_xlabel(xlabel)
    ax.set_ylabel(ylabel)
    ax.legend()
    if logy_scale:
        ax.set_yscale("log")
    if x_lim is not None:
        ax.set_xlim(x_lim)
    if y_lim is not None:
        ax.set_ylim(y_lim)
    return fig


import curses

"""
Author: Leah Hackman
Date: April 2, 2014
"""

class Hangman_v:

    empty_hangman = [" _______ ",
                    "|/     | ",
                    "|        ",
                    "|        ",
                    "|        ",
                    "|        ",
                    "|        ",
                    "---------"]

    man = [[(2,7, "O")],[(3,7, "|"), (4,7, "|")], [(3,6, "\\")], [(3,8, "/")],  [(5,8, "\\")], [(5,6, "/")]]
    full_hangman= [" _______ ",
                    "|/     | ",
                    "|      O ",
                    "|     \|/",
                    "|      | ",
                    "|     / \\",
                    "|        ",
                    "---------"]

    def __init__(self, game, term_width, term_height,stdscr):
        self.game = game
        self.stdscr = stdscr

        """
        break up the screen into four panes:
        ----------------
        | a     |b     |
        |       |      |
        |-------|      |
        | c     |      |
        ----------------
        | d            |
        ----------------
        window a is for drawing the hangman
        window b is for recording the guesses
        window c is for showing the word (blanks and correct letters)
        window d is for communicating with the user
        """
        self.a = curses.newwin(int(term_height*4/6), int(term_width/2), 0,0)
        self.b = curses.newwin(int(term_height*4/6), int(term_width/2),
                                0, int(term_width/2))
        self.c = curses.newwin(int(term_height*1/6), term_width,
                                int(term_height*4/6), 0)
        self.d = curses.newwin(int(term_height*1/6), term_width,
                                int(term_height*5/6), 0)

        #initialize magenta colour pairing for printing out guessed letters.
        curses.init_pair(1, curses.COLOR_MAGENTA,curses.COLOR_BLACK)

    def draw_hangman(self):
        window = self.a
        v_offset = 1
        h_offset = 3
        for line in enumerate(Hangman_v.empty_hangman):
            window.addstr(line[0] +v_offset, h_offset, line[1])
        for i in range(min(self.game.num_errors, self.game.max_guesses)):
           for j in Hangman_v.man[i]:
               window.addstr(j[0]+v_offset, j[1]+h_offset, j[2])
        window.noutrefresh()

    def draw_wrong_letters(self):
        """
        Print out the letters they've guessed wrong
        and information about the number of guesses they've made
        """
        window = self.b
        window.addstr(0,0, "You've made {} guesses and {} errors so far".format(self.game.num_guesses, self.game.num_errors))
        window.addstr(1,0, "Wrong Letters Guessed: ")
        window.addstr(2, 0,", ".join(self.game.wrong_guesses()))

        #need to put the changes up for drawing but do not redraw them yet.
        window.noutrefresh()

    def draw_word_field(self):
        """
        Print out word field. Print ___ for characters we haven't guessed yet.
        Othewise print the character. If the most recent guessed letter was a
        hit, print it in MAGENTA :O
        """
        window= self.c
        for letter in enumerate(self.game.secret_word):
            if letter[1] in self.game.guesses:
                window.addstr(0, letter[0]*5 + 1," {} ".format(letter[1]), curses.A_UNDERLINE | (curses.color_pair(0) if letter[1] != self.game.most_recent_guess else curses.color_pair(1)) )
            else: window.addstr(0,letter[0]*5 + 1, "   ", curses.A_UNDERLINE)

        #need to put the changes up for drawing but do not redraw them yet.
        window.noutrefresh()

    def draw_ui_field(self):
        """
        Print some text to communicate with the user.
        Either ask for more input OR let them know the game is over.
        """
        window = self.d
        if self.game.solved():
            window.clear()
            window.addstr(0,0, "You've won! Congratulations!")
        elif self.game.game_over():
            window.clear()
            window.addstr(0,0, "It looks as though you've lost. Sorry!")
        else:
            if not self.game.most_recent_guess is None:
                window.addstr(0,0, "You guessed the letter {}. There are {} {}'s in the secret word.".format(self.game.most_recent_guess, self.game.num_matches(self.game.most_recent_guess), self.game.most_recent_guess))
            window.addstr(1,0, "Please enter a character to guess")

        #need to put the changes up for drawing but do not redraw them yet.
        window.noutrefresh()

    def draw_screen(self):
        #each draw command updates what should be drawn and then
        #waits to be drawn. The doupdate call draws all the changes
        #at once.
        self.draw_hangman()
        self.draw_wrong_letters()
        self.draw_word_field()
        self.draw_ui_field()
        curses.doupdate()


from django.urls import path, re_path
from django.views.generic import TemplateView
from . import views

urlpatterns = [
    # Index page displays all posts + postform depending on whether user is authenticated. Renders Virtual DOM with React
    path("", TemplateView.as_view(
        template_name="network/index.html"), name="index"),

    # Login (display credentials form), logout and register
    path("login", views.login_view, name="login"),
    path("logout", views.logout_view, name="logout"),
    path("register", views.register, name="register"),

    # Almost all paths below render DOM with React (listingview does not)
    path("following", views.following, name="following"),
    path("<int:post_id>", views.viewpost, name="viewpost"),
    # regex to avoid matching api path
    re_path(r"^(?P<username>\w+(?<!api))$", views.profile, name="profile"),
    # Following and followers list page -- using regex to allow only /following or /followers and avoid media path
    re_path(r"^(?P<username>\w+(?<!api))\/(?P<list>follow(ers|ing))$",
            views.listingview, name="listing"),

    # API route
    path("api/update", views.update, name="update"),
    path("api/<str:filter>", views.send_data, name="send"),
    path("api", views.post, name="post"),

]


from pathlib import Path
from typing import AnyStr, Set

from datascience_model_commons.model import BaseModel, ModelException

import numpy as np
import pandas as pd
import tensorflow as tf
import yaml

DEFAULT_VALUES_DICT = {tf.string: (tf.string, ""), tf.float32: (tf.float32, 0.0)}

# Enable loading of metadata without model source code
class SafeLoaderIgnoreUnknown(yaml.SafeLoader):
    def ignore_unknown(self, node):
        return None

class Model(BaseModel):
    def __init__(self, model_path):
        model_directory = Path(__file__).parent
        with open(model_directory / "config.yaml", "r") as f:
            self.config_dict = yaml.load(f)
            self.test_description_prefix = self.config_dict.get(
                "TEST_DESCRIPTION_PREFIX"
            )

        from datascience_model_commons.transactions.business_rules.dutch_tax_rules import (
            apply_tax_rules as apply_nl_tax_rules,
        )
        from datascience_model_commons.transactions.business_rules.uk_tax_rules import (
            apply_tax_rules as apply_uk_tax_rules,
        )

        self.tax_function_dict = {
            "NL": apply_nl_tax_rules,  # function reference
            "GB": apply_uk_tax_rules,  # function reference
        }
        self.load_model(model_path=model_path)

    def load_model(self, model_path):
        def find_artifact(root):
            import os

            # Traverse root to find model signature files (.pb file extension)
            for root, dirs, files in os.walk(root):
                for file in files:
                    if file.endswith(".pb"):
                        return root

            raise ModelException("No .pb model artifact found!")

        try:
            artifact_location = find_artifact(root=model_path)
            model = tf.saved_model.load(artifact_location)
        except Exception as e:
            raise ModelException(f"Unable to load model {model_path}") from e

        metadata_path = f"{model_path}/training_metadata.yaml"
        try:
            with open(metadata_path, "r") as stream:
                SafeLoaderIgnoreUnknown.add_constructor(
                    None, SafeLoaderIgnoreUnknown.ignore_unknown
                )
                root = yaml.load(stream, Loader=SafeLoaderIgnoreUnknown)
                labels = root["labels"]
        except IOError as ioerr:
            raise ModelException(
                f'Error reading yaml config file: "{metadata_path}"'
            ) from ioerr
        except yaml.YAMLError as yerr:
            raise ModelException(f'Unable to parse "{metadata_path}"') from yerr

        self.model = model
        self.labels = np.array(labels)
        self.predict_function = self.model.signatures["serving_default"]
        self.input_tensors = [
            tensor
            for tensor in self.predict_function.inputs
            if "unknown" not in tensor.name
        ]
        self.input_description = {
            tensor.name.split(":")[0]: DEFAULT_VALUES_DICT.get(tensor.dtype)
            for tensor in self.input_tensors
        }

    def df_to_tensor_dict(self, df):
        N = df.shape[0]
        return {
            key: np.atleast_2d(
                df[key].fillna(default_value).values.astype(dtype=dtype.as_numpy_dtype)
                if key in df
                else np.repeat(default_value, N)
            ).T
            for key, (dtype, default_value) in self.input_description.items()
        }

    def _create_labels_set(self, trx_scores):
        return set(self.labels[trx_scores == 1.0])

    def scores_to_labels(self, scores):
        return np.apply_along_axis(self._create_labels_set, 1, scores)

    @staticmethod
    def append_labels(
        prefix: AnyStr,
        input_df: pd.DataFrame,
        add_label_column: AnyStr,
        label_column: AnyStr = "labels",
    ) -> pd.DataFrame:
        def map_to_label(record: pd.Series) -> Set[AnyStr]:
            current_labels = record[label_column]
            add_label = record[add_label_column]

            if type(add_label) == str:
                add_label = add_label.lower()
                add_label = f"{prefix}_{add_label}"
                current_labels.add(add_label)

            return current_labels

        input_df[label_column] = input_df.apply(map_to_label, axis=1)
        return input_df

    def add_tax_labels(self, input_df: pd.DataFrame):
        # Make sure country code is available
        if "country_code" not in input_df:
            input_df = input_df.assign(country_code="")
        else:
            input_df["country_code"].fillna("", inplace=True)

        # Add labels for all country_code tax rules
        for country_code, tax_func in self.tax_function_dict.items():
            country_code_match = input_df["country_code"] == country_code
            country_code_missing = input_df["country_code"] == ""
            subset_df = input_df.loc[country_code_match | country_code_missing]

            # Nothing to do continue to next country code
            if len(subset_df) == 0:
                continue

            # Temporarily assign country code if missing
            subset_df = tax_func(subset_df.assign(country_code=country_code))
            prefix = f"Tax_{country_code.lower()}"
            subset_df["labels"] = self.append_labels(
                prefix=prefix, input_df=subset_df, add_label_column="tax_label"
            )
        return input_df

    def apply_business_rules(self, input_df: pd.DataFrame) -> pd.DataFrame:
        """Apply business rules here, category column should be overwritten where business rules apply"""
        input_df = self.add_tax_labels(input_df=input_df)
        input_df = self.apply_test_description_rule(input_df=input_df)
        return input_df[["labels"]]

    def apply_test_description_rule(self, input_df: pd.DataFrame) -> pd.DataFrame:
        input_df.fillna("", inplace=True)
        is_test_transaction = input_df["description"].str.startswith(
            self.test_description_prefix
        )
        subset_df = input_df.loc[is_test_transaction]
        regex_pattern = "([A-Z][a-z_]+)"
        processed_labels = (
            subset_df["description"]
            .str.extractall(regex_pattern)
            .groupby(level=0)
            .agg(set)
        )
        input_df.loc[is_test_transaction, "labels"] = processed_labels.values
        return input_df

    def predict(self, *, df: pd.DataFrame):
        if df.empty:  # Nothing to predict so return empty dataframe
            scores = pd.Series(dtype=object)
        else:
            predictions = self.predict_function(**self.df_to_tensor_dict(df))
            scores = list(predictions.values())[0].numpy()

        return self.apply_business_rules(
            input_df=df.assign(
                labels=self.scores_to_labels(scores),
            )
        )


from bs4 import BeautifulSoup
import requests
from pprint import pprint
import os
import csv

def get_article_detail(url):#為了取得時間，必須先進入網頁在爬取當下標題的時間
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html.parser")
    time_element = soup.find_all("span", class_="article-meta-value")  # 找到包含完整時間的元素
    if time_element:
        return time_element[3].text.strip()
    else:
        return ""
def get_articles_data(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html.parser")
    articles = soup.find_all("div", class_="r-ent")#出來會是一團數據含有r-ent
    article_data = {}

    #用字典處理會比較容易，列表不好控制值
    #跑迴圈一一取出，如需要更深入的內容就繼續往下.find
    for article in articles:
        title_element = article.find("div", class_="title")
        if not title_element.a:  # 如果沒有超連結，跳過該篇文章
            continue

        title = title_element.text.strip()
        rate = article.find("div", class_="nrec").text.strip()
        article_url = "https://www.ptt.cc" + article.find("div", class_="title").a["href"]
        #以上取的方式也可以使用持續使用.find再利用屬性[""]，或是整理成一行
        #文章的時間卡在文章網頁裡頭，先透過func取的時間
        date = get_article_detail(article_url)#date的數據內容透過上面func幫我處理
        article_dict = {"title": title, "rate": rate, "date": date}
        article_data[title] = article_dict

    return article_data

base_url = "https://www.ptt.cc/bbs/movie/index.html"

# 先抓第一頁的東西及可以推演的內容
response = requests.get(base_url)
soup = BeautifulSoup(response.text, "html.parser")
prev_page_link = soup.find("a", string="‹ 上頁")["href"]#使用 ["href"] 就是從 <a> 元素中提取 href 屬性的值
page_index = int(prev_page_link[prev_page_link.index("index") + len("index"):prev_page_link.index(".html")])
#先抓index字的索引值然後針對字串[:]索引某數到某數，會求得一組index數字例如9640
article_data = get_articles_data(base_url)

# 拿可以推演的內容反推索引要更改的值依序扣一
for i in range(3):
    page_index -= 1
    prev_page_url = f"https://www.ptt.cc/bbs/movie/index{page_index}.html"
    article_data.update(get_articles_data(prev_page_url))#list時或是用+=類似extend
pprint(article_data)
print(article_data)
print(len(article_data))

output_file = "Week3/movies.csv"
fields=["title","rate","date"]
if not os.path.exists(output_file):
    with open(output_file, "w", newline="") as file:
        writer = csv.DictWriter(file, fieldnames=fields)
        writer.writeheader()
        for item in article_data.values():
            writer.writerow(item)
    print("movies.csv finished")
else:
    print("movies.csv exists")

output_file2= "Week3/movie.txt"
if not os.path.exists(output_file2):
    with open(output_file2, "w") as file:
        for value in article_data.values():
            line = f"{value['title']},{value['rate']},{value['date']}\n"
            file.write(line)
    print("movie.txt finished")
else:
    print("movie.txt exists")


import logging

import pygame

import environment
import globals
import script
from bush import animation, asset_handler, entity, physics, timer, util

loader = asset_handler.AssetHandler("sprites")
tile_loader = asset_handler.AssetHandler("tiled/tilesets")

logger = logging.getLogger(__name__)

class GameObject(entity.Actor):
    registry_groups = ("main",)

    def __init__(
        self,
        data,
        anim_dict=None,
        initial_state=None,
        physics_data=None,
        start_health=1,
        max_health=1,
        immunity=150,
        hit_effect=None,
    ):
        surface = data.surface
        if surface is None and anim_dict is not None:
            surface = anim_dict[min(anim_dict.keys())].image()
        super().__init__(
            data.pos,
            surface,
            (data.registry.get_group(i) for i in self.registry_groups),
            data.id,
            data.layer,
            data.topleft,
        )
        self.registry = data.registry
        self.anim_dict = {}
        if anim_dict is not None:
            self.anim_dict = anim_dict
        if initial_state is not None:
            self.state = initial_state
        elif self.anim_dict:
            self.state = "idle"
        else:
            self.state = None
        self.pushed_state = None
        if physics_data is not None:
            self.physics_data = physics_data
        else:
            self.physics_data = physics.PhysicsData(
                physics.TYPE_STATIC, pygame.sprite.Group()
            )
        self.current_health = start_health
        self.health_capacity = max_health
        self.desired_velocity = pygame.Vector2()
        self.move_state = "walk"
        self.idle_state = "idle"
        self.facing = "down"
        self.immunity_time = immunity
        self.immunity_timer = timer.DTimer(immunity)
        self.immunity_timer.finish()
        self.visual_effects = []
        self.hit_effect = hit_effect
        self.script_stack = []
        self.spawn_point = self.pos3.copy()
        self.spawn_groups = tuple(self.groups())

    def get_anim_key(self):
        return self.state

    def add_visual_effect(self, effect):
        self.visual_effects.append(effect)

    def heal(self, amount):
        self.current_health = min(self.health_capacity, self.current_health + amount)

    def respawn(self, health=None):
        self.pos3 = pygame.Vector3(self.spawn_point)
        if health is None:
            self.current_health = self.health_capacity
        else:
            self.current_health = pygame.math.clamp(health, 0, self.health_capacity)
        for group in self.spawn_groups:
            group.add(self)

    def hurt(self, amount):
        logger.info(f"Sprite {self._id} hurt by {amount}", self, amount)
        if self.immunity_timer.done():
            self.current_health -= amount
            if self.current_health <= 0:
                self.kill()
            self.immunify()

    def immunify(self):
        if self.hit_effect is not None:
            self.hit_effect.reset()
            self.add_visual_effect(self.hit_effect)
        self.immunity_timer.reset()

    def push_state(self, state):
        self.pushed_state = state

    def move(self, direction):
        self.desired_velocity = direction

    def stop(self):
        self.desired_velocity *= 0

    def run_script(self, script_name):
        self.script_stack.append(
            script.Script(self.get_id(), self.registry, script_name)
        )

    def update_state(self, dt):
        if self.desired_velocity:
            if self.state == self.idle_state:
                self.state = self.move_state
            self.facing = util.round_string_direction(
                util.string_direction(self.desired_velocity)
            )
        if not self.desired_velocity and self.state == self.move_state:
            self.state = self.idle_state
        if self.pushed_state:
            self.state = self.pushed_state

    def update_image(self, dt):
        if self.anim_dict:
            self.anim = self.anim_dict[self.get_anim_key()]
        if self.anim:
            self.image = self.anim.image()
        self.visual_effects = [e for e in self.visual_effects if not e.done()]
        for effect in self.visual_effects:
            effect.update(dt)
            self.image = effect.apply(self.image)

    def update_physics(self, dt):
        self.pos3 += self.velocity * dt
        self.update_rects()

    def update_behaviour(self, dt):
        pass

    def update_script(self, dt):
        if self.script_stack:
            self.script_stack[-1].update(dt)
            if self.script_stack[-1].finished():
                self.script_stack = self.script_stack[:-1]

    def update_rects(self):
        self.rect.center = self.pos

    def update(self, dt):
        self.update_state(dt)
        self.update_physics(dt)
        self.update_image(dt)
        self.update_behaviour(dt)
        self.update_script(dt)
        self.immunity_timer.update(dt)

    @staticmethod
    def spawn_particles(particles):
        globals.engine.stack.get_current().add_particles(particles)

class MobileGameObject(GameObject):
    def __init__(
        self,
        data,
        anim_dict=None,
        initial_state=None,
        physics_data=None,
        start_health=1,
        max_health=1,
        immunity=150,
        hit_effect=None,
        weight=0.5,
    ):
        super().__init__(
            data=data,
            anim_dict=anim_dict,
            initial_state=initial_state,
            physics_data=physics_data,
            start_health=start_health,
            max_health=max_health,
            immunity=immunity,
            hit_effect=hit_effect,
        )
        self.facing = "down"
        if physics_data is None:
            self.physics_data = physics.PhysicsData(
                physics.TYPE_STATIC, pygame.sprite.Group()
            )
        self.collision_rect = self.rect.copy()
        self.mobile = True
        self.knockback_remaining = pygame.Vector2()
        self.weight = weight
        self.force = pygame.Vector2()

    def knockback(self, from_pos, amount):
        self.force += (self.pos - from_pos).normalize() * amount * 200

    def get_current_environment(self):
        return environment.TERRAIN_DATA[
            self.registry.masks.collide_mask(
                pygame.Mask(self.collision_rect.size, True),
                self.collision_rect.topleft,
                *environment.TERRAIN_ORDER,
            )
            or "default"
        ]

    def mobilize(self):
        self.mobile = True

    def immobilize(self):
        self.mobile = False

    def face(self, dest):
        if isinstance(dest, str):
            self.facing = dest
        else:
            self.facing = util.string_direction(
                pygame.Vector2(util.direction_orthag(dest - self.pos))
            )

    def update_rects(self):
        self.collision_rect.center = self.pos
        self.rect.center = self.pos

    def get_anim_key(self):
        return f"{self.state} {self.facing}"

    def update_physics(self, dt):
        self.desired_velocity *= self.mobile
        self.update_rects()
        terrain = self.get_current_environment()
        # slippety-slide!
        if terrain.traction != 1:
            self.velocity += (self.desired_velocity * terrain.speed) * terrain.traction
        else:
            self.velocity = self.desired_velocity * terrain.speed
        self.force *= 1.5 - terrain.traction
        self.velocity += self.force

        physics.dynamic_update(self, dt)
        self.update_rects()

def get_anim_dict(path, size):
    frames = loader.load_spritesheet(path + ".png", size)
    return {
        "walk down": animation.Animation(frames[0:16:4], 150),
        "walk up": animation.Animation(frames[1:17:4], 150),
        "walk left": animation.Animation(frames[2:18:4], 150),
        "walk right": animation.Animation(frames[3:18:4], 150),
        "idle down": animation.Animation(frames[0:1]),
        "idle up": animation.Animation(frames[1:2]),
        "idle left": animation.Animation(frames[2:3]),
        "idle right": animation.Animation(frames[3:4]),
    }

def load_tile(tile_index, tileset="objects", tile_size=(16, 16)):
    return tile_loader.load_spritesheet(tileset + ".png", tile_size)[tile_index]

def load_tiles(tile_indices, tileset="objects", tile_size=(16, 16)):
    tiles = []
    for tile_index in tile_indices:
        tiles.append(load_tile(tile_index, tileset, tile_size))
    return tiles


import csv
import readline
import pandas as pd
import pymongo

conn  = pymongo.MongoClient('192.168.99.100', 32766)
db    = conn.get_database("mini_project") # 있으면 가져오고, 없으면 만들어
table = db.get_collection("korean_chulguk") # 있으면 가져오고, 없으면 만들어

f   = open('./resources/korean_chulguk.csv','r', encoding="euc-kr")

rdr = csv.reader(f)
print(rdr)

column = next(rdr) #[컬럼명 읽기]

for line in rdr:
    dict1 = dict()
    for idx, val in enumerate(line):
        dict1[column[idx]] = val
        print(line)

    table.insert_one(dict1)
conn.close()


# -*- coding: utf-8 -*-

from unittest.mock import patch

from tests.fixtures.benchmark import ComparativeBenchmarkMother
from bench.app.benchmark.domain.config import NotificationsConfig
from bench.app.benchmark.domain.events import ComparativeBenchmarkFinished
from bench.app.benchmark.infrastructure.repositories import ComparativeBenchmarkInMemoryRepository
from bench.app.benchmark.use_cases.listeners.email_alert_listener import ComparativeBenchmarkFinishedEmailAlertListener
from bench.app.notifications.use_cases.commands import SendEmailCommand

notifications_config = NotificationsConfig(notification_email='some email')

@patch('bench.app.benchmark.domain.specification.SubjectLoadedSlowerThanAtLeastOneOfCompetitorsSpecification')
def test_it_should_send_email_if_subject_url_loaded_slower_than_at_least_one_of_competitors(specification_mock):
    # given
    specification_mock.is_satisfied_by.return_value = True
    benchmark = ComparativeBenchmarkMother.create_any()

    # and
    repository = ComparativeBenchmarkInMemoryRepository()
    repository.add(benchmark)

    # when
    listener = ComparativeBenchmarkFinishedEmailAlertListener(notifications_config, specification_mock, repository)
    result = listener.execute(ComparativeBenchmarkFinished(benchmark.benchmark_id))

    # then
    expected_command = SendEmailCommand('Benchmark alert', [notifications_config.notification_email], 'Your site is slow')
    assert expected_command == result

@patch('bench.app.benchmark.domain.specification.SubjectLoadedSlowerThanAtLeastOneOfCompetitorsSpecification')
def test_it_should_not_send_email_if_subject_url_loaded_faster_than_competitors(specification_mock):
    # given
    specification_mock.is_satisfied_by.return_value = False
    benchmark = ComparativeBenchmarkMother.create_any()

    # and
    repository = ComparativeBenchmarkInMemoryRepository()
    repository.add(benchmark)

    # when
    listener = ComparativeBenchmarkFinishedEmailAlertListener(notifications_config, specification_mock, repository)
    result = listener.execute(ComparativeBenchmarkFinished(benchmark.benchmark_id))

    # then
    assert result is None


import logging
from pathlib import Path

from pyquery import PyQuery as pq

from config import config
from lian_jia import Community
from util.orm import Session

DATA_DIR = Path(__file__).parent.joinpath('../data/').resolve()

def parse_community_detail(community_id):
    save_file = DATA_DIR.joinpath(f'{community_id}.html')

    if save_file.exists():
        d = pq(save_file.read_text(encoding='utf-8'))

        keys = [e.text.rstrip('：') for e in d('span.hdic_key')]
        values = [e.text for e in d('span.hdic_value')]
        detail = dict(zip(keys, values))

        if not detail:
            logging.warning(f'# 页面已找到, 但其中未发现小区详情信息, community_id={community_id}')

        return detail

    else:
        logging.error(f'# 详情页面不存在, community_id={community_id}')

def parse_all_communities(city_id):
    db_session = Session()

    communities = db_session.query(Community).filter(
        Community.city_id == city_id,
        Community.detail == None,
        Community.page_fetched_at != None
    ).all()

    total_count = len(communities)
    logging.info(f'city_id={city_id}, 待分析={total_count}')

    for i, a_community in enumerate(communities):
        detail = parse_community_detail(a_community.id)

        if detail:
            a_community.detail = detail

        if (i + 1) % 100 == 0 or (i == total_count - 1):
            logging.info(f'进度={i + 1}/{total_count}, 剩余={total_count - i - 1}')
            db_session.commit()

    logging.info('已全部分析完成.')
    db_session.close()

def main():
    parse_all_communities(config.city_id)

if __name__ == '__main__':
    main()


'''
Downloads the desired number of images from the ImageNet database using a .txt file
with the urls of the images listed. Stores a testing set and a training set of these
images as numpy arrays in an .h5 file. Can be run from command line with click.
'''

import click
import h5py
from skimage import io
import os
import numpy as np
import urllib

'''
num_images - number of images to download
read_path - path to .txt file provided by ImageNet
returns - two lists with each entry being a url of an image, one for training
and the other for testing
'''

def choose_images(num_images, read_path, train_percent=0.8, val_percent=0.8):
    urls = []
    with open(read_path) as myfile:
        count = 0
        while count < num_images:
            x = next(myfile)
            if 'flickr' in x:
                count += 1
                url = x[x.find('h'):-1]
                urls.append(url)

    np.random.shuffle(urls)
    sep_index1 = int(num_images * train_percent)
    sep_index2 = int(sep_index1 * val_percent)

    return urls[:sep_index2], urls[sep_index2: sep_index1], urls[sep_index1:]

'''
Writes the images as a series of numpy arrays to an .h5 file

urls - list of urls
write_path - directory to write the .h5 file to
prefix - desired filename
'''

def image_to_h5(urls, write_path, prefix):
    if not os.path.exists(write_path):
        os.makedirs(write_path)

    filename = os.path.join(write_path, prefix + '.h5')

    with h5py.File(filename, "w") as f:
        for i in range(len(urls)):
            # some urls don't have images anymore and need to be skipped
            try:
                image = io.imread(urls[i])
                if i % 100 == 0:
                    print('Reached image {}'.format(i))
                name = 'img_{}'.format(i)
                f.create_dataset(name, data=image)
            except (urllib.error.URLError, ValueError):
                print('Could not download image {}'.format(i))

@click.command()
@click.option('--num_images', default=10000, help='Number of images to download')
@click.argument('read_path', type=click.Path(exists=True, readable=True))
@click.argument('write_path', type=click.Path(exists=False))
@click.argument('prefix')
def main(num_images, read_path, write_path, prefix):
    urls_train, urls_val, urls_test = choose_images(num_images, read_path)
    image_to_h5(urls_train, write_path, prefix + '_training')
    image_to_h5(urls_val, write_path, prefix + '_validation')
    image_to_h5(urls_test, write_path, prefix + '_testing')

if __name__ == '__main__':
    main()
    print('done')