import sys, time, xlsxwriter, os, openpyxl, keyboard
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from fake_useragent import UserAgent
from selenium.webdriver.common.action_chains import ActionChains
from os.path import join, dirname
from datetime import datetime
from openpyxl import load_workbook
from threading import Thread
from os.path import join, dirname, realpath
import http.client
# from dotenv import load_dotenv

class ZomatoThread(Thread):

    def __init__(self, locality_name, cur_row, url):
        Thread.__init__(self)
        self.locality_name = locality_name
        self.cur_row = cur_row
        self.url = url

    def run(self):
        global details, num_threads
        time_limit = 60
        start_time = datetime.now()
        ua = UserAgent()
        userAgent = ua.random
        userAgent = userAgent.split(" ")
        userAgent[0] = "Mozilla/5.0"
        userAgent = " ".join(userAgent)
        print("userAgent = " + userAgent)
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('user-agent={0}'.format(userAgent))
        chrome_options.add_argument("--headless")
        # chrome_options.add_argument("window-size=1280,800")

        chrome_options.add_argument('--log-level=0')
        path = join(dirname(__file__), 'webdriver', 'chromedriver.exe')
        driver = webdriver.Chrome (executable_path = path, options = chrome_options )
        # driver.maximize_window()
        driver.get(self.url)

        while True:
            try:
                rest_name = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//main/div/section[3]/section/section[1]/div/h1"))).text
                break
            except:
                if (datetime.now() - start_time).total_seconds() > time_limit:
                    details[self.cur_row] = {"rest_name": "", "rating": "", "commeters": "", "cuisine": "", "cost_alcohol": "", "cost": "", "address": "", "opening_hours": "", "company": ""}
                    num_threads -= 1
                    conn = http.client.HTTPConnection(driver.service.service_url.split("//")[1])
                    conn.request("GET", "/shutdown")
                    conn.close()
                    del driver
                    return
                time.sleep(0.1)
                pass

        company = driver.find_element_by_xpath("//main/div/section[3]/section/section[1]/section[1]/a").text
        rating = ''
        commeters = ''
        try:
            rating = driver.find_element_by_xpath("//main/div/section[3]/section/section[2]/section/div[1]/p").text
            commeters = driver.find_element_by_xpath("//main/div/section[3]/section/section[2]/section/div[2]/p").text
            commeters = driver.split(" ")[0]
        except:
            pass
        cuisine = ", ".join([e.text for e in driver.find_elements_by_xpath("//h3[contains(text(), 'Cuisines')]/following-sibling::section/a")])
        cost = ''
        cost_alcohol = ''
        try:
            cost_1 = driver.find_element_by_xpath("//h3[contains(text(), 'Average Cost')]/following-sibling::p[1]").text
            if cost_1.find("with alcohol") > -1:
                cost_alcohol = cost_1
            else:
                cost = cost_1
        except:
            pass

        try:
            cost_2 = driver.find_element_by_xpath("//h3[contains(text(), 'Average Cost')]/following-sibling::p[3]").text
            if cost_2.find("₺") > -1:
                if cost_2.find("with alcohol") > -1:
                    cost_alcohol = cost_2
                else:
                    cost = cost_2
        except:
            pass

        address = driver.find_element_by_xpath("//h5[contains(text(), 'Direction')]/following-sibling::p").text
        opening_hours = ''
        try:
            opening_hours = driver.find_element_by_xpath("//span[contains(text(), 'Close')]/following-sibling::span[1]").text
        except:
            try:
                opening_hours = driver.find_element_by_xpath("//span[contains(text(), 'Open')]/following-sibling::span[1]").text
            except:
                pass
        # xlsfile_name = "xls\\zomato\\" + self.locality_name + ".xlsx"
        # wb = load_workbook(xlsfile_name)
        # ws = wb.active
        # ws.cell(row=self.cur_row, column=2).value = rest_name
        # ws.cell(row=self.cur_row, column=3).value = rating
        # ws.cell(row=self.cur_row, column=4).value = commeters
        # ws.cell(row=self.cur_row, column=5).value = cuisine
        # ws.cell(row=self.cur_row, column=6).value = cost_alcohol
        # ws.cell(row=self.cur_row, column=7).value = cost
        # ws.cell(row=self.cur_row, column=8).value = address
        # ws.cell(row=self.cur_row, column=10).value = opening_hours
        details[self.cur_row] = {"rest_name": rest_name, "rating": rating, "commeters": commeters, "cuisine": cuisine, "cost_alcohol": cost_alcohol, "cost": cost, "address": address, "opening_hours": opening_hours, "company": company}
        # wb.save(xlsfile_name)
        # wb.close()
        # print("wrote at row " + str(self.cur_row), "  :: address=", address, ", opening_hours=", opening_hours)
        conn = http.client.HTTPConnection(driver.service.service_url.split("//")[1])
        conn.request("GET", "/shutdown")
        conn.close()
        del driver
        num_threads -= 1

def get_urls(locality_index):
    cur_row = 1
    start_time = datetime.now()
    ua = UserAgent()
    userAgent = ua.random
    userAgent = userAgent.split(" ")
    # userAgent[0] = "Mozilla/5.0"
    userAgent = " ".join(userAgent)
    print("userAgent = " + userAgent)
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument('user-agent={0}'.format(userAgent))
    chrome_options.add_argument("--headless")
    # chrome_options.add_argument("window-size=1280,800")

    chrome_options.add_argument('--log-level=0')
    path = join(dirname(__file__), 'webdriver', 'chromedriver.exe')
    driver = webdriver.Chrome (executable_path = path, options = chrome_options )
    # driver.maximize_window()

    #Remove navigator.webdriver Flag using JavaScript
    # driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
    driver.get("https://www.zomato.com/istanbul")

    while True:
        try:
            # popular_localities = driver.find_elements_by_xpath("//div[@class='title']/following-sibling::div/div/div")
            popular_localities = WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.XPATH, "//div[@class='title']/following-sibling::div/div/div")))
            print("get Popular localities: ", popular_localities)
            break
        except:
            time.sleep(0.1)
            pass
    popular_localities_url = []
    popular_localities_name = []
    for locality in popular_localities:
        popular_localities_url.append(locality.find_element_by_xpath("./a").get_attribute("href"))
        locality_name = locality.find_element_by_xpath("./a/div/div/h5").text
        locality_name= locality_name[:locality_name.find(" (")]
        popular_localities_name.append(locality_name)
    print("get Popular localities URL")
    print("popular_localities_url = ", popular_localities_url)

    # for locality_name, locality_url in zip(popular_localities_name, popular_localities_url):

    locality_name = popular_localities_name[locality_index]
    locality_url = popular_localities_url[locality_index]
    xlsfile_name = "xls\\zomato\\" + locality_name + ".xlsx"
    workbook = xlsxwriter.Workbook(xlsfile_name)
    worksheet = workbook.add_worksheet()
    col_headers = ['No', 'Restaurant', 'Rating', 'Number of Comments', 'Cuisine', 'Cost with alcohol', 'Cost without alcohol', 'Address', 'Address as coordinate', 'Opening hours', 'Locality', 'URL', 'Image URL']

    for c, i in zip(col_headers, range(len(col_headers))):
        worksheet.write(0, i, c)

    print("time = ",(datetime.now() - start_time).total_seconds())
    print("locality_name = ", locality_name)
    # rest_data = {}
    rest_urls = []
    img_urls = []
    count = 0

    driver.get(locality_url) # + "?sort=cd&category=2")
    print("go to new page '" + locality_url + "'")
    # while True:
    #     try:
    #         categories = driver.find_elements_by_xpath("(//section[@role='tablist'])[1]/div")
    #         print("get categories")
    #         break
    #     except:
    #         time.sleep(0.1)
    #         pass
    # is_dining_out = True
    # time.sleep(0.5)
    # print("len of categories = ", len(categories))
    # for category in categories:
    #     if not is_dining_out:
    #         pass
    restaurants = driver.find_elements_by_xpath("//div[@class='jumbo-tracker']")
    restaurants_count = len(restaurants)
    print(restaurants_count)
    restaurants_pre_count = 0

    while restaurants_pre_count != restaurants_count:
        driver.execute_script("window.scrollTo(0,document.body.scrollHeight)")
        ok = False
        restaurants_pre_count = restaurants_count
        restaurants_pre_count_2 = restaurants_count
        for i in range(5):
            time.sleep(1)
            restaurants = driver.find_elements_by_xpath("//div[@class='jumbo-tracker']")
            restaurants_count = len(restaurants)
            if restaurants_pre_count_2 != restaurants_count:
                restaurants_pre_count_2 = restaurants_count
                ok = True
            elif ok:
                break
        print("pre_count = " + str(restaurants_pre_count) + "  count = " + str(restaurants_count))

    try:
        end_of_results = driver.find_element_by_xpath("//h3[contains(text(), 'End of search results')]")
        print("####  End of search results  ####")
    except:
        pass

    cur_time = datetime.now()

    for restaurant in restaurants:
        # //div[@class='jumbo-tracker']/div/a/div/img
        rest_url = restaurant.find_element_by_xpath("./div/a[1]").get_attribute("href")
        img_url = restaurant.find_element_by_xpath("./div/a[1]/div[1]/img").get_attribute("src")

        if not rest_url in rest_urls:
            print("rest_url = ", rest_url)
            rest_urls.append(rest_url)
            img_urls.append(img_url)
            worksheet.write(cur_row, 0, cur_row)
            worksheet.write(cur_row, 10, locality_name)
            worksheet.write(cur_row, 11, rest_url)
            # worksheet.write(cur_row, 12, img_url)
            cur_row += 1

    workbook.close()

def get_details(locality_name, start_row, end_row, time_interval):
    global details, num_threads
    breaked = False
    xlsfile_name = "xls\\zomato\\" + locality_name + ".xlsx"
    wb = load_workbook(xlsfile_name)
    ws = wb.active
    total_rows = 2
    rest_urls = []
    while True:
        if ws.cell(row=total_rows, column=1).value == None: break
        url = ws.cell(row=total_rows, column=12).value
        rest_urls.append(url)
        total_rows += 1
    total_rows -= 1

    if end_row == -1:
        end_row = total_rows
    else:
        end_row += 1

    for i in range(start_row + 1, end_row + 1):
        t = ZomatoThread(locality_name, i, rest_urls[i - 2])
        t.start()
        num_threads += 1

        print("completed = ", len(details), " number of threads = ", num_threads, " total = ", end_row - start_row)
        time.sleep(time_interval)
        # for i in range(5):
        #     try:
        #         if keyboard.is_pressed(' '):  # if key 'q' is pressed
        #             print('You Pressed A Key!')
        #             breaked = True
        #             break
        #     except:
        #         pass
        #     time.sleep(time_interval / 5)
        # if breaked: break

    while True:
        if breaked or len(details) == end_row - start_row:
            for row in details:
                ws.cell(row=row, column=2).value = details[row]['rest_name']
                ws.cell(row=row, column=3).value = details[row]['rating']
                ws.cell(row=row, column=4).value = details[row]['commeters']
                ws.cell(row=row, column=5).value = details[row]['cuisine']
                ws.cell(row=row, column=6).value = details[row]['cost_alcohol']
                ws.cell(row=row, column=7).value = details[row]['cost']
                ws.cell(row=row, column=8).value = details[row]['address']
                ws.cell(row=row, column=9).value = details[row]['company']
                ws.cell(row=row, column=10).value = details[row]['opening_hours']
            wb.save(xlsfile_name)
            wb.close()
            break

        print("completed = ", len(details), " number of threads = ", num_threads, " total = ", end_row - start_row)
        for i in range(5):
            try:
                if keyboard.is_pressed(' '):  # if key 'q' is pressed
                    print('You Pressed A Key!')
                    breaked = True
                    break
            except:
                pass
            time.sleep(time_interval / 5)

def excel_merge():
    xlsfile_name = "xls\\zomato\\total.xlsx"
    # wb = load_workbook(xlsfile_name)
    wb = openpyxl.Workbook()
    ws = wb.active
    cur_row = 1

    directory = join(dirname(realpath(__file__)), "xls", "zomato")
    print("=##", directory, "##")
    urls = []
    for filename in os.listdir(directory):
        if filename.endswith(".xlsx"):
            wb_2 = load_workbook(join(directory, filename))
            ws_2 = wb_2.active
            if cur_row == 1:
                for c in range(1, 13):
                    ws.cell(row=1, column=c).value = ws_2.cell(row=1, column=c).value
                cur_row = 2
            cur_row_2 = 2
            while True:
                if ws_2.cell(row=cur_row_2, column=1).value == None: break
                url = ws_2.cell(row=cur_row_2, column=12).value
                if not url in urls:
                    urls.append(url)

                    ws.cell(row=cur_row, column=1).value = cur_row - 1
                    for c in range(2, 13):
                        ws.cell(row=cur_row, column=c).value = ws_2.cell(row=cur_row_2, column=c).value
                    print(cur_row)
                    cur_row = cur_row + 1

                cur_row_2 += 1
    wb.save(xlsfile_name)
    wb.close()

    # total_rows = 2
    # rest_urls = []
    # while True:
    #     if ws.cell(row=total_rows, column=1).value == None: break
    #     url = ws.cell(row=total_rows, column=12).value
    #     rest_urls.append(url)
    #     total_rows += 1
    # total_rows -= 1

    # for i in range(start_row + 1, total_rows + 1):
    #     t = ZomatoThread(locality_name, i, rest_urls[i - 2])
    #     t.start()
    #     time.sleep(time_interval)

    #     print(len(details))

    # while True:
    #     print(len(details))
    #     if len(details == tatal_rows - start_row):
    #         for row in details:
    #             ws.cell(row=row, column=2).value = details[row].rest_name
    #             ws.cell(row=row, column=3).value = details[row].rating
    #             ws.cell(row=row, column=4).value = details[row].commeters
    #             ws.cell(row=row, column=5).value = details[row].cuisine
    #             ws.cell(row=row, column=6).value = details[row].cost_alcohol
    #             ws.cell(row=row, column=7).value = details[row].cost
    #             ws.cell(row=row, column=8).value = details[row].address
    #             ws.cell(row=row, column=9).value = details[row].company
    #             ws.cell(row=row, column=10).value = details[row].opening_hours

    #     time.sleep(time_interval)

        # rest_name = restaurant.find_element_by_xpath("./div/a[2]/p[1]").text
        # print("Consuming Time: 2", (datetime.now() - start_time).total_seconds())
        # if rest_name in rest_data: continue
        # rest_rating = ''
        # rest_comments = ''
        # try:
        #     rest_rating = restaurant.find_element_by_xpath("./div/a[2]/div/section/div[1]/p").text
        #     rest_comments = restaurant.find_element_by_xpath("./div/a[2]/div/section/div[2]/p").text
        #     rest_comments = rest_comments.split(" ")[0][1:]
        # except:
        #     pass
        # print("Consuming Time: 3", (datetime.now() - start_time).total_seconds())

        # rest_cuisine = restaurant.find_element_by_xpath("./div/a[2]/p[2]").text
        # print("Consuming Time: 4", (datetime.now() - start_time).total_seconds())
        # rest_budget = restaurant.find_element_by_xpath("./div/a[2]/p[4]/span").text
        # print("Consuming Time: 5", (datetime.now() - start_time).total_seconds())
        # rest_data[rest_name] = {'rating': rest_rating, 'comments': rest_comments, 'cuisine': rest_cuisine, 'budget': rest_budget}
        # count += 1
        # print(count, rest_name, rest_rating, rest_comments, rest_cuisine, rest_budget)

        # is_dining_out = False

    # for rest_url, img_url in zip(rest_urls, img_urls):

details = {}
num_threads = 0
file_open_flag = False

if sys.argv[1] == "url":
    get_urls(int(sys.argv[2]))
elif sys.argv[1] == "detail":
    get_details(sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), int(sys.argv[5]) )
elif sys.argv[1] == "merge":
    excel_merge()

# End of search results h3


# (turn, hound1, hound2, hound3, hare, stalls)

HOUND_PLAYER = 0
HARE_PLAYER = 1

UNDECIDED = 0
HOUND_WIN = 1
HOUND_LOSS = 2

#   1 4 7
# 0 2 5 8 10
#   3 6 9

EDGES = {
    0: {1, 2, 3},
    1: {0, 2, 4, 5},
    2: {0, 1, 3, 5},
    3: {0, 2, 5, 6},
    4: {1, 5, 7},
    5: {1, 2, 3, 4, 6, 7, 8, 9},
    6: {3, 5, 9},
    7: {4, 5, 8, 10},
    8: {5, 7, 9, 10},
    9: {5, 6, 8, 10},
    10: {7, 8, 9},
}

def compress(turn, hound1, hound2, hound3, hare, stalls):
    hound_num = 0
    for i in range(hound1):
        hound_num += (9 - i) * (10 - i) // 2
    for i in range(hound1 + 1, hound2):
        hound_num += (10 - i)
    hound_num += hound3 - (hound2 + 1)

    left = turn * (2 ** 7)
    left += 11 * stalls + hare
    return left * (2 ** 8) + hound_num

def uncompress(num):
    left = num // (2 ** 8)
    hound_num = num % (2 ** 8)
    hound1 = 0
    while (9 - hound1) * (10 - hound1) // 2 <= hound_num:
        hound_num -= (9 - hound1) * (10 - hound1) // 2
        hound1 += 1
    hound2 = hound1 + 1
    while (10 - hound2) <= hound_num:
        hound_num -= (10 - hound2)
        hound2 += 1
    hound3 = hound2 + 1 + hound_num
    turn = left // (2 ** 7)
    stalls = (left % (2 ** 7)) // 11
    hare = (left % (2 ** 7)) % 11
    return (turn, hound1, hound2, hound3, hare, stalls)

def current_player(num):
    return uncompress(num)[0]

initial_position = compress(HOUND_PLAYER, 0, 1, 3, 10, 0) # 2561

def primitive(pos):
    turn, hound1, hound2, hound3, hare, stalls = uncompress(pos)
    if hare == 0: # Hare makes it to other side of board
        return HOUND_LOSS
    if turn == HARE_PLAYER:
        occupied = {hound1, hound2, hound3, hare}
        for new_space in EDGES[hare]:
            if new_space not in occupied:
                if stalls >= 10:
                    return HOUND_LOSS
                return UNDECIDED
        return HOUND_WIN # Hare has no moves
    else:
        if min(hound1, hound2, hound3, hare) >= 7:
            return HOUND_LOSS
    return UNDECIDED

def generate_moves(pos):
    turn, hound1, hound2, hound3, hare, stalls = uncompress(pos)
    occupied = {hound1, hound2, hound3, hare}
    if turn == HOUND_PLAYER:
        for init_space in (hound1, hound2, hound3):
            for new_space in EDGES[init_space]:
                if new_space not in occupied and \
                    ((init_space - 1) // 3 <= (new_space - 1) // 3):
                    yield (init_space, new_space)
    else:
        for new_space in EDGES[hare]:
            if new_space not in occupied:
                yield new_space

def do_move(pos, move):
    turn, hound1, hound2, hound3, hare, stalls = uncompress(pos)
    if turn == HOUND_PLAYER:
        init_space, new_space = move
        if (init_space - 1) // 3 == (new_space - 1) // 3:
            stalls += 1
        else:
            stalls = 0
        hounds = [hound1, hound2, hound3]
        hounds[hounds.index(init_space)] = new_space
        hounds.sort()
        hound1, hound2, hound3 = hounds
        turn = HARE_PLAYER
        new_pos = (turn, hound1, hound2, hound3, hare, stalls)
    else:
        hare = move
        turn = HOUND_PLAYER
        new_pos = (turn, hound1, hound2, hound3, hare, stalls)
    return compress(*new_pos)

import functools

@functools.lru_cache(maxsize=32768)
def solve_next_moves(current_pos):
    result = primitive(current_pos)
    if result != UNDECIDED:
        return []
    current_turn = uncompress(current_pos)[0]
    can_win = False
    fastest_win = float('inf')
    slowest_loss = 0
    best_move = None
    best_rest = None
    next_moves = []
    for move in generate_moves(current_pos):
        new_pos = do_move(current_pos, move)
        new_value, remoteness = solve(new_pos)
        next_moves.append({
            'move': move,
            'board': uncompress(new_pos), # ','.join([str(s) for s in uncompress(new_pos)]),
            'value': new_value,
            'remoteness': remoteness,
        })
    return next_moves

@functools.lru_cache(maxsize=32768)
def solve(current_pos):
    result = primitive(current_pos)
    if result != UNDECIDED:
        return result, 0
    current_turn = uncompress(current_pos)[0]
    can_win = False
    fastest_win = float('inf')
    slowest_loss = 0
    for move in generate_moves(current_pos):
        new_pos = do_move(current_pos, move)
        new_result, remoteness = solve(new_pos)
        if current_turn == HARE_PLAYER:
            if new_result == HOUND_LOSS:
                can_win = True
                if remoteness + 1 < fastest_win:
                    fastest_win = remoteness + 1
            elif not can_win:
                if remoteness + 1 > slowest_loss:
                    slowest_loss = remoteness + 1
        else:
            if new_result == HOUND_WIN:
                can_win = True
                if remoteness + 1 < fastest_win:
                    fastest_win = remoteness + 1
            elif not can_win:
                if remoteness + 1 > slowest_loss:
                    slowest_loss = remoteness + 1
    if current_turn == HOUND_PLAYER:
        if can_win:
            return HOUND_WIN, fastest_win
        return HOUND_LOSS, slowest_loss
    else:
        if can_win:
            return HOUND_LOSS, fastest_win
        return HOUND_WIN, slowest_loss

@functools.lru_cache(maxsize=32768)
def fancy_solve(current_pos):
    result = primitive(current_pos)
    if result != UNDECIDED:
        return result, 0, []
    current_turn = uncompress(current_pos)[0]
    can_win = False
    fastest_win = float('inf')
    slowest_loss = 0
    best_move = None
    best_rest = None
    for move in generate_moves(current_pos):
        new_pos = do_move(current_pos, move)
        new_result, remoteness, rest = solve(new_pos)
        if current_turn == HARE_PLAYER:
            if new_result == HOUND_LOSS:
                can_win = True
                if remoteness + 1 < fastest_win:
                    fastest_win = remoteness + 1
                    best_move = move
                    best_rest = rest
            elif not can_win:
                if remoteness + 1 > slowest_loss:
                    slowest_loss = remoteness + 1
                    best_move = move
                    best_rest = rest
        else:
            if new_result == HOUND_WIN:
                can_win = True
                if remoteness + 1 < fastest_win:
                    fastest_win = remoteness + 1
                    best_move = move
                    best_rest = rest
            elif not can_win:
                if remoteness + 1 > slowest_loss:
                    slowest_loss = remoteness + 1
                    best_move = move
                    best_rest = rest
    if current_turn == HOUND_PLAYER:
        if can_win:
            return HOUND_WIN, fastest_win, move_list
        return HOUND_LOSS, slowest_loss, move_list
    else:
        if can_win:
            return HOUND_LOSS, fastest_win, move_list
        return HOUND_WIN, slowest_loss, move_list

def main():
    pos = initial_position
    while primitive(pos) == UNDECIDED:
        # print(fancy_solve(pos))
        move = input('move: ')
        if ',' in move:
            move = move.split(',')
            move = (int(move[0].strip()), int(move[1].strip()))
        else:
            move = int(move.strip())
        pos = do_move(pos, move)

if __name__ == '__main__':
    # main()
    print(solve_next_moves(initial_position))


import json

import pandas
import urllib3
from classmerge import class_case
from dataclean import cleanall

df = pandas.read_csv("data/test.csv")
http = urllib3.PoolManager()
correct = 0
for index, row in df.iterrows():
    label = row[0]
    title = str(row[1])#.replace(".doc","").replace(".docx","")
    content = cleanall(str(row[2]))
    url = "http://192.168.0.161:58081/z?1={}&2={}".format(title, content)
    print(url)
    if len(url) > 9999:
        url = url[:9999]
    result = http.request('GET', url)
    result = json.loads(result.data)
    print(class_case[label-1], result['answer'][0])
    # df.at[index, 'category'] = result['answer'][0]
    # df.at[index, 'title'] = title
    # df.at[index, 'fulltext'] = content
    if class_case.index(result['answer'][0]) == label - 1:
        correct +=1
#df.to_csv("eval/test-bert.csv", index=False)
print('ACCURACY:{}%'.format(correct*100.0/len(df)))


"""Combines all components

The `sidebar` component combines all the inputs while other components potentially
have callbacks.

To add or remove components, adjust the `setup`.
If callbacks are present, also adjust `CALLBACK_INPUTS`, `CALLBACK_OUTPUTS` and
`callback_body`.
"""
from collections import OrderedDict

from dash_bootstrap_components import Row, Col
from dash_bootstrap_components.themes import BOOTSTRAP
from dash_html_components import Script, Div

from penn_chime.defaults import Constants
from penn_chime.models import SimSirModel

from chime_dash.app.components.base import Component, HTMLComponentError
from chime_dash.app.components.sidebar import Sidebar
from chime_dash.app.components.header import Header
from chime_dash.app.components.intro import Intro, ToolDetails
from chime_dash.app.components.additions import Additions
from chime_dash.app.components.visualizations import Visualizations
from chime_dash.app.components.definitions import Definitions
from chime_dash.app.components.footer import Footer
from chime_dash.app.components.navbar import Navbar

class Body(Component):
    """
    """

    external_stylesheets = [
        "https://www1.pennmedicine.org/styles/shared/penn-medicine-header.css",
        BOOTSTRAP,
    ]

    def __init__(self, language, defaults):
        """
        """
        super().__init__(language, defaults)
        self.components = OrderedDict(
            sidebar=Sidebar(language, defaults),
            header=Header(language, defaults),
            intro=Intro(language, defaults),
            tool_details=ToolDetails(language, defaults),
            visualizations=Visualizations(language, defaults),
            additions=Additions(language, defaults),
            definitions=Definitions(language, defaults),
            footer=Footer(language, defaults),
            navbar=Navbar(language, defaults),
        )
        self.callback_outputs = []
        self.callback_inputs = OrderedDict()
        self.callback_keys = []
        for component in self.components.values():
            self.callback_outputs += component.callback_outputs
            self.callback_inputs.update(component.callback_inputs)

    def get_html(self):
        """Glues individual setup components together
        """
        return Div(
            children=self.components["navbar"].html
            + [
                Row(
                    children=[
                        Col(
                            id="sidebar",
                            children=self.components["sidebar"].html,
                            width=3,
                            className="mt-4",
                        ),
                        Col(width=1),
                        Col(
                            self.components["header"].html
                            + self.components["intro"].html
                            + self.components["tool_details"].html
                            + self.components["visualizations"].html
                            + self.components["additions"].html
                            + self.components["definitions"].html
                            + self.components["footer"].html,
                            width=8,
                            className="mt-4",
                        ),
                    ],
                    className="container",
                ),
            ]
        )

    def callback(self, *args, **kwargs):
        """
        """
        kwargs = dict(zip(self.callback_inputs, args))
        pars = self.components["sidebar"].parse_form_parameters(**kwargs)
        kwargs["model"] = SimSirModel(pars)
        kwargs["pars"] = pars

        callback_returns = []
        for component in self.components.values():
            try:
                callback_returns += component.callback(**kwargs)
            except Exception as error:
                raise HTMLComponentError(component, error)

        return callback_returns


import matplotlib.pyplot as plt
import numpy as np
import uncertainties.unumpy as unp
import scipy.constants as con
from scipy import stats , integrate
from scipy.optimize import curve_fit
from scipy.integrate import quad, trapz, simps
from uncertainties import ufloat
from colorama import Fore, Back, Style
from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,
                               AutoMinorLocator, LogLocator)

################################################################################
# Funktionen
def f(x, a, b):
    return np.exp(a/x) * b

def bgr(x, a, b, c):
    return np.exp(a*x) * b + c

def better_fit(T, a, b):
    return b * np.exp(a/T)

def backgrd(T, a ,b):
  return a * np.exp(b * T)

# Konstanten
kB = con.k
#===============================================================================

# Methode 1
print(Back.YELLOW + Fore.BLACK + '\nBerechnung über Approximation' + Style.RESET_ALL)

t1, T1, I1 = np.genfromtxt('data/1.txt', unpack='True')
t2, T2, I2 = np.genfromtxt('data/2.txt', unpack='True')

T1, T2 = T1+273.15, T2+273.15 #in Kelvin

# Tbgr = [T1[0], T1[1], T1[2], T1[3], T1[4], T1[34], T1[35], T1[36], T1[37]]
# Ibgr = [I1[0], I1[1], I1[2], I1[3], I1[4], I1[34], I1[35], I1[36], I1[37]]
#
# paramsbgr, covbgr = curve_fit(bgr, Tbgr, Ibgr)
# errbgr = np.sqrt(np.diag(covbgr))
#===============================================================================
# Heizraten
H1 = []
H2 = []

for i in range (0, 66, 1):
    H1.append( T1[i] - T1[i+1])

for i in range (0, 88, 1):
    H2.append( T2[i] - T2[i+1])

# Mittelwert
H1 = ufloat(np.mean(H1), np.std(H1))
H2 = ufloat(np.mean(H2), np.std(H2))

print(Fore.YELLOW + '\nHeizraten:' + Style.RESET_ALL)
print('H_1 = ', '{:L}'.format(H1))
print('H_2 = ', '{:L}'.format(H2))
#===============================================================================

Tf1, If1 = T1[6:27], I1[6:27]

params1, cov1 = curve_fit(f, Tf1, If1)
err1 = np.sqrt(np.diag(cov1))

# Parameter
a1 = ufloat(params1[0], err1[0])
b1 = ufloat(params1[1], err1[1])

print(Fore.YELLOW + '\nFitparameter bei dT/dt = 2 K/min:' + Style.RESET_ALL)
print('a = ', '{:L}'.format(a1))
print('b = ', '{:L}'.format(b1))

# Linspace
Tt1 = np.linspace(200, 340, 1000)

# Eistellungen des Plots
fig = plt.figure()
ax = fig.add_axes([0.1, 0.13, 0.8, 0.83])
for axis in ['top','bottom','left','right']:
  ax.spines[axis].set_linewidth(0.3)

# Plot und Labels/ Legende
ax.plot(T1, I1, 'x', color='#31628c', label='Messwerte')
ax.plot(T1[6:27], I1[6:27], '^', markersize=4, color='r', label='Gefittete Messwerte')
ax.plot(Tt1, f(Tt1, *params1), '-', color='r', linewidth='0.75', label='Fit')
# ax.plot(Tt1, bgr(Tt1, *paramsbgr), '-', color='k', linewidth='0.75', label='Fit')
ax.set_xlabel(r'$T \:/\: \si{\kelvin}$')
ax.set_ylabel(r'$\ln(I)$')
leg1 = ax.legend(loc='best', fancybox=False, fontsize='small', edgecolor='None')
leg1.get_frame().set_linewidth(0.3)

# Einstellung der Achsen
ax.set_xlim(200, 340)
ax.set_ylim(10**(-2), 10)
ax.set_yscale('log')
ax.grid(linestyle='dotted', which="both")
ax.xaxis.set_minor_locator(AutoMinorLocator())
ax.yaxis.set_major_locator(LogLocator(base=10.0, numticks=15))
ax.tick_params(axis='both', direction='in')
ax.tick_params(which='major', direction='in', length=7, width=0.2)
ax.tick_params(which='minor', direction='in', length=4, width=0.2)

plt.savefig('plots/1.pdf')
plt.close()

Tf2, If2 = T2[21:36], I2[21:36]

params2, cov2 = curve_fit(f, Tf2, If2)
err2 = np.sqrt(np.diag(cov2))

# Parameter
a2 = ufloat(params2[0], err2[0])
b2 = ufloat(params2[1], err2[1])

print(Fore.YELLOW + '\nFitparameter bei dT/dt = 1.5 K/min:' + Style.RESET_ALL)
print('a = ', '{:L}'.format(a2))
print('b = ', '{:L}'.format(b2))

Tt2 = np.linspace(220, 340, 100)

fig = plt.figure()
ax2 = fig.add_axes([0.1, 0.13, 0.8, 0.83])
for axis in ['top','bottom','left','right']:
  ax2.spines[axis].set_linewidth(0.3)

# Plot und Labels/ Legende
ax2.plot(T2[14:], I2[14:], 'x', color='#36752b', label='Messwerte')
ax2.plot(T2[21:36], I2[21:36], '^', markersize=4, color='r', label='Gefittete Messwerte')
ax2.plot(Tt2, f(Tt2, *params2), '-', color='r', linewidth='0.75', label='Fit')
ax2.set_xlabel(r'$T \:/\: \si{\kelvin}$')
ax2.set_ylabel(r'$\ln(I)$')
leg2 = ax2.legend(loc='best', fancybox=False, fontsize='small', edgecolor='None')
leg2.get_frame().set_linewidth(0.3)

# Einstellung der Achsen
ax2.set_xlim(220, 340)
ax2.set_ylim(10**(-1), 2)
ax2.set_yscale('log')
ax2.grid(linestyle='dotted', which="both")
ax2.xaxis.set_minor_locator(AutoMinorLocator())
ax2.yaxis.set_major_locator(LogLocator(base=10.0, numticks=15))
ax2.tick_params(axis='both', direction='in')
ax2.tick_params(which='major', direction='in', length=7, width=0.2)
ax2.tick_params(which='minor', direction='in', length=4, width=0.2)

plt.savefig('plots/2.pdf')
plt.close()

#===============================================================================
# Aktivierungsenergien
W1 = -kB * a1
W2 = -kB * a2

print(Fore.YELLOW + '\nAktivierungsenergien [eV]:' + Style.RESET_ALL)
print('W1 = ', '{:L}'.format(W1 / con.e))
print('W2 = ', '{:L}'.format(W2 / con.e))

#===============================================================================
# Relaxationszeit
Tmax1 = [T1[27], T1[28]]
Tmax1 = ufloat(np.mean(Tmax1), np.std(Tmax1))

Tmax2 = T2[38]

print(Fore.YELLOW + '\nT_max [K]:' + Style.RESET_ALL)
print('T_max1 = ', '{:L}'.format(Tmax1))
print('T_max2 = ', Tmax2)

tau1 = H1 * a1 / (60 * Tmax1)
tau2 = H2 * a2 / (60 * Tmax2)

print(Fore.YELLOW + '\nRelaxationszeit tau_max = tau(T_max) [s]:' + Style.RESET_ALL)
print('tau_max1 = ', '{:L}'.format(tau1))
print('tau_max2 = ', '{:L}'.format(tau2))

tau01 = tau1 * unp.exp(-W1/(kB * Tmax1))
tau02 = tau2 * unp.exp(-W2/(kB * Tmax2))

print(Fore.YELLOW + '\nRelaxationszeit tau_0 [s]:' + Style.RESET_ALL)
print('tau_01 = ', '{:L}'.format(tau01))
print('tau_02 = ', '{:L}'.format(tau02))

################################################################################

# Methode 2
print('\n' + Back.BLUE + '\nBerechnung über Integration' + Style.RESET_ALL)

Ti1, Ii1 = T1[6:36], I1[6:36]
# Ti1, Ii1 = T1[6:27], I1[6:27]

#Methode 2 Integral
# IV1 = -integrate.simps(Ii1,Ti1) / (Ii1 * unp.nominal_values(H1)) #alte Position
print(Fore.BLUE + '\nIntegral 1, Parameter: ' + Style.RESET_ALL)
# print(IV1)

#Untergrundrate
FdatenT1,FdatenI1 = np.ones(7), np.ones(7)
FdatenT1[0:6], FdatenI1[0:6] = T1[1:7], I1[1:7]
FdatenT1[6], FdatenI1[6] = T1[35], I1[35]

Fparams1,Fcov1 = curve_fit(backgrd, FdatenT1, FdatenI1)
print('Untergrund\n', FdatenT1, '\n,', FdatenI1, '\na: ', Fparams1[0], '\nb: ', Fparams1[1])

# Ti1 -= Fparams1[1]
IIi1 = Ii1 - backgrd(Ti1,0.00001,0.039) ##!!!!! Es muss natürlich in I, nicht in T subtrahiert werden, II ist jetzt der bereinigte Wert
IIi1= np.abs(IIi1)
print('Ii1 = ', IIi1)

IV1 = -integrate.simps(IIi1,Ti1) / (IIi1 * unp.nominal_values(H1)) #neue Position
###
paramsi1, covi1 = curve_fit(better_fit, Ti1, IV1)
erri1 = np.sqrt(np.diag(covi1))

a3 = ufloat(paramsi1[0], erri1[0])
b3 = ufloat(paramsi1[1], erri1[1])

print('a = ', '{:L}'.format(a3))
print('b = ', '{:L}'.format(b3))

fig = plt.figure()
ax3 = fig.add_axes([0.1, 0.13, 0.8, 0.83])
for axis in ['top','bottom','left','right']:
  ax3.spines[axis].set_linewidth(0.3)

# Plot und Labels/ Legende
ax3.plot(T1, I1, 'x', color='#31628c', label='Messwerte')
ax3.plot(Ti1, IIi1, 'x', color='#363636', label='Bereinigt')
ax3.plot(Ti1, Ii1, '^', markersize=4, color='r', label='Integrierte Messwerte')
ax3.fill_between(Ti1, Ii1, backgrd(Ti1[:40], 0.00001, 0.039), color="red", alpha=0.3)
ax3.plot(T1[:40], backgrd(T1[:40], 0.00001, 0.039),label='Untergrund')
ax3.set_xlabel(r'$T \:/\: \si{\kelvin}$')
ax3.set_ylabel(r'$I(T) \:/\: \si{\pico\ampere}$')
leg3 = ax3.legend(loc='best', fancybox=False, fontsize='small', edgecolor='None')
leg3.get_frame().set_linewidth(0.3)

# Einstellung der Achsen
ax3.set_xlim(200, 340)
# ax3.set_ylim(10**(-2), 10)
# ax3.set_yscale('log')
ax3.grid(linestyle='dotted')
ax3.xaxis.set_minor_locator(AutoMinorLocator())
ax3.xaxis.set_minor_locator(AutoMinorLocator())
ax3.tick_params(axis='both', direction='in')
ax3.tick_params(which='major', direction='in', length=7, width=0.2)
ax3.tick_params(which='minor', direction='in', length=4, width=0.2)

plt.savefig('plots/3.pdf')
plt.close()

Ti2, Ii2 = T2[21:48], I2[21:48]

#Methode 2 Integral
#IV2 = -integrate.simps(Ii2,Ti2) / (Ii2 * unp.nominal_values(H2)) #alte Position

print(Fore.BLUE + '\nIntegral 2, Parameter:' + Style.RESET_ALL)
# print(IV2)

#Untergrundrate
FdatenT2, FdatenI2 = np.ones(7), np.ones(7)
FdatenT2[0:6], FdatenI2[0:6] = T2[1:7], I2[1:7]
FdatenT2[6], FdatenI2[6] = T2[46], I2[46]

Fparams2, Fcov2 = curve_fit(backgrd, FdatenT2, FdatenI2)
print('Untergrund\n', FdatenT2,'\n,', FdatenI2,'\na: ', Fparams2[0],'\nb: ', Fparams2[1])

# Ti2 -= Fparams2[1] #### Du ziehst hier von den x-Werten die Steigung der Untergrundfunktion ab, oder? Es soll der Funktionswert des Untergrunds vom Messwert subtrahiert werden.
IIi2 = Ii2 - backgrd(Ti2, 0.00001, 0.039) #Funktionswert des Untergrundfits abziehen
IIi2 = np.abs(IIi2) #Damit er mit ln kein Problem hat

IV2 = -integrate.simps(IIi2, Ti2) / (Ii2 * unp.nominal_values(H2)) #neue Position

paramsi2, covi2 = curve_fit(better_fit, Ti2, IV2)
erri2 = np.sqrt(np.diag(covi2))

a4 = ufloat(paramsi2[0], erri2[0])
b4 = ufloat(paramsi2[1], erri2[1])

print('a = ', '{:L}'.format(a4))
print('b = ', '{:L}'.format(b4))

fig = plt.figure()
ax4 = fig.add_axes([0.1, 0.13, 0.8, 0.83])
for axis in ['top','bottom','left','right']:
  ax4.spines[axis].set_linewidth(0.3)

# Plot und Labels/ Legende
ax4.plot(T2[14:], I2[14:], 'x', color='#36752b', label='Messwerte')
ax4.plot(Ti2[14:], IIi2[14:], 'x', color='#363636', label='Bereinigt')
ax4.plot(Ti2, Ii2, '^', markersize=4, color='r', label='Integrierte Messwerte')
ax4.fill_between(Ti2, Ii2, backgrd(Ti2, 0.00001, 0.039), color="red", alpha=0.3)
ax4.plot(T2[:55], backgrd(T2[:55], 0.00001, 0.039),label='Untergrund')
ax4.set_xlabel(r'$T \:/\: \si{\kelvin}$')
ax4.set_ylabel(r'$I(T) \:/\: \si{\pico\ampere}$')
leg4 = ax4.legend(loc='best', fancybox=False, fontsize='small', edgecolor='None')
leg4.get_frame().set_linewidth(0.3)

# Einstellung der Achsen
# ax4.set_yscale('log')
ax4.set_xlim(220, 340)
# ax4.set_ylim(0.1, 1.75)
ax4.grid(linestyle='dotted')
ax4.xaxis.set_minor_locator(AutoMinorLocator())
ax4.yaxis.set_minor_locator(AutoMinorLocator())
ax4.tick_params(axis='both', direction='in')
ax4.tick_params(which='major', direction='in', length=7, width=0.2)
ax4.tick_params(which='minor', direction='in', length=4, width=0.2)

plt.savefig('plots/4.pdf')
plt.close()

#===============================================================================
# Aktivierungsenergien
W12 = kB * a3
W22 = kB * a4

print(Fore.BLUE + '\nAktivierungsenergien [eV]:' + Style.RESET_ALL)
print('W1 = ', '{:L}'.format(W12 / con.e))
print('W2 = ', '{:L}'.format(W22 / con.e))

#===============================================================================
# Relaxationszeit
Tmax1 = [T1[27], T1[28]]
Tmax1 = ufloat(np.mean(Tmax1), np.std(Tmax1))

Tmax2 = T2[38]

tau1 = -H1 * a3 / (60 * Tmax1)
tau2 = -H2 * a4 / (60 * Tmax2)

print(Fore.BLUE + '\nRelaxationszeit tau_max = tau(T_max) [s]:' + Style.RESET_ALL)
print('tau_max1 = ', '{:L}'.format(tau1))
print('tau_max2 = ', '{:L}'.format(tau2))

tau012 = tau1 * unp.exp(-W1/(kB * Tmax1))
tau022 = tau2 * unp.exp(-W2/(kB * Tmax2))

print(Fore.BLUE + '\nRelaxationszeit tau_0 [s]:' + Style.RESET_ALL)
print('tau_01 = ', '{:L}'.format(tau012))
print('tau_02 = ', '{:L}'.format(tau022))

def tau(T, W, tau0):
    tau = unp.nominal_values(tau0) * np.exp(unp.nominal_values(W)/(kB*T))
    return tau

fig = plt.figure()
ax5 = fig.add_axes([0.1, 0.13, 0.8, 0.83])
for axis in ['top','bottom','left','right']:
  ax5.spines[axis].set_linewidth(0.3)

# Plot und Labels/ Legende
ax5.plot(T1, tau(T1, W1, tau01), 'x', label=r'Methode 1, $H_1$')
ax5.plot(T2, tau(T2, W2, tau02), 'v', markersize=4, label=r'Methode 1, $H_2$')
ax5.plot(T1, tau(T1, W12, tau012), '^', markersize=4, label=r'Methode 2, $H_1$')
ax5.plot(T2, tau(T2, W22, tau022), '1', label=r'Methode 2, $H_2$')
ax5.set_xlabel(r'$T \:/\: \si{\kelvin}$')
ax5.set_ylabel(r'$\ln[\tau(T)]')
leg5 = ax5.legend(loc='best', fancybox=False, fontsize='small', edgecolor='None')
leg5.get_frame().set_linewidth(0.3)

# Einstellung der Achsen
ax5.set_yscale('log')
# ax4.set_ylim(0.1, 1.75)
ax5.grid(linestyle='dotted', which='both')
ax5.xaxis.set_minor_locator(AutoMinorLocator())
ax5.yaxis.set_major_locator(LogLocator(base=10.0, numticks=15))
ax5.tick_params(axis='both', direction='in')
ax5.tick_params(which='major', direction='in', length=7, width=0.2)
ax5.tick_params(which='minor', direction='in', length=4, width=0.2)

plt.savefig('plots/5.pdf')
plt.close()

################################################################################

#Diskussion
print('\n' + Back.GREEN + '\nDiskussion' + Style.RESET_ALL)

tauTheo = ufloat(4, 2) * 10**(-14) #s
Wtheo = ufloat(0.66, 0.01) # eV

Dtau11 = (tau01 - tauTheo) / tauTheo * 100
Dtau21 = (tau02 - tauTheo) / tauTheo * 100

Dtau12 = (tau012 - tauTheo) / tauTheo * 100
Dtau22 = (tau022 - tauTheo) / tauTheo * 100

print(Fore.GREEN + '\nAbweichungen tau_0, Methode 1,  [%]:' + Style.RESET_ALL)
print('Delta tau1 = ', '{:L}'.format(Dtau11))
print('Delta tau2 = ', '{:L}'.format(Dtau21))

print(Fore.GREEN + '\nAbweichungen tau_0, Methode 2,  [%]:' + Style.RESET_ALL)
print('Delta tau1 = ', '{:L}'.format(Dtau12))
print('Delta tau2 = ', '{:L}'.format(Dtau22))

W1 /= con.e
W2 /= con.e
W12 /= con.e
W22 /= con.e

DW11 = (W1 - Wtheo) / Wtheo * 100
DW21 = (W2 - Wtheo) / Wtheo * 100
DW12 = (W12 - Wtheo) / Wtheo * 100
DW22 = (W22 - Wtheo) / Wtheo * 100

print(Fore.GREEN + '\nAbweichungen Aktivierungsenergien, Methode 1, [%]:' + Style.RESET_ALL)
print('Delta W1 = ', '{:L}'.format(DW11))
print('Delta W2 = ', '{:L}'.format(DW21))

print(Fore.GREEN + '\nAbweichungen Aktivierungsenergien, Methode 2, [%]:' + Style.RESET_ALL)
print('Delta W1 = ', '{:L}'.format(DW12))
print('Delta W2 = ', '{:L}'.format(DW22))


# 爬取糗事百科的页面内容——以文字部分为例\
# 此文件为个人尝试，出了一些问题，具体的展示以016-qiubai_spider.py为准
import requests
from lxml import etree

class QiubaiSpider(object):
    def __init__(self):
        self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"}
        self.temp_url = "https://www.qiushibaike.com/text/page/{}/"
        self.pre_url = "https://www.qiushibaike.com"

    def parse_url(self, url):
        response = requests.get(url, headers=self.headers)
        return response.content.decode()

    def get_content_list(self, html_str):
        html = etree.HTML(html_str)
        # content_list = html.xpath("//div[@class='content']/span/text()")
        # 通过xpath获取文章的链接，通过原文获取全部内容
        article_url_list = html.xpath("//a[@class='contentHerf']/@href")
        # print(article_url_list)
        return self.get_content(article_url_list)

    def save_content_list(self, content_list):
        with open("qiubai.txt", "a") as f:
            for content in content_list:
                f.write(content)
                f.write("\n\n")

    def get_content(self, url_list):
        content_list = []
        for url in url_list:
            article_url = self.pre_url + url
            print(article_url)
            html_str = self.parse_url(article_url)
            html = etree.HTML(html_str)
            content = html.xpath("//div[@class='content']/text()")
            content_list.append(content)
        print(content_list)
        return

    def run(self):
        num = 1
        while num <= 13:
            # 确定初始页面的url地址
            temp_url = self.temp_url.format(num)
            # 获取页面响应
            html_str = self.parse_url(temp_url)
            # print(html_str)
            # 提取所需内容
            content_list = self.get_content_list(html_str)
            print(content_list)
            # 将提取到的内容保存到本地
            # self.save_content_list(content_list)
            # 重复上方的循环，直到提取完全部页面
            num += 1

if __name__ == '__main__':
    qiubai_spider = QiubaiSpider()
    qiubai_spider.run()


"""
Incremental update pdf file
New in version 3
- add binary stream to object which have stream keyword
- Fuzz binary streams with mutation-based algorithms.
New in version 2
- attach multi-object to end of the pdf file. base on 'portion_of_rewrite_objects' in config.py
"""

__version__ = '0.3.2'
__author__ = 'Morteza'

import sys
import os

import math
import random
import datetime

import PyPDF2

from config import iu_config, learning_config
import pdf_object_preprocess as preprocess

class IncrementalUpdate(object):
    """
    Implement simple way to update a pdf file.
    """
    def __init__(self,
                 host_id=None,
                 object_file_path=iu_config['baseline_object_path'],
                 stream_directory_path=iu_config['stream_directory_path']):
        """

        :param host_id: Name of host file without postfix, e.g. host1_max, host2_min or host3_avg
        :param object_file_path: See iu_config, new_objects_path
        :param stream_directory_path: See iu_config, stream_directory_path
        """
        self.host_id = host_id

        self.object_file_path = object_file_path
        self.obj_list = preprocess.get_list_of_object(seq=preprocess.load_from_file(self.object_file_path),
                                                      is_sort=False)

        self.stream_directory_path = '../' + stream_directory_path
        self.stream_filename_list = os.listdir(self.stream_directory_path)

        # Creating new directory foreach time that program run and we want to generate new test data
        dt = datetime.datetime.now().strftime(self.host_id + '_date_%Y-%m-%d_%H-%M-%S')
        self.storage_dir_name = iu_config['new_pdfs_directory'] + self.host_id + '/' + dt + '/'
        if not os.path.exists(self.storage_dir_name):
            os.makedirs(self.storage_dir_name)
            print('new storage directory build.')

        self.obj_getter = self.obj_generator(self.obj_list)

    def read_pdf_file(self):
        with open(iu_config['raw_host_directory'] + self.host_id + '.pdf', 'rb') as f:
            data = f.read()
        return data

    def write_pdf_file(self, name_description, data):
        with open(self.storage_dir_name + self.host_id + name_description + '.pdf', 'wb') as f:
            f.write(data)

    def obj_generator(self, obj_list):
        i = 0
        while True:
            yield obj_list[i]
            i += 1
            if i >= len(obj_list):
                i = 0

    def get_one_object(self, getting_object_policy=iu_config['getting_object_policy']):
        """
        Provide one pdf data object whether an existing object in corpus or
        an online new generated object from learnt model
        this function is not complete yet!
        when complete it is expected to get object coming from deep model (learn_and_fuzz) algorithm
        """
        obj = ''
        if getting_object_policy == 'sequential':
            # For now using object getter generator
            obj = next(self.obj_getter)
            print(obj)
            # x = input()
        elif getting_object_policy == 'random':
            # For now randomly choose as object from given obj_list
            random_object_index = random.randint(0, len(self.obj_list) - 1)
            obj = self.obj_list[random_object_index]

        # Recently added (1397-01-16)
        # Check if selected object contain keyword 'stream' then add a random/ and fuzzed binary stream to
        # selected object.
        stream_index = obj.find('stream')
        obj = bytes(obj, encoding='ascii')
        if stream_index != -1:
            random_stream_index = random.randint(0, len(self.stream_filename_list)-1)
            with open(self.stream_directory_path+self.stream_filename_list[random_stream_index], mode='rb') as f:
                binary_stream = f.read()
            # Fuzz binary stream separately.
            # We use generation fuzzing for pdf data objects and mutation fuzzing for pdf binary streams that exist
            # within our pdf data objects
            # binary_stream = self.fuzz_binary_stream(binary_stream)
            obj = obj[:stream_index+7] + binary_stream + bytes('\nendstream\n', encoding='ascii') + obj[stream_index+7:]
            print('binary_stream added')
            # print(obj)

        return obj

    def get_last_object_id(self):
        with open(iu_config['raw_host_directory'] + self.host_id + '.pdf', 'br') as f:
            read_pdf = PyPDF2.PdfFileReader(f)
        last_object_id = read_pdf.trailer['/Size'] - 1  # size xref  - 1
        return last_object_id

    def incremental_update(self, sequential_number=0):
        """
        Shape the incremental update behaviour
        :param sequential_number: Number appear at the end of new pdf file name as identity number (e.g. 1,2 and 3)
        :return: Nothing
        """
        data = self.read_pdf_file()
        last_object_id = str(self.get_last_object_id())
        rewrite_object_content = self.get_one_object()  # Updated. Now include stream objects.

        if iu_config['single_object_update']:  # Just one object rewrite with new content
            if iu_config['update_policy'] == 'random':
                # Random choose between [2,:] because we don't want modify first object at any condition.
                rewrite_object_id = str(random.randint(2, int(last_object_id)))
            elif iu_config['update_policy'] == 'bottom_up':
                rewrite_object_id = last_object_id
            else:
                rewrite_object_id = last_object_id
            data = self.attach_new_object(data=data,
                                          rewrite_object_id=rewrite_object_id,
                                          rewrite_object_content=rewrite_object_content)
            # Set name for new pdf files like:
            # host1_sou_85_6_20180307_114117
            # dt = datetime.datetime.now().strftime('_%Y%m%d_%H%M%S')
            name_description = '_sou_' + str(sequential_number).zfill(4) + '_obj-' + str(rewrite_object_id).zfill(3)
            self.write_pdf_file(name_description, data)
            print('save new pdf file successfully')
        else:  # Multiple object rewrite with new content (base on 'portion_of_rewrite_objects') in config file
            number_of_rewrite_objects = math.ceil(iu_config['portion_of_rewrite_objects'] * int(last_object_id))
            # print(host_id, number_of_of_rewrite_objects)
            rewrite_object_id = last_object_id
            rewrite_object_ids = ''
            for i in range(int(number_of_rewrite_objects)):
                rewrite_object_content = self.get_one_object()
                if iu_config['update_policy'] == 'random':
                    # Random choose between [2,:] because we don't want modify first object at any condition.
                    rewrite_object_id = str(random.randint(2, int(last_object_id)))
                elif iu_config['update_policy'] == 'bottom_up':
                    rewrite_object_id = int(last_object_id) - i
                elif iu_config['update_policy'] == 'top-down':
                    # Not implement yet.
                    pass
                rewrite_object_ids += '-' + str(rewrite_object_id).zfill(3)
                data = self.attach_new_object(data=data,
                                              rewrite_object_id=rewrite_object_id,
                                              rewrite_object_content=rewrite_object_content)
            # Set name for new pdf files like:
            # host1_sou_85_6_20180307_114117
            # dt = datetime.datetime.now().strftime('_%Y%m%d_%H%M%S')
            name_description = '_mou_' + str(sequential_number).zfill(4) + '_objs' + str(rewrite_object_ids)
            self.write_pdf_file(name_description, data)
            print('save new pdf file successfully')

    def attach_new_object(self,
                          data=None,
                          rewrite_object_id=None,
                          rewrite_object_content=None):
        """
        Incremental update pdf file
        :param data:
        :param rewrite_object_id: pdf object id w want to update
        :param rewrite_object_content: new content fo pdf object
        :return:
        """

        # Find last trailer in a pdf file
        trailer_index = 0
        while data.find(b'trailer', trailer_index + 7) != -1:
            trailer_index = data.find(b'trailer', trailer_index + 7)
        print('trailer_index', trailer_index)

        trailer_index_dic_endof = data.find(b'>>', trailer_index)
        print('trailer_index_dic_endof', trailer_index_dic_endof)

        trailer_content = data[trailer_index: trailer_index_dic_endof + 2]
        print('trailer_content', trailer_content)

        # find last startxref offset in a pdf file
        startxref_index = trailer_index
        while data.find(b'startxref', startxref_index + 9) != -1:
            startxref_index = data.find(b'startxref', startxref_index + 9)
        # print('index ===', index_startxref)
        index_eof = data.find(b'%%EOF', startxref_index)
        # print('index 2===', index_eof)
        if data[startxref_index + 9] == b'\n' or b'\r':
            # print('yes', data[index_startxref+9])
            startxref_index += 10
        if data[index_eof - 1] == b'\n' or b'\r':
            index_eof -= 1
        startxref_offset = int(data[startxref_index: index_eof])
        print('startxref_offset', startxref_offset)

        # print(type(trailer_content))
        # remove all /Prev 1234 from trailer if exist
        # trailer_content = trailer_content.replace(b'/Prev', b'')
        # trailer_content = re.sub(r'/Prev \d+', b'', str(trailer_content))
        index_prev = trailer_content.find(b'/Prev')
        if index_prev != -1:
            index_curr = 0
            # print('##', trailer_content[index_prev+5], index_prev)
            # check whether a byte is ascii number or space
            eliminate_content = trailer_content[index_prev+5+index_curr]
            print('eliminate content', eliminate_content)
            while (48 <= eliminate_content <= 57) or (eliminate_content == 32):
                print('###', trailer_content[index_prev+5+index_curr])
                index_curr += 1
                eliminate_content = trailer_content[index_prev + 5 + index_curr]

            trailer_content = trailer_content[:index_prev] + trailer_content[index_prev+5+index_curr:]

        trailer_content_new = trailer_content[:-2] + b'   /Prev ' \
                              + bytes(str(startxref_offset), 'ascii') + b' \n>>'
        print('trailer_content_new', trailer_content_new)

        print('len_rewrite_object_content', len(rewrite_object_content))
        startxref_offset_new = len(data) + 1 + len(rewrite_object_id) + 3 + len(rewrite_object_content)  # if we attach just one obj
        print('startxref_offset_new', startxref_offset_new)

        attach_content = bytes(str(rewrite_object_id + ' 0 '), encoding='ascii')\
                         + rewrite_object_content\
                         + bytes('\nxref\n0 1\n0000000000 65535 f\n'
                                 + rewrite_object_id
                                 + ' 1\n'
                                 + str(len(data)).zfill(10)
                                 + ' 00000 n\n', encoding='ascii')
        attach_content = attach_content \
                         + trailer_content_new \
                         + b'\nstartxref\n'\
                         + bytes(str(startxref_offset_new), encoding='ascii')\
                         + b'\n%%EOF\n'

        # print('attach_content\n', attach_content)
        new_pdf_file = data + attach_content
        return new_pdf_file

    def fuzz_binary_stream(self, binary_stream):
        """
        Fuzzing fuzzing_policy for binary stream fuzz testing.
        Simple basic fuzzing fuzzing_policy:
        Reverse 1% of all bytes in stream randomly. Below code do this
        :param binary_stream:
        :return: fuzzed_binary_stream
        """
        if iu_config['stream_fuzzing_policy'] == 'basic_random':
            for i in range(math.ceil(len(binary_stream)/100)):
                # Choose one byte randomly
                byte_to_reverse_index = random.randint(0, len(binary_stream)-1)
                one_byte = binary_stream[byte_to_reverse_index]

                # Convert byte int representation to byte binary string representation (Step 2)
                eight_bit = "{0:b}".format(one_byte)
                # print('eight_bit of one_bye', eight_bit)

                # Reverse eight_bit e.g. 0110000 ==> 1001111
                eight_bit_reverse_str = ''
                for j in range(len(eight_bit)):
                    if eight_bit[j] == '1':
                        eight_bit_reverse_str += '0'
                    else:
                        eight_bit_reverse_str += '1'
                # print('eight_bit_reverse_str', eight_bit_reverse_str)

                # Back eight_bit_reverse_str to int representation (Reverse of step 2)
                eight_bit_reverse_int = int(eight_bit_reverse_str, 2)

                # Convert eight_bit_reverse_int to byte
                one_byte_reverse = \
                    eight_bit_reverse_int.to_bytes(1, 'little')  # 1 is for one byte as length, e.g 15 => 0x0f

                # Substitute one_byte with one_byte_reverse in the  input binary_stream
                binary_stream = binary_stream[0:byte_to_reverse_index]\
                                + one_byte_reverse \
                                + binary_stream[byte_to_reverse_index+1:]
        elif iu_config['stream_fuzzing_policy'] == 'other':
            # No other policy implement yet:)
            pass
        return binary_stream

def main(argv):
    host_id = 'host3_avg'
    amount_of_testdata = 1000
    iu = IncrementalUpdate(host_id=host_id)
    for i in range(amount_of_testdata):
        iu.incremental_update(sequential_number=i)

    print('%s test data was generate' % amount_of_testdata)

if __name__ == '__main__':
    main(sys.argv)


from django.test import TestCase

from .models import Campaign, Contributor
from .factories import CampaignFactory, ContributorFactory, CommitteeFactory, ContributionFactory

class CampaignTests(TestCase):
    def setUp(self):
        self.campaign = Campaign.objects.create(
            id="abc123", name="Bob 4 Prez", office="P", party="DEM")

    def test_create_campaigns(self):
        campaign = Campaign.objects.get(pk=self.campaign.id)

        self.assertEqual(campaign.name, "Bob 4 Prez")

    def test_party_abbreviation(self):
        self.assertEqual(self.campaign.party_abbreviation(), "D")

    def test_full_party(self):
        self.assertEqual(self.campaign.party_abbreviation(), "D")

    def test_search_empty(self):
        campaigns = Campaign.search("Nobody")
        assert not campaigns

    def test_search_by_name(self):
        campaigns = Campaign.search("Bob")

        self.assertEqual(campaigns[0], self.campaign)

    def test_search_by_id(self):
        campaigns = Campaign.search(self.campaign.id)

        self.assertEqual(campaigns[0], self.campaign)

    def test_similar_campaigns(self):
        alice = CampaignFactory.create(name="Alice")
        alex = CampaignFactory.create(name="Alex")
        bob = CampaignFactory.create(name="Bob")

        alice_committee = CommitteeFactory.create(
            name="alice 4 prez", campaign=alice)
        alex_committee = CommitteeFactory.create(
            name="alex 4 prez", campaign=alex)
        bob_committee = CommitteeFactory.create(
            name="bob 4 prez", campaign=bob)

        a_contributor_1 = ContributorFactory.create()
        a_contributor_2 = ContributorFactory.create()
        b_contributor_1 = ContributorFactory.create()
        b_contributor_2 = ContributorFactory.create()
        anti_alex_contributor = ContributorFactory.create()

        ContributionFactory.create(
            committee=alice_committee, contributor=a_contributor_1)

        ContributionFactory.create(
            committee=alex_committee, contributor=a_contributor_1)

        ContributionFactory.create(
            committee=alice_committee, contributor=a_contributor_2)

        ContributionFactory.create(
            committee=alex_committee, contributor=a_contributor_2)

        ContributionFactory.create(
            committee=bob_committee, contributor=b_contributor_1)

        ContributionFactory.create(
            committee=bob_committee, contributor=b_contributor_2)

        ContributionFactory.create(
            committee=alice_committee, contributor=anti_alex_contributor)

        ContributionFactory.create(
            committee=bob_committee, contributor=anti_alex_contributor)

        alice_committee.refresh_from_db()
        similar_campaigns = alice.similar_campaigns()

        self.assertEqual(similar_campaigns[0].name, alex.name)
        self.assertEqual(similar_campaigns[1].name, bob.name)

        self.assertEqual(similar_campaigns[0].contributor_count, 2)
        self.assertEqual(similar_campaigns[1].contributor_count, 1)

class ContributersTest(TestCase):
    def setUp(self):
        self.nickname = Contributor.objects.create(
            contributor_name='SMITH, JUDY MD.',
            contributor_city='Manchester',
            contributor_state='NH',
            contributor_zip='03104',
            contributor_employer='Self',
            contributor_occupation='PHYSICIAN')

    def test_nickname_matches_given(self):
        givenname = Contributor(
            contributor_name='SMITH MD., JUDITH R',
            contributor_city='Manchester',
            contributor_state='NH',
            contributor_zip='03104',
            contributor_employer='Self',
            contributor_occupation='PHYSICIAN')
        self.assertEqual(self.nickname.id, Contributor.search(givenname)[0].id)

    def test_nickname_matches_short(self):
        short = Contributor(
            contributor_name='SMITH, JUDY',
            contributor_city='Manchester',
            contributor_state='NH',
            contributor_zip='03104',
            contributor_employer='Self',
            contributor_occupation='PHYSICIAN')
        self.assertEqual(self.nickname.id, Contributor.search(short)[0].id)

    def test_nickname_matches_nickname(self):
        nickname2 = Contributor(
            contributor_name='SMITH, JUDY MD.',
            contributor_city='Manchester',
            contributor_state='NH',
            contributor_zip='03104',
            contributor_employer='Self',
            contributor_occupation='PHYSICIAN')
        self.assertEqual(self.nickname.id, Contributor.search(nickname2)[0].id)


import os
from torchvision.datasets import VisionDataset
from PIL import Image
import datasets.util.json as json

# label/ tag 구조가 small 데이터셋과 같음
from .fashion_product_small import (keys, productinfo, _read_styles_csv)

class FashionProduct(VisionDataset):
    """fashion-product dataset
    from https://www.kaggle.com/paramaggarwal/fashion-product-images-dataset
    Context:
        The growing e-commerce industry presents us with a large dataset waiting to be scraped and researched upon.
        In addition to professionally shot high resolution product images,
        we also have multiple label attributes describing the product
        which was manually entered while cataloging.
        To add to this, we also have descriptive text that comments on the product characteristics.

    Content:
        Each product is identified by an ID like 42431. You will find a map to all the products in styles.csv.
        From here, you can fetch the image for this product from images/42431.jpg and
        the complete metadata from styles/42431.json.

        To get started easily, we also have exposed some of the key product categories and
        it's display name in styles.csv.

        If this dataset is too large, you can start with a smaller (280MB) version
        here: https://www.kaggle.com/paramaggarwal/fashion-product-images-small

    Inspiration:
        So what can you try building? Here are some suggestions:

        Start with an image classifier. Use the masterCategory column from styles.csv and
        train a convolutional neural network.
        The same can be achieved via NLP.
        Extract the product descriptions from styles/42431.json and
        then run a classifier to get the masterCategory.
        Try adding more sophisticated classification by predicting the other category labels in styles.csv
        Transfer Learning is your friend and use it wisely.
        You can even take things much further from here:

        Is it possible to build a GAN that takes a category as input and outputs an image?
        Auto-encode the image attributes to be able to make a visual search engine
        that converts the image into a small encoding which is sent to the server to perform visual search?
        Visual similarity search? Given an image, suggest other similar images.
    """

    root = '/data/public/rw/datasets/fashion/fashion-product'
    image_path = f'{root}/images'
    json_path = f'{root}/styles'

    def __init__(self, train=None, transforms=None, transform=None, target_transform=None):
        super().__init__(root=self.root, transforms=transforms,
                         transform=transform, target_transform=target_transform)
        if train is not None:
            raise NotImplementedError('splits not available')
        self.products = _read_styles_csv(self.root)

    def __len__(self):
        return len(self.products)

    def __getitem__(self, index):
        label = self.products[index]
        file = self._imagepath(label.id)
        # assert os.path.exists(file)
        meta = self._jsontpath(label.id)
        img = Image.open(file)
        meta = json.load(open(meta, 'r'))

        if self.transforms:
            img, (label, meta) = self.transforms(img, meta)

        return img, (label, meta)

    def _imagepath(self, id):
        return f'{self.image_path}/{id}.jpg'

    def _jsontpath(self, id):
        return f'{self.json_path}/{id}.json'

def _read_images_csv(csv):
    with open(csv, 'r') as f:
        f.readline()  # pass first header
        for line in f.readlines():
            img, url = line.strip().split(',')
            yield img, url


# -----------------------------------------------------------------------------
# BSD 3-Clause License
#
# Copyright (c) 2021, Science and Technology Facilities Council.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# -----------------------------------------------------------------------------
# Author: Joerg Henrichs, Bureau of Meteorology

'''This module tests the Signature class.'''

from __future__ import absolute_import
import pytest

from psyclone.core import Signature
from psyclone.errors import InternalError

def test_signature():
    '''Test the Signature class.
    '''

    assert str(Signature("a")) == "a"
    assert str(Signature(("a",))) == "a"
    assert str(Signature(("a", "b", "c"))) == "a%b%c"
    assert repr(Signature("a")) == "Signature(a)"
    assert repr(Signature(("a",))) == "Signature(a)"
    assert repr(Signature(("a", "b", "c"))) == "Signature(a%b%c)"
    assert Signature("a") != "a"

def test_signature_errors():
    '''Tests error handling of Signature class.
    '''
    with pytest.raises(InternalError) as err:
        _ = Signature(1)

    assert "Got unexpected type 'int' in Signature" in str(err.value)

def test_signature_dict():
    '''Test that Signature instances work as expected as dictionary keys.
    '''
    sig1 = Signature("a")
    sig2 = Signature("a")
    assert sig1 is not sig2

    # Make sure that different instances representing the same signature
    # will have the same hash:
    test_dict = {}
    test_dict[sig1] = "a"
    assert test_dict[sig2] == "a"

    sig3 = Signature(("a", "b"))
    test_dict[sig3] = "ab"
    sig4 = Signature(("a", "c"))
    test_dict[sig4] = "ac"

    assert len(test_dict) == 3

def test_signature_sort():
    '''Test that signatures can be sorted.'''

    sig_list = [Signature("c"), Signature("a"), Signature("b"),
                Signature(("b", "a")),
                Signature(("a", "c")), Signature(("a", "b"))]

    assert str(sig_list) == "[Signature(c), Signature(a), Signature(b), " \
                            "Signature(b%a), Signature(a%c), Signature(a%b)]"

    sig_list.sort()
    assert str(sig_list) == "[Signature(a), Signature(a%b), Signature(a%c), " \
                            "Signature(b), Signature(b%a), Signature(c)]"


import detectspanish as spanish
import argparse

universo = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890 !?.'

def cifrado(mensaje, clave, universo):

  key = len(clave)

  translated = ''

  for symbol in mensaje:
    # Note: Only symbols in the `SYMBOLS` string can be encrypted/decrypted.
    if symbol in universo:
        symbolIndex = universo.find(symbol)
        translatedIndex = symbolIndex + key

        if translatedIndex >= len(universo):
            translatedIndex = translatedIndex - len(universo)
        elif translatedIndex < 0:
            translatedIndex = translatedIndex + len(universo)

        translated = translated + universo[translatedIndex]
    else:
        # Append the symbol without encrypting/decrypting:
        translated = translated + symbol

  return(translated)

def descifrado(mensaje, clave, universo):

 key = len(clave)

 translated = ''

 for symbol in mensaje:
    # Note: Only symbols in the `SYMBOLS` string can be encrypted/decrypted.
    if symbol in universo:
        symbolIndex = universo.find(symbol)
        translatedIndex = symbolIndex - key

        if translatedIndex >= len(universo):
            translatedIndex = translatedIndex - len(universo)
        elif translatedIndex < 0:
            translatedIndex = translatedIndex + len(universo)

        translated = translated + universo[translatedIndex]
    else:
        # Append the symbol without encrypting/decrypting:
        translated = translated + symbol

 return(translated)

def crackeo(mensaje, universo):
  print('Hacking...')
  for key in range(len(universo)):

      translated = ''

      for simbolo in mensaje:
          if simbolo in universo:
              symbolIndex = universo.find(simbolo)
              translatedIndex = symbolIndex - key

              if translatedIndex < 0:
                  translatedIndex = translatedIndex + len(universo)

              translated = translated + universo[translatedIndex]

          else:
              translated = translated + simbolo

      if spanish.isSpanish(translated):
        # Ask user if this is the correct decryption.
        print()
        print('Possible encryption hack:')
        print('Key %s: %s' % (key, translated[:100]))
        print()
        print('Enter D if done, anything else to continue hacking:')
        response = input('> ')

        if response.strip().upper().startswith('D'):
          return translated

      #print('Key #%s: %s' % (key, translated))

if __name__ == "__main__":
  description="Este script tiene la finalidad de cifrar   descifrar o crackear mensajes"
  parser = argparse.ArgumentParser(description="E12", epilog=description, formatter_class=argparse.RawDescriptionHelpFormatter)
  parser.add_argument('-modo', choices=['cifrado', 'descifrado', 'crackeo'] , help='Es necesario poner un modo para saber la finalidad para la cual se quiere utilizar el script.', required=True)
  parser.add_argument('-mensaje', type=str, help='Se debe poner un mensaje el cual se quiera cifrar o descifrar.',default = "Hola mundo mundial")
  parser.add_argument('-clave', type=str, help='Se utiliza para saber a base de cual palabra se quiere cifrar o descifrar un mensaje', default="mundo")
  params = parser.parse_args()
  mensaje = (params.mensaje)
  clave = (params.clave)
  modo = (params.modo)

if modo == 'cifrado':
  mensajecif = cifrado(mensaje, clave, universo)
  print(mensajecif)
elif modo == 'descifrado':
  mensajedes = descifrado(mensaje, clave, universo)
  print(mensajedes)
elif modo == 'crackeo':
  mensajecrack = crackeo(mensaje, universo)
  print(mensajecrack)
else:
  print("No se eligió una opción correcta")


from flask import Flask, jsonify, render_template
from flask_restful import Api
from flask_restful_swagger import swagger

from routes import admin, locks, users

from utils.exceptions import AppException

app = Flask(__name__)
api = swagger.docs(Api(app), apiVersion="0.1")

api.add_resource(admin.Locks, "/api/v1/admin/locks")
api.add_resource(locks.LockStatus, "/api/v1/locks/<lock_id>/lockStatus")
api.add_resource(locks.UserLock, "/api/v1/userLocks")
api.add_resource(users.User, "/api/v1/user")

@app.errorhandler(AppException)
def handle_error(error):
    response = jsonify(error.to_dict())
    response.status_code = error.status_code
    return response

# Return validation errors as JSON

@app.errorhandler(422)
@app.errorhandler(400)
def handle_error(err):
    headers = err.data.get("headers", None)
    messages = err.data.get("messages", ["Invalid request."])
    if headers:
        return jsonify({"errors": messages}), err.code, headers
    else:
        return jsonify({"errors": messages}), err.code

@app.route('/')
def index():
    return render_template('/index.html')


import pandas as pd
import numpy as np
import copy, time
import warnings
warnings.filterwarnings('ignore')
from IPython.display import display
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelEncoder

# 檢查 DataFrame 空缺值的狀態
def na_check(df_data):
    data_na = (df_data.isnull().sum() / len(df_data)) * 100
    data_na = data_na.drop(data_na[data_na == 0].index).sort_values(ascending=False)
    missing_data = pd.DataFrame({'Missing Ratio' :data_na})
    display(missing_data.head(10))

########## load data start
data_path = '../csv/3rd-ml100marathon-midterm/'
df_train = pd.read_csv(data_path + 'train_data.csv')
df_test = pd.read_csv(data_path + 'test_features.csv')

########## load data start end

########## 前處理理 (Processing) start
#薪水
var_salary = ['salary']

#獎金
var_bonus = ['bonus']

#業務報銷
var_business_expenses_group = ['long_term_incentive','deferred_income','loan_advances'
        ,'deferral_payments','other','expenses']

#董事费
var_director_fees = ['director_fees']

#總支出
var_total_payments = ['total_payments']

#股權
var_10_11_12 = ['exercised_stock_options','restricted_stock','restricted_stock_deferred']

#股權 total
var_total_stock_value = ['total_stock_value']

#email回覆
var_email_count_group = ['to_messages','from_poi_to_this_person','from_messages','from_this_person_to_poi','shared_receipt_with_poi']

#train_Y = df_train['poi']
train_Y = df_train['poi'].map(lambda x:1 if x==True else 0)

ids_train = df_train['name']
ids = df_test['name']
df_train = df_train.drop(['name', 'poi'] , axis=1)
df_test = df_test.drop(['name'] , axis=1)
df = pd.concat([df_train,df_test])
var_df_columns = df.columns
var_df_columns

var_df_dtypes = df.dtypes

df_cp = df.copy()

df_cp = df_cp.drop(['email_address'] , axis=1)

## 股權計算 start
#df_cp = df_cp.drop(var_10_11_12 , axis=1)
df_cp[var_10_11_12] = df[var_10_11_12].fillna(0)

df_cp[var_total_stock_value] = df[var_total_stock_value].fillna(0)

## 股權計算 end

## 公司支出計算 strat
df_cp[var_salary] = df_cp[var_salary].fillna(df_cp[var_salary].median())
df_cp[var_bonus] = df_cp[var_bonus].fillna(df_cp[var_bonus].median())
df_cp[var_director_fees] = df_cp[var_director_fees].fillna(0)

df_cp.dtypes
df_cp['total_payments_new']=np.zeros([146])

df_cp['total_payments_new']= df_cp['total_payments_new'] + df_cp['salary']
df_cp['total_payments_new'] = df_cp['total_payments_new'] + df_cp['bonus']

for col in var_business_expenses_group:
    df_cp[col] = df_cp[col].fillna(0)
    df_cp['total_payments_new'] = df_cp['total_payments_new'] + df_cp[col]

df_cp['total_payments_new'] = df_cp['total_payments_new'] + df_cp['director_fees']

df_cp = df_cp.drop('total_payments' , axis=1)

## 公司支出計算 end

## email 關係計算 start
for col_other in var_email_count_group:
    df_cp[col_other] = df_cp[col_other].fillna(df_cp[col_other].mean())

## email 關係計算 end

na_check(df_cp)

########## 前處理理 (Processing) end

# 將資料最大最小化
df_cp = MinMaxScaler().fit_transform(df_cp)

# 將前述轉換完畢資料 df , 重新切成 train_X, test_X
train_num = train_Y.shape[0]
train_X = df_cp[:train_num]
test_X = df_cp[train_num:]

# 使用三種模型 : 邏輯斯迴歸 / 梯度提升機 / 隨機森林, 參數使用 Random Search 尋找
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
lr = LogisticRegression(tol=0.001, penalty='l2', fit_intercept=True, C=1.0)
gdbt = GradientBoostingClassifier(tol=100, subsample=0.75, n_estimators=250, max_features=19,
                                  max_depth=3, learning_rate=0.03)
rf = RandomForestClassifier(n_estimators=100, min_samples_split=2, min_samples_leaf=1,
                            max_features='sqrt', max_depth=6, bootstrap=True)

# 線性迴歸預測檔 (結果有部分隨機, 請以 Kaggle 計算的得分為準, 以下模型同理)
lr.fit(train_X, train_Y)
lr_pred = lr.predict_proba(test_X)[:,1]
sub = pd.DataFrame({'name': ids, 'poi': lr_pred})

#sub['Survived'] = sub['Survived'].map(lambda x:1 if x>0.5 else 0)
sub.to_csv('test1023_lr.csv', index=False)

# 梯度提升機預測檔
gdbt.fit(train_X, train_Y)
gdbt_pred = gdbt.predict_proba(test_X)[:,1]
sub = pd.DataFrame({'name': ids, 'poi': gdbt_pred})
#sub['Survived'] = sub['Survived'].map(lambda x:1 if x>0.5 else 0)
sub.to_csv('test_gdbt1023--.csv', index=False)

# 隨機森林預測檔
rf.fit(train_X, train_Y)
rf_pred = rf.predict_proba(test_X)[:,1]
sub = pd.DataFrame({'name': ids, 'poi': rf_pred})
#sub['Survived'] = sub['Survived'].map(lambda x:1 if x>0.5 else 0)
sub.to_csv('test1023_rf.csv', index=False)

# 混合泛化預測檔
"""
Your Code Here
"""
blending_pred = lr_pred*0.3  + gdbt_pred*0.4 + rf_pred*0.3
sub = pd.DataFrame({'name': ids, 'poi': blending_pred})

sub.to_csv('test1023_blending.csv', index=False)


#coding=utf-8
"""
添加headers到你的HTTP请求。
"""

import requests
import time,os,sys,random
from lxml import etree
from threading import Thread
sys.path.append("..")
from func import *

headers = {
    'User-Agent': 'chrome' + str(random.randint(38,59)),
    'referer': 'https://www.douban.com',
    "Cookie":'ll="118282"; bid=_GpzEN0Zw7k; _ga=GA1.2.898129337.1495625264; dbcl2="4165793:mLMGUyh8k+Q"; ct=y'
}

url = 'https://movie.douban.com/top250'

def fetch_page(url):
    response = requests.get(url, headers = headers)
    text = response.text
    return text

def parse(url):
    fetch_list = []
    result = []
    xpath_pages = '//div[@class="paginator"]/a'
    xpath_movie = '//li/div[@class="item"]'
    xpath_title = './/span[@class="title"]'

    page = fetch_page(url)
    tree = etree.HTML(page)
    pages = tree.xpath(xpath_pages)

    for element_movie in tree.xpath(xpath_movie):
        result.append(element_movie)

    for p in pages:
        fetch_list.append(url + p.get('href'))

    print ( fetch_list )

    def fetch_content(url):
        page = fetch_page(url)
        tree = etree.HTML(page)
        for element_movie in tree.xpath(xpath_movie):
            result.append(element_movie)

    threads_run(fetch_content, fetch_list)
    # threadspool_run(fetch_content, fetch_list)

    for i, movie in enumerate(result, 1):
        title = movie.find(xpath_title).text
        print(i, title)

def main():
    start = time.time()
    parse(url)
    end = time.time()
    print ("time: %.5fs" % (end - start))

if __name__ == '__main__':
    main()


import argparse
import yaml
parser = argparse.ArgumentParser(description='parse train config')
parser.add_argument('-c', '--config', required=True)
args = parser.parse_args()
config = yaml.load(open(args.config, 'r'), Loader=yaml.FullLoader)
for c in config:
    print(c, config[c])
cuda_device = config['cuda_device']

import os
import numpy as np
import torch
import torchvision
from .VideoClsModel import TwoView_Classification_Model
from .functions import *
from .dataset import TwoViewDataset
from .transforms import *
from .utils import *
os.environ["CUDA_VISIBLE_DEVICES"] =cuda_device

# hyper-params
num_classes = config['num_classes']
batch_size = config['batch_size']
learning_rate = config['learning_rate']
total_epoch = config['total_epoch']
steps = config['steps']
seed = config['seed']
num_workers = config['num_workers']
momentum = config['momentum']
weight_decay = config['weight_decay']

view_dropout = config['view_dropout']
drop_rate = config['drop_rate']
add_bias= config['add_bias']
clip_size = config['clip_size']
train_tag = config['train_tag']

description = 'AS_2views_{}'.format(train_tag)
print('description： {}'.format(description))
save_dir = 'checkpoints/' + description

train_class_weight = config['train_class_weight']
print('train class weight', train_class_weight)
valid_class_weight = config['valid_class_weight']
print('valid class weight', valid_class_weight)

#define
fix_randomness(seed)
best_acc_path = ''
best_loss_path = ''
best_acc = 0
best_acc_epoch = -1
best_loss_epoch = -1
best_loss = 100

train_transform = torchvision.transforms.Compose([
    VideoArrayToPIL(convert_gray=True),
    GroupMultiScaleCrop(224,[1,0.9,0.9,0.9]),
    GroupRotation(),
    Stack(),
    ToTorchFormatTensor(div=True),
])
valid_transform = torchvision.transforms.Compose([
    VideoArrayToPIL(convert_gray=True),
    Stack(),
    ToTorchFormatTensor(div=True),
])

train_path = config['train_path']
valid_path = config['valid_path']

train_dataset = TwoViewDataset(train_path, train_transform, view_dropout=view_dropout, drop_rate=drop_rate, clip_size=clip_size, random_clip=True)
train_loader = torch.utils.data.DataLoader(train_dataset,
                                           batch_size=batch_size,
                                           shuffle=True,
                                           num_workers=num_workers,
                                           pin_memory=True,
                                           drop_last=True)

valid_dataset = TwoViewDataset(valid_path, valid_transform, clip_size=clip_size)
valid_loader = torch.utils.data.DataLoader(valid_dataset,
                                        batch_size=batch_size,
                                        shuffle=False,
                                        num_workers=num_workers,
                                        pin_memory=True,
                                        drop_last=False)

print('data size: training {}, validation {}.'.format(len(train_dataset), len(valid_dataset)))

model = TwoView_Classification_Model(class_num=num_classes, add_bias=add_bias)
model = torch.nn.DataParallel(model)
model = model.cuda()
train_criterion = torch.nn.CrossEntropyLoss(torch.Tensor(train_class_weight)).cuda()
valid_criterion = torch.nn.CrossEntropyLoss(torch.Tensor(valid_class_weight)).cuda()
optimizer = torch.optim.Adam(model.parameters(),
                            learning_rate,
                            weight_decay=weight_decay)

for epoch in range(total_epoch):
    adjust_learning_rate(learning_rate, optimizer, weight_decay, epoch, steps)

    train_loss, train_acc = train(train_loader, model, train_criterion, num_classes, optimizer, epoch)
    valid_loss, valid_acc, conf = validate(valid_loader, model, valid_criterion, num_classes, epoch)

    if epoch == 0:
        best_loss = valid_loss
        best_acc = valid_acc
        best_train_acc = train_acc
        best_acc_epoch = epoch
        best_loss_epoch = epoch
        save_checkpoint(model, epoch, prefix=save_dir+'/valid')
    else:
        if valid_loss <= best_loss:
            print('loss decreased from {best:.3f} to {loss:.3f} from epoch {epoch_num}'.format(best=best_loss, loss=valid_loss, epoch_num=best_loss_epoch))
            best_loss = valid_loss
            best_loss_epoch = epoch
            best_loss_path = save_checkpoint(model, epoch, prefix=save_dir+'/valid')
        else:
            print('loss did not decrease from {best:.3f} from epoch {epoch_num}'.format(best=best_loss, epoch_num=best_loss_epoch))
        if valid_acc >= best_acc:
            print('accuracy increased from {best:.3f} to {acc:.3f} from epoch {epoch_num}'.format(best=best_acc, acc=valid_acc, epoch_num=best_acc_epoch))
            best_acc = valid_acc
            best_acc_epoch = epoch
            best_acc_path = save_checkpoint(model, epoch, prefix=save_dir+'/valid')
        else:
            print('accuracy did not increase from {best:.3f} from epoch {epoch_num}'.format(best=best_acc, epoch_num=best_acc_epoch))
            if train_acc > best_train_acc:
                print('train accuracy improved from {best:.3f} to {acc:.3f}'.format(best=best_train_acc, acc=train_acc))
                best_train_model_path = save_checkpoint(model, epoch, prefix = save_dir+'/train')
                best_train_acc = train_acc

#best acc
print('Training complete. Now testing best acc')
model.load_state_dict(torch.load(best_acc_path))
print('model loaded from checkpoint:', best_acc_path)
v_loss, v_acc, conf = validate(valid_loader, model, valid_criterion, -1)

#best loss
print('Training complete. Now testing best loss')
model.load_state_dict(torch.load(best_loss_path))
print('model loaded from checkpoint:', best_loss_path)
test_loss, test_acc, test_conf = validate(valid_loader, model, valid_criterion, -1)