""" https://adventofcode.com/2020/day/17 """

from typing import List
from copy import deepcopy
from functools import lru_cache

def part1(data: List[str]) -> int:
    """ O(n) solution """

    size_x = len(data[0]) + 2 * CYCLES
    size_y = len(data) + 2 * CYCLES
    size_z = CYCLES * 2 + 1

    pocket = [[[False] * size_x for _ in range(size_y)] for _ in range(size_z)]

    for y in range(len(data)):
        for x in range(len(data[y])):
            pocket[CYCLES][CYCLES + y][CYCLES + x] = data[y][x] == "#"

    for _ in range(CYCLES):
        temp = deepcopy(pocket)
        for z, depth in enumerate(pocket):
            for y, row in enumerate(depth):
                for x, cube in enumerate(row):
                    position = (z, y, x)
                    adjacents = find_3d_adjacents(
                        position, len(pocket), len(pocket[0]))
                    active = sum([(pocket[i][j][k]) for i, j, k in adjacents])

                    if cube and active not in (2, 3):
                        temp[z][y][x] = False
                    elif not cube and active == 3:
                        temp[z][y][x] = True

        pocket = deepcopy(temp)

    return sum([x for z in pocket for y in z for x in y])

def part2(data: List[str]) -> int:
    """ O(?) solution """

    size_x = len(data[0]) + 2 * CYCLES
    size_y = len(data) + 2 * CYCLES
    size_z = CYCLES * 2 + 1
    size_w = CYCLES * 2 + 1

    pocket = [[[[False] * size_x for _ in range(size_y)]
               for _ in range(size_z)] for _ in range(size_w)]

    for y, _ in enumerate(data):
        for x, _ in enumerate(data[y]):
            pocket[CYCLES][CYCLES][CYCLES +
                                   y][CYCLES + x] = data[y][x] == "#"

    for _ in range(CYCLES):
        temp = deepcopy(pocket)
        for w, time in enumerate(pocket):
            for z, depth in enumerate(time):
                for y, row in enumerate(depth):
                    for x, cube in enumerate(row):
                        position = (w, z, y, x)
                        adjacents = find_4d_adjacents(position, len(
                            pocket), len(pocket[0]), len(pocket[0][0]))
                        active = sum([(pocket[i][j][k][l])
                                      for i, j, k, l in adjacents])

                        if cube and active not in (2, 3):
                            temp[w][z][y][x] = False
                        elif not cube and active == 3:
                            temp[w][z][y][x] = True

        pocket = deepcopy(temp)

    return sum([x for w in pocket for z in w for y in z for x in y])

@lru_cache(maxsize=None)
def find_3d_adjacents(pos, depth, width):
    z, y, x = pos

    adjacents = []
    for i in range(z - 1, z + 2):
        for j in range(y - 1, y + 2):
            for k in range(x - 1, x + 2):
                if (i, j, k) != pos and - 1 < i < depth and - 1 < j < width and - 1 < k < width:
                    adjacents.append((i, j, k))

    return adjacents

@lru_cache(maxsize=None)
def find_4d_adjacents(pos, time, depth, width):
    w, z, y, x = pos

    adjacents = []
    for i in range(w - 1, w + 2):
        for j in range(z - 1, z + 2):
            for k in range(y - 1, y + 2):
                for l in range(x - 1, x + 2):
                    if (i, j, k, l) != pos and - 1 < i < time and - 1 < j < depth and - 1 < k < width and - 1 < l < width:
                        adjacents.append((i, j, k, l))

    return adjacents

if __name__ == "__main__":
    TEST = [line.strip() for line in open("tests/d17.txt", "r")]
    PUZZLE = [line.strip() for line in open("puzzles/d17.txt", "r")]
    CYCLES = 6

    assert part1(TEST) == 112
    assert part2(TEST) == 848

    print(f"Part 1: {part1(PUZZLE)}")
    print(f"Part 2: {part2(PUZZLE)}")


import sys
import itertools

expected = open(sys.argv[1], 'r').readlines()
actual   = open(sys.argv[2], 'r').readlines()

# do a pc analysis. easiest bugs to find are the ones where we simply dont run a set of code that
# the expected output shows should be run.
expected_pcs = list(map(lambda x : x.strip().split(" ")[-1], expected))
actual_pcs   = list(map(lambda x : x.strip().split(" ")[-1], actual))

expected_pcs = list(set(expected_pcs))
actual_pcs   = list(set(actual_pcs))

print("Running PC Analysis...")
pcs_not_found = []
for pc in expected_pcs:
    if not pc in actual_pcs:
        # personal constraints:
        if (int(pc, 16) & 0xFF000000) == 0x00000000:
            continue

        if (int(pc, 16) & 0xFF000000) == 0x03000000:
            continue

        pcs_not_found.append(int(pc, 16))

pcs_not_found.sort()

ranged_pcs_not_found = []

previous_pc = -1
start_range = -1
range_len   = -1
for pc in pcs_not_found:
    if pc == previous_pc + 2 or pc == previous_pc + 4 or pc == previous_pc:
        range_len += (pc - previous_pc)
    else:
        ranged_pcs_not_found.append((start_range, range_len))

        range_len = 0
        start_range = pc

    previous_pc = pc
pcs_not_found = pcs_not_found[1:]

print("\nResults:")
for r in ranged_pcs_not_found:
    start = '{0:#010x}'.format(r[0])
    end   = '{0:#010x}'.format(r[0] + r[1])

    print("PCs in Range: {} to {} were not found.".format(start, end))

print("PC Analysis Complete")


import numpy as np
import dill
import math
import sys
sys.path.append('../')
sys.path.append('./')
from src.graph import Graph
from src.evolution_strategies import one_plus_lambda, tournament_selection
from src.population import Population
from src.arg_parser import parse_args
import cProfile
import pstats

def bool_and(x, y): return x and y
def bool_or(x, y): return x or y
def bool_nand(x, y): return not(x and y)
def bool_nor(x, y): return not(x or y)

Population.add_operation(arity=2, func=bool_and, string="AND")
Population.add_operation(arity=2, func=bool_or, string="OR")
Population.add_operation(arity=2, func=bool_nand, string="NAND")
Population.add_operation(arity=2, func=bool_nor, string="NOR")

def eleven_multiplexer(arr):
    if len(arr) != 11:
        print("AAAAAAAAAAAAAAAAAAAAAAA")
        raise
    d = arr[0:8]
    a = arr[8:11]
    index = (int(a[0]) * 4) + (int(a[1]) * 2) + (int(a[2]) * 1)
    if d[index] == "1":
        return True
    return False

def create_tests(n):
    tests = []

    for i in range(2**n):
        base_2_v = bin(i).replace("0b", "").zfill(n)
        cont = 0
        input_arr = []
        for c in base_2_v:
            inp = False
            if c == "1":
                cont += 1
                inp = True
            input_arr.append(inp)
        response = True if cont%2 == 0 else False
        tests.append((input_arr, [response]))
    return tests

def fitness_func(individual: Graph, gen: int, tests):
    fitness = 0
    for t in tests:
        inputs = t[0]
        expected_out = t[1]

        graph_out = individual.operate(inputs)

        for h, y in zip(graph_out, expected_out):
            if h == y:
                fitness += 1

    fitness = fitness/len(tests)
    return np.clip(fitness, -1*(10**10), 10**10)

def main():
    n = 5
    args = parse_args()
    Population.rng = np.random.default_rng(args["seed"])

    tests = create_tests(n)
    def fit_func(indv, gen): return fitness_func(indv, gen, tests)

    population = Population(
        population_size=args["pop_size"],
        n_in=n,
        n_out=1,
        n_middle=args["n_middle_nodes"]
    )

    def t_select(): return tournament_selection(
        population=population,
        generations=args["max_gens"],
        goal_fit=1,
        fitness_func=fit_func,
        minimize_fitness=False,
        fit_share=args["fit_share"],
        stagnation=args["stagnation"],
        stag_preservation=args["stag_preservation"],
        report=args["report"],
        mutate_active_only=args["mut_active_only"],
        mutation_rate=args["mut_rate"],
        elitism=args["elitism"],
        crossover_rate=args["crossover_rate"],
        tournament_size=args["tourney_size"],
        species_threshold=args["species_threshold"],
        n_threads=args["n_threads"],
        csv_file=args["csv"],
        fit_partition_size=args["fit_partition"]
    )

    def p_lambda(): return one_plus_lambda(
        population=population,
        generations=args["max_gens"],
        goal_fit=1,
        fitness_func=fit_func,
        minimize_fitness=False,
        fit_share=args["fit_share"],
        stagnation=args["stagnation"],
        stag_preservation=args["stag_preservation"],
        report=args["report"],
        n_champions=args["elitism"],
        mutate_active_only=args["mut_active_only"],
        mutation_rate=args["mut_rate"],
        species_threshold=args["species_threshold"],
        n_threads=args["n_threads"],
        csv_file=args["csv"],
        fit_partition_size=args["fit_partition"]
    )

    exec_func = t_select
    if args["selection_method"] == "lambda":
        exec_func = p_lambda

    # profile = cProfile.Profile()
    # profile.runcall(exec_func)
    # ps = pstats.Stats(profile)
    # ps.print_stats()
    # print()
    exec_func()

    if args["save_to"] is not None:
        dill.dump(population, open(args["save_to"], mode='wb'))

if __name__ == "__main__":
    main()


from rich import print
from napalm import get_network_driver
from my_devices import arista1, arista2, arista3, arista4

def main():
    for device in (arista1, arista2, arista3, arista4):

        driver = get_network_driver('eos')

        with driver(**device) as device:
            device.open()
            vlans = device.get_vlans()
            host = device.hostname

            device.load_merge_candidate(filename='vlans.cfg')

            diff = device.compare_config()
            print(f'diff for host {host}:')
            print(diff)
            print('-'*70)

            if diff:
                print('committing')
                device.commit_config()
            else:
                print('no changes')

        print()

if __name__ == '__main__':
    main()


#https://towardsdatascience.com/how-to-perform-lasso-and-ridge-regression-in-python-3b3b75541ad8
import numpy as np
import pandas as pd

#we only have three advertising mediums, and sales is our target variable.
DATAPATH = 'Advertising.csv'
data = pd.read_csv(DATAPATH)
print(data.head())
data.drop(['Unnamed: 0'], axis=1, inplace=True) #remove first column which have the record number

#Least square regression

from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LinearRegression

Xs = data.drop(['sales'], axis=1)
y = data['sales'].values.reshape(-1,1)

lin_reg = LinearRegression()

MSEs = cross_val_score(lin_reg, Xs, y, scoring='neg_mean_squared_error', cv=5)

mean_MSE = np.mean(MSEs)

print("Least square MSE  ",mean_MSE)

#Lasso regression
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import Lasso

lasso = Lasso()

parameters = {'alpha': [1e-15, 1e-10, 1e-8, 1e-4, 1e-3,1e-2, 1, 5, 10, 20]}

#GridSearchCV. This will allow us to automatically perform 5-fold cross-validation
# with a range of different regularization parameters in order to find the optimal value of alpha.

lasso_regressor = GridSearchCV(lasso, parameters, scoring='neg_mean_squared_error', cv = 5)

lasso_regressor.fit(Xs, y)

print("Lasso best alpth value ",lasso_regressor.best_params_)

print("Lasso MSE  score ",lasso_regressor.best_score_) #MSE


from pyspark.sql import *
from pyspark.sql.functions import *
from pyspark.sql.types import *
from job.config.ConfigStore import *
from job.udfs.UDFs import *
from job.graph import *

def pipeline(spark: SparkSession) -> None:
    df_Source_1 = Source_1(spark)
    df_FileOps_1 = FileOps_1(spark, df_Source_1)
    df_rank = rank(spark, df_Source_1)
    df_Reformat_5 = Reformat_5(spark, df_rank)
    df_Script_1 = Script_1(spark, df_Source_1)
    df_Aggregate_1 = Aggregate_1(spark, df_Source_1)
    df_Reformat_6 = Reformat_6(spark, df_Aggregate_1)
    df_Reformat_1 = Reformat_1(spark, df_Source_1)
    df_ref = ref(spark, df_Reformat_1)
    df_Reformat_3 = Reformat_3(spark, df_ref)
    df_Reformat_2 = Reformat_2(spark, df_Script_1)
    df_bug2 = bug2(spark, df_Source_1)
    df_Reformat_4 = Reformat_4(spark, df_bug2)
    df_bug = bug(spark, df_Source_1)

def main():
    Utils.initializeFromArgs(Utils.parseArgs())
    spark = SparkSession.builder\
                .config("spark.default.parallelism", "4")\
                .enableHiveSupport()\
                .appName("Prophecy Pipeline")\
                .getOrCreate()
    pipeline(spark)

if __name__ == "__main__":
    main()


import numpy as np

def MAPE(actual, forecast):
    return np.mean(np.abs((actual - forecast) / actual)) * 100

def double_exponential_smoothing(x, alpha=0.3, beta=0.5, l_zero=2, b_zero=0, mape=False):
    if not (0 <= alpha <= 1):
        raise ValueError("Invalid alpha")
    if not (0 <= beta <= 1):
        raise ValueError("Invalid beta")

    n = len(x)
    forecasts = np.zeros(n)
    l_prev = l_zero
    b_prev = b_zero

    for t in range(1, n):
        l_t = alpha * x[t] + (1 - alpha) * (l_prev + b_prev)
        b_t = beta * (l_t - l_prev) + (1 - beta) * b_prev
        forecasts[t] = l_t + b_t
        l_prev = l_t
        b_prev = b_t

    forecasts[0] = np.nan

    if mape:
        mape_value = MAPE(x[1:], forecasts[1:])
        return forecasts, mape_value
    else:
        return forecasts

# Sample input observations
x = np.array([2.92, 0.84, 2.69, 2.42, 1.83, 1.22,0.10,1.32,0.56,-0.35])

forecasts = double_exponential_smoothing(x)

forecasts, mape_value = double_exponential_smoothing(x, l_zero=2 , b_zero=0 ,alpha=0.3,beta=0.5)

print("Forecasts:", forecasts)
print("MAPE:", mape_value)


import csv

class NameDescriptor:
    """
    Дескриптор для проверки и хранения ФИО студента.
    Проверяет, что каждая часть ФИО содержит только буквы и начинается с заглавной буквы.
    """
    def __get__(self, instance, owner):
        return f'{instance._first_name} {instance._first_name} {instance._patronymic}'

    def __set__(self, instance, value):
        first_name, last_name, patronymic = value

        if not all(part.isalpha() and part.istitle() for part in [first_name, last_name, patronymic]):
            raise ValueError("Каждая часть ФИО должна содержать только буквы и начинаться с заглавной буквы")

        instance._first_name = first_name
        instance._last_name = last_name
        instance._patronymic = patronymic

class Subject:
    """
    Класс, представляющий предмет, с валидацией оценок и результатов тестов.
    """
    def __init__(self, name):
        self.name = name
        self.scores = []
        self.test_results = []

    def add_score(self, score):
        if 2 <= score <= 5:
            self.scores.append(score)
        else:
            raise ValueError("Оценка должна быть в диапазоне от 2 до 5")

    def add_test_result(self, result):
        if 0 <= result <= 100:
            self.test_results.append(result)
        else:
            raise ValueError("Результат теста должен быть в диапазоне от 0 до 100")

    def average_test_score(self):
        if self.test_results:
            return sum(self.test_results) / len(self.test_results)
        return 0

    def average_score(self):
        if self.scores:
            return sum(self.scores) / len(self.scores)
        return 0

class Student:
    """
    Класс, представляющий студента.
    """
    name = NameDescriptor()

    def __init__(self, first_name, last_name, patronymic, file):
        self.name = (first_name, last_name, patronymic)
        self.subjects = self.load_subjects(file)

    def load_subjects(self, csv_file):
        subjects = []
        with open(csv_file, 'r') as file:
            reader = csv.reader(file)
            subjects = [Subject(row[0]) for row in reader]
        return subjects

    def add_score(self, subject_name, score):
        subject = self.find_subject(subject_name)
        if subject:
            subject.add_score(score)
        else:
            raise ValueError("Предмет не найден")

    def add_test_result(self, subject_name, result):
        subject = self.find_subject(subject_name)
        if subject:
            subject.add_test_result(result)
        else:
            raise ValueError("Предмет не найден")

    def find_subject(self, subject_name):
        for subject in self.subjects:
            if subject.name == subject_name:
                return subject
        return None

    def average_test_score_per_subject(self):
        averages = {}
        for subject in self.subjects:
            averages[subject.name] = subject.average_test_score()
        return averages

    def average_score_all_subjects(self):
        total_scores = [score for subject in self.subjects for score in subject.scores]
        if total_scores:
            return sum(total_scores) / len(total_scores)
        return 0

subjects = 'subjects.csv'
student = Student('Иванов', 'Иван', 'Иванович', subjects)

student.add_score('Математика', 5)
student.add_score('Математика', 4)
student.add_test_result('Математика', 85)
student.add_test_result('Математика', 90)

student.add_score('Физика', 3)
student.add_score('Физика', 4)
student.add_test_result('Физика', 70)
student.add_test_result('Физика', 80)

print("Средний балл по тестам для каждого предмета:", student.average_test_score_per_subject())
print("Средний балл по оценкам всех предметов:", student.average_score_all_subjects())


#Pasos a seguir:
#1) Installar el repostorio de proyecto EELabs, con .env  y credentials.json en \resources_folder\google, y el entorno conda
#2) Pegar este script en dentro del repositorio \
#3) Ejecutar desde su ubicacion
#IMPORTANTE no permite realizar actualizaciones de fecha, está pensado para una descarga única no para realizar descargas actualizadas
import select
from utils.devices_api.eelabs_devices_api import EELabsDevicesApi
from utils.devices.tess import TESS
from utils.devices.skyglow import SkyGlow
from utils.devices.sqm import SQM
from utils.devices.astmon import ASTMON
import pandas as pd
import numpy as np
from datetime import datetime
from utils.my_utils import Utils

import config as conf #Configuration variables
from utils.filter import Filter
from datetime import date

import argparse
import os

#Input of the script
parser = argparse.ArgumentParser()
parser.add_argument('--f', '--from',  type=int, help='Year from')
parser.add_argument('--to',  type=int, help='Year to')
parser.add_argument('--out','--output', required=True, type=str, help='Output filename')
parser.add_argument('--filter', type=str, help='Data filtering: sun, moon, clouds, galaxy, zodiacal, sigma. Format example: [sun,galaxy,zodaical] Write all without brackets for all filters. Format example: all')
parser.add_argument('--device',type=str,help='Format exalmple: [LPL1_001,LPL2_033,stars1] rite all without brackets for ones device. Format example: stars1')
parser.add_argument('--ephemeris',type=bool,help='True for ephemeris included')
args = parser.parse_args()

initial_year = args.f
final_year = args.to
output = args.out
filter = args.filter
ephemerides = args.ephemeris
if filter:
    if filter[0]=='[':
        filter=filter[1:-1].split(',')
    else:
        filter=filter
else:
    filter=[]
select_devices=args.device
if select_devices:
    if select_devices[0]=='[':
        select_devices=set(select_devices[1:-1].split(','))
    else:
        select_devices=set([select_devices])

#Create the save folder
output=output+'\Photometer_data'
if not os.path.exists(output):
    os.mkdir(output)

#Photometers dataset
devices=EELabsDevicesApi().get_all_devices_info()
devices=pd.DataFrame(devices)
devices=devices.drop(['sg_type','lpl','zero_point','filters','mov_sta_position','local_timezone','location','info_img','info_tess','place','tester','info_org','__v','latitude','longitude','country','city'],axis=1)
localizacion=pd.DataFrame(list(devices['info_location'])).drop(['latitude_hide','longitude_hide'],axis=1)
devices['place']=localizacion['place']
devices['town']=localizacion['town']
devices['sub_region']=localizacion['sub_region']
devices['region']=localizacion['region']
devices['country']=localizacion['country']
devices['latitude']=localizacion['latitude']
devices['longitude']=localizacion['longitude']
devices['elevation']=localizacion['elevation']
devices=devices.drop(['info_location'],axis=1)
devices.to_csv(output+'\All_devices.csv', index = False)

#Folder to save records
if not os.path.exists(output+'\Records'):
    os.mkdir(output+'\Records')

#Obtain the device class regardless of its type
def Device(device_name):
    devices=pd.read_csv(output+'\All_devices.csv')
    type=devices[devices['name']==device_name]['TYPE'].values[0]
    if type==TESS.TYPE:
        device_obj=TESS(name=device_name)
    elif type==SkyGlow.TYPE:
        device_obj=SkyGlow(name=device_name)
    elif type==SQM.TYPE:
        device_obj=SQM(name=device_name)
    elif type==ASTMON.TYPE:
        device_obj=ASTMON(name=device_name)
    return device_obj
#Obtain filtered data for a device and year
def Data(device_name,year,filter): #filter: data vector such as ['sun', 'moon'] for example.
    device_obj=Device(device_name)
    FIRST_DATE=pd.Timestamp(datetime(year, 1, 1, 0, 0), tz='UTC')
    LAST_DATE=pd.Timestamp(datetime(year+1, 1, 1, 0, 0), tz='UTC')
    df_all=None
    try:
        df_all = device_obj.get_all_data(date_from=FIRST_DATE, date_to=LAST_DATE,force=False)
        No_data=False
    except:
        df_all=None
        No_data=True
    if No_data:
        print('The device '+device_name+' not responded due to an error')
    df_all=df_all[(df_all['mag']>conf.MAG_MIN) & (df_all['mag']<conf.MAG_MAX)] #Filter for extreme magnitudes
    if __name__ == '__main__':
        df_all = Utils().add_ephems(df_all, device_obj.getObserver(), parallelize=False) # The parallelize option is causing issues
    V=[]
    if 'sun' in filter or filter=='all':
        df_all = Filter().filter_sun(df_all, max_sun_alt=conf.SUN_ALT_MAX)
    else:
        df_filter=Filter().filter_sun(df_all, max_sun_alt=conf.SUN_ALT_MAX)
        F=np.array([True]*(df_all.index[-1]+1)) #Vector with all True for all indices
        F[df_filter.index]=False #Replace remaining indices to False after filtering
        df_all['sun']=F[df_all.index] #Retrieve data according to the original index
        V=V+['sun']
    if 'moon' in filter or filter=='all':
        df_all = Filter().filter_moon(df_all, max_moon_alt=conf.MOON_ALT_MAX)
    else:
        df_filter=Filter().filter_moon(df_all, max_moon_alt=conf.MOON_ALT_MAX)
        F=np.array([True]*(df_all.index[-1]+1))
        F[df_filter.index]=False
        df_all['moon']=F[df_all.index]
        V=V+['moon']
    if 'clouds' in filter or filter=='all':
        clouds_threshold=conf.CLOUD_STD_FREQ
        df_all = Filter().filter_column(df_all, device_obj.getMagSTDColname(), max=clouds_threshold)
    else:
        clouds_threshold=conf.CLOUD_STD_FREQ
        df_filter=Filter().filter_column(df_all, device_obj.getMagSTDColname(), max=clouds_threshold)
        F=np.array([True]*(df_all.index[-1]+1))
        F[df_filter.index]=False
        df_all['clouds']=F[df_all.index]
        V=V+['clouds']
    if 'galaxy' in filter or filter=='all':
        df_all = Filter().filter_galactic_abs_lat(df_all, min_lat=conf.GALACTIC_LAT_MIN, max_lat=180)
    else:
        df_filter=Filter().filter_galactic_abs_lat(df_all, min_lat=conf.GALACTIC_LAT_MIN, max_lat=180)
        F=np.array([True]*(df_all.index[-1]+1))
        F[df_filter.index]=False
        df_all['galaxy']=F[df_all.index]
        V=V+['galaxy']
    if 'zodiacal' in filter or filter=='all':
        df_all = Filter().filter_column(df_all, col_name='ecliptic_f', max=conf.ECLIPTIC_F_MAX)
    else:
        df_filter=Filter().filter_column(df_all, col_name='ecliptic_f', max=conf.ECLIPTIC_F_MAX)
        F=np.array([True]*(df_all.index[-1]+1))
        F[df_filter.index]=False
        df_all['zodiacal']=F[df_all.index]
        V=V+['zodiacal']
    if 'sigma' in filter or filter=='all':
        sigma=conf.NSIGMA
        df_all = Filter().filter_nsigma(df_all, col_name='mag', sigma=sigma)
    else:
        sigma=conf.NSIGMA
        df_filter=Filter().filter_nsigma(df_all, col_name='mag', sigma=sigma)
        F=np.array([True]*(df_all.index[-1]+1))
        F[df_filter.index]=False
        df_all['sigma']=F[df_all.index]
        V=V+['sigma']
    if ephemerides:
        df=pd.DataFrame({'time':df_all['time'],'mag':df_all['mag'],'name':device_name,'moon_phase':df_all['moon_phase'],'moon_alt':df_all['moon_alt'],'galactic_lat':df_all['galactic_lat'],'galactic_lon':df_all['galactic_lon'],'helioecliptic_lon_abs':df_all['helioecliptic_lon_abs'],'ecliptic_lat_abs':df_all['ecliptic_lat_abs']})
    else:
        df=pd.DataFrame({'time':df_all['time'],'mag':df_all['mag'],'name':device_name})
    for ii in V:
        df[ii]=df_all[ii]
    return df
#Obtain all data between two years
def Data_download(V,initial_year=None,final_year=None,iterate=True): #Iterate to prompt for enter key per iteration
    #Downloaded devices
    Downloaded_devices=set()
    for j in range(0,1000):
        try:
            df_records=pd.read_csv(output+'\Records\Records_'+str(j)+'.csv')
            Downloaded_devices=Downloaded_devices|set(df_records['name'])
        except:
            Downloaded_devices=Downloaded_devices

    #Log devices
    try:
        df_log=pd.read_csv(output+'\Log.csv')
        Log_devices=set(df_log['Devices'])
        Log_exists=True
    except:
        Log_devices=set()
        Log_exists=False
    diff=Downloaded_devices-Log_devices
    Downloaded_devices=Downloaded_devices|Log_devices
    print(Downloaded_devices)
    df_all_devices=pd.read_csv(output+'\All_devices.csv')
    All_devices=set(df_all_devices['name'])
    if select_devices:
        Missing_devices=select_devices-Downloaded_devices
    else:
        Missing_devices=All_devices-Downloaded_devices #To know which devices need to be download
    n_missing_devices=len(Missing_devices)
    if initial_year:
        i_year=initial_year
    else:
        i_year=2010
    if final_year:
        f_year=final_year
    else:
        f_year=date.today().year

    Downloaded_missing_devices=[]
    v_empty=[]
    v_time=[]
    #Loop where it goes device by device and then year by year
    for i in Missing_devices:
        df=pd.DataFrame()
        empty=True
        for ii in range(i_year,f_year+1):
            try:
                dat=Data(i,ii,V)
                df=pd.concat([df,dat])
                if list(dat.values)!=[]:
                    empty=False
            except:
                df=df
            print('Year: '+str(ii))
        #Save
        #Saving with files limited to 1 GB
        try:
            df_records=pd.read_csv(output+'\Records\Records_1.csv')
            Records_exist=True
        except:
            df_final=df
            df_final.to_csv(output+'\Records\Records_1.csv', index = False)
            Records_exist=False
        if Records_exist==True:
            counter=0
            for j in range(1,1000):
                try:
                    df_records=pd.read_csv(output+'\Records\Records_'+str(j)+'.csv')
                    if os.stat(output+'\Records\Records_'+str(j)+'.csv').st_size<1000000000:
                        df_final=pd.concat([df_records,df])
                        df_final.to_csv(output+'\Records\Records_'+str(j)+'.csv', index = False)
                        counter=1
                except:
                    if counter==0:
                        df_final=df
                        df_final.to_csv(output+'\Records\Records_'+str(j)+'.csv', index = False)
                        counter=1

        time=datetime.now()
        v_empty=v_empty+[empty]
        v_time=v_time+[time]
        Downloaded_missing_devices=Downloaded_missing_devices+[i]

        Log_downloaded_devices=pd.DataFrame({'Devices':Downloaded_missing_devices,'Time':v_time,'Empty':v_empty})
        Log_downloaded_devices_2=pd.DataFrame({'Devices':list(diff),'Time':None,'Empty':False})
        Log=pd.concat([Log_downloaded_devices_2,Log_downloaded_devices])
        #Save log
        if Log_exists:
            Log_2=pd.concat([df_log,Log])
        else:
            Log_2=Log
        Log_2.to_csv(output+'\Log.csv', index = False)
        n_no_downloaded_missing_devices=n_missing_devices-len(Downloaded_missing_devices)
        print(str(n_no_downloaded_missing_devices)+' are still pending for download')
        if iterate:
            if input('Downloaded device:'+i+'\n')=='exit':
                break
        else:
            print('Downloaded device:'+i+'\n')
#Run
Data_download(filter,initial_year,final_year,iterate=False)


#!/usr/bin/python3
"""Use reddit api to get info about subredit subscribers"""

def number_of_subscribers(subreddit):
    """Return number of subscribers in subreddit given as argument"""
    import requests

    url = 'https://www.reddit.com/r/{}/about.json'.format(subreddit)
    headers = {'user-agent': 'andy'}
    res = requests.get(url, headers=headers, allow_redirects=False)

    if res.status_code != 200:
        return (0)
    else:
        return (res.json()['data']['subscribers'])


"""Parsing url to check its SEO and availability"""
from datetime import date
from bs4 import BeautifulSoup

def get_page_data(response):
    """Check SEO functionality of url"""

    result = {'status_code': response.status_code}

    page = BeautifulSoup(response.text, 'html.parser')

    result['h1'] = page.h1.get_text() if page.h1 else ''

    result['title'] = page.title.get_text() if page.title else ''

    result['description'] = page.find(
        'meta', {'name': 'description'}
    ).get('content') if page.find('meta', {'name': 'description'}) else ''

    result['created_at'] = date.today()

    return result


from __future__ import annotations

import contextlib
import inspect
import os
import time
import warnings
from collections import OrderedDict
from importlib import import_module
from typing import Any, Callable, Dict, Optional, Union

import numpy as np
import torch
import torch.cuda.amp as amp
from tqdm import tqdm

from .tensorboard_logger import TensorboardLogger
from .wandb_logger import WandbLogger
from ..util import auto_compile, get_constructor_arguments, is_compiled

class DefaultTrainer:
    """Trainer class for 2d/3d training on a single GPU."""

    def __init__(
        self,
        name: Optional[str],
        train_loader: torch.utils.data.DataLoader,
        val_loader: torch.utils.data.DataLoader,
        model: torch.nn.Module,
        loss,
        optimizer,
        metric,
        device: Union[str, torch.device],
        lr_scheduler=None,
        log_image_interval=100,
        mixed_precision=True,
        early_stopping=None,
        logger=TensorboardLogger,
        logger_kwargs: Optional[Dict[str, Any]] = None,
        id_: Optional[str] = None,
        save_root: Optional[str] = None,
        compile_model: Optional[Union[bool, str]] = None,
    ):
        if name is None and not issubclass(logger, WandbLogger):
            raise TypeError("Name cannot be None if not using the WandbLogger")

        if not all(hasattr(loader, "shuffle") for loader in [train_loader, val_loader]):
            raise ValueError(f"{self.__class__} requires each dataloader to have 'shuffle' attribute.")

        self._generate_name = name is None
        self.name = name
        self.id_ = id_ or name
        self.train_loader = train_loader
        self.val_loader = val_loader
        self.model = model
        self.loss = loss
        self.optimizer = optimizer
        self.metric = metric
        self.device = device
        self.lr_scheduler = lr_scheduler
        self.log_image_interval = log_image_interval
        self.save_root = save_root
        self.compile_model = compile_model

        self._iteration = 0
        self._epoch = 0
        self._best_epoch = 0

        self.mixed_precision = mixed_precision
        self.early_stopping = early_stopping

        self.scaler = amp.GradScaler() if mixed_precision else None

        self.logger_class = logger
        self.logger_kwargs = logger_kwargs
        self.log_image_interval = log_image_interval

    @property  # because the logger may generate and set trainer.id on logger.__init__
    def checkpoint_folder(self):
        assert self.id_ is not None
        # save_root enables saving the checkpoints somewhere else than in the local
        # folder. This is handy for filesystems with limited space, where saving the checkpoints
        # and log files can easily lead to running out of space.
        save_root = getattr(self, "save_root", None)
        return os.path.join("./checkpoints", self.id_) if save_root is None else\
            os.path.join(save_root, "./checkpoints", self.id_)

    @property
    def iteration(self):
        return self._iteration

    @property
    def epoch(self):
        return self._epoch

    class Deserializer:
        """Determines how to deserialize the trainer kwargs from serialized 'init_data'

        Examples:
            To extend the initialization process you can inherite from this Deserializer in an inherited Trainer class.
            Note that `DefaultTrainer.Deserializer.load_generic()` covers most cases already.

            This example adds `the_answer` kwarg, which requires 'calculations' upon initialization:
            >>> class MyTrainer(DefaultTrainer):
            >>>     def __init__(self, *args, the_answer: int, **kwargs):
            >>>         super().__init__(*args, **kwargs)
            >>>         self.the_answer = the_answer  # this allows the default Serializer to save the new kwarg,
            >>>                                       # see DefaultTrainer.Serializer
            >>>
            >>>     class Deserializer(DefaultTrainer.Deserializer):
            >>>         def load_the_answer(self):
            >>>             generic_answer = self.init_data["the_answer"]
            >>>             # (device dependent) special deserialization
            >>>             if self.trainer_kwargs["device"].type == "cpu":  # accessing previously deserialized kwarg
            >>>                 self.trainer_kwargs["the_answer"] = generic_answer + 1
            >>>             else:
            >>>                 self.trainer_kwargs["the_answer"] = generic_answer * 2
        """

        def __init__(self, init_data: dict, save_path: str, device: Union[str, torch.device]):
            self.init_data = init_data
            self.save_path = save_path
            # populate with deserialized trainer kwargs during deserialization; possibly overwrite 'device'
            self.trainer_kwargs: Dict[str, Any] = dict(
                device=torch.device(self.init_data["device"]) if device is None else torch.device(device)
            )

        def load(self, kwarg_name: str, optional):
            """`optional` is True if self.trainer.__class__.__init__ specifies a default value for 'kwarg_name'"""

            if kwarg_name == "device":
                pass  # deserialized in __init__
            elif kwarg_name.endswith("_loader"):
                self.load_data_loader(kwarg_name, optional)
            else:
                load = getattr(self, f"load_{kwarg_name}", self.load_generic)
                load(kwarg_name, optional=optional)

        def load_data_loader(self, loader_name, optional) -> None:
            ds = self.init_data.get(loader_name.replace("_loader", "_dataset"))
            if ds is None and optional:
                return

            loader_kwargs = self.init_data[f"{loader_name}_kwargs"]
            loader = torch.utils.data.DataLoader(ds, **loader_kwargs)
            # monkey patch shuffle loader_name to the loader
            loader.shuffle = loader_kwargs.get("shuffle", False)
            self.trainer_kwargs[loader_name] = loader

        def load_generic(
            self,
            kwarg_name: str,
            *dynamic_args,
            optional: bool,
            only_class: bool = False,
            dynamic_kwargs: Optional[Dict[str, Any]] = None,
        ) -> None:
            if kwarg_name in self.init_data:
                self.trainer_kwargs[kwarg_name] = self.init_data[kwarg_name]
                return

            this_cls = self.init_data.get(f"{kwarg_name}_class", None)
            if this_cls is None:
                if optional:
                    return
                else:
                    raise RuntimeError(f"Could not find init data for {kwarg_name} in {self.save_path}")

            assert isinstance(this_cls, str), this_cls
            assert "." in this_cls, this_cls
            cls_p, cls_m = this_cls.rsplit(".", 1)
            this_cls = getattr(import_module(cls_p), cls_m)
            if only_class:
                self.trainer_kwargs[kwarg_name] = this_cls
            else:
                self.trainer_kwargs[kwarg_name] = this_cls(
                    *dynamic_args, **self.init_data.get(f"{kwarg_name}_kwargs", {}), **(dynamic_kwargs or {})
                )

        def load_name(self, kwarg_name: str, optional: bool):
            self.trainer_kwargs[kwarg_name] = os.path.split(os.path.dirname(self.save_path))[1]

        def load_optimizer(self, kwarg_name: str, optional: bool):
            self.load_generic(kwarg_name, self.trainer_kwargs["model"].parameters(), optional=optional)

        def load_lr_scheduler(self, kwarg_name: str, optional: bool):
            self.load_generic(kwarg_name, self.trainer_kwargs["optimizer"], optional=optional)

        # todo: remove and rename kwarg 'logger' to 'logger_class'
        def load_logger(self, kwarg_name: str, optional: bool):
            assert kwarg_name == "logger"
            self.load_generic("logger", optional=optional, only_class=True)

    @staticmethod
    def _get_save_dict(save_path, device):
        if not os.path.exists(save_path):
            raise ValueError(f"Cannot find checkpoint {save_path}")
        return torch.load(save_path, map_location=device)

    @classmethod
    def from_checkpoint(cls, checkpoint_folder, name="best", device=None):
        save_path = os.path.join(checkpoint_folder, f"{name}.pt")
        # make sure the correct device is set if we don't have access to CUDA
        if not torch.cuda.is_available():
            device = "cpu"
        save_dict = cls._get_save_dict(save_path, device)
        deserializer = cls.Deserializer(save_dict["init"], save_path, device)

        has_kwargs = False
        deserialized = []
        for name, parameter in inspect.signature(cls).parameters.items():
            if name == "kwargs":
                has_kwargs = True
                continue
            deserializer.load(name, optional=parameter.default is not inspect.Parameter.empty)
            deserialized.append(name)

        # to deserialze kwargs we can't rely on inspecting the signature, so we
        # go through the remaning kwarg names in init data instead
        if has_kwargs:
            kwarg_names = list(set(deserializer.init_data.keys()) - set(deserialized))
            for name in kwarg_names:
                if name.endswith("_kwargs"):
                    continue
                elif name.endswith("_dataset"):
                    deserializer.load(name.replace("dataset", "loader"), optional=False)
                elif name.endswith("_class"):
                    deserializer.load(name.replace("_class", ""), optional=False)
                else:
                    deserializer.load(name, optional=False)

        trainer = cls(**deserializer.trainer_kwargs)
        trainer._initialize(0, save_dict)
        trainer._is_initialized = True
        return trainer

    class Serializer:
        """Implements how to serialize trainer kwargs from a trainer instance

        Examples:
            To extend the serialization process you can inherite from this Serializer in a derived Trainer class.
            Note that the methods `dump_generic_builtin()`, `dump_generic_class()` and `dump_generic_instance()`
            called by the `dump()` method when appropriate cover most cases already.

            This example adds `the_answer` kwarg, which requires extra steps on dumping only because we don't keep a
            'the_answer' attribute:
            >>> class MyTrainer(DefaultTrainer):
            >>>     def __init__(self, *args, the_answer: int, **kwargs):
            >>>         super().__init__(*args, **kwargs)
            >>>         # self.the_answer = the_answer  # this would allow the default Serializer to save the new kwarg,
            >>>         # but let's make things more interesting...
            >>>         self.the = the_answer // 10
            >>>         self.answer = the_answer % 10
            >>>
            >>>     class Serializer(DefaultTrainer.Serializer):
            >>>         trainer: MyTrainer
            >>>         def dump_the_answer(self, kwarg_name: str) -> None:  # custom dump method for 'the_answer' kwarg
            >>>             assert kwarg_name == "the_answer"
            >>>             # populate self.init_data with the serialized data required by Deserializer
            >>>             # to restore the trainer kwargs
            >>>             self.init_data["the_answer"] = self.trainer.the * 10 + self.trainer.answer

            This example with both Serializer and Deserializer adds `the_answer` kwarg,
            while saving it in two separate entries 'the' and 'answer'
            >>> class MyTrainer(DefaultTrainer):
            >>>     def __init__(self, *args, the_answer: int, **kwargs):
            >>>         super().__init__(*args, **kwargs)
            >>>         self.the_answer = the_answer
            >>>
            >>>     class Serializer(DefaultTrainer.Serializer):
            >>>         trainer: MyTrainer
            >>>         def dump_the_answer(self, kwarg_name: str):
            >>>             assert kwarg_name == "the_answer"
            >>>             self.init_data.update({
            >>>                 "the": self.trainer.the_answer // 10,
            >>>                 "answer": self.trainer.the_answer % 10
            >>>             })
            >>>
            >>>     class Deserializer(DefaultTrainer.Deserializer):
            >>>         def load_the_answer(self, kwarg_name: str, optional: bool):
            >>>             assert kwarg_name == "the_answer"
            >>>             # 'optional' is True if MyTrainer.__init__ specifies a default value for 'kwarg_name'
            >>>             self.trainer_kwargs[kwarg_name] = self.init_data["the"] * 10 + self.init_data["answer"]
        """

        def __init__(self, trainer: DefaultTrainer):
            self.trainer = trainer
            self.init_data = {}  # to be populated during serialization process

        def dump(self, kwarg_name: str) -> None:
            dumper = getattr(self, f"dump_{kwarg_name}", None)
            if dumper is not None:
                dumper(kwarg_name)
            elif kwarg_name.endswith("_loader"):
                self.dump_data_loader(kwarg_name)
            elif kwarg_name.endswith("_class"):
                self.dump_generic_class(kwarg_name)
            elif not hasattr(self.trainer, kwarg_name):
                raise AttributeError(
                    f"{self.trainer.__class__} missing attribute '{kwarg_name}' "
                    f"or special dump method {self.trainer.__class__}.Serializer.dump_{kwarg_name}()"
                )
            else:
                assert hasattr(self.trainer, kwarg_name)
                obj = getattr(self.trainer, kwarg_name)
                if obj is None or type(obj) in (
                    bool,
                    bytearray,
                    bytes,
                    dict,
                    float,
                    frozenset,
                    int,
                    list,
                    set,
                    str,
                    tuple,
                ):
                    self.dump_generic_builtin(kwarg_name)
                else:
                    self.dump_generic_instance(kwarg_name)

        def dump_generic_builtin(self, kwarg_name: str) -> None:
            assert hasattr(self.trainer, kwarg_name)
            self.init_data[kwarg_name] = getattr(self.trainer, kwarg_name)

        def dump_generic_class(self, kwarg_name: str) -> None:
            assert hasattr(self.trainer, kwarg_name)
            assert kwarg_name.endswith("_class")
            obj = getattr(self.trainer, kwarg_name)
            self.init_data[kwarg_name] = None if obj is None else f"{obj.__module__}.{obj.__name__}"

        def dump_generic_instance(self, kwarg_name: str) -> None:
            assert hasattr(self.trainer, kwarg_name)
            instance = getattr(self.trainer, kwarg_name)
            self.init_data.update(
                {
                    f"{kwarg_name}_class": f"{instance.__class__.__module__}.{instance.__class__.__name__}",
                    f"{kwarg_name}_kwargs": get_constructor_arguments(instance),
                }
            )

        def dump_device(self, kwarg_name: str):
            assert hasattr(self.trainer, kwarg_name)
            self.init_data[kwarg_name] = str(getattr(self.trainer, kwarg_name))

        def dump_data_loader(self, kwarg_name: str) -> None:
            assert hasattr(self.trainer, kwarg_name)
            loader = getattr(self.trainer, kwarg_name)
            if loader is None:
                return
            self.init_data.update(
                {
                    f"{kwarg_name.replace('_loader', '_dataset')}": loader.dataset,
                    f"{kwarg_name}_kwargs": get_constructor_arguments(loader),
                }
            )

        def dump_logger(self, kwarg_name: str):  # todo: remove and rename kwarg 'logger' to 'logger_class'
            self.dump_generic_class(f"{kwarg_name}_class")

        def dump_model(self, kwarg_name: str):
            if is_compiled(self.trainer.model):
                self.init_data.update(
                    {
                        "model_class": self.trainer._model_class,
                        "model_kwargs": self.trainer._model_kwargs,
                    }
                )
            else:
                self.dump_generic_instance("model")

    def _build_init(self) -> Dict[str, Any]:
        serializer = self.Serializer(self)
        for name in inspect.signature(self.__class__).parameters:
            # special rules to serialize kwargs
            # if a trainer class inherits from DefaultTrainer and has **kwargs
            # they need to be saved in self._kwargs
            if name == "kwargs":
                if not hasattr(self, "_kwargs"):
                    msg = "The trainer class has **kwargs in its signature, but is missing the _kwargs attribute. " +\
                          "Please add self._kwargs to its __init__ function"
                    raise RuntimeError(msg)
                kwargs = getattr(self, "_kwargs")
                for kwarg_name in kwargs:
                    serializer.dump(kwarg_name)
                continue
            serializer.dump(name)

        return serializer.init_data

    def _initialize(self, iterations, load_from_checkpoint, epochs=None):
        assert self.train_loader is not None
        assert self.val_loader is not None
        assert self.model is not None
        assert self.loss is not None
        assert self.optimizer is not None
        assert self.metric is not None
        assert self.device is not None

        if load_from_checkpoint is not None:
            self.load_checkpoint(load_from_checkpoint)

        if sum((iterations is not None, epochs is not None)) != 1:
            raise ValueError(
                "Exactly one of 'iterations' or 'epochs' has to be specified to initialize the trainer."
                f"You have passed 'iterations'={iterations} and 'epochs'={epochs}"
            )

        if epochs is None:
            epochs = int(np.ceil(float(iterations) / len(self.train_loader)))
        else:
            iterations = epochs * len(self.train_loader)

        self.max_iteration = self._iteration + iterations
        self.max_epoch = self._epoch + epochs

        if not getattr(self, "_is_initialized", False):
            # check if we compile the model (only supported by pytorch 2)
            # to enable (de)serialization of compiled models, we keep track of the model class and kwargs
            if is_compiled(self.model):
                warnings.warn(
                    "You have passed a compiled model to the trainer."
                    "It will not be possible to (de)serialize the trainer with it."
                    "If you want to be able to do this please pass the normal model."
                    "It can be automatically compiled by setting 'compile_model' to True"
                )
            self._model_class = f"{self.model.__class__.__module__}.{self.model.__class__.__name__}"
            self._model_kwargs = get_constructor_arguments(self.model)
            self.model = auto_compile(self.model, self.compile_model)

            self.model.to(self.device)
            self.loss.to(self.device)

            # this saves all the information that is necessary
            # to fully load the trainer from the checkpoint
            self.init_data = self._build_init()

            if self.logger_class is None:
                self.logger = None
            else:
                # may set self.name if self.name is None
                save_root = getattr(self, "save_root", None)
                self.logger = self.logger_class(self, save_root, **(self.logger_kwargs or {}))

            try:
                os.makedirs(self.checkpoint_folder, exist_ok=True)
            except PermissionError:
                warnings.warn(
                    f"The checkpoint folder at {self.checkpoint_folder} could not be created."
                    "The most likely reason for this is that you copied the checkpoint somewhere else,"
                    "so we skip this error to enable loading the model from this checkpoint."
                )
                pass

        best_metric = np.inf
        return best_metric

    def save_checkpoint(self, name, best_metric, **extra_save_dict):
        save_path = os.path.join(self.checkpoint_folder, f"{name}.pt")
        extra_init_dict = extra_save_dict.pop("init", {})
        save_dict = {
            "iteration": self._iteration,
            "epoch": self._epoch,
            "best_epoch": self._best_epoch,
            "best_metric": best_metric,
            "model_state": self.model.state_dict(),
            "optimizer_state": self.optimizer.state_dict(),
            "init": self.init_data | extra_init_dict,
        }
        save_dict.update(**extra_save_dict)
        if self.scaler is not None:
            save_dict.update({"scaler_state": self.scaler.state_dict()})
        if self.lr_scheduler is not None:
            save_dict.update({"scheduler_state": self.lr_scheduler.state_dict()})
        torch.save(save_dict, save_path)

    def load_checkpoint(self, checkpoint="best"):
        if isinstance(checkpoint, str):
            save_path = os.path.join(self.checkpoint_folder, f"{checkpoint}.pt")
            if not os.path.exists(save_path):
                warnings.warn(f"Cannot load checkpoint. {save_path} does not exist.")
                return
            save_dict = torch.load(save_path)
        elif isinstance(checkpoint, dict):
            save_dict = checkpoint
        else:
            raise RuntimeError

        self._iteration = save_dict["iteration"]
        self._epoch = save_dict["epoch"]
        self._best_epoch = save_dict["best_epoch"]
        self.best_metric = save_dict["best_metric"]

        model_state = save_dict["model_state"]
        # to enable loading compiled models
        compiled_prefix = "_orig_mod."
        model_state = OrderedDict(
            [(k[len(compiled_prefix):] if k.startswith(compiled_prefix) else k, v) for k, v in model_state.items()]
        )
        self.model.load_state_dict(model_state)
        # we need to send the network to the device before loading the optimizer state!
        self.model.to(self.device)

        self.optimizer.load_state_dict(save_dict["optimizer_state"])
        if self.scaler is not None:
            self.scaler.load_state_dict(save_dict["scaler_state"])
        if self.lr_scheduler is not None:
            self.lr_scheduler.load_state_dict(save_dict["scheduler_state"])

        return save_dict

    def fit(self, iterations=None, load_from_checkpoint=None, epochs=None, save_every_kth_epoch=None):
        """Run neural network training.

        Exactly one of 'iterations' or 'epochs' has to be passed.

        Parameters:
            iterations [int] - how long to train, specified in iterations (default: None)
            load_from_checkpoint [str] - path to a checkpoint from where training should be continued (default: None)
            epochs [int] - how long to train, specified in epochs (default: None)
            save_every_kth_epoch [int] - save checkpoints after every kth epoch separately.
                The corresponding checkpoints will be saved with the naming scheme 'epoch-{epoch}.pt'. (default: None)
        """
        best_metric = self._initialize(iterations, load_from_checkpoint, epochs)
        print(
            "Start fitting for",
            self.max_iteration - self._iteration,
            "iterations / ",
            self.max_epoch - self._epoch,
            "epochs",
        )
        print("with", len(self.train_loader), "iterations per epoch")

        if self.mixed_precision:
            train_epoch = self._train_epoch_mixed
            validate = self._validate_mixed
            print("Training with mixed precision")
        else:
            train_epoch = self._train_epoch
            validate = self._validate
            print("Training with single precision")

        progress = tqdm(
            total=epochs * len(self.train_loader) if iterations is None else iterations,
            desc=f"Epoch {self._epoch}", leave=True
        )
        msg = "Epoch %i: average [s/it]: %f, current metric: %f, best metric: %f"

        train_epochs = self.max_epoch - self._epoch
        for _ in range(train_epochs):

            # run training and validation for this epoch
            t_per_iter = train_epoch(progress)
            current_metric = validate()

            # perform all the post-epoch steps:

            # apply the learning rate scheduler
            if self.lr_scheduler is not None:
                self.lr_scheduler.step(current_metric)

            # save this checkpoint as the new best checkpoint if
            # it has the best overall validation metric
            if current_metric < best_metric:
                best_metric = current_metric
                self._best_epoch = self._epoch
                self.save_checkpoint("best", best_metric)

            # save this checkpoint as the latest checkpoint
            self.save_checkpoint("latest", best_metric)

            # if we save after every k-th epoch then check if we need to save now
            if save_every_kth_epoch is not None and (self._epoch + 1) % save_every_kth_epoch == 0:
                self.save_checkpoint(f"epoch-{self._epoch + 1}", best_metric)

            # if early stopping has been specified then check if the stopping condition is met
            if self.early_stopping is not None:
                epochs_since_best = self._epoch - self._best_epoch
                if epochs_since_best > self.early_stopping:
                    print("Stopping training because there has been no improvement for", self.early_stopping, "epochs")
                    break

            self._epoch += 1
            progress.set_description(msg % (self._epoch, t_per_iter, current_metric, best_metric), refresh=True)

        print(f"Finished training after {self._epoch} epochs / {self._iteration} iterations.")
        print(f"The best epoch is number {self._best_epoch}.")

        if self._generate_name:
            self.name = None

        # TODO save the model to wandb if we have the wandb logger
        if isinstance(self.logger, WandbLogger):
            self.logger.get_wandb().finish()

    def _backprop(self, loss):
        loss.backward()
        self.optimizer.step()

    def _backprop_mixed(self, loss):
        self.scaler.scale(loss).backward()
        self.scaler.step(self.optimizer)
        self.scaler.update()

    def _train_epoch(self, progress):
        return self._train_epoch_impl(progress, contextlib.nullcontext, self._backprop)

    def _train_epoch_mixed(self, progress):
        return self._train_epoch_impl(progress, amp.autocast, self._backprop_mixed)

    def _forward_and_loss(self, x, y):
        pred = self.model(x)
        if self._iteration % self.log_image_interval == 0:
            if pred.requires_grad:
                pred.retain_grad()

        loss = self.loss(pred, y)
        return pred, loss

    def _train_epoch_impl(self, progress, forward_context, backprop: Callable[[torch.Tensor], None]):
        self.model.train()

        n_iter = 0
        t_per_iter = time.time()
        for x, y in self.train_loader:
            x, y = x.to(self.device), y.to(self.device)

            self.optimizer.zero_grad()

            with forward_context():
                pred, loss = self._forward_and_loss(x, y)

            backprop(loss)

            lr = [pm["lr"] for pm in self.optimizer.param_groups][0]
            if self.logger is not None:
                self.logger.log_train(self._iteration, loss, lr, x, y, pred, log_gradients=True)

            self._iteration += 1
            n_iter += 1
            if self._iteration >= self.max_iteration:
                break
            progress.update(1)

        t_per_iter = (time.time() - t_per_iter) / n_iter
        return t_per_iter

    def _validate(self):
        return self._validate_impl(contextlib.nullcontext)

    def _validate_mixed(self):
        return self._validate_impl(amp.autocast)

    def _validate_impl(self, forward_context):
        self.model.eval()

        metric_val = 0.0
        loss_val = 0.0

        with torch.no_grad():
            for x, y in self.val_loader:
                x, y = x.to(self.device), y.to(self.device)
                with forward_context():
                    pred, loss = self._forward_and_loss(x, y)
                    metric = self.metric(pred, y)

                loss_val += loss.item()
                metric_val += metric.item()

        metric_val /= len(self.val_loader)
        loss_val /= len(self.val_loader)
        if self.logger is not None:
            self.logger.log_validation(self._iteration, metric_val, loss_val, x, y, pred)
        return metric_val


# Aditya Halder // @AdityaHalder

import os
import aiofiles
import aiohttp
import ffmpeg
import requests
from os import path
from asyncio.queues import QueueEmpty
from typing import Callable
from pyrogram import Client, filters
from pyrogram.types import Message, Voice, InlineKeyboardButton, InlineKeyboardMarkup
from pyrogram.errors import UserAlreadyParticipant
from modules.cache.admins import set
from modules.clientbot import clientbot, queues
from modules.clientbot.clientbot import client as USER
from modules.helpers.admins import get_administrators
from youtube_search import YoutubeSearch
from modules import converter
from modules.downloaders import youtube
from modules.config import que, SUDO_USERS
from modules.cache.admins import admins as a
from modules.helpers.command import commandpro
from modules.helpers.filters import command, other_filters
from modules.helpers.decorators import errors, sudo_users_only
from modules.helpers.errors import DurationLimitError
from modules.helpers.gets import get_url, get_file_name
from pytgcalls import StreamType
from pytgcalls.types.input_stream import InputStream
from pytgcalls.types.input_stream import InputAudioStream

# plus
chat_id = None
useer = "NaN"

def transcode(filename):
    ffmpeg.input(filename).output(
        "input.raw", format="s16le", acodec="pcm_s16le", ac=2, ar="48k"
    ).overwrite_output().run()
    os.remove(filename)

# Convert seconds to mm:ss
def convert_seconds(seconds):
    seconds = seconds % (24 * 3600)
    seconds %= 3600
    minutes = seconds // 60
    seconds %= 60
    return "%02d:%02d" % (minutes, seconds)

# Convert hh:mm:ss to seconds
def time_to_seconds(time):
    stringt = str(time)
    return sum(int(x) * 60 ** i for i, x in enumerate(reversed(stringt.split(":"))))

@Client.on_message(
    commandpro(["ply"])
    & filters.group
    & ~filters.edited
    & ~filters.forwarded
    & ~filters.via_bot
)
@errors
@sudo_users_only
async def play(_, message: Message):
    global que
    global useer
    await message.delete()
    lel = await message.reply("**ðŸ”„ Æ¤É¤Ã¸ÆˆÉ˜ssÉ©É³Ê› ...**")

    administrators = await get_administrators(message.chat)
    chid = message.chat.id

    audio = (
        (message.reply_to_message.audio or message.reply_to_message.voice)
        if message.reply_to_message
        else None
    )
    url = get_url(message)

    if audio:

        file_name = get_file_name(audio)
        title = file_name
        thumb_name = "https://te.legra.ph/file/ed6920a2f0ab5af3fd55d.png"
        thumbnail = thumb_name
        duration = round(audio.duration / 60)
        views = "Locally added"

        requested_by = message.from_user.first_name
        file_path = await converter.convert(
            (await message.reply_to_message.download(file_name))
            if not path.isfile(path.join("downloads", file_name))
            else file_name
        )

    elif url:
        try:
            results = YoutubeSearch(url, max_results=1).to_dict()
            # print results
            title = results[0]["title"]
            duration = results[0]["duration"]
            url_suffix = results[0]["url_suffix"]
            views = results[0]["views"]
            durl = url
            durl = durl.replace("youtube", "youtubepp")

            secmul, dur, dur_arr = 1, 0, duration.split(":")
            for i in range(len(dur_arr) - 1, -1, -1):
                dur += int(dur_arr[i]) * secmul
                secmul *= 60

        except Exception as e:
            title = "NaN"
            thumb_name = "https://te.legra.ph/file/ed6920a2f0ab5af3fd55d.png"
            duration = "NaN"
            views = "NaN"

        requested_by = message.from_user.first_name
        file_path = await converter.convert(youtube.download(url))
    else:
        if len(message.command) < 2:
           return await lel.edit(
                "**ðŸ¤– WÉ¦É‘Êˆ ðŸ™ƒ YÃ¸Ê‹ ðŸ’¿ WÉ‘Å‹Êˆ ðŸ˜�\nðŸ’ž Æ¬Ã¸ ðŸ”Š Æ¤ÉÉ‘yâ�“ ...**"
            )

        await lel.edit("**ðŸ”Ž SÉ˜É‘É¤ÆˆÉ¦É©É³Ê› ...**")
        query = message.text.split(None, 1)[1]
        # print(query)
        await lel.edit("**ðŸ”„ Æ¤É¤Ã¸ÆˆÉ˜ssÉ©É³Ê› ...**")
        try:
            results = YoutubeSearch(query, max_results=1).to_dict()
            url = f"https://youtube.com{results[0]['url_suffix']}"
            # print results
            title = results[0]["title"]
            duration = results[0]["duration"]
            url_suffix = results[0]["url_suffix"]
            views = results[0]["views"]
            durl = url
            durl = durl.replace("youtube", "youtubepp")

            secmul, dur, dur_arr = 1, 0, duration.split(":")
            for i in range(len(dur_arr) - 1, -1, -1):
                dur += int(dur_arr[i]) * secmul
                secmul *= 60

        except Exception as e:
            await lel.edit(
                "**ðŸ”Š â±®Ê‹sÉ©Æˆ ðŸ˜• Æ�Ã¸Êˆ ðŸ“µ FÃ¸Ê‹É³É—â�—ï¸�\nðŸ’ž Æ¬É¤y â™¨ï¸� É…É³Ã¸ÊˆÉ¦É˜É¤ ðŸŒ·...**"
            )
            print(str(e))
            return

        requested_by = message.from_user.first_name
        file_path = await converter.convert(youtube.download(url))
    ACTV_CALLS = []
    chat_id = message.chat.id
    for x in clientbot.pytgcalls.active_calls:
        ACTV_CALLS.append(int(x.chat_id))
    if int(chat_id) in ACTV_CALLS:
        position = await queues.put(chat_id, file=file_path)
        await lel.edit("**ðŸ’¥ Æ˜É‘É‘ÉðŸ¤žÉ…É—É—É˜É— ðŸ’¿ SÃ¸É³Ê›â�—ï¸�\nðŸ”Š É…Êˆ ðŸ’ž Æ¤Ã¸sÉ©ÊˆÉ©Ã¸É³ Â» `{}` ðŸŒ· ...**".format(position),
    )
    else:
        await clientbot.pytgcalls.join_group_call(
                chat_id,
                InputStream(
                    InputAudioStream(
                        file_path,
                    ),
                ),
                stream_type=StreamType().local_stream,
            )

        await lel.edit("**ðŸ’¥ Æ˜É‘É‘ÉðŸ¤žMÊ‹sÉ©Æˆ ðŸŽ¸ NÃ¸Ï‰ ðŸ’ž\nðŸ”Š Æ¤ÉÉ‘yÉ©É³Ê› ðŸ˜� Ã˜Æ¤ ðŸ¥€ ...**".format(),
        )

    return await lel.delete()

@Client.on_message(commandpro(["pse"]) & other_filters)
@errors
@sudo_users_only
async def pause(_, message: Message):
    await message.delete()
    await clientbot.pytgcalls.pause_stream(message.chat.id)
    pase = await message.reply_text("**â–¶ï¸� Æ¤É‘Ê‹sÉ˜É— ðŸŒ· ...**")
    await pase.delete()

@Client.on_message(commandpro(["rsm"]) & other_filters)
@errors
@sudo_users_only
async def resume(_, message: Message):
    await message.delete()
    await clientbot.pytgcalls.resume_stream(message.chat.id)
    rsum = await message.reply_text("**â�¸ Æ¦É˜sÊ‹É±É˜É— ðŸŒ· ...**")
    await rsum.delete()

@Client.on_message(commandpro(["skp", "nxt"]) & other_filters)
@errors
@sudo_users_only
async def skip(_, message: Message):
    global que
    await message.delete()
    ACTV_CALLS = []
    chat_id = message.chat.id
    for x in clientbot.pytgcalls.active_calls:
        ACTV_CALLS.append(int(x.chat_id))
    if int(chat_id) not in ACTV_CALLS:
       novc = await message.reply_text("**ðŸ’¥ Æ�Ã¸ÊˆÉ¦É©É³Ê› ðŸ”‡ Æ¤ÉÉ‘yÉ©É³Ê› ðŸŒ· ...**")
       await novc.delete()
    else:
        queues.task_done(chat_id)

        if queues.is_empty(chat_id):
            empt = await message.reply_text("**ðŸ¥€ Empty Queue, Leaving VC âœ¨ ...**")
            await empt.delete()
            await clientbot.pytgcalls.leave_group_call(chat_id)
        else:
            next = await message.reply_text("**â�© SÆ™É©Æ¥Æ¥É˜É— ðŸŒ· ...**")
            await next.delete()
            await clientbot.pytgcalls.change_stream(
                chat_id,
                InputStream(
                    InputAudioStream(
                        clientbot.queues.get(chat_id)["file"],
                    ),
                ),
            )

@Client.on_message(commandpro(["end", "stp"]) & other_filters)
@errors
@sudo_users_only
async def stop(_, message: Message):
    await message.delete()
    try:
        clientbot.queues.clear(message.chat.id)
    except QueueEmpty:
        pass

    await clientbot.pytgcalls.leave_group_call(message.chat.id)
    leav = await message.reply_text("**â�Œ SÊˆÃ¸Æ¥Æ¥É˜É— ðŸŒ· ...**")
    await leav.delete()


import matplotlib.pyplot as plt
import pandas as pd
from matplotlib.cm import get_cmap

data = pd.read_csv("./output_geo.csv")
df = pd.DataFrame(data)
fig, ax = plt.subplots()

# get a color map
cmap = get_cmap("tab20", 28)  # type: matplotlib.colors.ListedColormap
colors = cmap.colors  # type: list

ips = df['ip']
mean_rtt = df['mean_rtt']

ax.bar(ips, mean_rtt, color=colors)
ax.set_ylabel('Mean RTT in ms')
ax.set_title('Mean RTT for each hop between ips')

plt.xticks(rotation=15)
plt.show()


# coding=utf-8
from django.contrib.admin import ModelAdmin, site
from models import News

class NewsAdmin(ModelAdmin):
    list_display = ('id', 'match_type', 'game_start_time', 'end_score', 'middle_score', 'status', 'team1', 'score',
                    'team2',  'yapan','yapanSB', 'daxiaopan','daxiaopanSB', 'findex', 'create_time')
    ording = ('id',)
    list_per_page = 300
    list_filter = ('create_time',)
    search_fields = ['team1', 'team2', 'findex', 'score']

site.register(News, NewsAdmin)


import math
from Solution import Solution

class P003(Solution):
    def is_prime(self, number):
        if number % 2 == 0:
            return False
        upper_limit = math.floor(math.sqrt(number))
        if upper_limit % 2 == 0:
            upper_limit -= 1
        for i in range(upper_limit, 1, -2):
            if number % i == 0:
                return False
        return True

    def solve(self):
        self.problem_number = 3
        number = 600851475143
        upper_limit = math.floor(math.sqrt(number))
        if upper_limit % 2 == 0:
            upper_limit -= 1

        possible_factor = upper_limit
        result = number
        while possible_factor > 1:
            if (number % possible_factor == 0) and self.is_prime(possible_factor):
                result = possible_factor
                break
            possible_factor -=2
        return result

def main():
	P003().run()

if __name__ == "__main__":
	main()


import functools
from enum import Enum
import json
import re

class SCJDEntry:
	def __init__(self):
		self.data = {}

	def set_title(self, title):
		self.data['title'] = title

	def set_id(self, idx):
		self.data['idx'] = idx

	def glue_pronounce(self, prn):
		if 'prn' in self.data:
			self.data['prn'] += prn
		else:
			self.data['prn'] = prn

	def push_kanji(self, kanji):
		if 'kanjis' in self.data:
			self.data['kanjis'].append(kanji)
		else:
			self.data['kanjis'] = [kanji]

	def push_void_definition(self):
		if 'defs' in self.data:
			self.data['defs'].append({})
		else:
			self.data['defs'] = [{}]

	def set_definition_order(self, order):
		self.data['defs'][-1]['order'] = order

	def glue_definition_indicator(self, ind):
		if 'defs' in self.data:
			if 'ind' in self.data['defs'][-1]:
				self.data['defs'][-1]['ind'] += ind
			else:
				self.data['defs'][-1]['ind'] = ind

	def push_definition_chinese_translation(self, trans):
		if 'chi_transs' in self.data['defs'][-1]:
			self.data['defs'][-1]['chi_transs'].append(trans)
		else:
			self.data['defs'][-1]['chi_transs'] = [trans]

	def push_definition_english_translation(self, trans):
		if 'eng_transs' in self.data['defs'][-1]:
			self.data['defs'][-1]['eng_transs'].append(trans)
		else:
			self.data['defs'][-1]['eng_transs'] = [trans]

	def push_void_definition_sentence_example(self):
		if 'sent_exs' in self.data['defs'][-1]:
			self.data['defs'][-1]['sent_exs'].append({})
		else:
			self.data['defs'][-1]['sent_exs'] = [{}]

	def glue_definition_sentence_example(self, jpn_sent, chi_sent):
		if jpn_sent is not None:
			if 'jpn_sent' in self.data['defs'][-1]['sent_exs'][-1]:
				self.data['defs'][-1]['sent_exs'][-1]['jpn_sent'] += jpn_sent
			else:
				self.data['defs'][-1]['sent_exs'][-1]['jpn_sent'] = jpn_sent

		if chi_sent is not None:
			if 'chi_sent' in self.data['defs'][-1]['sent_exs'][-1]:
				self.data['defs'][-1]['sent_exs'][-1]['chi_sent'] += chi_sent
			else:
				self.data['defs'][-1]['sent_exs'][-1]['chi_sent'] = chi_sent

	def push_void_phrase(self):
		if 'phrs' in self.data:
			self.data['phrs'].append({})
		else:
			self.data['phrs'] = [{}]

	def set_phrase_idx(self, idx):
		self.data['phrs'][-1]['idx'] = idx

	def glue_phrase_first_title(self, title):
		if 'titles' in self.data['phrs'][-1]:
			self.data['phrs'][-1]['titles'][0] += title
		else:
			self.data['phrs'][-1]['titles'] = [title]

	def push_phrase_title(self, title):
		self.data['phrs'][-1]['titles'].append(title)

	def push_void_phrase_section(self):
		if 'secs' in self.data['phrs'][-1]:
			self.data['phrs'][-1]['secs'].append({})
		else:
			self.data['phrs'][-1]['secs'] = [{}]

	def push_phrase_section_translation(self, trans):
		if 'transs' in self.data['phrs'][-1]['secs'][-1]:
			self.data['phrs'][-1]['secs'][-1]['transs'].append(trans)
		else:
			self.data['phrs'][-1]['secs'][-1]['transs'] = [trans]

	def push_void_phrase_section_sentence_example(self):
		if 'sent_exs' in self.data['phrs'][-1]['secs'][-1]:
			self.data['phrs'][-1]['secs'][-1]['sent_exs'].append({})
		else:
			self.data['phrs'][-1]['secs'][-1]['sent_exs'] = [{}]

	def glue_phrase_section_sentence_example(self, jpn_sent, chi_sent):
		if jpn_sent is not None:
			if 'jpn_sent' in self.data['phrs'][-1]['secs'][-1]['sent_exs'][-1]:
				self.data['phrs'][-1]['secs'][-1]['sent_exs'][-1]['jpn_sent'] += jpn_sent
			else:
				self.data['phrs'][-1]['secs'][-1]['sent_exs'][-1]['jpn_sent'] = jpn_sent

		if chi_sent is not None:
			if 'chi_sent' in self.data['phrs'][-1]['secs'][-1]['sent_exs'][-1]:
				self.data['phrs'][-1]['secs'][-1]['sent_exs'][-1]['chi_sent'] += chi_sent
			else:
				self.data['phrs'][-1]['secs'][-1]['sent_exs'][-1]['chi_sent'] = chi_sent

	def get_data(self):
		return self.data

class SCJDStateMachine:
	class Node(Enum):
		IGNORE = 0
		ROOT = 1
		D_ENTRY = 2
		D_PRN = 3
		D_DEF = 4
		A = 5

		HWG_SPAN = 10
		HW_SPAN = 11
		HV_SPAN = 12
		GRAMB_SPAN = 13
		SEMB_SPAN = 14
		TRG_SPAN = 15
		OUP_LABEL_SPAN = 16
		TRANS_SPAN = 17
		IDMB_SPAN = 18
		IDMSEC_SPAN = 19
		IDM_SPAN = 20
		EXG_SPAN = 21
		EX_SPAN = 22
		IND_SPAN = 23
		CB_SPAN = 24
		CSEC_SPAN = 25
		CW_SPAN = 26
		CV_SPAN = 27

	class State(Enum):
		NUMBNESS = 0
		BEGIN = 1

		GRAMB = 10
		GRAMB_SEMB = 11
		GRAMB_OUP_LABEL = 12
		GRAMB_TRG_AFTER_OUP_LABEL = 13
		GRAMB_EXG = 14

		IDMB = 20
		IDMB_FIRST_IDM = 21
		IDMB_NOT_FIRST_IDM = 22
		IDMB_SEMB = 23
		IDMB_EXG = 24

		CB = 30
		CB_GRAMB = 31
		CB_SEMB = 32
		CB_EXG = 33

		def get_gramb_cluster():
			return [SCJDStateMachine.State.GRAMB, SCJDStateMachine.State.GRAMB_SEMB, SCJDStateMachine.State.GRAMB_OUP_LABEL, SCJDStateMachine.State.GRAMB_EXG, SCJDStateMachine.State.GRAMB_TRG_AFTER_OUP_LABEL]

		def get_idmb_cluster():
			return [SCJDStateMachine.State.IDMB, SCJDStateMachine.State.IDMB_FIRST_IDM, SCJDStateMachine.State.IDMB_NOT_FIRST_IDM, SCJDStateMachine.State.IDMB_SEMB, SCJDStateMachine.State.IDMB_EXG]

		def get_cb_cluster():
			return [SCJDStateMachine.State.CB, SCJDStateMachine.State.CB_GRAMB, SCJDStateMachine.State.CB_SEMB, SCJDStateMachine.State.CB_EXG]

	IGNORE_SPAN = {'hvg', 'gp', 'x_xoh', 'ty_pinyin', 'x_xdh', 'sn', 'gl', 'cwg', 'cvg', 'tail', 'ty_日中比較', 'x_xopt', 'pr', 'ty_参考', 'ty_参考参照', 'ty_項目参照', 'ty_注意', 'gr', 'ty_文化', 'ph', 'xr', 'xrlabelGroup', 'xrlabel', 'underline'}
	INHERIT_SPAN = {'rf', 'tg_ind', 't_fld', 'subEnt'}

	def __init__(self):
		self.reinit()

	def reinit(self):
		self.stk = [SCJDStateMachine.Node.ROOT]
		self.sta = SCJDStateMachine.State.BEGIN

	def get_node(self):
		return self.stk[-1]

	def get_state(self):
		return self.sta

	def push_node(self, node):
		self.stk.append(node)

	def pop_node(self):
		return self.stk.pop()

	def numb(self):
		self.sta = SCJDStateMachine.State.NUMBNESS

	def is_numb(self):
		return self.get_state() == SCJDStateMachine.State.NUMBNESS

	def startelement_move(self, tag, attrs):

		if tag == 'd:entry':
			self.push_node(SCJDStateMachine.Node.D_ENTRY)

		elif tag == 'span':
			attrs_keys = attrs.getQNames()
			if 'class' in attrs_keys:
				attrs_class_values = attrs['class'].split(' ')

				if not SCJDStateMachine.INHERIT_SPAN.isdisjoint(attrs_class_values):
					self.push_node(self.get_node())

				elif not SCJDStateMachine.IGNORE_SPAN.isdisjoint(attrs_class_values):
					self.push_node(SCJDStateMachine.Node.IGNORE)

				elif 'hwg' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.HWG_SPAN)

				elif 'hw' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.HW_SPAN)

				elif 'hv' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.HV_SPAN)

				elif 'gramb' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.GRAMB_SPAN)
					if self.get_state() == SCJDStateMachine.State.BEGIN:
						self.sta = SCJDStateMachine.State.GRAMB

					elif self.get_state() in SCJDStateMachine.State.get_cb_cluster():
						self.sta = SCJDStateMachine.State.CB_GRAMB

					else:
						raise RuntimeError(f'{self.get_state()}')

				elif 'semb' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.SEMB_SPAN)

					if self.get_state() in SCJDStateMachine.State.get_gramb_cluster():
						self.sta = SCJDStateMachine.State.GRAMB_SEMB

					elif self.get_state() in SCJDStateMachine.State.get_idmb_cluster():
						self.sta = SCJDStateMachine.State.IDMB_SEMB

					elif self.get_state() in SCJDStateMachine.State.get_cb_cluster():
						self.sta = SCJDStateMachine.State.CB_SEMB

					else:
						raise RuntimeError(f'{self.get_state()}')

				elif 'trg' in attrs_class_values:
					"""
					Generally, there is only one "oup_label" span in "gramb-semb" span, except in some rare cases.
					This happens when there is more than one kinds of translation in one "gramb-semb" span.
					An example is where id = j_CRJC000115, and title = 相手役.
					That "[芝居など] 配角" and "[ダンス] 舞伴" show up here is weird.
					And this is the reason why I put "tg_ind" span into INHERIT_SPAN instead of IGNORE_SPAN.
					Otherwise, key of ind in entry will become "芝居などダンス" instead of "[芝居など][ダンス]".
					"""
					if 'x_xd2' in attrs_class_values:
						if self.get_state() == SCJDStateMachine.State.GRAMB_OUP_LABEL:
							self.sta = SCJDStateMachine.State.GRAMB_TRG_AFTER_OUP_LABEL

						elif self.get_state() == SCJDStateMachine.State.GRAMB_EXG:
							pass

						elif self.get_state() == SCJDStateMachine.State.GRAMB_SEMB:
							pass

						elif self.get_state() == SCJDStateMachine.State.GRAMB_TRG_AFTER_OUP_LABEL:
							pass

						else:
							raise RuntimeError(f'{self.get_state()}')

					self.push_node(SCJDStateMachine.Node.TRG_SPAN)

				elif 'oup_label' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.OUP_LABEL_SPAN)

					if self.get_state() == SCJDStateMachine.State.GRAMB_SEMB:
						self.sta = SCJDStateMachine.State.GRAMB_OUP_LABEL

					elif self.get_state() == SCJDStateMachine.State.GRAMB_TRG_AFTER_OUP_LABEL:
						self.sta = SCJDStateMachine.State.GRAMB_OUP_LABEL

						"""
						Generally, there is no "oup_label" span in "idmb" span, except in some rare cases.
						An example is where id = j_CRJC010600, and title = 塞翁が馬.
						"""
					elif self.get_state() == SCJDStateMachine.State.IDMB_SEMB:
						pass

					else:
						raise RuntimeError(f'{self.get_state()}')

				elif 'trans' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.TRANS_SPAN)

				elif 'idmb' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.IDMB_SPAN)
					self.sta = SCJDStateMachine.State.IDMB

				elif 'idmsec' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.IDMSEC_SPAN)

				elif 'idm' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.IDM_SPAN)

					if self.get_state() == SCJDStateMachine.State.IDMB or self.get_state() == SCJDStateMachine.State.IDMB_EXG or self.get_state() == SCJDStateMachine.State.IDMB_SEMB:
						self.sta = SCJDStateMachine.State.IDMB_FIRST_IDM

					elif self.get_state() == SCJDStateMachine.State.IDMB_FIRST_IDM:
						self.sta = SCJDStateMachine.State.IDMB_NOT_FIRST_IDM

				elif 'exg' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.EXG_SPAN)

					if self.get_state() in SCJDStateMachine.State.get_gramb_cluster():
						self.sta = SCJDStateMachine.State.GRAMB_EXG

					elif self.get_state() in SCJDStateMachine.State.get_idmb_cluster():
						self.sta = SCJDStateMachine.State.IDMB_EXG

					elif self.get_state() in SCJDStateMachine.State.get_cb_cluster():
						self.sta = SCJDStateMachine.State.CB_EXG

					else:
						raise RuntimeError(f'{self.get_state()}')

				elif 'ex' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.EX_SPAN)

				elif 'ind' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.IND_SPAN)

				elif 'fld' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.IND_SPAN)

				elif 'cb' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.CB_SPAN)
					self.sta = SCJDStateMachine.State.CB

				elif 'csec' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.CSEC_SPAN)

				elif 'cw' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.CW_SPAN)

				elif 'cv' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.CV_SPAN)

				else:
					raise RuntimeError(f"SPAN with {attrs_class_values} in class key is not defined")

			else:
				raise RuntimeError(f"SPAN with {attrs_keys} key is not defined")

		elif tag == 'd:prn':
			self.push_node(SCJDStateMachine.Node.D_PRN)

		elif tag == 'd:def':
			self.push_node(SCJDStateMachine.Node.D_DEF)

		elif tag == 'a':
			self.push_node(SCJDStateMachine.Node.A)

		else:
			raise RuntimeError(f"TAG {tag} is not defined")

	def endelement_move(self, tag):
		self.pop_node()

	def startdocument_move(self):
		self.reinit()

	def enddocument_move(self):
		pass

def log(func):
	@functools.wraps(func)
	def wrapper(*args, **kwargs):
		if args[0].debug:
			if func.__name__ == 'startElement':
				print(f'SM: {args[0].sm.stk}, State: {args[0].sm.sta}')
				print(f'Tag: {args[1]}, class: {args[2].getValueByQName("class") if "class" in args[2].getQNames() else None}')
				print(f'{args[0].entry.get_data()}')
				print(f'==========')
			elif func.__name__ == 'endElement':
				print(f'Tag: {args[1]}')
				print(f'==========')
			elif func.__name__ == 'characters':
				print(f'Chars: {args[1]}')
				print(f'==========')
		return func(*args, **kwargs)
	return wrapper

class SCJDController:
	def __init__(self, debug=False):
		self.reinit()
		self.debug = debug

	def reinit(self):
		self.sm = SCJDStateMachine()
		self.entry = SCJDEntry()

	def get_entry(self):
		return self.entry.data

	@log
	def startElement(self, tag, attrs):
		if self.sm.is_numb():
			return

		self.sm.startelement_move(tag, attrs)

		if self.sm.get_node() == SCJDStateMachine.Node.D_ENTRY:
			if re.search("^j_CRJC.*", attrs['id']) is not None:
				self.entry.set_title(attrs['d:title'])
				self.entry.set_id(attrs['id'])
			else:
				self.sm.numb()

		elif self.sm.get_node() == SCJDStateMachine.Node.SEMB_SPAN:
			if self.sm.get_state() in SCJDStateMachine.State.get_gramb_cluster():
				self.entry.push_void_definition()
				if 'ord' in attrs.getQNames():
					self.entry.set_definition_order(attrs['ord'])
				else:
					self.entry.set_definition_order('1')

			elif self.sm.get_state() in SCJDStateMachine.State.get_idmb_cluster():
				self.entry.push_void_phrase_section()

			elif self.sm.get_state() in SCJDStateMachine.State.get_cb_cluster():
				self.entry.push_void_phrase_section()

			else:
				raise RuntimeError(f"Node {self.sm.get_node()} with State {self.sm.get_state()} startElement function is not defined")

		elif self.sm.get_node() == SCJDStateMachine.Node.IDMSEC_SPAN:
			self.entry.push_void_phrase()
			self.entry.set_phrase_idx(attrs['id'])

		elif self.sm.get_node() == SCJDStateMachine.Node.EXG_SPAN:
			if self.sm.get_state() in SCJDStateMachine.State.get_idmb_cluster():
				self.entry.push_void_phrase_section_sentence_example()

			elif self.sm.get_state() in SCJDStateMachine.State.get_gramb_cluster():
				self.entry.push_void_definition_sentence_example()

			elif self.sm.get_state() in SCJDStateMachine.State.get_cb_cluster():
				self.entry.push_void_phrase_section_sentence_example()

			else:
				raise RuntimeError(f"Node {self.sm.get_node()} with State {self.sm.get_state()} startElement function is not defined")

		elif self.sm.get_node() == SCJDStateMachine.Node.CSEC_SPAN:
			self.entry.push_void_phrase()
			self.entry.set_phrase_idx(attrs['id'])

	@log
	def endElement(self, tag):
		if self.sm.is_numb():
			return
		self.sm.endelement_move(tag)

	@log
	def characters(self, chars):
		if self.sm.is_numb():
			return
		def process_chars(chars):
			return chars.strip()

		if self.sm.get_node() == SCJDStateMachine.Node.HW_SPAN:
			self.entry.glue_pronounce(process_chars(chars))

		elif self.sm.get_node() == SCJDStateMachine.Node.HV_SPAN:
			self.entry.push_kanji(process_chars(chars))

		elif self.sm.get_node() == SCJDStateMachine.Node.TRANS_SPAN:
			if self.sm.get_state() == SCJDStateMachine.State.GRAMB_SEMB:
				self.entry.push_definition_chinese_translation(process_chars(chars))

			elif self.sm.get_state() == SCJDStateMachine.State.GRAMB_TRG_AFTER_OUP_LABEL:
				self.entry.push_definition_chinese_translation(process_chars(chars))

			elif self.sm.get_state() == SCJDStateMachine.State.GRAMB_OUP_LABEL:
				self.entry.push_definition_english_translation(process_chars(chars))

			elif self.sm.get_state() == SCJDStateMachine.State.GRAMB_EXG:
				self.entry.glue_definition_sentence_example(None, process_chars(chars))

			elif self.sm.get_state() == SCJDStateMachine.State.IDMB_SEMB:
				self.entry.push_phrase_section_translation(chars.strip())

			elif self.sm.get_state() == SCJDStateMachine.State.IDMB_EXG:
				self.entry.glue_phrase_section_sentence_example(None, chars.strip())

			elif self.sm.get_state() == SCJDStateMachine.State.CB_SEMB:
				self.entry.push_phrase_section_translation(chars.strip())

			elif self.sm.get_state() == SCJDStateMachine.State.CB_EXG:
				self.entry.glue_phrase_section_sentence_example(None, chars.strip())

			else:
				raise RuntimeError(f"Node {self.sm.get_node()} with State {self.sm.get_state()} characters function is not defined")

		elif self.sm.get_node() == SCJDStateMachine.Node.IDM_SPAN:
			if self.sm.get_state() == SCJDStateMachine.State.IDMB_FIRST_IDM:
				self.entry.glue_phrase_first_title(process_chars(chars))

			elif self.sm.get_state() == SCJDStateMachine.State.IDMB_NOT_FIRST_IDM:
				self.entry.push_phrase_title(process_chars(chars))

			else:
				raise RuntimeError(f"Node {self.sm.get_node()} with State {self.sm.get_state()} characters function is not defined")

		elif self.sm.get_node() == SCJDStateMachine.Node.EX_SPAN:
			if self.sm.get_state() == SCJDStateMachine.State.IDMB_EXG:
				self.entry.glue_phrase_section_sentence_example(process_chars(chars), None)

			elif self.sm.get_state() == SCJDStateMachine.State.CB_EXG:
				self.entry.glue_phrase_section_sentence_example(process_chars(chars), None)

			elif self.sm.get_state() == SCJDStateMachine.State.GRAMB_EXG:
				self.entry.glue_definition_sentence_example(process_chars(chars), None)

			else:
				raise RuntimeError(f"Node {self.sm.get_node()} with State {self.sm.get_state()} characters function is not defined")

		elif self.sm.get_node() == SCJDStateMachine.Node.IND_SPAN:
			self.entry.glue_definition_indicator(process_chars(chars))

		elif self.sm.get_node() == SCJDStateMachine.Node.CW_SPAN:
			self.entry.glue_phrase_first_title(process_chars(chars))

		elif self.sm.get_node() == SCJDStateMachine.Node.CV_SPAN:
			self.entry.push_phrase_title(process_chars(chars))

	def startDocument(self):
		self.sm.startdocument_move()
		self.reinit()

	def endDocument(self):
		if self.sm.is_numb():
			return
		self.sm.enddocument_move()
		# print(self.entry.data['title'])


import cv2

import numpy as np

from scipy import signal

import math

import matplotlib.pyplot as plt

if __name__ == "__main__":

    gauss_blur_filter = [[0 for x in range(3)] for y in range(3)]

    gauss_blur_filter[0][0] = 1/16
    gauss_blur_filter[0][1] = 1/8
    gauss_blur_filter[0][2] = 1/16
    gauss_blur_filter[1][0] = 1/8
    gauss_blur_filter[1][1] = 1/4
    gauss_blur_filter[1][2] = 1/8
    gauss_blur_filter[2][0] = 1/16
    gauss_blur_filter[2][1] = 1/8
    gauss_blur_filter[2][2] = 1/16

    image = cv2.imread('point.jpg',0)

    kernel = np.array([[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]], dtype = np.float)

    replicate = cv2.copyMakeBorder(image,20,20,20,20,cv2.BORDER_REPLICATE)

    resultant_image = cv2.blur(replicate,(5,5))

    cv2.imwrite('gauss-blue.jpg',resultant_image)

    resultant_image_1 = signal.convolve2d(image,kernel,'same')

    rows,columns = resultant_image_1.shape

    for i in range(rows):
        for j in range(columns):

            resultant_image_1[i][j] = abs(resultant_image_1[i][j])

    cv2.imwrite('mask-application.jpg',resultant_image_1)

    print(resultant_image_1.max())

    for i in range(rows):
        for j in range(columns):

            if resultant_image_1[i][j] >= 2024:

                print(i,j)

            else:

                resultant_image_1[i][j] = 0

    cv2.imwrite('point-detection.jpg',resultant_image_1)

    image_segment = cv2.imread('segment.jpg',0)

    rows,columns = image_segment.shape

    '''x = np.zeros(255)

    y = np.arange(0,255,1)
    for i in range(rows):
        for j in range(columns):
            if image_segment[i][j] != 0:
                x[image_segment[i][j]] += 1

    hist, bins = np.histogram(x, bins=y)
    width = 1.0 * (bins[1] - bins[0])
    center = (bins[:-1] + bins[1:]) / 2
    plt.bar(center, hist, align='center', width=width)

    #plt.plot(y,x)
    #plt.bar(np.arange(len(y)),y)
    plt.show()'''

    for i in range(rows):
        for j in range(columns):

            if image_segment[i][j] > 208 or image_segment[i][j] < 200 :
                image_segment[i][j] = 0

    cv2.imwrite('segemented.jpg',image_segment)


# https://www.codewars.com/kata/58558673b6b0e5a16b000028

def fight_resolve(defender, attacker):
    if (defender.lower() == defender) == (attacker.lower() == attacker):
        return -1
    defender_win = { 'a': 's', 'k':'a', 'p': 'k', 's': 'p'}
    a = attacker.lower()
    d = defender.lower()
    if defender_win[d] == a:
        return defender
    else:
        return attacker


import pygame

from configs import ColorConfig

class Button(object):

    def __init__(self, x_coordinate: int, y_coordinate: int, button_width: int,
                 button_height: int, text_font: str, text_size: str,
                 button_name: str, onclick_function=None):
        self.x = x_coordinate
        self.y = y_coordinate
        self.width = button_width
        self.height = button_height
        self.function_by_click = onclick_function
        self.buttonSurface = pygame.Surface((self.width, self.height))
        self.buttonRect = pygame.Rect(self.x, self.y, self.width, self.height)
        self.buttonSurf = pygame.font.SysFont(
            text_font, int(text_size)).render(button_name, True, (20, 20, 20))

    def process(self, game_window: pygame.display, parameter: str):
        self.buttonSurface.fill(ColorConfig.WHITE)
        if self.buttonRect.collidepoint(pygame.mouse.get_pos()):
            self.buttonSurface.fill(ColorConfig.GREY)
            if pygame.mouse.get_pressed(num_buttons=3)[0]:
                self.buttonSurface.fill(ColorConfig.GREEN)
                self.function_by_click(parameter)
                return True
        self.buttonSurface.blit(self.buttonSurf, [
            self.buttonRect.width / 2 - self.buttonSurf.get_rect().width / 2,
            self.buttonRect.height / 2 - self.buttonSurf.get_rect().height / 2
        ])
        game_window.blit(self.buttonSurface, self.buttonRect)