import random
import imageio
import pandas as pd
import numpy as np
from PIL import Image
from sklearn.model_selection import train_test_split
import csv
import matplotlib.pyplot as plt
import matplotlib.pyplot as plt
import pandas as pd
from numpy.random import RandomState
from utils.data_collection.img_plots import img_plots
from utils.data_collection.convert import convert_img

# -----------------------------------------------------------
# all functions that evaluate the image files
#
# (C) 2022 Felix Nahrstedt, Berlin, Germany
# email contact@felixnahrstedt.com
# -----------------------------------------------------------

def calcBrightness(img_arr):
    sum = 0
    for pixelArr in img_arr:
      for pixel in pixelArr:
         sum = sum+pixel
    return sum

def normalizeImages(minBrightness, image):
    faktor = minBrightness/calcBrightness(image)
    newImage = (faktor*image).astype(np.uint8)
    return newImage

def maxMeanBrightness(args):
    meanBrightness = []
    stdBrightness = []
    for element in args:
        meanBrightness.append(element.mean(axis=0).mean())
        stdBrightness.append(element.std(axis=0).std())
    #mean = / amount of pixel and / maximum grayscale val --> 255
    return max(meanBrightness)/255, max(stdBrightness)

def evaluate_images(path_images, id, date, bands):
    img_Band_arr = []
    for band in bands:
        img_Band_arr.append(imageio.imread(f'{path_images}/{id}-{date}-{band}.jpg'))
    maxMeanBright, maxStdBright = maxMeanBrightness(img_Band_arr)
    obj = img_plots(path_images,id,bands,date)
    histArr = obj.histogram_matching(img_Band_arr)
    for idx,band in enumerate(bands):
        imageio.imsave(f'{path_images}/Matched-{id}-{date}-{band}.jpg',histArr[idx])
    return maxMeanBright, maxStdBright, img_Band_arr
def NormalizeData(data):
    return (data - np.min(data)) / (np.max(data) - np.min(data))
def display_data(path_data):
    file = open(path_data)
    csvreader = csv.reader(file)
    header = []
    header = next(csvreader)
    rows = []
    for row in csvreader:
        rows.append(row)
    file.close()
    # Define Data
    #0 = Drehendes Windrad,
    goodQual = []
    goodQual2 = []

    medQual1 = []
    medQual2 = []

    badQual1 = []
    badQual2 = []
    print(header)
    amountSpin = 0
    amountNotSpin = 0
    amountUndetected = 0
    goodVisibleSpin = 0
    print(f'Insgesamt: {len(rows)}')
    for row in rows:
        if(row[3] =='0'):
            amountSpin = amountSpin+1
            goodQual.append(row[1])
            goodQual2.append(row[2])
        if(row[3] =='1'):
            amountNotSpin = amountNotSpin +1
            medQual1.append(row[1])
            medQual2.append(row[2])
        if(row[3] =='2'):
            amountUndetected = amountUndetected +1
            badQual1.append(row[1])
            badQual2.append(row[2])
        # if(row[3] == '0' and (row[4] == '1' or row[4] == '2' )):
        #     goodVisibleSpin = goodVisibleSpin +1
        # if(row[4] =='1' or row[4] =='2'):
        #     goodQual.append(row[5])
        #     goodQual2.append(row[6])

        # elif(row[4] =='3' or row[4] == '4'):
        #     medQual1.append(row[5])
        #     medQual2.append(row[6])
        # else:
        #     badQual1.append(row[5])
        #     badQual2.append(row[6])
    #1 = Stehendes Windrad,
    #2 = Nicht erkennbar
    print(f'Of these Spinning: {amountSpin}, Not Spinning: {amountNotSpin}, undetectable: {amountUndetected}')
    print(f'spinning and in class 1-2: {goodVisibleSpin}')
    # Create Figure

    fig = plt.figure(figsize = (7,8))
    plt.title('Relationships Shadow Recognizability')
    plt.ylabel('longitude')
    plt.xlabel('latitude')
    # Create Plot
    plt.scatter(np.asarray(goodQual2).astype(np.float16),np.asarray(goodQual).astype(np.float16), s=20, color='green',label='Spinning',alpha = 0.2)
    plt.scatter(np.asarray(medQual2).astype(np.float16),np.asarray(medQual1).astype(np.float16),  s=20, color='blue',label='Not Spinning',alpha = 0.2)
    plt.scatter(np.asarray(badQual2).astype(np.float16),np.asarray(badQual1).astype(np.float16), s=20, color='red',label='Undetected',alpha = 0.2)

    plt.legend(loc='lower right')
    plt.savefig("Data/data_science/PLOTS/Relationships_Shadow_Recognizability.png")

    # Show plot

    plt.show()

#! MOVE TO load_data.py !! to-do
# -----------------------------------------------------------
# all functions for saving/loading data
#
# (C) 2022 Felix Nahrstedt, Berlin, Germany
# email contact@felixnahrstedt.com
# -----------------------------------------------------------

def appendCsv(pathCsv, id, lat, lon, std, mean, label, date, qual, region):
    with open(pathCsv, 'a', encoding='UTF8', newline='') as f:
        writer = csv.writer(f)
        # write the header

        # write the data
        writer.writerow([id,lat,lon,label,qual,mean,std,date,region])

def appendCsv_open(pathCsv, allData):
    with open(pathCsv, 'a', encoding='UTF8', newline='') as f:
        writer = csv.writer(f)
         # write the data
        writer.writerow(allData)

def createCsv(pathCsv, header):
    with open(pathCsv, 'w', encoding='UTF8', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(header)

def unspin_turbines(path_data, path_jpg,path_unspinned):
    file = open(path_data)
    csvreader = csv.reader(file)
    header = []
    header = next(csvreader)
    rows = []
    for row in csvreader:
        rows.append(row)
    file.close()
    print(header)
    goodQualSpin =  []
    for row in rows:
         if(row[3] == '0' and (row[4] == '1' or row[4] == '2' )):
            goodQualSpin.append([row[0],row[7]])

    for satImg in goodQualSpin:

        bands = ["B2","B3","B4"]
        imgArr = []
        originalFile = f'{satImg[0]}-{satImg[1]}'
        fakeFile = f'{satImg[0]}-unspin-{satImg[1]}'
        for band in bands:
            imgArr.append(np.asarray(Image.open(f'{path_jpg}/{originalFile}-{band}.jpg')))

        id = "280498836"
        bands = ["B2","B3","B4"]
        date = "2022-03-14"
        newImgPlot = img_plots(path_jpg,id,bands, date)
        def saltPepperChannel(frame,size):
                for pixelRow in range(size-1):
                    for pixel in range(size-1):
                        newVal = frame[pixelRow,pixel]
                        movementInt = random.randint(0,10)

                        chooseRand = random.randint(0,80)

                        #every second pixel either gets down or upscaled in order to add Noise
                        if(chooseRand >0 and chooseRand < 20):
                            newVal = newVal - movementInt
                        elif(chooseRand >20 and chooseRand < 40):
                            newVal = newVal + movementInt

                        if(newVal>=253):
                            newVal = 255

                        if(newVal <= 2):
                            newVal = 1

                        frame[pixelRow,pixel] = newVal

                return frame

        transformed = newImgPlot.histogram_Transform(np.asarray(imgArr))
        imgArr = []
        imgArr.append(saltPepperChannel(transformed[0],40))
        imgArr.append(saltPepperChannel(transformed[1],40))
        imgArr.append(saltPepperChannel(transformed[2],40))
        convertTransform = convert_img(id,date,imgArr)
        #convertTransform.saveToGif(path_jpg,imgArr)
        convertTransform.saveGrayScaleFromRGB(path_unspinned,imgArr,fakeFile,bands)

def createSubCsv(turbine_data,undetectedData,csvPath,subsetSize = 1600 ):
    header = ['id', 'latitude','longitude','label','quality','max_mean-bright', 'max_std_bright', 'date','region']
    #not_spinning = csvData+"not_spinning.csv"
    spinningAndUndetectedNew = csvPath + "spinningAndUndetectedNew-"+str(subsetSize)+".csv"
    # spinningTest = csvPath + "spinTest-"+str(subsetSize)+".csv"

    # undetected = csvPath+ "undetected-"+str(subsetSize)+".csv"
    # undetectedTest = csvPath+ "undetectedTest-"+str(subsetSize)+".csv"

    #createCsv(not_spinning, header)
    createCsv(spinningAndUndetectedNew, header)
    # createCsv(spinningTest, header)

    # createCsv(undetected, header)
    # createCsv(undetectedTest, header)

    file = open(turbine_data)
    fileUndetected = open(undetectedData)
    csvreader = csv.reader(file)
    csvreaderUndetected = csv.reader(fileUndetected)
    rowsSpin = []
    rowsUndetected = []

    for row in csvreader:
        if(row[0]!="id"):
            rowsSpin.append(row)
    for row in csvreaderUndetected:
        rowsUndetected.append(row)
    file.close()
    fileUndetected.close()
    print(len(rowsSpin))
    spines = 0
    spinesTest = 0
    undetectedes =0
    undetectedesTest = 0
    for row in rowsUndetected:
        if(undetectedes < subsetSize):
            undetectedes +=1
            appendCsv_open(spinningAndUndetectedNew, row)
    for row in rowsSpin:
        if(row[3] == '0' and (int(row[4])<4)):

            if(spines < subsetSize):
                spines = spines +1
                #append date and ID for later image loading
                appendCsv_open(spinningAndUndetectedNew,row)
            # if(row[4] == '1' or row[4] == '2'):
            #     if(nospins<subsetSize):
            #         nospins = nospins+1
            #         row[0] = row[0]+"-unspin"
            #         row[3] = 1
            #         appendCsv_open(not_spinning,row)
            # elif(spinesTest<750):
            #     spinesTest +=1
            #     appendCsv_open(spinningTest,row)

        # if(row[3] == '2'):
        #     if(undetectedes<subsetSize):
        #         undetectedes = undetectedes +1
        #         appendCsv_open(undetected,row)
        #     elif(undetectedesTest<750):
        #         undetectedesTest +=1
        #         appendCsv_open(undetectedTest,row)
        # if(row[3] == '1'):
        #     if(nospins<1064):
        #         nospins = nospins +1
        #         appendCsv_open(not_spinning,row)
    print(spines,undetectedes)
    pathTrain = csvPath+"train-"+str(subsetSize)+".csv"
    pathTest = csvPath+"test-"+str(subsetSize)+".csv"
    csvToTrainTest(spinningAndUndetectedNew,pathTrain,pathTest)
    #return train, test
    #print(spines, nospins, undetectedes)
    #return spinning,spinningTest, undetected,undetectedTest

def splitTrainTest(spinningCsv,spinningTestCsv,UndetectedCsv,UndetectedTestCsv, TrainCsv, TestCsv,notSpinningCsv="",size=1600):
    csvreader = csv.reader(spinningCsv)

    dfSpinTrain = pd.read_csv(spinningCsv)
    dfSpinTest = pd.read_csv(spinningTestCsv)
    dfUndetectedTrain =  pd.read_csv(UndetectedCsv)
    #dfNoSpinTrain,dfNoSpinTest = csvToTrainTest(notSpinningCsv)
    dfUndetectedTest =  pd.read_csv(UndetectedTestCsv)

    dfTrain = pd.concat([dfSpinTrain,dfUndetectedTrain])
    dfTest = pd.concat([dfSpinTest,dfUndetectedTest])
    print(f"Train: {len(dfTrain)}")
    print(f"Test: {len(dfTest)}")

    #shuffle
    dfTrainShuffled = dfTrain.sample(frac=1)
    dfTestShuffled = dfTest.sample(frac=1)
    dfTrainShuffled.to_csv(TrainCsv, index=False)
    dfTestShuffled.to_csv(TestCsv, index=False)

def csvToTrainTest(filePath, newTrainPath, newTestPath):
    df = pd.read_csv(filePath)
    df = df.sample(frac=1).reset_index(drop=True)
    train=df.sample(frac=0.8,random_state=200) #random state is a seed value
    test=df.drop(train.index)
    train.to_csv(newTrainPath, index=False)
    test.to_csv(newTestPath, index=False)


#
# Created by Maksim Eremeev (mae9785@nyu.edu)
#

from ..elsa import Preprocessing
from tqdm.auto import tqdm
import argparse

def prepare_fasttext_cnn(preprocessor, dataset_path, save_path):
    with open(dataset_path, 'r') as f:
        for line in tqdm(f, total=221000):
            doc = preprocessor.preproc(line, check_stopwords=False, check_length=True,
                                       use_lemm=False, use_stem=False, include_tf=False)
            with open(save_path, 'a') as fw:
                fw.write(f'{doc}\n')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--stopwords', nargs='*', help='path to stopwords.txt')
    parser.add_argument('-d', '--dataset', nargs='*', help='path to the dataset')
    parser.add_argument('-o', '--output', nargs='*', help='output path')
    args = parser.parse_args()

    preprocessor = Preprocessing(stopwords=args.stopwords[0])
    dataset_path = args.dataset[0]
    save_path = args.output[0]

    prepare_fasttext_cnn(preprocessor, dataset_path, save_path)


import torch

class FGM():
    def __init__(self, model):
        self.model = model
        self.backup = {}

    def attack(self, epsilon=1., emb_name='bert.embeddings.word_embeddings'):
        # emb_name这个参数要换成你模型中embedding的参数名
        for name, param in self.model.named_parameters():
            if param.requires_grad and emb_name in name:
                self.backup[name] = param.data.clone()
                norm = torch.norm(param.grad)
                if norm != 0 and not torch.isnan(norm):
                    r_at = epsilon * param.grad / norm
                    param.data.add_(r_at)

    def restore(self, emb_name='char_emb.'):
        # emb_name这个参数要换成你模型中embedding的参数名
        for name, param in self.model.named_parameters():
            if param.requires_grad and emb_name in name:
                assert name in self.backup
                param.data = self.backup[name]
        self.backup = {}

class PGD():
    def __init__(self, model):
        self.model = model
        self.emb_backup = {}
        self.grad_backup = {}

    def attack(self, epsilon=1., alpha=0.3, emb_name='char_emb.', is_first_attack=False):
        # emb_name这个参数要换成你模型中embedding的参数名
        for name, param in self.model.named_parameters():
            if param.requires_grad and emb_name in name:
                if is_first_attack:
                    self.emb_backup[name] = param.data.clone()
                norm = torch.norm(param.grad)
                if norm != 0 and not torch.isnan(norm):
                    r_at = alpha * param.grad / norm
                    param.data.add_(r_at)
                    param.data = self.project(name, param.data, epsilon)

    def restore(self, emb_name='char_emb.'):
        # emb_name这个参数要换成你模型中embedding的参数名
        for name, param in self.model.named_parameters():
            if param.requires_grad and emb_name in name:
                assert name in self.emb_backup
                param.data = self.emb_backup[name]
        self.emb_backup = {}

    def project(self, param_name, param_data, epsilon):
        r = param_data - self.emb_backup[param_name]
        if torch.norm(r) > epsilon:
            r = epsilon * r / torch.norm(r)
        return self.emb_backup[param_name] + r

    def backup_grad(self):
        for name, param in self.model.named_parameters():
            if param.requires_grad:
                self.grad_backup[name] = param.grad.clone()

    def restore_grad(self):
        for name, param in self.model.named_parameters():
            if param.requires_grad:
                param.grad = self.grad_backup[name]


import json
import os
import sys
import time as counter
from datetime import datetime

def readJSON(infile):
    with open(infile, "r") as f:
        data = f.read()
    return json.loads(data)

def writeMarkdown(processed, outfile):
    with open(outfile, "w+") as f:
        f.write(processed)

def setModTime(path, mod_time):
    # Convert to epoch (seconds)
    # Remove the 'Z' at end of ISO input: https://bugs.python.org/issue35829
    mtime = datetime.fromisoformat(mod_time[:-1])
    return os.utime(path, times=(datetime.now().timestamp(), mtime.timestamp()))

def main():
    if len(sys.argv) <= 2:
        print(
            "Missing arguments.\nUsage: python convert.py [JSON input filename] [output dirname]\nExample: python convert.py export.json notes\n"
        )
        sys.exit(1)

    infile = sys.argv[1]
    dirname = sys.argv[2]

    # Open the file
    try:
        raw_dict = readJSON(infile)
    except json.decoder.JSONDecodeError:
        print("Input file must be JSON format.")
        sys.exit(1)

    # Create directory for notes
    try:
        os.mkdir(dirname)
    except FileExistsError:
        print(
            f"A directory called {dirname} already exists. Stopping to avoid accidental overwriting.\nPlease provide a different directory name."
        )
        sys.exit(1)

    # Initialize stats
    count = 0
    start = counter.time()
    print("Starting the tedious work for you...")

    # Standard Notes export has a top-level "items" key
    items = raw_dict["items"]

    notes_list = []
    # Loop through the exported items. Not all of these are your notes.
    items: list
    obj: dict
    for obj in items:
        # Get just the notes in the notes_list of dicts
        if obj["content_type"] == "Note":
            # We only want these keys in the new dicts
            fields = ["uuid", "created_at", "updated_at", "content"]
            note: dict = {key: value for key, value in obj.items() if key in fields}
            notes_list.append(note)

    # Tag notes
    tags_list = []
    for obj in items:
        if obj["content_type"] == "Tag":
            fields = ["content"]
            tag: dict = {key: value for key, value in obj.items() if key in fields}
            tags_list.append(tag)

    # Loop the notes list
    note: dict
    notes_list: list
    for note in notes_list:
        tag_text = "\n" # Reset for each note
        # Optionally, update this to add a bit of the uuid to mitigate duplicate filenames
        if note["content"]["title"] == "":
            # If no title exists, use the creation date
            filename = f'{note["created_at"][:10]}.md'
        else:
            name = (
                f'{note["content"]["title"]}_{note["created_at"][:10]}'.replace("/", "or")
                .replace(" ", "_")
                .replace(".", "")
                .replace(":", "")
                .replace("'", "")
                .replace("?", "")
                .replace('"', "")
            )
            filename = f'{name}.md'

        # Loop the tags list
        tag: dict
        tags_list: list
        for tag in tags_list:
            if str(note["uuid"]) in str(tag["content"]["references"]):
                tag_text = tag_text + f'#{tag["content"]["title"]} '
        # Create a new .md file
        outpath = os.path.join(dirname, filename)
        # The str method renders newlines
        content = str(note["content"]["text"])
        processed = f'{content}\n{tag_text}'
        writeMarkdown(processed, outpath)
        # Preserve the note's last modified time
        setModTime(outpath, note["updated_at"])
        count += 1

    end = counter.time()
    time = end - start
    print(f"Created {count} notes in {time} seconds. You're welcome.")

if __name__ == "__main__":
    main()


#!/usr/bin/python
# -*- coding: utf-8 -*-

import numpy as np
import sys
import os

from matplotlib import pyplot as plt
import matplotlib.cm as cm
import matplotlib.patheffects as PathEffects

import netCDF4 as nc
import cartopy.crs as ccrs
import pandas as pd

from numpy import linalg as LA
from eofs.standard import Eof
from scipy import stats
from scipy import interpolate as itrp
import itertools as itt

from sklearn.cluster import KMeans

from datetime import datetime
import pickle

import climtools_lib as ctl
import climdiags as cd

from copy import deepcopy as cp

##############################################
### constants
# cp = 4185.5 # J kg-1 K-1 QUESTA E L'ACQUA COJOOOO
cp = 1005.0 # specific enthalpy dry air - J kg-1 K-1
cpw = 1840.0 # specific enthalpy water vapor - J kg-1 K-1
# per la moist air sarebbe cpm = cp + q*cpw, ma conta al massimo per un 3 %
L = 2501000.0 # J kg-1
Lsub = 2835000.0 # J kg-1
g = 9.81 # m s-2
Rearth = 6371.0e3 # mean radius
#####################################

#plt.ion()
shortnam = {'mshf': 'SH', 'mpef': 'PE', 'mlhf': 'LH'}
fluxnames = ['mshf', 'mpef', 'mlhf']
fluxlongnames = {'mshf':'Sensible Heat', 'mpef':'Potential Energy', 'mlhf':'Latent Heat'}
factors = [cp/g, 1., L/g]

cart_out = '/home/fabiano/Research/lavori/SPHINX_for_lisboa/heat_flux/'

figure_file_all = cart_out+'heat_fluxes_lcs0_vs_lcb0.pdf'
figures_all = []

zonal_margins = dict()
zonal_margins['tot'] = (-3.5e16, 4.5e16)
zonal_margins['mshf'] = (-3.5e16, 4.5e16)
zonal_margins['mpef'] = (-3.5e16, 4.5e16)
zonal_margins['mlhf'] = (-7e15, 9.e15)
map_margins = dict()
map_margins['mshf'] = (-1.5e10, 1.5e10)
map_margins['mpef'] = (-4.e9, 4.e9)
map_margins['mlhf'] = (-7.e8, 7.e8)
map_margins['tot'] = (-2.e10, 2.e10)

# Loading reference pressure file
pressurefile = '/data-hobbes/fabiano/SPHINX/heat_flux/1988_daily/lcs0_day_1988_ps.nc'
press0row, lat, lon, datespress, time_units, var_units = ctl.read3Dncfield(pressurefile)
press0 = dict()
press0['DJF'] = np.mean(ctl.sel_season(press0row, datespress, 'DJF', cut = False)[0], axis = 0)
press0['JJA'] = np.mean(ctl.sel_season(press0row, datespress, 'JJA', cut = False)[0], axis = 0)
press0['year'] = np.mean(press0row, axis = 0)

# Loading ERA reference
cart_era = '/data-hobbes/fabiano/OBS/ERA/ERAInterim/'
era_fi = 'prova_heatflux_1988_MM.nc'
cpc = nc.Dataset(cart_era+era_fi)
era_lat = cpc.variables['latitude'][:]
era_lon = cpc.variables['longitude'][:]
era_zonal_factor = 2*np.pi*Rearth*np.cos(np.deg2rad(era_lat))

era_fluxes_maps = dict()
era_fluxes_zonal = dict()
era_fluxes_zonal_itrp = dict()

okmon = [0,1,11]
era_fluxes_maps[('DJF','tot')] = np.mean(list(cpc.variables.values())[-1][okmon, ...], axis = 0)
era_fluxes_maps[('DJF','mpef')] = np.mean(list(cpc.variables.values())[-2][okmon, ...], axis = 0)
era_fluxes_maps[('DJF','mshf')] = np.mean(list(cpc.variables.values())[-3][okmon, ...], axis = 0)
okmon = [5,6,7]
era_fluxes_maps[('JJA','tot')] = np.mean(list(cpc.variables.values())[-1][okmon, ...], axis = 0)
era_fluxes_maps[('JJA','mpef')] = np.mean(list(cpc.variables.values())[-2][okmon, ...], axis = 0)
era_fluxes_maps[('JJA','mshf')] = np.mean(list(cpc.variables.values())[-3][okmon, ...], axis = 0)

era_fluxes_maps[('year','tot')] = np.mean(list(cpc.variables.values())[-1][:], axis = 0)
era_fluxes_maps[('year','mpef')] = np.mean(list(cpc.variables.values())[-2][:], axis = 0)
era_fluxes_maps[('year','mshf')] = np.mean(list(cpc.variables.values())[-3][:], axis = 0)

fi = 'northward_water_flux.nc'
cpc = nc.Dataset(cart_era+fi)
okmon = [0,1,11]
era_fluxes_maps[('DJF','mlhf')] = L*np.mean(list(cpc.variables.values())[-1][okmon, ...], axis = 0)
okmon = [5,6,7]
era_fluxes_maps[('JJA','mlhf')] = L*np.mean(list(cpc.variables.values())[-1][okmon, ...], axis = 0)
era_fluxes_maps[('year','mlhf')] = L*np.mean(list(cpc.variables.values())[-1][:], axis = 0)

for fu in era_fluxes_maps:
     era_fluxes_zonal[fu] = np.mean(era_fluxes_maps[fu], axis = 1)*era_zonal_factor

# ERA ref figures:
exp = 'ERA'
year = 1988
figures_era = []
for seas in press0:
    fig = plt.figure()
    plt.title('Meridional heat fluxes - {} {} {}'.format(exp, year, seas))
    for flun in fluxnames:
        plt.plot(era_lat, era_fluxes_zonal[(seas, flun)], label = shortnam[flun])
    total = np.sum([era_fluxes_zonal[(seas, flun)] for flun in fluxnames], axis = 0)
    plt.plot(era_lat, total, label = 'Total')
    plt.legend()
    plt.grid()
    plt.xlabel('Latitude')
    plt.ylabel('Integrated Net Heat Flux (W)')
    figures_era.append(fig)

    for flun in fluxnames:
        fig = ctl.plot_map_contour(era_fluxes_maps[(seas, flun)], era_lat, era_lon, title = fluxlongnames[flun]+' - {} {} {}'.format(exp, year, seas))
        figures_era.append(fig)

    total = np.sum([era_fluxes_maps[(seas, flun)] for flun in fluxnames], axis = 0)
    fig = ctl.plot_map_contour(total, era_lat, era_lon, title = 'Total meridional flux - {} {} {}'.format(exp, year, seas))
    figures_era.append(fig)

file_era = cart_out+'ERA_reference_fluxes_1988.pdf'
ctl.plot_pdfpages(file_era, figures_era)

ally = [1975, 2000, 2025, 2050, 2075, 2090, 2100]
# exp = 'lcs0'
freq = '6hrs'
# year = 1988
# Ora per gli exps
#cart_in = '/data-hobbes/fabiano/SPHINX/heat_flux/lcs0_1988_6hrs/'
cart_in = '/data-hobbes/fabiano/SPHINX/heat_flux/data_6hrs/'

allfluxes = dict()
# for exp in ['lcb0', 'lcs0']:
#     for year in [1975, 2000, 2025, 2050, 2075, 2090, 2100]:
for exp in ['las1']:
    for year in [1988]:
        pressurefile = cart_in+'{}_day_{}_ps.nc'.format(exp, year)
        press0row, latdad, londsad, datespress, time_units, var_units = ctl.read3Dncfield(pressurefile)
        press0 = dict()
        press0['DJF'] = np.mean(ctl.sel_season(press0row, datespress, 'DJF', cut = False)[0], axis = 0)
        press0['JJA'] = np.mean(ctl.sel_season(press0row, datespress, 'JJA', cut = False)[0], axis = 0)
        press0['year'] = np.mean(press0row, axis = 0)

        figure_file_exp = cart_out+'hf_{}_{}.pdf'.format(exp, year)
        figures_exp = []
        figure_file_exp_maps = cart_out+'hf_{}_{}_maps.pdf'.format(exp, year)
        figures_exp_maps = []

        fluxes_row = dict()
        fluxes_levels = dict()
        fluxes_cross = dict()
        fluxes_maps = dict()
        fluxes_zonal = dict()

        vars = dict()
        varnames = ['va', 'ta', 'zg', 'hus']
        fils = ['{}_{}_{}_{}.nc'.format(exp, freq, year, varna) for varna in varnames]

        var, level, lat, lon, dates, time_units, var_units, time_cal = ctl.read4Dncfield(cart_in+fils[0])
        vars[varnames[0]] = var

        var, level, lat, lon, dates, time_units, var_units, time_cal = ctl.read4Dncfield(cart_in+fils[1])
        vars[varnames[1]] = var
        flun = 'mshf'
        fluxes_row[flun] = factors[0]*vars['va']*vars['ta']
        del vars['ta']
        fluxes_levels[('DJF',flun)] = np.mean(ctl.sel_season(fluxes_row[flun], dates, 'DJF', cut = False)[0], axis = 0)
        fluxes_levels[('JJA',flun)] = np.mean(ctl.sel_season(fluxes_row[flun], dates, 'JJA', cut = False)[0], axis = 0)
        fluxes_levels[('year',flun)] = np.mean(fluxes_row[flun], axis = 0)
        del fluxes_row[flun]

        var, level, lat, lon, dates, time_units, var_units, time_cal = ctl.read4Dncfield(cart_in+fils[2])
        vars[varnames[2]] = var
        flun = 'mpef'
        fluxes_row[flun] = vars['va']*vars['zg']
        del vars['zg']
        fluxes_levels[('DJF',flun)] = np.mean(ctl.sel_season(fluxes_row[flun], dates, 'DJF', cut = False)[0], axis = 0)
        fluxes_levels[('JJA',flun)] = np.mean(ctl.sel_season(fluxes_row[flun], dates, 'JJA', cut = False)[0], axis = 0)
        fluxes_levels[('year',flun)] = np.mean(fluxes_row[flun], axis = 0)
        del fluxes_row[flun]

        var, level, lat, lon, dates, time_units, var_units, time_cal = ctl.read4Dncfield(cart_in+fils[3])
        vars[varnames[3]] = var
        flun = 'mlhf'
        fluxes_row[flun] = factors[2]*vars['va']*vars['hus']
        del vars['hus']

        fluxes_levels[('DJF',flun)] = np.mean(ctl.sel_season(fluxes_row[flun], dates, 'DJF', cut = False)[0], axis = 0)
        fluxes_levels[('JJA',flun)] = np.mean(ctl.sel_season(fluxes_row[flun], dates, 'JJA', cut = False)[0], axis = 0)
        fluxes_levels[('year',flun)] = np.mean(fluxes_row[flun], axis = 0)
        del fluxes_row[flun]

        # for flun in fluxes_row:
        #     fluxes_levels[('DJF',flun)] = np.mean(ctl.sel_season(fluxes_row[flun], dates, 'DJF')[0], axis = 0)
        #     fluxes_levels[('JJA',flun)] = np.mean(ctl.sel_season(fluxes_row[flun], dates, 'JJA')[0], axis = 0)
        #     fluxes_levels[('year',flun)] = np.mean(fluxes_row[flun], axis = 0)

        del fluxes_row, vars

        zonal_factor = 2*np.pi*Rearth*np.cos(np.deg2rad(lat))
        for flun in fluxes_levels:
            fluxes_maps[flun] = np.zeros(fluxes_levels[flun].shape[1:])
            fluxes_cross[flun] = np.mean(fluxes_levels[flun], axis = -1)*zonal_factor

        print('Starting vertical integration\n')
        for seas in press0:
            for ila in range(len(lat)):
                for ilo in range(len(lon)):
                    #print(lat[ila], lon[ilo])
                    p0 = press0[seas][ila, ilo]

                    # Selecting pressure levels lower than surface pressure
                    exclude_pres = level > p0
                    if np.any(exclude_pres):
                        lev0 = np.argmax(exclude_pres)
                    else:
                        lev0 = None
                    levels_ok = np.append(level[:lev0], p0)

                    if not np.all(np.diff(levels_ok) > 0):
                        print(levels_ok)
                        raise ValueError('x not increasing')

                    for flun in fluxnames:
                        coso = fluxes_levels[(seas, flun)][:, ila, ilo]
                        coso = np.append(coso[:lev0], np.interp(p0, level, coso))
                        fluxes_maps[(seas, flun)][ila, ilo] = np.trapz(coso, x = levels_ok)
                        #print(levels_ok, coso.mean(), coso[-1], fluxes_maps[(seas, flun)][ila, ilo])

        print('done\n')
        if not os.path.exists(cart_out+'Out_fluxes_nc/'): os.mkdir(cart_out+'Out_fluxes_nc/')

        for fu in fluxes_maps:
            seas, flun = fu
            namefi = '{}_{}_flux_{}_{}.nc'.format(exp, year, flun, seas)
            ctl.save2Dncfield(lat,lon,fluxes_maps[fu],flun,cart_out+'Out_fluxes_nc/'+namefi)
        # for seas in ['DJF', 'JJA', 'year']:
        #     for flun in fluxnames:
        #         namefi = '{}_{}_flux_{}_{}.nc'.format(exp, year, flun, seas)
        #         fluxes_maps[(seas, flun)] = ctl.read2Dncfield(cart_out+'Out_fluxes_nc/'+namefi)[0]

        zonal_factor = 2*np.pi*Rearth*np.cos(np.deg2rad(lat))
        for fu in fluxes_maps:
            fluxes_zonal[fu] = np.mean(fluxes_maps[fu], axis = 1)*zonal_factor

        for fu in era_fluxes_zonal:
            #print(lat, era_lat, era_fluxes_zonal[fu])
            era_fluxes_zonal_itrp[fu] = np.interp(lat, era_lat[::-1], era_fluxes_zonal[fu][::-1])
            print('{:12.3e} - {:12.3e}\n'.format(era_fluxes_zonal_itrp[fu].min(), era_fluxes_zonal_itrp[fu].max()))

        # Single exps, single season figures:
        for seas in press0:
            fig = plt.figure()
            plt.title('SPHINX Meridional heat fluxes - {} {} {}'.format(exp, year, seas))
            for flun in fluxnames:
                plt.plot(lat, fluxes_zonal[(seas, flun)], label = shortnam[flun])
            total = np.sum([fluxes_zonal[(seas, flun)] for flun in fluxnames], axis = 0)
            fluxes_zonal[(seas, 'tot')] = total
            plt.plot(lat, total, label = 'Total')
            plt.legend()
            plt.grid()
            plt.ylim(zonal_margins['tot'])
            plt.xlabel('Latitude')
            plt.ylabel('Integrated Net Heat Flux (W)')
            figures_exp.append(fig)

            fig = plt.figure()
            plt.title('ERA Meridional heat fluxes - {} {}'.format(year, seas))
            for flun in fluxnames:
                plt.plot(lat, era_fluxes_zonal_itrp[(seas, flun)], label = shortnam[flun])
            total_era = np.sum([era_fluxes_zonal_itrp[(seas, flun)] for flun in fluxnames], axis = 0)
            plt.plot(lat, total_era, label = 'Total')
            plt.legend()
            plt.grid()
            plt.ylim(zonal_margins['tot'])
            plt.xlabel('Latitude')
            plt.ylabel('Integrated Net Heat Flux (W)')
            figures_exp.append(fig)

            fig = plt.figure()
            plt.title('SPHINX - ERA diff - {} {}'.format(year, seas))
            for flun in fluxnames:
                plt.plot(lat, fluxes_zonal[(seas, flun)]-era_fluxes_zonal_itrp[(seas, flun)], label = shortnam[flun])
            plt.plot(lat, total-total_era, label = 'Total')
            plt.legend()
            plt.grid()
            plt.ylim(zonal_margins['tot'])
            plt.xlabel('Latitude')
            plt.ylabel('Integrated Net Heat Flux (W)')
            figures_exp.append(fig)

            for flun in fluxnames:
                fig = ctl.plot_map_contour(fluxes_maps[(seas, flun)], lat, lon, title = fluxlongnames[flun]+' - {} {} {}'.format(exp, year, seas), cbar_range = map_margins[flun])
                figures_exp_maps.append(fig)

            total = np.sum([fluxes_maps[(seas, flun)] for flun in fluxnames], axis = 0)
            fig = ctl.plot_map_contour(total, lat, lon, title = 'Total meridional flux - {} {} {}'.format(exp, year, seas), cbar_range = map_margins['tot'])
            figures_exp_maps.append(fig)

            pinotot = itrp.RectBivariateSpline(lat, lon, total)
            total_eragrid = np.zeros(era_fluxes_maps[(seas, 'tot')].shape)
            for lae, eral in enumerate(era_lat):
                for loe, erol in enumerate(era_lon):
                    total_eragrid[lae, loe] = pinotot(eral, erol)
            diff = total_eragrid - era_fluxes_maps[(seas, 'tot')]
            fig = ctl.plot_map_contour(diff, era_lat, era_lon, title = 'Difference vs ERA flux - {} {} {}'.format(exp, year, seas), plot_anomalies = True)
            figures_exp_maps.append(fig)

        for flun in fluxnames:
            fig = plt.figure()
            plt.title('{} fluxes - {}'.format(shortnam[flun], year))
            cset = ctl.color_set(3, bright_thres = 1)
            for seas, col in zip(press0.keys(), cset):
                plt.plot(lat, fluxes_zonal[(seas, flun)], label = seas, color = col, linewidth = 2.)
                plt.plot(lat, era_fluxes_zonal_itrp[(seas, flun)], label = 'ERA '+seas, color = col, linewidth = 0.7, linestyle = '--')
            plt.legend()
            plt.grid()
            plt.ylim(zonal_margins[flun])
            plt.xlabel('Latitude')
            plt.ylabel('Integrated Net Heat Flux (W)')
            figures_exp.append(fig)

        flun = 'tot'
        fig = plt.figure()
        plt.title('{} fluxes - {}'.format('tot', year))
        cset = ctl.color_set(3, bright_thres = 1)
        for seas, col in zip(press0.keys(), cset):
            plt.plot(lat, fluxes_zonal[(seas, flun)], label = seas, color = col, linewidth = 2.)
            plt.plot(lat, era_fluxes_zonal_itrp[(seas, flun)], label = 'ERA '+seas, color = col, linewidth = 0.7, linestyle = '--')
        plt.legend()
        plt.grid()
        plt.ylim(zonal_margins[flun])
        plt.xlabel('Latitude')
        plt.ylabel('Integrated Net Heat Flux (W)')
        figures_exp.append(fig)

        print('pitototot')
        ctl.plot_pdfpages(figure_file_exp, figures_exp)
        ctl.plot_pdfpages(figure_file_exp_maps, figures_exp_maps)

        plt.close('all')
        allfluxes[(exp, year, 'cross')] = fluxes_cross
        allfluxes[(exp, year, 'maps')] = fluxes_maps
        allfluxes[(exp, year, 'zonal')] = fluxes_zonal

pickle.dump(allfluxes, open(cart_out+'allfluxes_las1_26112018.p','wb'))
sys.exit()

# pickle.dump(allfluxes, open(cart_out+'allfluxes_23112018.p','wb'))
allfluxes = pickle.load(open(cart_out+'allfluxes_23112018.p','rb'))
# comparison between exps

for fu in era_fluxes_zonal:
    era_fluxes_zonal_itrp[fu] = np.interp(lat, era_lat[::-1], era_fluxes_zonal[fu][::-1])

figures = []
figure_file = cart_out + 'lcb0_vs_lcs0_fluxes.pdf'
seasons = ['year', 'JJA', 'DJF']
level = np.array([92500, 85000, 70000, 60000, 50000, 40000, 25000, 10000])
level = level[::-1]

# figura yearly/JJA/DJF total con tutti i base e tutti gli stoc per tutti gli anni
for flun in fluxnames:
    for year in ally:
        okflubase = allfluxes[('lcb0', year, 'zonal')]
        okflustoc = allfluxes[('lcs0', year, 'zonal')]
        fig = plt.figure()
        plt.title('{} fluxes - {}'.format(shortnam[flun], year))
        cset = ctl.color_set(3, bright_thres = 1)
        for seas, col in zip(seasons, cset):
            plt.plot(lat, okflubase[(seas, flun)], label = 'base '+seas, color = col, linewidth = 1.5)
            plt.plot(lat, okflustoc[(seas, flun)], label = 'stoc '+seas, color = col, linewidth = 1.5, linestyle = '--')
            plt.plot(lat, era_fluxes_zonal_itrp[(seas, flun)], label = 'ERA '+seas, color = col, linewidth = 0.7, linestyle = ':')
        plt.legend(fontsize = 'small')
        plt.grid()
        plt.ylim(zonal_margins[flun])
        plt.xlabel('Latitude')
        plt.ylabel('Integrated Net Heat Flux (W)')
        figures.append(fig)

flun = 'tot'
for year in ally:
    okflubase = allfluxes[('lcb0', year, 'zonal')]
    okflustoc = allfluxes[('lcs0', year, 'zonal')]
    fig = plt.figure()
    plt.title('{} fluxes - {}'.format('total', year))
    cset = ctl.color_set(3, bright_thres = 1)
    for seas, col in zip(seasons, cset):
        plt.plot(lat, okflubase[(seas, flun)], label = 'base '+seas, color = col, linewidth = 1.5)
        plt.plot(lat, okflustoc[(seas, flun)], label = 'stoc '+seas, color = col, linewidth = 1.5, linestyle = '--')
        plt.plot(lat, era_fluxes_zonal_itrp[(seas, flun)], label = 'ERA '+seas, color = col, linewidth = 0.7, linestyle = ':')
    plt.legend(fontsize = 'small')
    plt.grid()
    plt.ylim(zonal_margins[flun])
    plt.xlabel('Latitude')
    plt.ylabel('Integrated Net Heat Flux (W)')
    figures.append(fig)

# stesso per SH e PE
# poi mappe zonal di confronto con anno base (2100 vs 1975)
y1 = 1975
y2 = 2100
for exp in ['lcb0', 'lcs0']:
    okflubase1 = allfluxes[(exp, y1, 'maps')]
    okflubase2 = allfluxes[(exp, y2, 'maps')]
    print(okflubase1.keys())
    for flun in ['mshf', 'mpef']:
        for seas in seasons:
            diff = okflubase2[(seas, flun)]-okflubase1[(seas, flun)]
            fig = ctl.plot_map_contour(diff, lat, lon, title = '{} - {} diff - {} - {} vs {}'.format(shortnam[flun], exp, seas, y1, y2), plot_anomalies = True, cbar_range = map_margins[flun])
            figures.append(fig)

for y1 in [1975, 2100]:
    okflubase = allfluxes[('lcb0', y1, 'maps')]
    okflustoc = allfluxes[('lcs0', y1, 'maps')]
    for flun in ['mshf', 'mpef']:
        for seas in seasons:
            diff = okflustoc[(seas, flun)]-okflubase[(seas, flun)]
            fig = ctl.plot_map_contour(diff, lat, lon, title = '{} lcb0 vs lcs0 diff - {} - {}'.format(shortnam[flun], seas, y1), plot_anomalies = True, cbar_range = map_margins[flun])
            figures.append(fig)

# mappa di confronto fra base e stoc tra anni base e fut
# cross di confronto stesso tra anno base e fut e tra base e stoc

y1 = 1975
y2 = 2100
for exp in ['lcb0', 'lcs0']:
    okflubase1 = allfluxes[(exp, y1, 'cross')]
    okflubase2 = allfluxes[(exp, y2, 'cross')]
    for flun in ['mshf', 'mpef']:
        for seas in seasons:
            diff = okflubase2[(seas, flun)]-okflubase1[(seas, flun)]
            fig = ctl.plot_lat_crosssection(diff, lat, level, title = '{} {} - {} cross section {} vs {}'.format(shortnam[flun], seas, exp, y1, y2), plot_anomalies = True)
            figures.append(fig)

for y1 in [1975, 2100]:
    okflubase = allfluxes[('lcb0', y1, 'cross')]
    okflustoc = allfluxes[('lcs0', y1, 'cross')]
    for flun in ['mshf', 'mpef']:
        for seas in seasons:
            diff = okflustoc[(seas, flun)]-okflubase[(seas, flun)]
            fig = ctl.plot_lat_crosssection(diff, lat, level, title = '{} {} - lcb0 vs lcs0 cross section {}'.format(shortnam[flun],seas, y1), plot_anomalies = True)
            figures.append(fig)

ctl.plot_pdfpages(figure_file, figures)


import random
import requests
import threading

from lxml import etree
from queue import Queue
from fake_useragent import UserAgent
from multiprocessing import cpu_count
from datetime import datetime, timedelta

from setting import MAX_PROXY_REQUESTS_NUM, MAX_PROXY_POOL_NUM
from utils.utils import request_message, get_amazon_domain, is_robot, wait
from utils.api import amazon_header
from utils.log import log

proxy_url = 'http://ip.ipjldl.com/index.php/api/entry?method=proxyServer.generate_api_url&packid=0&fa=0&fetch_key=&' \
            'groupid=0&qty={num}&time=1&pro=&city=&port=1&format=json&ss=5&css=&ipport=1&et=1&dt=1&specialTxt=3&' \
            'specialJson=&usertype=2'

proxies_mate = 'http://%(host)s:%(port)s'

class Proxy:
    _instance_lock = threading.Lock()

    def __new__(cls):
        if not hasattr(cls, '_instance'):
            with Proxy._instance_lock:
                if not hasattr(cls, '_instance'):
                    Proxy._instance = super().__new__(cls)
        return Proxy._instance

    def __init__(self):
        self.agents = {}
        self.proxies_num = 0

    def agent_pool(self, country=None, proxy_num=1):
        proxies_array = []
        log('开始请求代理')
        session, response = self.request(requests.session(), proxy_url
                                         .format(num=min(proxy_num, cpu_count()) if country else 1), types='json')
        if response['success'] == 'true':
            log('请求代理成功')
            self.proxies_num = 0
            for item in response['data']:
                host, port = item['IP'].split(':')
                proxies = {
                    'http': proxies_mate % {'host': host, 'port': port},
                    'https': proxies_mate % {'host': host, 'port': port}
                }
                proxies_array.append({'session': session, 'proxies': proxies, 'expire_time': item['ExpireTime']})
            if country:
                log('有国家参数, 进行amazon访问处理，判断ip是否有效')
                proxies_data = {}
                thread_list = []
                q = Queue()
                for proxies_item in proxies_array:
                    t = threading.Thread(target=self.amazon_robot_check, args=(proxies_item, country, q))
                    t.setDaemon(True)
                    thread_list.append(t)
                    t.start()

                for thread_item in thread_list:
                    thread_item.join()
                    results = q.get()
                    if results:
                        if country in proxies_data:
                            proxies_data[country].append(results)
                        else:
                            proxies_data[country] = [results]

                log('获取有效代理对象: ', proxies_data if proxies_data else None)
                return proxies_data if proxies_data else None
            else:
                log('无国家参数, 不需要处理, 直接返回代理')
                return proxies_array.pop()
        else:
            self.proxies_num += 1
            log('请求代理失败: ', response['msg'])
            log('正在重试...重试次数为: ', self.proxies_num)
            if self.proxies_num < MAX_PROXY_REQUESTS_NUM:
                wait()
                return self.agent_pool(country, proxy_num)
            else:
                log('请求代理失败, 重试已达最大次数')
                return None

    def add_agent(self):
        # add_agent_arr 需要添加代理的国家
        add_agent_arr = []
        log('检测代理是否需要添加')
        for item in MAX_PROXY_POOL_NUM:
            if MAX_PROXY_POOL_NUM[item] == 0:
                continue
            cur_proxy_num = len(self.agents[item]) if item in self.agents else 0
            if cur_proxy_num >= MAX_PROXY_POOL_NUM[item]:
                continue
            add_agent_arr.append({'country': item, 'proxy_num': (MAX_PROXY_POOL_NUM[item] - cur_proxy_num)})
        if add_agent_arr:
            log('代理需要添加的国家有: ', ','.join([item['country'] for item in add_agent_arr]))
            for item in add_agent_arr:
                results = self.agent_pool(item['country'], item['proxy_num'])
                if results:
                    if item['country'] in self.agents:
                        self.agents[item['country']].extend(results[item['country']])
                    else:
                        self.agents[item['country']] = results[item['country']]
                else:
                    continue
            wait()
            self.add_agent()
        else:
            log('代理以达到设定数量, 不在进行添加')
            return None

    def remove_expired(self):
        log('正在移除过期代理')
        for country in self.agents:
            for index, item in enumerate(self.agents[country]):
                if self.compare_time(item['expire_time']):
                    log('存在过期代理进行移除')
                    self.agents[country].pop(index)
                    self.add_agent()

    def get_proxies(self, country=None):
        if country:
            country = country.upper()
            if country in self.agents:
                index = random.randint(0, len(self.agents[country]) - 1)
                agent = self.agents[country][index]
                if self.compare_time(agent['expire_time']):
                    log('取出当前代理发现过期')
                    self.agents[country].pop(index)
                    self.add_agent()
                    return self.get_proxies(country)
                else:
                    log('直接取出代理')
                return agent['session'], agent['proxies']
            else:
                results = self.agent_pool(country, proxy_num=1)
                if results:
                    agent = random.choice(results[country])
                    return agent['session'], agent['proxies']
                wait()
                return self.get_proxies(country)
        else:
            return self.agent_pool()

    def amazon_robot_check(self, data, country, q):
        log('正在进行amazon机器人验证')
        try:
            cur_header = amazon_header.copy()
            cur_header['user-agent'] = UserAgent().random
            session, response = self.request(data['session'], get_amazon_domain(country), headers=cur_header,
                                             proxies=data['proxies'])
            if not is_robot(etree.HTML(response)):
                data['session'] = session
                q.put(data)
            else:
                log('验证结果: 机器人')
                q.put(None)
        except Exception as e:
            log(e)
            q.put(None)

    @staticmethod
    def request(session, url, headers=None, proxies=None, types='txt'):
        response = session.get(url, headers=headers, proxies=proxies, timeout=10)
        response.encoding = 'utf-8'
        return session, request_message(response, types)

    @staticmethod
    def compare_time(date):
        # 2020-04-14 11:44:46
        now = (datetime.now() + timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S')
        return datetime.strptime(now, '%Y-%m-%d %H:%M:%S') > datetime.strptime(date, '%Y-%m-%d %H:%M:%S')


import cv2
import numpy as np
import classes.system_utilities.image_utilities.ImageUtilities as IU
import time

from KalmanFilter2D import KalmanFilter2D

scale_percentage = 0.5

cap = cv2.VideoCapture("..\\data\\reference footage\\videos\\Leg_2_Starting_Full.mp4")

feature_params = dict(maxCorners=1,
                      qualityLevel=0.01,
                      minDistance=7,
                      blockSize=3)

lk_params = dict(winSize=(15, 15),
                 maxLevel=3,
                 criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))

_, frame = cap.read()
frame = IU.RescaleImageToScale(frame, scale_factor=scale_percentage)
old_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

bbox = [[170, 207], [644,395]] #car_leg_full
bbox_tl = [170, 207]
bbox_br = [644, 395]

# initialize array
old_points = np.zeros((1, 2))

#mouse function
def SelectPoint(event, x, y, flags, params):
    global selectedPoint, point_selected, old_points
    if event == cv2.EVENT_LBUTTONDOWN:
        selectedPoint = (int(x), int(y))
        point_selected = True
        old_points = np.array([[x, y]], dtype=np.float32)

cv2.namedWindow("Frame")
cv2.setMouseCallback("Frame", SelectPoint)
point_selected = False
selectedPoint = ()

cv2.imshow('Frame', frame)
cv2.waitKey(0)

def GetBboxGoodFeatures(bbox, gray):
    cropped_frame = IU.CropImage(gray, bbox)

    points = cv2.goodFeaturesToTrack(cropped_frame, **feature_params)

    # translate old_points of bbox to the non-cropped frame
    for i in range(len(points)):
        points[i][0][0] += bbox[0][0]
        points[i][0][1] += bbox[0][1]
        if points[i][0][0] > frame.shape[1]:
            points[i][0][0] = frame.shape[1]

        if points[i][0][1] > frame.shape[0]:
            points[i][0][1] = frame.shape[0]

    return points

#old_points = GetBboxGoodFeatures(bbox, old_gray)

# Create a random color
color = np.random.randint(0, 255, (100, 3))

# Create a mask img to see optical flow drawing
mask = np.zeros_like(frame)

# Instantiate Kalman Filter
KF = KalmanFilter2D(0.1, 1, 1, 1, 0.1 ,0.1)

start_time = time.time()
seconds_before_display = 1  # displays the frame rate every 1 second
counter = 0

frames_num = 1
x_old = 0; y_old = 0
ctr = 0
while True:
    frames_num+=1
    _, frame = cap.read()
    frame = IU.RescaleImageToScale(frame, scale_factor=scale_percentage)

    new_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    if point_selected is False:
        continue

    cv2.circle(frame, selectedPoint, 5, (0, 0, 255), 2)

    #Predict KF
    if ctr == 0:
        x = [355.505, 285.805]
        y = [355.505, 285.805]
        ctr+=1
    else:
        (x, y) = KF.predict()

    print(x, y)

    if(frames_num==2): x = x[0] ; y = y[0]
    else: x = x[0] ; y = y[1]

    # Draw a rectangle as the predicted object position

    print("bbox before: TL - ", bbox_tl, "| BR - ", bbox_br)
    print("predicted estimates:", (int(x), int(y)))
    bbox_tl[0] = int(bbox_tl[0] + (int(x) - x_old))
    bbox_tl[1] = int(bbox_tl[1] + (int(y) - y_old))
    bbox_br[0] = int(bbox_br[0] + (int(x) - x_old))
    bbox_br[1] = int(bbox_br[1] + (int(y) - y_old))
    print("bbox after: TL - ", bbox_tl, "| BR - ", bbox_br)
    #cv2.rectangle(frame, (bbox_tl[0], bbox_tl[1]), (bbox_br[0], bbox_br[1]), (255, 0, 0), 2)
    frame = IU.DrawBoundingBoxes(frame, [[bbox_tl, bbox_br]])

    cv2.imshow("Frame bbox", frame)
    cv2.waitKey(100)

    #Lucas Kenade
    new_points, status, error = cv2.calcOpticalFlowPyrLK(old_gray, new_gray, old_points, None, **lk_params)

    #bbox = [[170, 207], [644, 395]]

    # draw the tracks
    for i in range(len(new_points)):
        a, b = new_points.ravel()
        c, d = old_points.ravel()
        a = int(a);
        b = int(b);
        c = int(c);
        d = int(d)
        # draws a line connecting old point with new point
        mask = cv2.line(mask, (a, b), (c, d), (0, 0, 255), 1)
        # draws new point
        frame = cv2.circle(frame, (a, b), 2, (0, 255, 0), 2)

       #print("a: ", a, "| b: ", b)
        point = KF.update((a, b))
        x1 = point[0][0]; y1 = point[1][1]
        #print("updated points: ", x1, "| ", y1)
        # Draw a rectangle as the estimated object position
        cv2.rectangle(frame, (int(x1) - 15, int(y1) - 15), (int(x1) + 15, int(y1) + 15), (0, 0, 255), 2)
        cv2.putText(frame, "Estimated Position", (int(x1) + 15, int(y1) + 10), 0, 0.5, (0, 0, 255), 2)
        cv2.putText(frame, "Predicted Position", (int(x) + 15, int(y)), 0, 0.5, (255, 0, 0), 2)
        cv2.putText(frame, "Measured Position", (int(a) + 15, int(b) - 15), 0, 0.5, (0, 191, 255), 2)

    frame = cv2.add(frame, mask)
    cv2.imshow("Frame1 mask added", frame)

    # update previous frame and old points
    old_gray = new_gray.copy()
    old_points = new_points.reshape(-1, 1, 2)
    x_old = int(x)
    y_old = int(y)

    counter += 1
    if (time.time() - start_time) > seconds_before_display:
        print("FPS: ", counter / (time.time() - start_time))
        counter = 0
        start_time = time.time()

    if cv2.waitKey(1) == 27:
        cap.release()
        cv2.destroyAllWindows()
        break


### 003 load data from csv file
import numpy as np
import pandas as pd

print('### 003')
df = pd.read_csv('./movie_data.csv')
print(df.head(3))
print()

### 004 transform words into feature vectors
print('### 004')
from sklearn.feature_extraction.text import CountVectorizer

# 1-gram representation is used by default, we could switch to a 2-gram
# representation by initializing a new CountVectorizer instance with
# ngram_range=(2,2).
count = CountVectorizer()
docs = np.array([
       'The sun is shining',
       'The weather is sweet',
       'The sun is shining and the weather is sweet'])
bag = count.fit_transform(docs)
print(count.vocabulary_)
# feature vectors, are also called the raw term frequencies: tf(t,d)
# the number of times a term t occurs in a document d
print(bag.toarray())
print()

### 005 td-idf
print('### 005')
from sklearn.feature_extraction.text import TfidfTransformer

tfidf = TfidfTransformer()
np.set_printoptions(precision=2)
print(tfidf.fit_transform(count.fit_transform(docs)).toarray())
print()

### 006 clean text data by regexp
print('### 006')
print(df.loc[0, 'review'][-50:])

import re

def preprocessor(text):
    text = re.sub('<[^>]*>', '', text)
    emoticons = re.findall('(?::|;|=)(?:-)?(?:\)|\(|D|P)', text)
    # Although the addition of the emoticon characters to the end of the cleaned document
    # strings may not look like the most elegant approach, the order of the words doesn't
    # matter in our bag-of-words model if our vocabulary only consists of 1-word tokens.
    text = re.sub('[\W]+', ' ', text.lower()) + ' '.join(emoticons).replace('-', '')
    return text

print(preprocessor(df.loc[0, 'review'][-50:]))
print(preprocessor("</a>This :) is :( a test :-)!"))
print()

# now apply our preprocessor function to all movie reviews in our DataFrame
df['review'] = df['review'].apply(preprocessor)

### 007 process documents into tokens
print('### 007')

# One way to tokenize documents is to split them into individual words
# by splitting the cleaned document at its whitespace characters
def tokenizer(text):
    return text.split()

print(tokenizer('runners like running and thus they run'))

# another useful technique is word stemming, which is the process of
# transforming a word into its root form that allows us to map related
# words to the same stem.
from nltk.stem.porter import PorterStemmer

porter = PorterStemmer()
def tokenizer_porter(text):
    return [porter.stem(word) for word in text.split()]

print(tokenizer_porter('runners like running and thus they run'))
print()

### 008 stop-word removal
print('### 008')

from nltk.corpus import stopwords

stop = stopwords.words('english')
print([w for w in tokenizer_porter('a runner likes running and runs a lot')[-10:] if w not in stop])
print()

### 009 train a logistic regression model for document classification
print('### 009')
print('This step will take nearly 2 hours running in a desktop with 16G RAM and E5-2670 CPU')
X_train = df.loc[:25000, 'review'].values
y_train = df.loc[:25000, 'sentiment'].values
X_test = df.loc[25000:, 'review'].values
y_test = df.loc[25000:, 'sentiment'].values

from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer

# TfidVectorizer is equivalent to CountVectorizer followed by TfidfTransformer.
tfidf = TfidfVectorizer(strip_accents=None, lowercase=False, preprocessor=None)
# The first dictionary uses the TfidfVectorizer with its default settings
# (use_idf=True, smooth_idf=True, and norm='l2') to calculate the tf-idfs;
param_grid = [{'vect__ngram_range': [(1,1)],
               'vect__stop_words': [stop, None],
               'vect__tokenizer': [tokenizer, tokenizer_porter],
               'clf__penalty': ['l1', 'l2'],
               'clf__C': [1.0, 10.0, 100.0]},
              {'vect__ngram_range': [(1,1)],
               'vect__stop_words': [stop, None],
               'vect__tokenizer': [tokenizer, tokenizer_porter],
               'vect__use_idf':[False],
               'vect__norm':[None],
               'clf__penalty': ['l1', 'l2'],
               'clf__C': [1.0, 10.0, 100.0]}
             ]
lr_tfidf = Pipeline([('vect', tfidf), ('clf', LogisticRegression(random_state=0))])
gs_lr_tfidf = GridSearchCV(lr_tfidf, param_grid, scoring='accuracy', cv=5, verbose=1, n_jobs=-1)
gs_lr_tfidf.fit(X_train, y_train)

print('Best parameter set: %s ' % gs_lr_tfidf.best_params_)
print('CV Accuracy: %.3f' % gs_lr_tfidf.best_score_)
clf = gs_lr_tfidf.best_estimator_
print('Test Accuracy: %.3f' % clf.score(X_test, y_test))


from hypothesis import strategies, given, settings, example
import binascii

@given(data=strategies.binary())
@settings(max_examples=30000, max_iterations=200000)
def test_base64_pack_unpack(data):
    assert binascii.a2b_base64(binascii.b2a_base64(data)) == data

@given(binary_data=strategies.binary())
@settings(max_examples=30000, max_iterations=200000)
def test_hexlify(binary_data):
    assert binascii.unhexlify(binascii.hexlify(binary_data)) == binary_data


import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from scipy.cluster.hierarchy import dendrogram, linkage
from sklearn import preprocessing
from sklearn.cluster import KMeans

data = pd.read_csv('Data/30_128_X.csv').values
label = pd.read_csv('Data/30_y.csv').values.astype(np.int).squeeze()

print(data.shape)
print(label.shape)

# plt.plot(np.mean(data[label == 0], axis=0), color='g', label='cylider')
# plt.plot(np.mean(data[label == 1], axis=0), color='b', label='bell')
# plt.plot(np.mean(data[label == 2], axis=0), color='r', label='funnel')
# plt.legend()
# plt.xlabel('Date')
# plt.show()

# linked = linkage(data, 'single')
# plt.figure(figsize=(10, 7))
# dendrogram(linked,
#            orientation='top',
#            labels=np.array([i for i in range(len(data))]),
#            distance_sort='descending',
#            show_leaf_counts=True,
#            leaf_font_size=14)
# plt.show()

# km = KMeans(
#     n_clusters=3, init='random',
#     n_init=10, max_iter=300,
#     tol=1e-04, random_state=0
# )
# y_km = km.fit_predict(data)
#
# plt.plot(np.mean(data[y_km == 0], axis=0), color='g', label='Cluster 1 with {} sample'.format(len(data[y_km == 0])))
# plt.plot(np.mean(data[y_km == 1], axis=0), color='b', label='Cluster 2 with {} sample'.format(len(data[y_km == 1])))
# plt.plot(np.mean(data[y_km == 2], axis=0), color='r', label='Cluster 3 with {} sample'.format(len(data[y_km == 2])))
# plt.legend()
# plt.xlabel('Date')
# plt.show()

data_cat = data.ravel()
print(data_cat.shape)

data_conv = []
pace = 128
for i in range(len(data_cat) - pace):
    data_conv.append(data_cat[i:i+pace])

data_conv = np.array(data_conv)
print(data_conv.shape)
data_conv = preprocessing.minmax_scale(data_conv, axis=1)

# labels = np.random.randint(3584, size=50)
# linked = linkage(data_conv[labels, :], 'single')
#
# plt.figure(figsize=(10, 7))
# dendrogram(linked,
#            orientation='top',
#            labels=labels,
#            distance_sort='descending',
#            show_leaf_counts=True,
#            leaf_font_size=14)
# plt.show()

km = KMeans(
    n_clusters=4, init='random',
    n_init=10, max_iter=300,
    tol=1e-04, random_state=0
)
y_km = km.fit_predict(data_conv)

plt.plot(np.mean(data_conv[y_km == 0], axis=0), color='g', label='Cluster 1 with {} sample'.format(len(data_conv[y_km == 0])))
plt.plot(np.mean(data_conv[y_km == 1], axis=0), color='b', label='Cluster 2 with {} sample'.format(len(data_conv[y_km == 1])))
plt.plot(np.mean(data_conv[y_km == 2], axis=0), color='r', label='Cluster 3 with {} sample'.format(len(data_conv[y_km == 2])))
plt.plot(np.mean(data_conv[y_km == 3], axis=0), color='orange', label='Cluster 4 with {} sample'.format(len(data_conv[y_km == 3])))
plt.legend()
plt.show()


from __future__ import print_function

import argparse
import os
import random
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.utils as vutils
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from IPython.display import HTML
import torchsummary

# Set random seem for reproducibility
manualSeed = 999
#manualSeed = random.randint(1, 10000) # use if you want new results
random.seed(manualSeed)
torch.manual_seed(manualSeed)

# Root directory for dataset
dataroot = "split/DSMWRAPPER"

# Number of workers for dataloader
workers = 2

# Batch size during training
batch_size = 128

# Spatial size of training images. All images will be resized to this
#   size using a transformer.
image_size = 64

# Number of channels in the training images. For color images this is 3
nc = 1

# Size of z latent vector (i.e. size of generator input)
nz = 100

# Size of feature maps in generator
ngf = 64

# Size of feature maps in discriminator
ndf = 64

# Number of training epochs
num_epochs = 5

# Learning rate for optimizers
lr = 0.0002

# Beta1 hyperparam for Adam optimizers
beta1 = 0.5

# Number of GPUs available. Use 0 for CPU mode.
ngpu = 1

# We can use an image folder dataset the way we have it setup.
# Create the dataset
dataset = dset.ImageFolder(root=dataroot,
                           transform=transforms.Compose([
                               transforms.Grayscale(num_output_channels=1),
                               transforms.Resize(image_size),
                               transforms.CenterCrop(image_size),
                               transforms.ToTensor(),
                               #transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
                           ]))

# Create the dataloader
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size,
                                         shuffle=True, num_workers=workers)

# Decide which device we want to run on
device = torch.device("cuda:0" if (torch.cuda.is_available() and ngpu > 0) else "cpu")

# Plot some training images
real_batch = next(iter(dataloader))
plt.figure(figsize=(8,8))
plt.axis("off")
plt.title("Training Images")
plt.imshow(np.transpose(vutils.make_grid(real_batch[0].to(device)[:64], padding=2, normalize=True).cpu(),(1,2,0)))

netG = Generator(1).to(device)

netG.apply(weights_init)

torchsummary.summary(netG,input_size=(100,1,1))

netD = Discriminator(ngpu).to(device)

netD.apply(weights_init)

torchsummary.summary(netD,input_size=(1,64,64))

# Initialize BCELoss function
criterion = nn.BCELoss()

# Create a batch of latent vectors that we will use to visuallize
# as we train the generator
fixed_noise = torch.randn(64, nz, 1, 1, device=device)

# Set labelling convention
real_label = 1
fake_label = 0

# Set up the two optimizers for G and D, we use ADAM since it's in the paper
optimizerD = optim.Adam(netD.parameters(),lr=lr, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters(),lr=lr, betas=(beta1, 0.999))

# Training Loop

# lists to keep track of losses and progress
img_list = []
G_losses = []
D_losses = []
iters = 0

print("Starting Training Loop")

#for epoch in range(25):

for epoch in range(num_epochs):

    # for each batch in the dataloader
    for i, data in enumerate(dataloader,0):

        ####
        # Train the discriminator: Maximize log(D(x) + log(1- D(G(z)))
        ####

        ## Train with all real batch
        netD.zero_grad()

        # format the batch
        real_cpu = data[0].to(device)
        b_size = real_cpu.size(0) #checks to see uniform batch size
        label = torch.full((b_size,), real_label, device=device)
        # Forward pass for real batch through the Discriminator
        output = netD(real_cpu).view(-1)

        # Calculate loss
        errD_real = criterion(output, label)

        # Calculate gradients for D in backward pass
        errD_real.backward()

        D_x = output.mean().item()

        ## Train with all fake batch
        noise = torch.randn(b_size, nz, 1, 1, device=device)

        # Generate Fake images using the Generator
        fake = netG(noise)
        label.fill_(fake_label)

        # Classify fake images
        output = netD(fake.detach()).view(-1)

        # Calculate the Discriminators loss on the fake batch
        errD_fake = criterion(output,label)

        # Calculate the gradients of the backward pass for the Generator
        errD_fake.backward()
        D_G_z1 = output.mean().item()

        # Add the gradients from the real and fake batches
        errD = errD_real + errD_fake
        optimizerD.step()

        ####
        # training the generator: Maximize log(D(G(z)))
        ###

        netG.zero_grad()
        label.fill_(real_label) # fake labels are real for the generator cost

        # since we just updated D, we perform another forward pass of all fake through D
        output = netD(fake).view(-1)

        # Calculate generator loss basd on the output
        errG = criterion(output,label)

        # Calculate gradients
        errG.backward()
        D_G_z2 = output.mean().item()

        # update G
        optimizerG.step()

        # Output stats during training
        if i%50 == 0:
            print('[%d/%d][%d/%d]\tLoss_D; %.4f\tLoss_g:%.4f\tD(x): %.4f\tD(G(z)): %.4f / %.4f'
                 % (epoch, num_epochs, i, len(dataloader),
                   errD.item(), errG.item(), D_x, D_G_z1, D_G_z2))

        G_losses.append(errG.item())
        D_losses.append(errD.item())

        # Check how the generator is doing by saving G's output on fixed_noise
        if (iters % 500 == 0) or ((epoch == num_epochs-1) and (i == len(dataloader)-1)):
            with torch.no_grad():
                fake = netG(fixed_noise).detach().cpu()
            img_list.append(vutils.make_grid(fake, padding=2, normalize=True))

        iters += 1

# Grab a batch of real images from the dataloader
real_batch = next(iter(dataloader))

# Plot the real images
plt.figure(figsize=(30,30))
plt.subplot(1,2,1)
plt.axis("off")
plt.title("Real Images")
plt.imshow(np.transpose(vutils.make_grid(real_batch[0].to(device)[:64], padding=5, normalize=True).cpu(),(1,2,0)))

# Plot the fake images from the last epoch
plt.subplot(1,2,2)
plt.axis("off")
plt.title("Fake Images")
plt.imshow(np.transpose(img_list[-1],(1,2,0)))
plt.show()

torch.save(netD,"discriminator.pt")
torch.save(netG,"generator.pt")


import numpy as np
import h5py as h5
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.cm as cm

prefix = "/media/user/9EAEE48CAEE45DF1/cyl_temp/p-bed-data/1e4/"
prefix = "../../post/"
prefix = "../"
prefix = prefix + "test_pbed_r1_"
ppy =  131
R = 8
num = np.arange(0,730+1)
Dm = 1.15e-6;

name = prefix + str(num[0]).zfill(4) + ".h5"
print(name)
f = h5.File(name)
PX = np.array(f['RWPposition'])
px = PX[0:-2:3]
py = PX[1:-1:3]
Ga = np.array(f['Gamma'])
Nx = int(f['Nx'][0])
Ny = int(f['Ny'][0])
ga = Ga.reshape((Ny,Nx))
ga[np.where(ga>1.0)] = 1.0
gga = np.sum(ga,axis=1)
# print(gga)
print(ppy-1,gga[ppy-1])
print(ppy,gga[ppy])

layer = np.array([ppy-6*(R+2),ppy-4*(R+2),ppy-2*(R+2),ppy])
layer = np.array([66,88,110,131]);
M0 = np.zeros((len(layer))) + np.nan
kkk = []
for i in range(len(layer)):
	kkk.append(np.where(py<=layer[i]))
	M0[i] = np.size(np.where(py<=layer[i]))
colors = cm.rainbow(np.linspace(0,1,len(kkk)))

print('initial num ',M0)

nn = np.zeros(len(num)) + np.nan
M = np.zeros((len(num),len(layer))) + np.nan
for i in range(len(num)):
	nn[i] = i
	name = prefix + str(num[i]).zfill(4) + ".h5"
	print("start process ", name)
	f = h5.File(name)
	PX = np.array(f['RWPposition'])
	px = PX[0:-2:3]
	py = PX[1:-1:3]

	for ii in range(len(layer)):
		M[i,ii] = float(np.size(np.where(py[kkk[ii]]<=layer[ii])))/M0[ii]

k1 = np.zeros(len(layer)) + np.nan
Vs = np.zeros(len(layer)) + np.nan
Vt = np.zeros(len(layer)) + np.nan
Vv = np.zeros(len(layer)) + np.nan
As = np.zeros(len(layer)) + np.nan

tt = nn*1e2
for ii,c in zip(range(len(layer)),colors):
	plt.plot(np.sqrt(tt),M[:,ii],'o',color=c,label=layer[ii])
	Z = np.polyfit(np.sqrt(tt),M[:,ii],1)
	k1[ii] = -Z[0]
	plt.plot(np.sqrt(tt),Z[0]*np.sqrt(tt)+Z[1],color=c)
	Vs[ii] = np.sum(np.sum(ga[1:layer[ii],:]))
	Vt[ii] = (layer[ii]*1.0-1.0)*Nx*1.0
	Vv[ii] = Vt[ii] - Vs[ii]
	As[ii] = Nx - np.sum(ga[layer[ii],:])

plt.legend()
plt.savefig('M.png',dpi=500)
plt.clf()

Deff = np.pi*(0.5*Vv/As*k1)**2/Dm
print('Deff/Dm ', Deff)
plt.plot(Deff,layer,'*')
plt.ylabel('Depth')
plt.xlabel('Deff/Dm')

plt.savefig('Deff',dpi=500)
plt.clf()

f.close()

with open("Deff_1e4.txt","w") as f:
	f.write(str(Dm)+ "\t Dm\n")
	for l,d in zip(layer, Deff):
		f.write(str(l)+" "+str(d)+"\t pos, Deff/Dm\n")


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import requests
import time
from datetime import datetime
import pandas as pd
from bs4 import BeautifulSoup
from operator import itemgetter

# NAVITIME 住所検索TOP
def crawl():
  init()
  prefectures()

def init():
  if not os.path.exists('data'):
    os.mkdir('data')

def prefectures():
  print(f"prefectures start : {now()}")
  TOP_URL = 'https://www.navitime.co.jp/?ctl=0050'
  response = requests.get(TOP_URL)
  soup = BeautifulSoup(response.text, 'lxml')
  sub_urls = []
  prefectures = []
  for li in soup.find_all("li", attrs={"class": "pref-item"}):
    target = li.find("a", reversed=False)
    url = target.get("href")
    sub_urls.append(url)
    data = itemgetter(4, 5)(url.split("/"))
    prefectures.append(data)

  df = pd.DataFrame(prefectures, columns=['code', 'name'])
  df.to_csv("data/prefectures.csv", index=False)
  print(f"prefectures end : {now()}")

  cities(sub_urls)

def cities(urls: list):
  print(f"cities start : {now()}")
  sub_urls = []
  cities = []
  for url in urls:
    prefecture_code =  url.split("/")[4]
    print(f"----- prefecture_code={prefecture_code} start : {now()} -----")
    # if not prefecture_code == '13':
    #   continue
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'lxml')
    for div in soup.find_all("div", attrs={"class": "address_list"}):
      for li in div.find_all("li", attrs={"class": "left"}):
        ruby = li.find("span").getText()
        url = 'https://www.navitime.co.jp' + li.find("a").get("href")
        name = li.find("a").getText()
        sub_urls.append(url)
        code = url.split("/")[4]
        data = (code, name, ruby, prefecture_code)
        cities.append(data)

    time.sleep(1)

  df = pd.DataFrame(cities, columns=['code', 'name', 'ruby', 'prefecture_code'])
  df.to_csv("data/cities.csv", index=False)
  print(f"cities end : {now()}")

  towns(sub_urls)

def towns(urls: list):
  print(f"towns start : {now()}")
  sub_urls = []
  towns = []
  for url in urls:
    city_code =  url.split("/")[4]
    print(f"----- city_code={city_code} start : {now()} -----")
    # if not city_code == '13113':
    #   continue
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'lxml')
    for div in soup.find_all("div", attrs={"class": "address_list"}):
      for li in div.find_all("li", attrs={"class": "left"}):
        ruby = li.find("span").getText()
        url = 'https://www.navitime.co.jp' + li.find("a").get("href")
        name = li.find("a").getText()
        sub_urls.append(url)
        code = url.split("/")[4]
        data = (code, name, ruby, city_code)
        towns.append(data)

    time.sleep(1)

  df = pd.DataFrame(towns, columns=['code', 'name', 'ruby', 'city_code'])
  df.to_csv("data/towns.csv", index=False)
  print(f"towns end : {now()}")

def now():
  return datetime.now().strftime("%Y/%m/%d %H:%M:%S")

if __name__ == "__main__":
  crawl()


from typing import Dict, List

from rest_framework import serializers

from apps.users.models import Profile

META_FIELDS: Dict[str, List[str]] = {
    'PROFILE_SERIALIZER_FIELDS': ['url', 'id', 'avatar_image'],
}

class ProfileSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = Profile
        fields = META_FIELDS['PROFILE_SERIALIZER_FIELDS']
        extra_kwargs = {
            'url': {'view_name': 'users:profile', 'lookup_field': 'pk'}
        }


# main.py
import torch
from data_loader import load_data, preprocess_data, create_datasets
from train import train_model, compute_metrics
from inference import load_trained_model, correct_grammar
import config
from transformers import T5Tokenizer

if __name__ == "__main__":
    # Load data
    data = load_data(config.DATA_FILE_PATH)
    print('01')

    # Preprocess data
    preprocessed_data = preprocess_data(data)
    print('02')

    # Create datasets
    tokenizer = T5Tokenizer.from_pretrained(config.MODEL_NAME)
    print('03')

    # Create datasets, passing the tokenizer argument
    train_dataset, test_dataset = create_datasets(preprocessed_data, tokenizer)
    print('04')

    # Train the model
    trained_model = train_model(train_dataset, test_dataset)
    print('05')

    # Save the trained model
    #save_model(trained_model, config.MODEL_OUTPUT_DIR)

    # Load the trained model for inference
    inference_model, inference_tokenizer = load_trained_model(config.INFERENCE_MODEL_PATH, config.TORCH_DEVICE)
    print('06')

    # Perform inference to correct grammar
    input_text = 'He are moving here.'  # Replace with your input sentence
    num_return_sequences = 2  # Number of corrected sequences to generate
    corrected_sentences = correct_grammar(inference_model, inference_tokenizer, input_text, num_return_sequences)

    # Print corrected sentences
    for i, corrected_sentence in enumerate(corrected_sentences):
        print(f'Correction {i+1}: {corrected_sentence}')


import requests
from bs4 import BeautifulSoup
import csv

url = "https://apps.acgme.org/ads/Public/Programs/Search"

r = requests.get(url).text

soup = BeautifulSoup(r,"lxml")

states = len(soup.find("div", class_="listview-options-bar-filters listview-options-bar-filters-required").find("select", id="stateFilter").find_all("option")[1:])

search = "https://apps.acgme.org/ads/Public/Programs/Search?stateId="
for st in range(1,states+1):
    state = search + "{}".format(st)
    r = requests.get(state).text
    soup = BeautifulSoup(r, "lxml")
    pages = soup.find("tbody").find_all("td", class_="listview-cell listview-command-cell")

    for page in pages:
        page_url = page.find("a").get("href")

        data_page= "https://apps.acgme.org" + page_url

        r = requests.get(data_page).text

        soup = BeautifulSoup(r, "lxml")

        try:
            specialty = soup.find("div").find("dl", class_="dl-horizontal").find("dd").text.strip()
        except:
            specialty = None
        try:
            sponsoring = soup.find("div").find("dl", class_="dl-horizontal").find_all("dd")[-1].text.split("]")[1].strip()
        except:
            sponsoring = None

        try:
            address = soup.find("address").text.split(",")[1].strip()
        except:
            address = None

        try:
            web_address = soup.find("div", id="content-panel").find("a", target="_blank").get("href")
        except:
            web_address = None

        try:
            instut_email = soup.find("div",id="content-panel").find_all("div")[2].find_all("a")[-1].text
        except:
            instut_email= "None"

        with open("acgme.csv","a") as f:
            write = csv.writer(f)
            write.writerow(["specialty", "sponsoring", "address", "web_address","instut_email"])
            write.writerow([specialty, sponsoring, address, web_address,instut_email])

    print("qutardim")
    break


from typing import List, Optional

import faiss

from pydantic import BaseModel, Field
from transformers import PreTrainedModel, PreTrainedTokenizer

UID = str

# See: https://fastapi.tiangolo.com/tutorial/body/ for more details on creating a Request Body.

class Document(BaseModel):
    uid: UID
    text: Optional[str] = None

class Search(BaseModel):
    query: Document
    documents: List[Document] = []
    top_k: int = Field(10, gt=0, description="top_k must be greater than 0")
    docs_only: bool = False

    class Config:
        schema_extra = {
            "example": {
                "query": {
                    "uid": "0",
                    "text": "It has recently been shown that Craf is essential for Kras G12D-induced NSCLC.",
                },
                "documents": [
                    {
                        "uid": "1",
                        "text": "Craf is essential for the onset of Kras-driven non-small cell lung cancer.",
                    },
                    {
                        "uid": "2",
                        "text": "Tumorigenesis is a multistage process that involves multiple cell types.",
                    },
                    {
                        "uid": "3",
                        "text": "Only concomitant ablation of ERK1 and ERK2 impairs tumor growth.",
                    },
                ],
                "top_k": 3,
            }
        }

class TopMatch(BaseModel):
    uid: UID
    score: float

class Model(BaseModel):
    tokenizer: PreTrainedModel = None
    model: PreTrainedTokenizer = None
    index: faiss.Index = None

    class Config:
        arbitrary_types_allowed = True


# montecarlo_parallel.py
#
# Omkar H. Ramachandran
# omkar.ramachandran@colorado.edu
#
# Parellelized montecarlo designed to run on CU's Summit supercomputer
#

import numpy as np
import montecarlo_isometric_headers as head
from astropy.io import fits

Nruns = 50
Nrays1 = 6055
Nrays2 = 3000
Nrays3 = 1000
DEGTORAD = np.pi/180

print("**********Montecarlo_Parallel***********")
print("Simulation Paramters")
print("Nruns=",Nruns)
print("N(E_1)=",Nrays1,",N(E_2)=",Nrays2,",N(E_3)=",Nrays3)

hdul_he = fits.open("data/gll_psch_v07.fit")
data_he = hdul_he[1].data
data_he_lon = data_he.field(4)[abs(data_he.field(4))> 0]*DEGTORAD
data_he_lat = data_he.field(3)[abs(data_he.field(4))> 0]*DEGTORAD

hdul_jpd = fits.open("data/gtselected_exp_map_50_60.fits")

Jpd = (hdul_jpd[0].data)[0]

print(Jpd.sum())

Jpd1 = Jpd / Jpd.sum()

R = np.linspace(2,20,10)

data = []

Q = np.zeros([Nruns,len(R)])
Qnorth = np.zeros([Nruns,len(R)])
Qsouth = np.zeros([Nruns,len(R)])
std = np.zeros([Nruns,len(R)])
stdnorth = np.zeros([Nruns,len(R)])
stdsouth = np.zeros([Nruns,len(R)])

for j in range(Nruns):

    theta1_all,phi1_all,theta2_all,phi2_all,theta3_all,phi3_all = head.genRays(Nrays1,Nrays2,Nrays3,Jpd1)
    print("Masking Data around 1FHL sources")
    for i in range(len(data_he_lon)):
        v = np.array([np.cos(data_he_lat[i])*np.cos(data_he_lon[i]),np.cos(data_he_lat[i])*np.sin(data_he_lon[i]),np.sin(data_he_lat[i])])
        # Mask to 2 degrees
        target = np.cos(2*DEGTORAD)

        theta3_new = theta3_all[np.dot(v,np.array([np.cos(theta3_all)*np.sin(phi3_all),np.cos(theta3_all)*np.cos(phi3_all),np.sin(theta3_all)])) - target <= 0]
        phi3_new = phi3_all[np.dot(v,np.array([np.cos(theta3_all)*np.sin(phi3_all),np.cos(theta3_all)*np.cos(phi3_all),np.sin(theta3_all)])) - target <= 0]
        theta2_new = theta2_all[np.dot(v,np.array([np.cos(theta2_all)*np.sin(phi2_all),np.cos(theta2_all)*np.cos(phi2_all),np.sin(theta2_all)])) - target <= 0]
        phi2_new = phi2_all[np.dot(v,np.array([np.cos(theta2_all)*np.sin(phi2_all),np.cos(theta2_all)*np.cos(phi2_all),np.sin(theta2_all)])) - target <= 0]
        theta1_new = theta1_all[np.dot(v,np.array([np.cos(theta1_all)*np.sin(phi1_all),np.cos(theta1_all)*np.cos(phi1_all),np.sin(theta1_all)])) - target <= 0]
        phi1_new = phi1_all[np.dot(v,np.array([np.cos(theta1_all)*np.sin(phi1_all),np.cos(theta1_all)*np.cos(phi1_all),np.sin(theta1_all)])) - target <= 0]

        theta3_all = theta3_new
        phi3_all = phi3_new
        theta2_all = theta2_new
        phi2_all = phi2_new
        theta1_all = theta1_new
        phi1_all = phi1_new

    phi3 = phi3_all[abs(theta3_all) > 80*DEGTORAD]
    theta3 = theta3_all[abs(theta3_all) > 80*DEGTORAD]

    Q[j],std[j] = head.computeQ(R,theta1_all,phi1_all,theta2_all,phi2_all,theta3,phi3)

    phi3 = phi3_all[(theta3_all) > 80*DEGTORAD]
    theta3 = theta3_all[(theta3_all) > 80*DEGTORAD]

    Qnorth[j],stdnorth[j] = head.computeQ(R,theta1_all,phi1_all,theta2_all,phi2_all,theta3,phi3)

    phi3 = phi3_all[-(theta3_all) > 80*DEGTORAD]
    theta3 = theta3_all[-(theta3_all) > 80*DEGTORAD]

    Qsouth[j],stdsouth[j] = head.computeQ(R,theta1_all,phi1_all,theta2_all,phi2_all,theta3,phi3)

Qmean = np.zeros(len(R))
stdmean = np.zeros(len(R))
QmeanN = np.zeros(len(R))
stdmeanN = np.zeros(len(R))
QmeanS = np.zeros(len(R))
stdmeanS = np.zeros(len(R))
for j in range(len(R)):
    Qmean[j] = np.mean(Q[:,j])
    stdmean[j] = np.sqrt(np.mean(std[:,j]**2))
    QmeanN[j] = np.mean(Qnorth[:,j])
    stdmeanN[j] = np.sqrt(np.mean(stdnorth[:,j]**2))
    QmeanS[j] = np.mean(Qsouth[:,j])
    stdmeanS[j] = np.sqrt(np.mean(stdsouth[:,j]**2))


# Standard Library
import os
import textwrap
from math import sqrt

# First Party
from utils import read_input

# Type

Position = tuple[int, int]
Grid = dict[Position, int]

def get_grid(input: str) -> Grid:
    grid: Grid = {}
    for y, row in enumerate(input.splitlines()):
        for x, cell in enumerate(row):
            grid[(x, y)] = int(cell)

    return grid

def neighbors(position: Position) -> list[Position]:
    x, y = position
    return [(x + dx, y + dy) for dx in [-1, 0, 1] for dy in [-1, 0, 1]]

def flash(grid: Grid, position: Position, flashed: set[Position]) -> tuple[Grid, set[Position]]:
    if position not in flashed:
        flashed.add(position)
        for neighbor in [n for n in neighbors(position) if n in grid]:
            grid[neighbor] += 1
            if grid[neighbor] >= 10:
                flash(grid, neighbor, flashed)

    return grid, flashed

def flattenForTesting(grid: Grid) -> str:
    flat = "".join(map(str, grid.values()))
    lines = textwrap.wrap(flat, int(sqrt(len(flat))))
    return "\n".join(lines)

def step(grid: Grid) -> tuple[Grid, int]:
    flashed: set[Position] = set()
    for position in grid:
        grid[position] += 1

    for position in grid:
        if grid[position] == 10:
            grid, newFlashed = flash(grid, position, flashed)
            flashed |= newFlashed

    for position in flashed:
        grid[position] = 0

    return grid, len(flashed)

def part_1(input: str) -> int:
    grid: Grid = get_grid(input)

    flashes = 0
    for _ in range(100):
        grid, stepFlashes = step(grid)
        flashes += stepFlashes

    return flashes

def part_2(input: str) -> int:
    grid: Grid = get_grid(input)

    flashes = 0
    octopuses = len(grid)
    steps = 0
    while flashes != octopuses:
        grid, flashes = step(grid)
        steps += 1

    return steps

# -- Tests

def get_example_input() -> str:
    return """5483143223
2745854711
5264556173
6141336146
6357385478
4167524645
2176841721
6882881134
4846848554
5283751526"""

def test_part_1():
    input = get_example_input()
    assert part_1(input) == 1656

def test_step_function():
    grid = get_grid(
        """11111
19991
19191
19991
11111"""
    )

    grid, _ = step(grid)
    assert flattenForTesting(grid) == "34543\n40004\n50005\n40004\n34543"

    grid, _ = step(grid)
    assert flattenForTesting(grid) == "45654\n51115\n61116\n51115\n45654"

def test_steps():
    with open(os.path.join(os.path.dirname(__file__), "..", "inputs", "day_11_test.txt")) as f:
        test_data = f.read().strip().strip("\n\r")
    test_sections = [line.split("\n") for line in test_data.split("\n\n")]
    tests = {}
    for section in test_sections:
        stepNumber = int([i for i in section[0].replace(":", "").split() if i.isdigit()][0])
        tests[stepNumber] = section[1:]

    grid = get_grid(get_example_input())

    for stepNumber in range(1, max(tests.keys()) + 1):
        grid, _ = step(grid)
        if stepNumber in tests:
            assert flattenForTesting(grid) == "\n".join(tests[stepNumber])

def test_part_2():
    input = get_example_input()
    assert part_2(input) == 195

def test_part_1_real():
    input = read_input(__file__)
    assert part_1(input) == 1588

def test_part_2_real():
    input = read_input(__file__)
    assert part_2(input) == 517

# -- Main

if __name__ == "__main__":
    input = read_input(__file__)

    print(f"Part1: {part_1(input)}")
    print(f"Part2: {part_2(input)}")