import ROOT
import uproot
from hipe4ml.tree_handler import TreeHandler
import numpy as np

import argparse
import yaml

import sys
sys.path.append('utils')
import utils as utils

utils.set_style()
kBlueC = ROOT.TColor.GetColor('#1f78b4')
kOrangeC  = ROOT.TColor.GetColor('#ff7f00')
ROOT.gROOT.SetBatch()

## create signal extraction class
class SignalExtraction:
    def __init__(self, input_data_hdl, input_mc_hdl=None): ## could be either a pandas or a tree handler

        self.data_hdl = input_data_hdl
        self.mc_hdl = input_mc_hdl
        self.is_3lh = True
        self.bins = 40
        self.n_evts = 1e9
        self.is_matter = False
        self.signal_fit_func = 'dscb'
        self.bkg_fit_func = 'pol1'
        self.performance = False
        self.additional_pave_text = '' ## additional text to be added to the ALICE performance pave

        ## variables
        self.pdf = None
        self.roo_dataset = None

        ### frames to be saved to file
        self.mc_frame_fit = None
        self.data_frame_fit = None
        self.local_pvalue_graph = None

    def process_fit(self, extended_likelihood=True, rooworkspace_path=None):

        if self.is_3lh:
            self.inv_mass_string = '#it{M}_{^{3}He+#pi^{-}}' if self.is_matter else '#it{M}_{^{3}#bar{He}+#pi^{+}}'
            decay_string = '{}^{3}_{#Lambda}H #rightarrow ^{3}He+#pi^{-}' if self.is_matter else '{}^{3}_{#bar{#Lambda}}#bar{H} #rightarrow ^{3}#bar{He}+#pi^{+}'
            tree_var_name = 'fMassH3L'
        else:
            self.inv_mass_string = '#it{M}_{^{4}He+#pi^{-}}' if self.is_matter else '#it{M}_{^{4}#bar{He}+#pi^{+}}'
            decay_string = '{}^{4}_{#Lambda}H #rightarrow ^{4}He+#pi^{-}' if self.is_matter else '{}^{4}_{#bar{#Lambda}}#bar{H} #rightarrow ^{4}#bar{He}+#pi^{+}'
            tree_var_name = 'fMassH4L'

        # define signal and bkg variables
        if self.is_3lh:
            mass = ROOT.RooRealVar('m', self.inv_mass_string, 2.96, 3.04, 'GeV/c^{2}')
            mu = ROOT.RooRealVar('mu', 'hypernucl mass', 2.97, 2.992, 'GeV/c^{2}')
        else:
            mass = ROOT.RooRealVar('m', self.inv_mass_string, 3.89, 3.97, 'GeV/c^{2}')
            mu = ROOT.RooRealVar('mu', 'hypernucl mass', 3.9, 3.95, 'GeV/c^{2}')

        sigma = ROOT.RooRealVar('sigma', 'hypernucl width', 0.001, 0.004, 'GeV/c^{2}')
        a1 = ROOT.RooRealVar('a1', 'a1', 0, 5.)
        a2 = ROOT.RooRealVar('a2', 'a2', 0, 5.)
        n1 = ROOT.RooRealVar('n1', 'n1', 1, 5.)
        n2 = ROOT.RooRealVar('n2', 'n2', 1, 5.)
        c0 = ROOT.RooRealVar('c0', 'constant c0', -1., 1)
        c1 = ROOT.RooRealVar('c1', 'constant c1', -1., 1)

        if self.signal_fit_func == 'dscb':
            signal = ROOT.RooCrystalBall('cb', 'cb', mass, mu, sigma, a1, n1, a2, n2)
        elif self.signal_fit_func == 'gaus':
            signal = ROOT.RooGaussian('gaus', 'gaus', mass, mu, sigma)
        else:
            raise ValueError(f'Invalid signal fit function. Expected one of: dscb, gaus')

        # define background pdf
        if self.bkg_fit_func == 'pol1':
            background = ROOT.RooChebychev('bkg', 'pol1 bkg', mass, ROOT.RooArgList(c0))
        elif self.bkg_fit_func == 'pol2':
            background = ROOT.RooChebychev('bkg', 'pol2 bkg', mass, ROOT.RooArgList(c0, c1))
        else:
            raise ValueError(f'Invalid background fit function. Expected one of: pol1, pol2')

        if extended_likelihood:
            n_signal = ROOT.RooRealVar('n_signal', 'n_signal', 10., 1e6)
            n_background = ROOT.RooRealVar('n_background', 'n_background', 0., 1e6)
        else:
            f = ROOT.RooRealVar('f', 'fraction of signal', 0., 0.4)

        # fix DSCB parameters to MC
        if self.mc_hdl != None:
            mass_roo_mc = utils.ndarray2roo(np.array(self.mc_hdl['fMassH3L'].values, dtype=np.float64), mass, 'histo_mc')
            signal.fitTo(mass_roo_mc, ROOT.RooFit.Range(2.97, 3.01), ROOT.RooFit.PrintLevel(-1))
            a1.setConstant()
            a2.setConstant()
            n1.setConstant()
            n2.setConstant()
            sigma.setRange(sigma.getVal(), sigma.getVal()*1.5)
            self.mc_frame_fit = mass.frame(80)
            self.mc_frame_fit.SetName('mc_frame_fit')
            mass_roo_mc.plotOn(self.mc_frame_fit)
            signal.plotOn(self.mc_frame_fit)
            fit_param = ROOT.TPaveText(0.6, 0.6, 0.9, 0.9, 'NDC')
            fit_param.SetBorderSize(0)
            fit_param.SetFillStyle(0)
            fit_param.SetTextAlign(12)
            fit_param.AddText('#mu = ' + f'{mu.getVal()*1e3:.2f} #pm {mu.getError()*1e3:.2f}' + ' MeV/#it{c}^{2}')
            fit_param.AddText('#sigma = ' + f'{sigma.getVal()*1e3:.2f} #pm {sigma.getError()*1e3:.2f}' + ' MeV/#it{c}^{2}')
            self.mc_frame_fit.addObject(fit_param)

        # define the fit function and perform the actual fit
        if extended_likelihood:
            self.pdf = ROOT.RooAddPdf('total_pdf', 'signal + background', ROOT.RooArgList(signal, background), ROOT.RooArgList(n_signal, n_background))
        else:
            self.pdf = ROOT.RooAddPdf('total_pdf', 'signal + background', ROOT.RooArgList(signal, background), ROOT.RooArgList(f))

        mass_array = np.array(self.data_hdl[tree_var_name].values, dtype=np.float64)
        self.roo_dataset = utils.ndarray2roo(mass_array, mass)
        self.pdf.fitTo(self.roo_dataset, ROOT.RooFit.Extended(extended_likelihood), ROOT.RooFit.Save(True), ROOT.RooFit.PrintLevel(-1))
        ## get fit parameters
        fit_pars = self.pdf.getParameters(self.roo_dataset)
        sigma_val = fit_pars.find('sigma').getVal()
        sigma_val_error = fit_pars.find('sigma').getError()
        mu_val = fit_pars.find('mu').getVal()
        mu_val_error = fit_pars.find('mu').getError()

        if extended_likelihood:
            signal_counts = n_signal.getVal()
            signal_counts_error = n_signal.getError()
            background_counts = n_background.getVal()
            background_counts_error = n_background.getError()

        else:
            signal_counts = (1-f.getVal())*self.roo_dataset.sumEntries()
            signal_counts_error = (1-f.getVal()) * self.roo_dataset.sumEntries()*f.getError()/f.getVal()
            background_counts = f.getVal()*self.roo_dataset.sumEntries()
            background_counts_error = f.getVal() * self.roo_dataset.sumEntries()*f.getError()/f.getVal()

        self.data_frame_fit = mass.frame(self.n_bins)
        self.data_frame_fit.SetName('data_frame_fit')

        self.roo_dataset.plotOn(self.data_frame_fit, ROOT.RooFit.Name('data'), ROOT.RooFit.DrawOption('p'))
        self.pdf.plotOn(self.data_frame_fit, ROOT.RooFit.Components('bkg'), ROOT.RooFit.LineStyle(ROOT.kDashed), ROOT.RooFit.LineColor(kOrangeC))
        self.pdf.plotOn(self.data_frame_fit, ROOT.RooFit.LineColor(kBlueC), ROOT.RooFit.Name('fit_func'))

        self.data_frame_fit.GetYaxis().SetTitleSize(0.06)
        self.data_frame_fit.GetYaxis().SetTitleOffset(0.9)
        self.data_frame_fit.GetYaxis().SetMaxDigits(2)
        self.data_frame_fit.GetXaxis().SetTitleOffset(1.1)

        # signal within 3 sigma
        mass.setRange('signal', mu_val-3*sigma_val, mu_val+3*sigma_val)
        signal_int = signal.createIntegral(ROOT.RooArgSet(mass), ROOT.RooArgSet(mass), 'signal')
        signal_int_val_3s = signal_int.getVal()*signal_counts
        signal_int_val_3s_error = signal_int_val_3s*signal_counts_error/signal_counts
        # background within 3 sigma
        mass.setRange('bkg', mu_val-3*sigma_val, mu_val+3*sigma_val)
        bkg_int = background.createIntegral(ROOT.RooArgSet(mass), ROOT.RooArgSet(mass), 'bkg')
        bkg_int_val_3s = bkg_int.getVal()*background_counts
        bkg_int_val_3s_error = bkg_int_val_3s*background_counts_error/background_counts
        significance = signal_int_val_3s / np.sqrt(signal_int_val_3s + bkg_int_val_3s)
        significance_err = utils.significance_error(signal_int_val_3s, bkg_int_val_3s, signal_int_val_3s_error, bkg_int_val_3s_error)
        s_b_ratio_err = np.sqrt((signal_int_val_3s_error/signal_int_val_3s)**2 + (bkg_int_val_3s_error/bkg_int_val_3s)**2)*signal_int_val_3s/bkg_int_val_3s

        # add pave for stats
        pinfo_vals = ROOT.TPaveText(0.632, 0.5, 0.932, 0.85, 'NDC')
        pinfo_vals.SetBorderSize(0)
        pinfo_vals.SetFillStyle(0)
        pinfo_vals.SetTextAlign(11)
        pinfo_vals.SetTextFont(42)
        pinfo_vals.AddText(f'Signal (S): {signal_counts:.0f} #pm {signal_counts_error:.0f}')
        pinfo_vals.AddText(f'S/B (3 #sigma): {signal_int_val_3s/bkg_int_val_3s:.1f} #pm {s_b_ratio_err:.1f}')
        pinfo_vals.AddText('S/#sqrt{S+B} (3 #sigma): ' + f'{significance:.1f} #pm {significance_err:.1f}')
        pinfo_vals.AddText('#mu = ' + f'{mu_val*1e3:.2f} #pm {mu.getError()*1e3:.2f}' + ' MeV/#it{c}^{2}')
        pinfo_vals.AddText('#sigma = ' + f'{sigma_val*1e3:.2f} #pm {sigma.getError()*1e3:.2f}' + ' MeV/#it{c}^{2}')

        ## add pave for ALICE performance
        if self.performance:
            pinfo_alice = ROOT.TPaveText(0.6, 0.5, 0.93, 0.85, 'NDC')
        else:
            pinfo_alice = ROOT.TPaveText(0.14, 0.6, 0.42, 0.85, 'NDC')
        pinfo_alice.SetBorderSize(0)
        pinfo_alice.SetFillStyle(0)
        pinfo_alice.SetTextAlign(11)
        pinfo_alice.SetTextFont(42)
        pinfo_alice.AddText('ALICE Performance')
        pinfo_alice.AddText('Run 3, pp #sqrt{#it{s}} = 13.6 TeV')
        if not self.performance:
            pinfo_alice.AddText('N_{ev} = ' f'{self.n_evts:.0f} '  '#times 10^{9}')
        pinfo_alice.AddText(decay_string)
        if self.additional_pave_text != '':
            pinfo_alice.AddText(self.additional_pave_text)

        if not self.performance:
            self.data_frame_fit.addObject(pinfo_vals)
        self.data_frame_fit.addObject(pinfo_alice)

        fit_stats = {'signal': [signal_counts, signal_counts_error],
                     'significance': [significance, significance_err], 's_b_ratio': [signal_int_val_3s/bkg_int_val_3s, s_b_ratio_err]}

        if rooworkspace_path != None:
            w = ROOT.RooWorkspace('w')
            sb_model = ROOT.RooStats.ModelConfig('sb_model', w)
            sb_model.SetPdf(self.pdf)
            sb_model.SetParametersOfInterest(ROOT.RooArgSet(n_signal))
            sb_model.SetObservables(ROOT.RooArgSet(mass))
            getattr(w, 'import')(sb_model)
            getattr(w, 'import')(self.roo_dataset)
            w.writeToFile(rooworkspace_path + '/rooworkspace.root', True)

        return fit_stats

    def compute_significance_asymptotic_calc(self, rooworkspace_path, do_local_p0plot=False):
        print("-----------------------------------------------")
        print("Computing significance with asymptotic calculator")
        ## get saved workspace
        workspace_file = ROOT.TFile(rooworkspace_path + '/rooworkspace.root', 'READ')
        w = workspace_file.Get('w')
        roo_abs_data = w.data('data')
        sb_model = w.obj('sb_model')
        poi = sb_model.GetParametersOfInterest().first()
        sb_model.SetSnapshot(ROOT.RooArgSet(poi))
        ## create the b-only model
        b_model = sb_model.Clone()
        b_model.SetName('b_model')
        poi.setVal(0)
        b_model.SetSnapshot(poi)
        b_model.Print()
        # w.var('sigma').setConstant(True)
        w.var('mu').setConstant(True)

        asymp_calc = ROOT.RooStats.AsymptoticCalculator(roo_abs_data, sb_model, b_model)
        asymp_calc.SetPrintLevel(0)
        asymp_calc_result = asymp_calc.GetHypoTest()
        null_p_value = asymp_calc_result.NullPValue()
        null_p_value_err = asymp_calc_result.NullPValueError()
        significance = asymp_calc_result.Significance()
        significance_err = asymp_calc_result.SignificanceError()

        if do_local_p0plot:
            ### perform a scan in mass and compute the significance
            masses = []
            p0_values = []
            p0_values_expected = []
            mass_array = np.linspace(w.var('mu').getMin(), w.var('mu').getMax(), 100)
            for mass in mass_array:

                w.var('mu').setVal(mass)
                w.var('mu').setConstant(True)
                asymp_calc_scan = ROOT.RooStats.AsymptoticCalculator(roo_abs_data, sb_model, b_model)
                asymp_calc_scan.SetOneSidedDiscovery(True)
                asym_calc_result_scan = asymp_calc_scan.GetHypoTest()
                null_p_value_scan = asym_calc_result_scan.NullPValue()
                masses.append(mass)
                p0_values.append(null_p_value_scan)

                print(f"Mass: {mass} MeV/c^2, p0: {null_p_value_scan:.10f}")

            ## create a graph with the p0 values
            self.local_pvalue_graph = ROOT.TGraph(len(masses), np.array(masses), np.array(p0_values))
            self.local_pvalue_graph.SetName('p0_values')
            self.local_pvalue_graph.GetXaxis().SetTitle(self.inv_mass_string)
            self.local_pvalue_graph.GetYaxis().SetTitle('Local p-value')
            # log Y axis
            self.local_pvalue_graph.SetMarkerStyle(20)
            self.local_pvalue_graph.SetMarkerColor(kBlueC)
            self.local_pvalue_graph.SetMarkerSize(0)
            self.local_pvalue_graph.SetLineColor(kBlueC)
            self.local_pvalue_graph.SetLineWidth(2)

        print("****************************************************")
        print(f'p0: {null_p_value:.3E} +/- {null_p_value_err:.3E}')
        print(f'significance: {significance:.5f} +/- {significance_err:.5f}')
        print("****************************************************")

if __name__ == '__main__':

    # set parameters
    parser = argparse.ArgumentParser(
        description='Configure the parameters of the script.')
    parser.add_argument('--config-file', dest='config_file', default='',
                        help='path to the YAML file with configuration.')
    parser.add_argument('--nbins', dest='n_bins', default=30,
                        help='number of bins in the final plot.')
    parser.add_argument('--performance', action='store_true',
                        help="True for performance plot", default=False)
    args = parser.parse_args()

    config_file = open(args.config_file, 'r')
    config = yaml.full_load(config_file)

    input_parquet_data = config['input_parquet_data']
    input_analysis_results = config['input_analysis_results']

    input_parquet_mc = config['input_parquet_mc']

    output_dir = config['output_dir']
    output_file = config['output_file']

    is_4lh = config['is_4lh']
    matter_type = config['matter_type']
    n_bins = config['n_bins']

    performance = args.performance
    data_hdl = TreeHandler(input_parquet_data)
    mc_hdl =  TreeHandler(input_parquet_mc)

    an_vtx_z = uproot.open(input_analysis_results)['hyper-reco-task']['hZvtx']
    n_evts = an_vtx_z.values().sum() / 1e9
    n_evts = round(n_evts, 0)

    signal_extraction = SignalExtraction(data_hdl, mc_hdl)
    signal_extraction.n_bins = n_bins
    signal_extraction.n_evts = n_evts
    signal_extraction.matter_type = matter_type
    signal_extraction.performance = performance
    signal_extraction.is_3lh = not is_4lh
    signal_extraction.bkg_fit_func = 'pol1'

    signal_extraction.process_fit(extended_likelihood=True, rooworkspace_path="../results")
    signal_extraction.compute_significance_asymptotic_calc(rooworkspace_path="../results", do_local_p0plot=True)

    # create output file and save frames
    out_file = ROOT.TFile(f'{output_dir}/{output_file}', 'recreate')
    out_file.cd()
    signal_extraction.data_frame_fit.Write()
    signal_extraction.mc_frame_fit.Write()
    signal_extraction.local_pvalue_graph.Write()
    out_file.Close()

    if is_4lh:
        state_label = '4lh'
    else:
        state_label = '3lh'

    cSignalExtraction = ROOT.TCanvas('cSignalExtraction', 'cSignalExtraction', 800, 600)
    signal_extraction.data_frame_fit.SetTitle('')
    signal_extraction.data_frame_fit.Draw()
    cSignalExtraction.SaveAs(f'{output_dir}/cSignalExtraction_{matter_type}_{state_label}.pdf')


import asyncio
import multiprocessing as mp
import os
import time

from bs4 import BeautifulSoup

from src.download_utils import download_page

class DownloadRepository:
    def __init__(self, address: str):
        self.files = []
        self.address = address
        self.queue = mp.Queue()
        self.process = mp.Process(target=self.start_download)

    async def start_collect(self, start_page: str) -> None:
        self.process.start()
        await self.search_repository(start_page)
        time.sleep(2)
        self.process.terminate()

    async def search_repository(self, url: str) -> None:
        raw_html = await download_page(self.address, url)
        is_file = self.check_page(raw_html)
        if not is_file:
            embedded_links = self.parse_html_code(raw_html)
            for link in embedded_links:
                await self.search_repository(link)

    def check_page(self, html_text: str) -> bool:
        soup = BeautifulSoup(html_text, 'lxml')
        tool_bar = soup.find(
            name='h4',
            attrs='file-header ui top attached header df ac sb',
        )
        if tool_bar:
            tag = tool_bar.find('div', class_='ui buttons mr-2')
            if tag is None:
                return False
            link = tag.find('a', string='Raw').get('href')
            print(f'[+]Found file -> {self.address}{link}')
            self.queue.put(link)
            return True
        return False

    def parse_html_code(self,  html_code: str) -> list[str]:
        soup = BeautifulSoup(html_code, 'lxml')
        links = []
        if soup.tbody:
            for tag in soup.tbody.find_all('a'):
                link = tag.get('href')
                title = tag.get('title')
                if link and title:
                    links.append(link)
            return links
        return []

    def start_download(self):
        asyncio.run(self.save_file())

    @staticmethod
    def create_temp_folder():
        if not os.path.exists('repository'):
            os.mkdir('repository')

    @staticmethod
    def create_saving_directory(path: str) -> None:
        if not os.path.exists(path):
            os.makedirs(path)

    @staticmethod
    def extract_paths(link: str) -> tuple[str, str]:
        paths = link.split('/')
        project = paths[2]
        directory = '/'.join(paths[6:-1])
        filename = paths[-1]
        folder = f'repository/{project}/{directory}'
        full_path = f'{folder}/{filename}'

        return full_path, folder

    async def save_file(self):
        """Download and save downloaded files to repository directory."""
        self.create_temp_folder()

        while True:
            if not self.queue.empty():
                link = self.queue.get()
                text = await download_page(self.address, link)
                full_path, directory = self.extract_paths(link)
                self.create_saving_directory(directory)

                with open(f'{full_path}', 'w') as repo_file:
                    repo_file.write(text)
                print(f'[+]File saved in {full_path}')


from gtts import gTTS
from pydub import AudioSegment
from pydub.playback import play
from screeninfo import get_monitors
import numpy as np
import cv2
import json
import tempfile
import os
import time

def get_screen_resolution(use_default):
    if use_default:
        return 640, 480  # Default resolution

    monitors = get_monitors()
    if monitors:
        return monitors[0].width, monitors[0].height
    else:
        print("Unable to determine screen resolution.")
        return None

def text_to_speech(text, lang='en'):
    tts = gTTS(text=text, lang=lang, slow=False)
    with tempfile.NamedTemporaryFile(delete=False) as temp_audio:
        tts.save(temp_audio.name)
        audio_segment = AudioSegment.from_file(temp_audio.name)
        os.remove(temp_audio.name)
    return audio_segment

input_file = 'results.json'
with open(input_file, 'r') as f:
    results_data = json.load(f)

use_default_resolution = 1  # 0 to use device's resolution
screen_width, screen_height = get_screen_resolution(use_default_resolution)

# Total duration of audio (in milliseconds)
total_duration = 10000  # 10sec

# Silent audio segment
audio = AudioSegment.silent(duration=total_duration)

for frame_idx, frame_results in enumerate(results_data):
    frame_duration = total_duration / len(results_data)
    frame_audio = AudioSegment.silent(duration=frame_duration)

    for obj_result in frame_results:
        bounding_box = obj_result['bounding_box']
        class_name = obj_result['class_name']

        # Adjusting amplitude based on distance between y-coordinates
        distance_y = bounding_box['y'] + bounding_box['height'] - bounding_box['y']
        amplitude = np.clip(1 - distance_y / screen_height, 0.1, 1)  # Amplitude range = [0.1, 1]

        # Midpoint of bounding box for panning audio
        midpoint_x = bounding_box['x'] + bounding_box['width'] / 2
        midpoint_y = bounding_box['y'] + bounding_box['height'] / 2

        # Distance between midpoints and pan accordingly
        distance_to_center = midpoint_x - screen_width / 2
        pan = np.clip(distance_to_center / (screen_width / 2), -1, 1)  # Pan range = [-1, 1]

        voice_audio = text_to_speech(class_name)
        voice_audio = voice_audio - (1 - amplitude) * 50
        voice_audio = voice_audio.pan(pan)
        frame_audio = frame_audio.overlay(voice_audio)

        # print(f"Frame {frame_idx + 1}, Object: {class_name}, Pan: {pan}, Amplitude: {amplitude:.2f}")

    audio = audio.overlay(frame_audio, position=int(frame_idx * frame_duration))
    if (frame_idx + 1) % 3 == 0:    # Speak out the class name every 3 seconds
        text_to_speech("Attention, " + class_name).play()

# Export the stereo audio to a file
audio.export('stereo_audio_with_speech_amplitude.wav', format='wav')


import shutil
import zipfile
import os
import sys
import warnings
from urllib.request import urlretrieve
from tqdm import tqdm
from zrp import about

# This is used to show progress when downloading.
# see here: https://github.com/tqdm/tqdm#hooks-and-callbacks
class TqdmUpTo(tqdm):
    """Provides `update_to(n)` which uses `tqdm.update(delta_n)`."""
    def update_to(self, b=1, bsize=1, tsize=None):
        """
        b  : int, optional
            Number of blocks transferred so far [default: 1].
        bsize  : int, optional
            Size of each block (in tqdm units) [default: 1].
        tsize  : int, optional
            Total size (in tqdm units). If [default: None] remains unchanged.
        """
        if tsize is not None:
            self.total = tsize
        self.update(b * bsize - self.n)  # will also set self.n = b * bsize

def download_progress(url, fname):
    """
    Download a file and show a progress bar.
    :param url: A string for the url of the release zip to download.
    :param fname: A string for the local file name under which the downloaded file can be found.
    :return:
    """

    with TqdmUpTo(unit='B', unit_scale=True, miniters=1,
              desc=url.split('/')[-1]) as t:  # all optional kwargs

        print(f"Retrieving url at {url}")
        urlretrieve(url, filename=fname, reporthook=t.update_to, data=None)
        t.total = t.n
    return fname

def download_and_clean_lookup_tables(url, lookup_tables_output_fname, lookup_tables_output_zip_fname, geo_yr="2019",
                                     acs_yr="2019", acs_range="5yr"):
    """
    Downloads look up tables and file them within the module.
    This downloads the zip file from the repository, extracts it, renames it, then moves the
    tables to the correct directory, and removes large files not used at runtime.
    :param lookup_tables_output_fname: A string for the name of the file downloaded after unzipping.
    :param acs_range: A string for the year range the acs lookup table data will be from.
    :param acs_yr: A string for the year the acs lookup table data will be from.
    :param geo_yr: A string for the year the geo lookup table data will be from.
    :param url: A string for the url of the release zip to download.
    :param lookup_tables_output_zip_fname: A string for the name of the zip file downloaded.
    :return:
    """
    cwd = os.path.dirname(os.path.abspath(__file__))
    path_to_lt_zip = os.path.join(cwd, lookup_tables_output_zip_fname)
    print("Downloading zrp release...", file=sys.stderr)
    download_progress(url, path_to_lt_zip)
    print("Finished download")
    print("\n")

    print("Filing extras...")
    with zipfile.ZipFile(path_to_lt_zip, 'r') as zf:
        zf.extractall(cwd)
    os.remove(path_to_lt_zip)

    # Get rid of prefix that unzipping prepends
    # curr_folder = cwd.split("/")[-1]
    # unzipped_src_fname = curr_folder + "-" + lookup_tables_output_fname
    # path_to_unzipped_src = os.path.join(cwd, unzipped_src_fname)
    path_to_lookup_tables = os.path.join(cwd, lookup_tables_output_fname)
    # os.rename(path_to_unzipped_src, path_to_lookup_tables)

    # Clear old look up table directories
    data_dir = os.path.join(cwd, 'data')
    geo_data_dir = os.path.join(data_dir, f'processed/geo/{geo_yr}')
    acs_data_dir = os.path.join(data_dir, f'processed/acs/{acs_yr}/{acs_range}')

    if os.path.isdir(geo_data_dir):
        shutil.rmtree(geo_data_dir)
    if os.path.isdir(acs_data_dir):
        shutil.rmtree(acs_data_dir)

    print("Old geo lookup table data cleared out.")

    # Migrate lookup tables
    dl_geo_dir = os.path.join(cwd, lookup_tables_output_fname, f'geo/{geo_yr}')
    dl_acs_dir = os.path.join(cwd, lookup_tables_output_fname, f'acs/{acs_yr}/{acs_range}')

    if os.path.isdir(dl_geo_dir):
        shutil.move(dl_geo_dir, geo_data_dir)
        print(f"New geo lookup tables successfully migrated from {dl_geo_dir} to {geo_data_dir}.")
    else:
        warnings.warn(f"The geo lookup data was not found in {dl_geo_dir}. Ensure you're requesting a valid year. "
                      "Consult the lookup_tables release to troubleshoot.")
    if os.path.isdir(dl_acs_dir):
        shutil.move(dl_acs_dir, acs_data_dir)
        print(f"New geo lookup tables successfully migrated from {dl_acs_dir} to {acs_data_dir}.")
    else:
        warnings.warn(f"The acs lookup data was not found in {dl_acs_dir}. Ensure you're requesting a valid year and"
                      "year range. Consult the lookup_tables release to troubleshoot.")

    # Remove rest of lookup table folder
    shutil.rmtree(path_to_lookup_tables)

    # save a version file so we can tell what it is
    vpath = os.path.join(data_dir, 'version')
    with open(vpath, 'w') as vfile:
        vfile.write('zrp release --> {}'.format(lookup_tables_output_fname))

    print("Filed lookup tables successfully,", file=sys.stderr)
    print("   to", data_dir)

def download_and_clean_pipelines(url, pipelines_output_fname, pipelines_output_zip_fname):
    """
    Downloads pipeline pickle files and file them within the module.
    This downloads the zip file from the repository, extracts it, renames it, then moves the
    tables to the correct directory, and removes large files not used at runtime.
    :param pipeline_output_fname: A string for the name of the file downloaded after unzipping.
    :param url: A string for the url of the release zip to download.
    :param pipelines_output_zip_fname: A string for the name of the zip file downloaded.
    :return:
    """
    cwd = os.path.dirname(os.path.abspath(__file__))
    path_to_ppln_zip = os.path.join(cwd, pipelines_output_zip_fname)
    print("Downloading zrp release...", file=sys.stderr)
    download_progress(url, path_to_ppln_zip)
    print("Finished download")
    print("\n")

    print("Filing extras...")
    with zipfile.ZipFile(path_to_ppln_zip, 'r') as zf:
        zf.extractall(cwd)
    os.remove(path_to_ppln_zip)

    # Get rid of prefix that unzipping prepends
    # curr_folder = cwd.split("/")[-1]
    # unzipped_src_fname = curr_folder + "-" + pipelines_output_fname
    # path_to_unzipped_src = os.path.join(cwd, unzipped_src_fname)
    path_to_pipelines = os.path.join(cwd, pipelines_output_fname)
    # os.rename(path_to_unzipped_src, path_to_pipelines)

    # Clear old look up table directories
    model_dir = os.path.join(cwd, 'modeling/models')
    block_group_dir = os.path.join(model_dir, 'block_group')
    census_tract_dir = os.path.join(model_dir, 'census_tract')
    zip_code_dir = os.path.join(model_dir, 'zip_code')

    block_group_pipeline = os.path.join(block_group_dir, 'pipe.pkl')
    census_tract_pipeline = os.path.join(census_tract_dir, 'pipe.pkl')
    zip_code_pipeline = os.path.join(zip_code_dir, 'pipe.pkl')

    if os.path.isfile(block_group_pipeline):
        os.remove(block_group_pipeline)
    if os.path.isfile(census_tract_pipeline):
        os.remove(census_tract_pipeline)
    if os.path.isfile(zip_code_pipeline):
        os.remove(zip_code_pipeline)
    print("Old pipelines cleared out.")

    # Migrate pipelines
    dl_bg_pipe_file = os.path.join(path_to_pipelines, 'block_group_pipe.pkl')
    dl_ct_pipe_file = os.path.join(path_to_pipelines, 'census_tract_pipe.pkl')
    dl_zp_pipe_file = os.path.join(path_to_pipelines, 'zip_code_pipe.pkl')

    if os.path.isfile(dl_bg_pipe_file):
        shutil.move(dl_bg_pipe_file, os.path.join(block_group_dir, 'pipe.pkl'))
        print("Block group pipeline successfully migrated.")
    else:
        warnings.warn(f"The block group pipeline was not found in {dl_bg_pipe_file}."
                      "Consult the pipelines release to troubleshoot.")

    if os.path.isfile(dl_ct_pipe_file):
        shutil.move(dl_ct_pipe_file, os.path.join(census_tract_dir, 'pipe.pkl'))
        print("Census tract pipeline successfully migrated.")
    else:
        warnings.warn(f"The census tract pipeline was not found in {dl_ct_pipe_file}."
                      "Consult the pipelines release to troubleshoot.")

    if os.path.isfile(dl_zp_pipe_file):
        shutil.move(dl_zp_pipe_file, os.path.join(zip_code_dir, 'pipe.pkl'))
        print("Zip code pipeline successfully migrated.")
    else:
        warnings.warn(f"The zip code pipeline was not found in {dl_zp_pipe_file}."
                      "Consult the pipelines release to troubleshoot.")

    # Remove rest of pipelines folder
    shutil.rmtree(path_to_pipelines)

    # save a version file so we can tell what it is
    data_dir = os.path.join(cwd, 'data')
    vpath = os.path.join(data_dir, 'version')
    with open(vpath, 'w') as vfile:
        vfile.write('zrp release --> {}'.format(pipelines_output_fname))

    print("Filed pipelines successfully", file=sys.stderr)
    print("   to", model_dir)

def get_release():
    version = about.__version__
    dl_tpl = "{m}-{v}"
    return dl_tpl.format(m="zrp", v=version)

def download():
    release_pkg = get_release()

    # lookup_tables_output_fname = release_pkg + "_lookup_tables"
    lookup_tables_output_fname = "lookup_tables"
    lookup_tables_output_zip_fname = release_pkg + "_lookup_tables" + ".zip"
    lookup_table_url = about.__download_url_prefix__ + release_pkg + "/lookup_tables.zip"
    download_and_clean_lookup_tables(lookup_table_url, lookup_tables_output_fname, lookup_tables_output_zip_fname)

    pipelines_output_fname = "pipelines"
    pipelines_output_zip_fname = release_pkg + "_pipelines" + ".zip"
    pipelines_url = about.__download_url_prefix__ + release_pkg + "/pipelines.zip"
    download_and_clean_pipelines(pipelines_url, pipelines_output_fname, pipelines_output_zip_fname)


import sys
import pdb

def count_primes(n):
    isPrime = [False] * 2 + [True] * (n - 2)
    i = 2
    while i * i < n:
        if not isPrime[i]:
            i += 1
            continue
        j = i * i
        while j < n:
            isPrime[j] = False
            j += i
        i += 1
    print(isPrime)
    return isPrime.count(True)

count_primes(10)


class NrPdu:
    def __init__(self, byteStream=None):
        if byteStream==None:
            self.PduByteArray = bytearray()
        else:
            if isinstance(byteStream, str):
                self.PduByteArray = bytearray.fromhex(byteStream)
            else:
                if hasattr(byteStream, 'decode'):
                    self.PduByteArray = byteStream
                    print(self.PduByteArray)
                else:
                    print("argument must be hex string or bytearray")

    def getBitField(self, inputByte, bitMask, bitOffset):
        print('getBitField\t : inputByte = 0x{:02x} '.format(inputByte) \
        + 'bitMask = ' + '0x{:02x} '.format(bitMask) \
        + 'bitOffset = ' + '0x{:02x} '.format(bitOffset))
        return (inputByte & bitMask) >> bitOffset


#!/bin/python3

import re
import getopt, sys
from kazoo.client import KazooClient
import json

def getRollupRules(zookeeperHostsIn, zNodePath):
    zk = KazooClient(hosts=zookeeperHostsIn);
    zk.start();
    result = {};

    if zk.exists(zNodePath):
        for zookeeperChild in zk.get_children(zNodePath):
            zookeeperChildPath = zNodePath + "/" + zookeeperChild
            if zk.exists(zookeeperChildPath):
                Data, Stat = zk.get(zookeeperChildPath)
                result[zookeeperChild] = json.loads(Data)
            else:
                print("Reported ZK path no longer exists: " + Zookeeper_Child_Path)

    zk.stop();
    return result;


#!/usr/bin/env python3
import random
import sys

STRENGTH = {
    "All Star": 2,
    "Angry Bird": 1,
    "GOAT": 1,
    "MKatz": 3,
    "Pegasus": 1,
    "Piston": 4,
}

CARDIO = {
    22: 2,
    "Abacus": 3,
    "Empire": 2,
    "Mont Blanc": 1,
    "MVP": 4,
    "Quarterbacks": 1,
}

def pick_random_vid(videos: dict) -> tuple[str, int]:
    key = random.choice(list(videos.keys()))
    idx = random.randint(0, videos[key] - 1)
    return key, idx

args = sys.argv

if len(args) == 2:
    arg = args[1]

    if arg == "cardio":
        video, idx = pick_random_vid(CARDIO)
        print(f"{video} video {idx + 1}")

    if arg == "strength":
        video, idx = pick_random_vid(STRENGTH)
        print(f"{video} video {idx + 1}")

else:
    video, idx = pick_random_vid({**STRENGTH, **CARDIO})
    print(f"{video} video {idx + 1}")


import turtle as t
import random

t.colormode(255)

def random_color_finder():
    r = random.randint(0, 255)
    g = random.randint(0, 255)
    b = random.randint(0, 255)
    random_color = (r, g, b)
    return random_color

directions = [0, 90, 180, 270]
t.pensize(15)
t.speed('fastest')

for _ in range(200):
    t.color(random_color_finder())
    t.forward(30)
    t.setheading(random.choice(directions))

my_screen = t.Screen()
my_screen.bgcolor('black')
my_screen.exitonclick()


from NETWORKS import *

numAgentes = 100
interaction = 'Ising'
topologia = 'Circular'
iteracoes = numAgentes*10
gamma = np.linspace(0.0001,10,100)
valoresFinais = np.zeros((np.size(gamma),1))
for i in range(0,np.size(gamma)):
    current = np.zeros((iteracoes, 1))
    Grid = Network(numAgentes, interaction, topologia, iteracoes)
    for j in range (0,iteracoes):
        Grid.IsingInteraction(gamma[i],0,j)
        current[j] = Grid.Expectative
    valoresFinais[i] = np.mean(current[int(np.size(current)/2-1) :])
fig1 = plt.figure(1)
plt.plot(gamma, valoresFinais)
plt.ylabel('Magnetization')
plt.xlabel('Gamma')
plt.title('')
plt.show()


import random
import numpy as np
from scipy.stats import entropy

class CNF:
    def __init__(self, path=None, clauses=None):
        if path:
            with open(path, 'r') as cnf:
                formula = cnf.read()
                formula = formula.split('\n')
                start_index = 0
                while formula[start_index][0] != 'p':
                    start_index += 1
                self.n = int(formula[start_index].split()[2])
                self.variables = [i+1 for i in range(self.n)]
                self.m = int(formula[start_index].split()[3])
                self.clauses = [list(map(int, formula[start_index + 1 + i].split()[:-1])) for i in range(self.m)]
        else:
            variables = set()
            for clause in clauses:
                for var in clause:
                    variables.add(abs(var))
            self.n = len(variables)
            self.variables = list(variables)
            self.m = len(clauses)
            self.clauses = clauses

    def get_size(self):
        return [self.m, self.n, float(self.m) / self.n]

    def get_vc(self):
        nodes = {i: set() for i in self.variables}
        for j in range(self.m):
            for var in self.clauses[j]:
                nodes[abs(var)].add(j)
        nodes = [len(nodes.get(i)) for i in nodes]
        nodes_np = np.array(nodes)
        nodes_proba = np.unique(nodes_np, return_counts=True)[1]/float(len(nodes_np))
        nodes_entropy = entropy(list(nodes_proba))
        clause = []
        for j in range(self.m):
            cl = set()
            for i in range(len(self.clauses[j])):
                cl.add(abs(self.clauses[j][i]))
            clause.append(len(cl))
        clause_np = np.array(clause)
        clause_proba = np.unique(clause_np, return_counts=True)[1]/float(len(clause_np))
        clause_entropy = entropy(list(clause_proba))
        return [nodes_np.mean(), nodes_np.std()/nodes_np.mean(), nodes_np.min(), nodes_np.max(), nodes_entropy,
                clause_np.mean(), clause_np.std()/clause_np.mean(), clause_np.min(), clause_np.max(), clause_entropy]

    def get_v(self):
        variables = {i: set() for i in self.variables}
        for j in range(self.m):
            for var in self.clauses[j]:
                for var_o in self.clauses[j]:
                    if abs(var_o) != abs(var):
                        variables[abs(var)].add(abs(var_o))
        var_deg = [len(variables.get(i)) for i in variables]
        var_deg_np = np.array(var_deg)
        return [var_deg_np.mean(), var_deg_np.std()/var_deg_np.mean(), var_deg_np.min(), var_deg_np.max()]

    def get_balance(self):
        ratio_clause = []
        for clause in self.clauses:
            pos, neg = 0, 0
            for var in clause:
                if var > 0:
                    pos += 1
                else:
                    neg += 1
            ratio_clause.append(float(pos) / (pos + neg))
        ratio_clause_np = np.array(ratio_clause)
        ratio_clause_proba = np.unique(ratio_clause_np, return_counts=True)[1] / float(len(ratio_clause_np))
        ratio_clause_entropy = entropy(list(ratio_clause_proba))
        ration_var = {i: [0, 0] for i in self.variables}
        for j in range(self.m):
            for var in self.clauses[j]:
                if var > 0:
                    ration_var.get(abs(var))[0] += 1
                else:
                    ration_var.get(abs(var))[1] += 1
        ration_var = [float(ration_var.get(i)[0]) / (ration_var.get(i)[0] + ration_var.get(i)[1]) for i in ration_var]
        ration_var_np = np.array(ration_var)
        ration_var_proba = np.unique(ration_var_np, return_counts=True)[1] / float(len(ration_var_np))
        ration_var_entropy = entropy(list(ration_var_proba))
        binary, ternary = 0, 0
        for clause in self.clauses:
            if len(clause) == 2:
                binary += 1
            elif len(clause) == 3:
                ternary += 1
        return [ratio_clause_np.mean(), ratio_clause_np.std()/ratio_clause_np.mean(), ratio_clause_entropy,
                ration_var_np.mean(), ration_var_np.std()/ration_var_np.mean(), ration_var_np.min(),
                ration_var_np.max(), ration_var_entropy, float(binary)/self.m, float(ternary)/self.m]

    def get_horn(self):
        num_of_horns = 0
        horn_var = {i: 0 for i in self.variables}
        for clause in self.clauses:
            horn = True
            cnt = 0
            for var in clause:
                if var > 0:
                    cnt += 1
                if cnt > 1:
                    horn = False
                    break
            if horn:
                num_of_horns += 1
                for vr in clause:
                    horn_var[abs(vr)] += 1
        horn_var = [horn_var.get(i) for i in horn_var]
        horn_var_np = np.array(horn_var)
        horn_var_proba = np.unique(horn_var_np, return_counts=True)[1] / float(len(horn_var_np))
        horn_var_entropy = entropy(list(horn_var_proba))
        return [float(num_of_horns) / self.m, horn_var_np.mean(), horn_var_np.std()/horn_var_np.mean(),
                horn_var_np.min(), horn_var_np.max(), horn_var_entropy]

    def get_features(self):
        size = self.get_size()
        vc = self.get_vc()
        v = self.get_v()
        balance = self.get_balance()
        horn = self.get_horn()
        return size + vc + v + balance + horn

    def set_var(self, var=None):
        if not var:
            var = random.choice(self.variables + [-i for i in self.variables])
        new_clauses = [[i for i in clause if i != -var] for clause in self.clauses if var not in clause]
        return CNF(clauses=new_clauses)


from typing import List

class Solution:

    def factorial(self, n: int) -> int:
        fact = 1
        for i in range(1, n+1):
            fact *= i
        return fact

    def combination(self, n: int, r: int) -> int:
        return self.factorial(n) // (self.factorial(n-r) * self.factorial(r))

    def generate(self, numRows: int) -> List[List[int]]:
        pascal_triangles = list()
        for i in range(numRows):
            inner_list = []
            for j in range(i+1):
                inner_list.append(self.combination(i,j))
            pascal_triangles.append(inner_list)
        return pascal_triangles

print(Solution().generate(5))


#! /usr/bin/env python
"""
A very simple program to print the triplet primes less than n.

Leon Hostetler, Jan. 26, 2017

USAGE: python primes_triplets.py
"""
from __future__ import division, print_function

n = 1000

# Checks to see if a number is prime
def is_prime(n):
    for i in range(2, n):
        if n % i == 0:
            return False    # Return false if divisible by any smaller number
            break
    return True             # Return true if not divisible by any smaller number

# Print all the triplet primes less than a million
counter = 0
for i in range(2, n):
    if is_prime(i) and is_prime(i+2) and is_prime(i+6):
        print(i, ", ", i+2, ", ", i+6, sep="")
        counter += 1
    if is_prime(i) and is_prime(i+4) and is_prime(i+6):
        print(i, ", ", i+4, ", ", i+6, sep="")
        counter += 1

print()
print("There are ", counter, " prime triplets less than ", n, ".")


# _*_ coding: utf-8 _*_
# @author: anniequ
# @file: datapre.py
# @time: 2020/11/12 11:07
# @Software: PyCharm

import os
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np
import torch
import torchvision.transforms as tfs
from torch.utils.data import DataLoader

from torch import nn
import torch.nn.functional as f
import torchvision
from torch.autograd import Variable
import torchvision.models as models

from datetime import datetime

voc_root = os.path.join("data", "VOC2012")
np.seterr(divide='ignore',invalid='ignore')
# 读取图片
def read_img(root=voc_root, train=True):
    txt_frame = root + '/ImageSets/Segmentation/' + ('train.txt' if train else 'val.txt')

    with open(txt_frame, 'r') as f:
        images = f.read().split()
    data = [os.path.join(root, 'JPEGImages', i + '.jpg') for i in images]
    label = [os.path.join(root, 'SegmentationClass', i + '.png') for i in images]
    return data, label

# 图片大小不同，同时裁剪data and label
def crop(data, label, height, width):
    'data and label both are Image object'
    box = (0, 0, width, height)
    data = data.crop(box)
    label = label.crop(box)
    return data, label

# VOC数据集中对应的标签
classes = ['background', 'aeroplane', 'bicycle', 'bird', 'boat',
           'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable',
           'dog', 'horse', 'motorbike', 'person', 'potted plant',
           'sheep', 'sofa', 'train', 'tv/monitor']

# 各种标签所对应的颜色
colormap = [[0, 0, 0], [128, 0, 0], [0, 128, 0], [128, 128, 0], [0, 0, 128],
            [128, 0, 128], [0, 128, 128], [128, 128, 128], [64, 0, 0], [192, 0, 0],
            [64, 128, 0], [192, 128, 0], [64, 0, 128], [192, 0, 128],
            [64, 128, 128], [192, 128, 128], [0, 64, 0], [128, 64, 0],
            [0, 192, 0], [128, 192, 0], [0, 64, 128]]
cm2lbl = np.zeros(256 ** 3)

# 枚举的时候i是下标，cm是一个三元组，分别标记了RGB值
for i, cm in enumerate(colormap):
    cm2lbl[(cm[0] * 256 + cm[1]) * 256 + cm[2]] = i

# 将标签按照RGB值填入对应类别的下标信息
def image2label(im):
    data = np.array(im, dtype="int32")
    idx = (data[:, :, 0] * 256 + data[:, :, 1]) * 256 + data[:, :, 2]
    return np.array(cm2lbl[idx], dtype="int64")

def image_transforms(data, label, height, width):
    data, label = crop(data, label, height, width)
    # 将数据转换成tensor，并且做标准化处理
    im_tfs = tfs.Compose([
        tfs.ToTensor(),
        tfs.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

    data = im_tfs(data)
    label = image2label(label)
    label = torch.from_numpy(label)
    return data, label

class VOCSegDataset(torch.utils.data.Dataset):

    # 构造函数
    def __init__(self, train, height, width, transforms=image_transforms):
        self.height = height
        self.width = width
        self.fnum = 0  # 用来记录被过滤的图片数
        self.transforms = transforms
        data_list, label_list = read_img(train=train)
        self.data_list = self._filter(data_list)
        self.label_list = self._filter(label_list)
        if train == True:
            print("训练集：加载了 " + str(len(self.data_list)) + " 张图片和标签" + ",过滤了" + str(self.fnum) + "张图片")
        else:
            print("测试集：加载了 " + str(len(self.data_list)) + " 张图片和标签" + ",过滤了" + str(self.fnum) + "张图片")

    # 过滤掉长小于height和宽小于width的图片
    def _filter(self, images):
        img = []
        for im in images:
            if (Image.open(im).size[1] >= self.height and
                    Image.open(im).size[0] >= self.width):
                img.append(im)
            else:
                self.fnum = self.fnum + 1
        return img

    # 重载getitem函数，使类可以迭代
    def __getitem__(self, idx):
        img = self.data_list[idx]
        label = self.label_list[idx]
        img = Image.open(img)
        label = Image.open(label).convert('RGB')
        img, label = self.transforms(img, label, self.height, self.width)
        return img, label

    def __len__(self):
        return len(self.data_list)

height = 224
width = 224
voc_train = VOCSegDataset(True, height, width)
voc_test = VOCSegDataset(False, height, width)

train_data = DataLoader(voc_train, batch_size=8, shuffle=True)
valid_data = DataLoader(voc_test, batch_size=8)

# 下面就构建一个基于resnet34的fcn网络

# 初始化转置卷积卷积核的函数
def bilinear_kernel(in_channels, out_channels, kernel_size):
    factor = (kernel_size + 1) // 2
    if kernel_size % 2 == 1:
        center = factor - 1
    else:
        center = factor - 0.5
    og = np.ogrid[:kernel_size, :kernel_size]
    filt = (1 - abs(og[0] - center) / factor) * \
           (1 - abs(og[1] - center) / factor)
    weight = np.zeros((in_channels, out_channels, kernel_size, kernel_size),
                      dtype='float32')
    weight[range(in_channels), range(out_channels), :, :] = filt
    return torch.from_numpy(np.array(weight))

# 加载预训练的resnet34网络
model_root = "./model/resnet34-333f7ec4.pth"
pretrained_net = models.resnet34(pretrained=False)
pre = torch.load(model_root)
pretrained_net.load_state_dict(pre)
# 分类的总数
num_classes = len(classes)
print(num_classes)

class fcn(nn.Module):
    def __init__(self, num_classes):
        super(fcn, self).__init__()

        # 第一段，通道数为128，输出特征图尺寸为28*28
        self.stage1 = nn.Sequential(*list(pretrained_net.children())[:-4])
        # 第二段，通道数为256，输出特征图尺寸为14*14
        self.stage2 = list(pretrained_net.children())[-4]
        # 第三段，通道数为512，输出特征图尺寸为7*7
        self.stage3 = list(pretrained_net.children())[-3]

        # 三个1*1的卷积操作，各个通道信息融合
        self.scores1 = nn.Conv2d(512, num_classes, 1)
        self.scores2 = nn.Conv2d(256, num_classes, 1)
        self.scores3 = nn.Conv2d(128, num_classes, 1)

        # 将特征图尺寸放大八倍
        self.upsample_8x = nn.ConvTranspose2d(num_classes, num_classes, kernel_size=16, stride=8, padding=4, bias=False)
        self.upsample_8x.weight.data = bilinear_kernel(num_classes, num_classes, 16)  # 使用双线性 kernel
        # 这是放大了四倍，下同
        self.upsample_4x = nn.ConvTranspose2d(num_classes, num_classes, kernel_size=4, stride=2, padding=1, bias=False)
        self.upsample_4x.weight.data = bilinear_kernel(num_classes, num_classes, 4)  # 使用双线性 kernel

        self.upsample_2x = nn.ConvTranspose2d(num_classes, num_classes, kernel_size=4, stride=2, padding=1, bias=False)
        self.upsample_2x.weight.data = bilinear_kernel(num_classes, num_classes, 4)  # 使用双线性 kernel

    def forward(self, x):
        x = self.stage1(x)
        s1 = x  # 224/8 = 28

        x = self.stage2(x)
        s2 = x  # 224/16 = 14

        x = self.stage3(x)
        s3 = x  # 224/32 = 7

        s3 = self.scores1(s3)  # 将各通道信息融合
        s3 = self.upsample_2x(s3)  # 上采样
        s2 = self.scores2(s2)
        s2 = s2 + s3  # 14*14

        s1 = self.scores3(s1)
        s2 = self.upsample_4x(s2)  # 上采样，变成28*28
        s = s1 + s2  # 28*28

        s = self.upsample_8x(s2)  # 放大八倍，变成224*224
        return s  # 返回特征图

# 计算混淆矩阵
def _fast_hist(label_true, label_pred, n_class):
    # mask在和label_true相对应的索引的位置上填入true或者false
    # label_true[mask]会把mask中索引为true的元素输出
    mask = (label_true >= 0) & (label_true < n_class)
    # np.bincount()会给出索引对应的元素个数
    """
    hist是一个混淆矩阵
    hist是一个二维数组，可以写成hist[label_true][label_pred]的形式
    最后得到的这个数组的意义就是行下标表示的类别预测成列下标类别的数量
    比如hist[0][1]就表示类别为1的像素点被预测成类别为0的数量
    对角线上就是预测正确的像素点个数
    n_class * label_true[mask].astype(int) + label_pred[mask]计算得到的是二维数组元素
    变成一位数组元素的时候的地址取值(每个元素大小为1)，返回的是一个numpy的list，然后
    np.bincount就可以计算各中取值的个数
    """
    hist = np.bincount(
        n_class * label_true[mask].astype(int) +
        label_pred[mask], minlength=n_class ** 2).reshape(n_class, n_class)
    return hist

"""
label_trues 正确的标签值
label_preds 模型输出的标签值
n_class 数据集中的分类数
"""

def label_accuracy_score(label_trues, label_preds, n_class):
    """Returns accuracy score evaluation result.
      - overall accuracy
      - mean accuracy
      - mean IU
      - fwavacc
    """
    hist = np.zeros((n_class, n_class))
    # 一个batch里面可能有多个数据
    # 通过迭代器将一个个数据进行计算
    for lt, lp in zip(label_trues, label_preds):
        # numpy.ndarray.flatten将numpy对象拉成1维
        hist += _fast_hist(lt.flatten(), lp.flatten(), n_class)

    # np.diag(a)假如a是一个二维矩阵，那么会输出矩阵的对角线元素
    # np.sum()可以计算出所有元素的和。如果axis=1，则表示按行相加
    """
    acc是准确率 = 预测正确的像素点个数/总的像素点个数
    acc_cls是预测的每一类别的准确率(比如第0行是预测的类别为0的准确率)，然后求平均
    iu是召回率Recall，公式上面给出了
    mean_iu就是对iu求了一个平均
    freq是每一类被预测到的频率
    fwavacc是频率乘以召回率，我也不知道这个指标代表什么
    """
    acc = np.diag(hist).sum() / hist.sum()
    acc_cls = np.diag(hist) / hist.sum(axis=1)
    # nanmean会自动忽略nan的元素求平均
    acc_cls = np.nanmean(acc_cls)
    iu = np.diag(hist) / (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist))
    mean_iu = np.nanmean(iu)
    freq = hist.sum(axis=1) / hist.sum()
    fwavacc = (freq[freq > 0] * iu[freq > 0]).sum()
    return acc, acc_cls, mean_iu, fwavacc

net = fcn(num_classes)
if torch.cuda.is_available():
    net = net.cuda()
criterion = nn.NLLLoss()
basic_optim = torch.optim.SGD(net.parameters(), lr=1e-2, weight_decay=1e-4)
optimizer = basic_optim

# 训练网络
EPOCHES = 20

# 训练时的数据
train_loss = []
train_acc = []
train_acc_cls = []
train_mean_iu = []
train_fwavacc = []

# 验证时的数据
eval_loss = []
eval_acc = []
eval_acc_cls = []
eval_mean_iu = []
eval_fwavacc = []

for e in range(EPOCHES):

    _train_loss = 0
    _train_acc = 0
    _train_acc_cls = 0
    _train_mean_iu = 0
    _train_fwavacc = 0

    prev_time = datetime.now()
    net = net.train()
    for img_data, img_label in train_data:

        if torch.cuda.is_available:
            im = Variable(img_data).cuda()
            label = Variable(img_label).cuda()
        else:
            im = Variable(img_data)
            label = Variable(img_label)
        # 前向传播
        out = net(im)
        out = f.log_softmax(out, dim=1)
        loss = criterion(out, label)

        # 反向传播
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        _train_loss += loss.item()

        # label_pred输出的是21*224*224的向量，对于每一个点都有21个分类的概率
        # 我们取概率值最大的那个下标作为模型预测的标签，然后计算各种评价指标
        label_pred = out.max(dim=1)[1].data.cpu().numpy()
        label_true = label.data.cpu().numpy()

        for lbt, lbp in zip(label_true, label_pred):
            acc, acc_cls, mean_iu, fwavacc = label_accuracy_score(lbt, lbp, num_classes)
            _train_acc += acc
            _train_acc_cls += acc_cls
            _train_mean_iu += mean_iu
            _train_fwavacc += fwavacc

    # 记录当前轮的数据
    train_loss.append(_train_loss / len(train_data))
    train_acc.append(_train_acc / len(voc_train))
    train_acc_cls.append(_train_acc_cls)
    train_mean_iu.append(_train_mean_iu / len(voc_train))
    train_fwavacc.append(_train_fwavacc)

    net = net.eval()

    _eval_loss = 0
    _eval_acc = 0
    _eval_acc_cls = 0
    _eval_mean_iu = 0
    _eval_fwavacc = 0

    for img_data, img_label in valid_data:
        if torch.cuda.is_available():
            im = Variable(img_data).cuda()
            label = Variable(img_label).cuda()
        else:
            im = Variable(img_data)
            label = Variable(img_label)

        # forward
        out = net(im)
        out = f.log_softmax(out, dim=1)
        loss = criterion(out, label)
        _eval_loss += loss.item()

        label_pred = out.max(dim=1)[1].data.cpu().numpy()
        label_true = label.data.cpu().numpy()
        for lbt, lbp in zip(label_true, label_pred):
            acc, acc_cls, mean_iu, fwavacc = label_accuracy_score(lbt, lbp, num_classes)
            _eval_acc += acc
            _eval_acc_cls += acc_cls
            _eval_mean_iu += mean_iu
            _eval_fwavacc += fwavacc

    # 记录当前轮的数据
    eval_loss.append(_eval_loss / len(valid_data))
    eval_acc.append(_eval_acc / len(voc_test))
    eval_acc_cls.append(_eval_acc_cls)
    eval_mean_iu.append(_eval_mean_iu / len(voc_test))
    eval_fwavacc.append(_eval_fwavacc)

    # 打印当前轮训练的结果
    cur_time = datetime.now()
    h, remainder = divmod((cur_time - prev_time).seconds, 3600)
    m, s = divmod(remainder, 60)
    epoch_str = ('Epoch: {}, Train Loss: {:.5f}, Train Acc: {:.5f}, Train Mean IU: {:.5f}, \
Valid Loss: {:.5f}, Valid Acc: {:.5f}, Valid Mean IU: {:.5f} '.format(
        e, _train_loss / len(train_data), _train_acc / len(voc_train), _train_mean_iu / len(voc_train),
           _eval_loss / len(valid_data), _eval_acc / len(voc_test), _eval_mean_iu / len(voc_test)))
    time_str = 'Time: {:.0f}:{:.0f}:{:.0f}'.format(h, m, s)
    print(epoch_str + time_str)

# 绘图
epoch = np.array(range(EPOCHES))
plt.plot(epoch, train_loss, label="train_loss")
plt.plot(epoch, train_loss, label="valid_loss")
plt.title("loss during training")
plt.legend()
plt.grid()
plt.show()

plt.plot(epoch, train_acc, label="train_acc")
plt.plot(epoch, eval_acc, label="valid_acc")
plt.title("accuracy during training")
plt.legend()
plt.grid()
plt.show()

plt.plot(epoch, train_mean_iu, label="train_mean_iu")
plt.plot(epoch, eval_mean_iu, label="valid_mean_iu")
plt.title("mean iu during training")
plt.legend()
plt.grid()
plt.show()

# 测试模型性能
# 保存模型
PATH = "./model/fcn-resnet34.pth"
torch.save(net.state_dict(), PATH)

# 加载模型
# model.load_state_dict(torch.load(PATH))

cm = np.array(colormap).astype('uint8')

def predict(img, label):  # 预测结果
    img = Variable(img.unsqueeze(0)).cuda()
    out = net(img)
    pred = out.max(1)[1].squeeze().cpu().data.numpy()
    # 将pred的分类值，转换成各个分类对应的RGB值
    pred = cm[pred]
    # 将numpy转换成PIL对象
    pred = Image.fromarray(pred)
    label = cm[label.numpy()]
    return pred, label

size = 224
num_image = 10
_, figs = plt.subplots(num_image, 3, figsize=(12, 22))
for i in range(num_image):
    img_data, img_label = voc_test[i]
    pred, label = predict(img_data, img_label)
    img_data = Image.open(voc_test.data_list[i])
    img_label = Image.open(voc_test.label_list[i]).convert("RGB")
    img_data, img_label = crop(img_data, img_label, size, size)
    figs[i, 0].imshow(img_data)  # 原始图片
    figs[i, 0].axes.get_xaxis().set_visible(False)  # 去掉x轴
    figs[i, 0].axes.get_yaxis().set_visible(False)  # 去掉y轴
    figs[i, 1].imshow(img_label)  # 标签
    figs[i, 1].axes.get_xaxis().set_visible(False)  # 去掉x轴
    figs[i, 1].axes.get_yaxis().set_visible(False)  # 去掉y轴
    figs[i, 2].imshow(pred)  # 模型输出结果
    figs[i, 2].axes.get_xaxis().set_visible(False)  # 去掉x轴
    figs[i, 2].axes.get_yaxis().set_visible(False)  # 去掉y轴

# 在最后一行图片下面添加标题
figs[num_image - 1, 0].set_title("Image", y=-0.2)
figs[num_image - 1, 1].set_title("Label", y=-0.2)
figs[num_image - 1, 2].set_title("fcns", y=-0.2)


# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):
    dependencies = [("orgs", "0014_auto_20150722_1419")]

    operations = [
        migrations.CreateModel(
            name="ContactField",
            fields=[
                ("id", models.AutoField(verbose_name="ID", serialize=False, auto_created=True, primary_key=True)),
                ("label", models.CharField(max_length=36, verbose_name="Label")),
                ("key", models.CharField(max_length=36, verbose_name="Key")),
                ("value_type", models.CharField(max_length=1, verbose_name="Field Type")),
                (
                    "org",
                    models.ForeignKey(
                        related_name="contactfields", on_delete=models.PROTECT, verbose_name="Org", to="orgs.Org"
                    ),
                ),
            ],
        )
    ]


#!/usr/bin/python3
"""
base module contains the Base class
"""
import json

class Base:
    """
    Base class: manage id attribute in all the subclass

    Attributes:
        __nb_objects - class attribute initialized with 0
        __init__ - class constructor
    """

    __nb_objects = 0

    def __init__(self, id=None):
        """assign the public instance attribute id"""

        if id:
            self.id = id
        else:
            type(self).__nb_objects += 1
            self.id = type(self).__nb_objects

    @staticmethod
    def to_json_string(list_dictionaries):
        """returns the JSON string repr"""
        if list_dictionaries:
            return json.dumps(list_dictionaries)
        return "[]"

    @classmethod
    def save_to_file(cls, list_objs):
        """writes the JSON string repr of list_objs to a file"""
        list_dict = []
        if list_objs:
            for i in list_objs:
                list_dict.append(i.to_dictionary())
        objs_json = cls.to_json_string(list_dict)
        filename = cls.__name__ + ".json"
        with open(filename, 'w', encoding="utf-8") as f:
            f.write(objs_json)

    @staticmethod
    def from_json_string(json_string):
        """deserialises the json obj and returns the python object"""
        import json

        if not json_string:
            return []
        return json.loads(json_string)

    @classmethod
    def create(cls, **dictionary):
        """returns an instance with all attr already set"""
        if not dictionary:
            return cls
        if cls.__name__ == "Rectangle":
            rectangle = cls(1, 1)
            rectangle.update(**dictionary)
            return rectangle
        square = cls(1)
        square.update(**dictionary)
        return square

    @classmethod
    def load_from_file(cls):
        """returns a list of instances"""
        import os

        filename = cls.__name__ + ".json"
        if not os.path.exists(filename):
            return []
        with open(filename, "r", encoding='utf-8') as f:
            json_str = f.read()
            list_dict = cls.from_json_string(json_str)
        list_obj = []
        for item in list_dict:
            instance = cls.create(**item)
            list_obj.append(instance)
        return list_obj

    @classmethod
    def save_to_file_csv(cls, list_objs):
        """parse list_objs to csv"""
        import csv

        """if not list_objs:
            return None"""
        if cls.__name__ == "Rectangle":
            fields = ["id", "width", "height", "x", "y"]
        elif cls.__name__ == "Square":
            fields = ["id", "size", "x", "y"]
        filename = cls.__name__ + ".csv"
        with open(filename, 'w', newline='', encoding='utf-8') as f:
            csvwriter = csv.writer(f)
            csvwriter.writerow(fields)
            list_dict = []
            for item in list_objs:
                cls_dict = item.to_dictionary()
                instance_value = []
                for key in fields:
                    instance_value.append(cls_dict[key])
                list_dict.append(instance_value)
            csvwriter.writerows(list_dict)

    @classmethod
    def load_from_file_csv(cls):
        """load a csv to list_obj"""
        import csv

        filename = cls.__name__ + ".csv"
        list_objs = []
        with open(filename, 'r', newline='', encoding='utf-8') as f:
            csvreader = csv.reader(f)
            fields = next(csvreader)
            key_value = {}
            for row in csvreader:
                i = 0
                for attr in fields:
                    key_value[attr] = int(row[i])
                    i += 1
                python_obj = cls.create(**key_value)
                list_objs.append(python_obj)
        return list_objs


import logging
import json
import gzip
import ipaddress
import datetime

from c99api import EndpointClient
from typing import List, Dict, Optional
from os.path import exists

from pydantic import BaseModel

logger = logging.getLogger()

def enrich_object_c99(object, c99_key:str=""):
    c99 = EndpointClient
    c99.key = c99_key
    ip = object["IPAddress"]
    resp = c99.gethostname(ip)
    if resp["success"] and ip != resp["hostname"] and resp["hostname"] not in object["hostname_list"]:
        logging.info(f"gethostname: {resp['hostname']}")
        object["hostname_list"].append(resp["hostname"])
    resp = c99.ip2domains(ip)
    if resp["success"] and resp["count"] >= 1:
        logging.info(f"ip2domains: {resp['data']}")
        object["domain_list"].extend([hname for hname in resp["data"] if hname not in object["domain_list"]])

def merge_config(current_config: Dict[int, str] = {}, custom_config: Dict[int, str] = {}):
    for key, value in custom_config.items():
        if key in current_config.keys():
            if isinstance(value, (list,)):
                current_config[key] = list(set(current_config[key].extend(current_config[key])))
            elif isinstance(value, (dict,)):
                current_config[key] = merge_config(current_config[key], custom_config[key])
            else:
                current_config[key] = value
        else:
            current_config.update({key: value})
    return current_config

def load_config(default_config: str = "config.default.json", override_config: str = ""):
    config_builder = {}
    if exists(default_config):
        with open(default_config, "r", encoding="utf-8") as config_file:
            config_builder = json.load(config_file)
    else:
        raise ValueError("config file not found")
    if exists(override_config):
        with open(override_config, "r", encoding="utf-8") as config_file:
            try:
                configData = json.load(config_file)
                config_builder = merge_config(current_config=config_builder, custom_config=configData)
            except Exception as e:
                logger.error(f"Error adding override config\n{e}")
    return config_builder

def decode_shodan(obj:dict={}):
    try:
        parsed_object = {
            "domain_list": obj["domains"] if "domains" in obj else [],
            "hostname_list": [obj["_shodan"]["options"]["hostname"]] if "hostname" in obj["_shodan"]["options"] else [],
            "cloud_provider": None,
            "operating_system": obj["os"],
            "product": obj["product"] if "product" in obj else "",
            "IPAddress": ipaddress.ip_address(obj["ip_str"]),
            "timestamp": datetime.datetime.fromisoformat(obj["timestamp"]),
            "protocol": obj["transport"] if "transport" in obj else "",
            "internet_service_provider": obj["isp"],
            "version": obj["version"] if "version" in obj else "",
            "organisation": obj["org"],
            "country": obj["location"]["country_name"] if "country_name" in obj["location"] else "",
            "city": obj["location"]["city"] if "city" in obj["location"] else "",
            "port": obj["port"]
        }
        parsed_object["hostname_list"].extend([hname.strip() for hname in obj["hostnames"]])
    except Exception as e:
        logging.error(e)
        return {}
    try:
        if "ssl" in obj and "cert" in obj["ssl"]:
            cert = obj["ssl"]
            #parsed_object["ssl_fingerprint"] = cert["cert"]["fingerprint"]["sha256"]
            #parsed_object["ssl_serial"] = cert["cert"]["serial"]
            parsed_object["ssl_SAN"] = [cert["cert"]["subject"]["CN"]] if "CN" in cert["cert"]["subject"]["CN"] else []
            for alt in cert["cert"]["extensions"]:
                if alt["name"]=="subjectAltName" and alt["data"]:
                    i = 0
                    while i < len(alt["data"]):
                        if alt["data"][i] == "\\":
                            i += 4
                            continue
                        next_slash = alt["data"][i:].find("\\")
                        if next_slash >= 0:
                            parsed_object["ssl_SAN"].append(alt["data"][i:i+next_slash])
                            i += next_slash
                        else:
                            parsed_object["ssl_SAN"].append(alt["data"][i:])
                            i = len(alt["data"])
                        if parsed_object["ssl_SAN"][-1] == "0.":
                            parsed_object["ssl_SAN"].pop()
            parsed_object["ssl_SAN"] = list(set(parsed_object["ssl_SAN"]))
            parsed_object["ssl_issuer"] = cert["cert"]["issuer"]["O"] if "O" in cert["cert"]["issuer"] else cert["cert"]["issuer"]["CN"]
            #parsed_object["ssl_ja3"] = cert["ja3s"]
            #parsed_object["ssl_jarm"] = cert["jarm"]
            parsed_object["ssl_expiration"] = datetime.datetime.strptime(cert["cert"]["expires"], "%Y%m%d%H%M%SZ")
        else:
            #parsed_object["ssl_fingerprint"] = ""
            #parsed_object["ssl_serial"] = -1
            parsed_object["ssl_SAN"] = []
            parsed_object["ssl_issuer"] = ""
            #parsed_object["ssl_ja3"] = ""
            #parsed_object["ssl_jarm"] = ""
            parsed_object["ssl_expiration"] = datetime.datetime.fromordinal(1)
    except Exception as e:
        #parsed_object["ssl_fingerprint"] = ""
        #parsed_object["ssl_serial"] = -1
        parsed_object["ssl_SAN"] = []
        parsed_object["ssl_issuer"] = ""
        #parsed_object["ssl_ja3"] = ""
        #parsed_object["ssl_jarm"] = ""
        parsed_object["ssl_expiration"] = datetime.datetime.fromordinal(1)
        logging.error(e)
    return parsed_object

def load_shodan_files(filename:str="", config:Dict={}):
    if not exists(filename):
        logging.error(f"File not found: {filename}")
        raise FileNotFoundError
    logging.info(f"Loading file: {filename}")
    if filename.endswith(".json.gz"):
        with gzip.open(filename, "rb") as archive:
            lines = archive.readlines()
    else:
        with open(filename, "rb") as raw_file:
            lines = raw_file.readlines()
    data = []
    error_count = 0
    for line in lines:
        try:
            json_obj = json.loads(line)
            try:
                obj = decode_shodan(obj=json_obj)
                data.append(obj)
            except Exception as e:
                logger.warning(f"JSON data could not be parsed")
                logger.warning(e)
        except:
            error_count += 1
            continue
    if error_count > 0:
        logging.error(f"{filename} - Errors occurred during loading of data: {error_count}")
    return data

if __name__ == "__main__":
    configFileName = "../../config/config.default.json"
    loaded_config = load_config(default_config=configFileName)
    logConfig = loaded_config["logging"]
    logging.basicConfig(
        level=logConfig["log_level"],
        format=logConfig["log_fstr_std"],
        datefmt=logConfig["log_date_formt"]
    )
    pass


class Node:
    def __init__(self, value):
        self.value = value
        self.next = None

    def __str__(self):
        return str(self.value)

class LinkedList:
    def __init__(self):
        self.First = None
        self.Size = 0

    def append(self, value):
        myNode = Node(value)
        if self.Size == 0:
            self.First = myNode
            self.Last = myNode
        else:
            currentNode = self.First
            while currentNode.next is not None:
                currentNode = currentNode.next
            currentNode.next = myNode
            self.Last = myNode
        self.Size +=1

        return myNode

    def remove(self, value):
        if self.Size == 0:
            return False
        else:
            currentNode = self.First
            try:
                while currentNode.next.value != value:
                    currentNode = currentNode.next
                deleteNode = currentNode.next
                currentNode.next = deleteNode.next
                deleteNode.value = None

            except AttributeError:
                return False

        self.Size -= 1
        return deleteNode

    def pop(self):
        currentNode = self.First
        i=0
        while i < len(self):
            i+=1
            currentNode = currentNode.next
        if i==0:
            print("No existen Nodos")
        elif i == 1:
            del self.First
            print("Ya no hay Nodos")
            exit()
        else:
            cont = 1
            nodo = self.First
            while cont < len(self)-1:
                nodo = nodo.next
                cont += 1
            self.Last = nodo
            #print(f"{nodo} ____{nodo.next}")
            self.Last.next = None
            self.Size -= 1

    def prepend(self, value):
        myNode = Node(value)
        if self.Size == 0:
            self.First = myNode
            self.Last = myNode
            self.Size = 1
        else:
            myNode.next = self.First
            self.First = myNode
            self.Size +=1

    def popfisrt(self):
        if len(self)==0:
            print("No hay nodos")
            return False
        elif len(self)==1:
            emptyNode = Node(" ")
            self.First=emptyNode
            self.Last = emptyNode
            self.Size -= 1
            return 0
        else:
            aux=self.First.next
            self.First=aux
            self.Size -= 1

    def get(self, index):
        currentNode = self.First
        if index>self.Size:
            print("Valor no permitido")
        else:
            for i in range(index):
                currentNode = currentNode.next
            return currentNode

    def insert (self, index, value):
        myNode = Node(value)
        currentNode = self.First
        if index>self.Size:
            print("Valor no permitido")
        elif index == 0:
            self.First = myNode
            myNode.next=currentNode
            self.Size +=1
        else:
            for i in range(index-1):
                currentNode = currentNode.next
            aux=currentNode.next
            currentNode.next=myNode
            myNode.next=aux
            self.Size+=1

    def set (self, index, value):
        myNode = self.get(index)
        myNode.value=value

    def removeIndex(self, index):
        currentNode = self.First
        if index == len(self):
            self.pop
        else:
            for i in range(index-1):
                currentNode=currentNode.next

            deletedNode = currentNode.next
            currentNode.next = deletedNode.next
            deletedNode.value = None
            self.Size-=1

    def __len__(self):
        return self.Size

    def __str__(self):
        String = "["
        currentNode = self.First
        for i in range(len(self)):
            String += str(currentNode)
            if i is not len(self)-1:
                String += str(", ")
            currentNode = currentNode.next
        String += "]"
        return String

myList = LinkedList()


from django.urls import path
from . import views

app_name = 'orders'

urlpatterns = [
    path('checkout', views.checkout, name='checkout'),
    path('order_details', views.order_detail, name='order_details'),
    path('orders', views.orders, name='orders'),
    path('create_order/', views.create_order, name='create_order'),

]


# Obj: Data persistance

# Opt1: External files
# Opt2: DB

# Procedure:
# Create the external file.
# Open the file
# Manipulate the file
# Close the file

from io import open

# First parameter file name, second parameter mode to open (read, write)
textFile = open('file.txt', 'w')

line = 'Great day to code Python \nIsn\'t it?'
textFile.write(line)  # writing on the file
textFile.close()  # closing the file

textFile = open('file.txt', 'r')  # Opens the file on read mode

text = textFile.read()  # reads the file

textFile.close()  # closing the file

print(text)

textFile = open('file.txt', 'r')  # Opens the file on read mode

# reads the file line by line saving each one of themn on a list
textLines = textFile.readlines()

textFile.close()  # closes the file

print(textLines[0])

# a parameter allows to append lines to the text file
textFile = open('file.txt', 'a')

textFile.write('\nEveryday it\'s a god day to code')

textFile.close()

textFile = open('file.txt', 'r')

print(textFile.read())
print(textFile.read())  # After executing the first read command, the pointer stays at the end of the file, so the second time it's executed there are no more lines ahead and it won't print anything
# seek sets the pointer to the given position, in this case index = 0
textFile.seek(0)
print(textFile.read())
print(textFile.read(11))  # Starts reading on the given position (11)
textFile.close()

# Writing and reading mode, sets the pointer on the first postion
textFile = open('file.txt', 'r+')


import tensorflow as tf
import numpy as np
from typing import Union, Optional, Sequence
from pathlib import Path

from dui.datasets.hdf5datasetfactory import HDF5DatasetFactory
from dui.utils.signal import compress_db
from dui.layers.utils import get_channel_axis

def create_image_dataset(
        path: Union[str, Path],
        name: str,
        factor: Union[str, float] = '0db',
        # TODO: None as default or 1?
        signal_type: str = 'rf',
        # TODO: None or 'raw' as default?
        data_format: str = 'channels_last',
        # TODO: patch paddings typing elsewhere if validated
        # paddings: Optional[Union[Sequence[int], np.ndarray]] = None,
        paddings: Optional[Union[Sequence[Sequence[int]], np.ndarray]] = None,
        start: int = 0,
        stop: Optional[int] = None,
        step: int = 1,
        slicer: Optional[Sequence[slice]] = None,
        batch_size: int = 1,
        shuffle: bool = False,
        num_parallel_calls: Optional[int] = None,
        seed: Optional[int] = None,
) -> tf.data.Dataset:

    # Factory
    dataset_factory = HDF5DatasetFactory(
        path=path,
        name=name,
        start=start,
        stop=stop,
        step=step,
        slicer=slicer,
        shuffle=shuffle,
        seed=seed
    )

    # Check sample shape
    base_sample_shape = dataset_factory._output_sample_shape
    if len(base_sample_shape) != 2:
        raise ValueError(
            "Dataset sample must be a 2D array. Current shape: {}".format(
                base_sample_shape
            )
        )

    # Normalization factor
    if isinstance(factor, str):
        attr_key = factor
        factor = dataset_factory.attrs.get(attr_key)
        if factor is None:
            raise ValueError(
                "No attribute '{}' for dataset '{}' in '{}'".format(
                    attr_key, dataset_factory._dataset.name,
                    dataset_factory._dataset.file.filename
                )
            )
    elif type(factor) in (int, float):
        pass
    else:
        raise TypeError("Unsupported type for 'factor'")

    # Create dataset
    dataset = dataset_factory.create_dataset()
    # TODO: include factor directly and specialize the pre-processing
    #  for US-specific only?

    # Hack to avoid having an <unknown> shape (probably unsafe)
    # TODO: handle this in factory or by sub-classing tf.data.Dataset
    #  Note: Probably below some Dataset._element_structure properties
    #  Note: most probably not compatible with 1.15
    dataset._element_structure._shape = tf.TensorShape(base_sample_shape)

    # Pre-processing
    dataset = dataset.batch(batch_size=batch_size)
    # TODO: use `dataset.padded_batch` instead and remove following
    #  `paddings` option from following pre-processing
    # TODO: apply normalization factor before
    dataset = _preprocess_image_dataset(
        dataset=dataset,
        factor=factor,
        data_format=data_format,
        signal_type=signal_type,
        paddings=paddings,
        num_parallel_calls=num_parallel_calls
    )

    return dataset

def _preprocess_image_dataset(
        dataset: tf.data.Dataset,
        factor: Optional[float] = None,
        data_format: str = 'channels_last',
        signal_type: Optional[str] = None,
        paddings: Optional[Union[Sequence[int], np.ndarray]] = None,
        num_parallel_calls: Optional[int] = None
) -> tf.data.Dataset:

    # Specify pre-processing function as a mapping function
    def map_func(x: tf.Tensor) -> tf.Tensor:
        return _image_preproc_fun(
            x,
            factor=factor,
            data_format=data_format,
            signal_type=signal_type,
            paddings=paddings
        )

    return dataset.map(
        map_func=map_func,
        num_parallel_calls=num_parallel_calls
    )

def _image_preproc_fun(
        x: tf.Tensor,
        factor: Optional[float] = None,
        data_format: str = 'channels_last',
        signal_type: Optional[str] = None,
        paddings: Optional[Union[Sequence[int], np.ndarray]] = None,
) -> tf.Tensor:

    # TODO: check inputs
    x = tf.convert_to_tensor(x)

    # Normalization factor
    if factor:
        # TODO: apply factor before and keep this pre-proc
        #  function only for US-specific transformations?
        x /= factor

    # Paddings
    if paddings is not None:
        # TODO: would probably make more sense to remove paddings
        #  from this US-specific pre-processing function
        # x = _batched_pad(x, paddings=paddings)
        paddings = np.array(paddings)
        valid_pad_shape = 2, 2
        pad_shape = paddings.shape
        # TODO: this test is too restrictive in general (e.g. 3D)
        #  but ok for now as we only work on 2D images
        if pad_shape != valid_pad_shape:
            raise ValueError(
                "Incompatible 'paddings' shape. Current: {}. "
                "Expected {}".format(pad_shape, valid_pad_shape)
            )
        paddings = [[0, 0], *paddings.tolist()]
        pad_kwargs = {
            'paddings': tf.constant(paddings, dtype='int32'),
            'mode': 'CONSTANT',
            'constant_values': 0
        }
        x = tf.pad(x, **pad_kwargs)

    # Channel axis
    channel_axis = get_channel_axis(data_format=data_format)

    # Signal type
    if signal_type is not None:
        if signal_type == 'rf':
            x = tf.math.real(x)
        elif signal_type == 'iq':
            # Stack complex components in channels
            x = tf.stack((tf.math.real(x), tf.math.imag(x)), axis=channel_axis)
        elif signal_type == 'env':
            # Takes modulus of complex IQ signal
            x = tf.math.abs(x)
        elif signal_type == 'bm':
            # Takes modulus of complex IQ signal
            x = tf.math.abs(x)
            # Compress to dB
            x = compress_db(tensor=x)
        elif signal_type == 'raw':
            pass
        else:
            raise ValueError("Invalid signal type")

    # Expand dimension
    if signal_type != 'iq':
        x = tf.expand_dims(x, axis=channel_axis)

    return x


class Solution:
    def findLongestChain(self, pairs: List[List[int]]) -> int:
        pairs.sort(key = lambda a:a[0])
        dp = [1] * len(pairs)
        ans = 1
        for i in range(len(pairs)):
            for j in range(i):
                if pairs[i][0] > pairs[j][1]:
                    dp[i] = max(dp[j]+1, dp[i])
                    ans = max(dp[i], ans)
        return ans

temp/toy/python/646. Maximum Length of Pair Chain.py


#!/usr/bin/env python3

""" Using p022_names.txt, a 46K text file containing over five-thousand first names,
    begin by sorting it into alphabetical order. Then working out the
    alphabetical value for each name, multiply this value by its alphabetical position in the list
    to obtain a name score.

    For example, when the list is sorted into alphabetical order, COLIN, which is worth
    3 + 15 + 12 + 9 + 14 = 53, is the 938th name in the list. So, COLIN would obtain a score of
    938 × 53 = 49714.

    What is the total of all the name scores in the file?
"""

import csv

alphabet = "_ABCDEFGHIJKLMNOPQRSTUVWXYZ"

with open('p022_names.txt', newline='') as f:
    reader = csv.reader(f)
    name_list = next(reader)
    name_list.sort()

    total = 0
    for name in name_list:
        name_score = 0
        for letter in name:
            name_score += alphabet.index(letter)
        name_score *= (name_list.index(name) + 1)
        total += name_score

    print(f'Total score {total}')