import os
import numpy as np
import librosa
import librosa.display
import matplotlib.pyplot as plt
from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.operators.python import PythonOperator
from datetime import datetime, timedelta
from skimage.transform import resize

## 로컬에 저장한 후
## 해당 내용을 s3에 업로드하게 하기

# train data가 저장되어 있는 장소
source_path = '/Users/yoohajun/Desktop/grad_audio/source'
# output mfcc,spectrogram이 저장될 장소
output_path = '/Users/yoohajun/Desktop/grad_audio/output'

# Define the output directory for the spectrogram, mfcc images
mfcc_dir = os.path.join(output_path, 'mfcc_fixed')

default_args = {
    'owner': 'airflow',
    'depends_on_past': False,
    'catchup': False,
    'schedule_interval': '* */2 * * *',
    # 'schedule_interval': '@daily',
    'start_date': datetime(2023, 3, 8),
    'retries': 1,
    'retry_delay': timedelta(minutes=3)
}

dag = DAG('audio_to_mfcc_v2', default_args=default_args)

def random_pad(mels, pad_size, mfcc=True):
    pad_width = pad_size - mels.shape[1]
    rand = np.random.rand()
    left = int(pad_width * rand)
    right = pad_width - left

    if left < 0:
        right += abs(left)
        left = 0

    if right < 0:
        left += abs(right)
        right = 0

    if mfcc:
        mels = np.pad(mels, pad_width=((0, 0), (left, right)), mode='constant')
        local_max, local_min = mels.max(), mels.min()
        mels = (mels - local_min) / (local_max - local_min)

    else:
        local_max, local_min = mels.max(), mels.min()
        mels = (mels - local_min) / (local_max - local_min)
        mels = np.pad(mels, pad_width=((0, 0), (left, right)), mode='constant')

    return mels

# Load the audio files
def generate_mfcc(subdir, audio_dir, img_height, img_width):
    # Create the output directory for the spectrogram images under the subdir/data directory

    mfcc_dir_2 = os.path.join(mfcc_dir, subdir)
    if not os.path.exists(mfcc_dir_2):
        os.makedirs(mfcc_dir_2)
        print(f"{mfcc_dir_2} : vacant created")

    target_shape = (img_height, img_width)
    size = 128
    pad_size = 128
    repeat_size = 2

    # Loop over all the audio files in the input directory
    for idx, filename in enumerate(os.listdir(audio_dir)):
        if filename.endswith('.wav'):
            # Load the audio file
            filepath = os.path.join(audio_dir, filename)
            y, sr = librosa.load(filepath, sr=44100)

            mfcc = librosa.feature.mfcc(y=y, sr=sr, n_mfcc=size)

            for i in range(repeat_size):
                mfcc_filename = f'{filename[:-4]}_{i + 1}.png'

                if mfcc_filename in os.listdir(mfcc_dir_2):
                    print(f'{subdir} : {idx + 1}, {mfcc_filename} -> exists')
                    continue
                else:
                    ## padding
                    mfcc = random_pad(mfcc, pad_size=pad_size, mfcc=True)

                    # Resize the mfcc to the fixed shape
                    mfcc_resized = resize(mfcc, target_shape)

                    # Save the MFCC as an image file
                    mfcc_filepath = os.path.join(mfcc_dir_2, mfcc_filename)
                    plt.imsave(mfcc_filepath, mfcc_resized)

                    print(f'{subdir} : {idx + 1}, {mfcc_filename} -> saved')

    print(f'Finished processing audio files in directory {audio_dir}')

# Define the BashOperator that creates the output directory
create_mfcc_dir = BashOperator(
    task_id='create_mfcc_dir',
    bash_command=f'mkdir -p {mfcc_dir}',
    dag=dag
)

# Define the PythonOperator that creates the data directories and runs the generate_spectrogram function
folders = ['robbery', 'theft', 'exterior', 'interior', 'sexual', 'violence', 'help']

for folder in folders:
    data_dir = os.path.join(source_path, folder, 'train')

    create_source_dir = BashOperator(
        task_id=f'create_source_dir_{folder}',
        bash_command=f'mkdir -p {data_dir} && echo "vacant created"',
        dag=dag
    )

    # Define the directory for the audio files
    audio_dir = os.path.join(source_path, folder, 'train')
    # Define the task ID for the PythonOperator
    task_id = f'generate_mfcc_{folder}'

    # Define the PythonOperator
    generate_mfcc_task = PythonOperator(
        task_id=task_id,
        python_callable=generate_mfcc,
        op_kwargs={
            'subdir': folder,
            'audio_dir': audio_dir,
            'img_height': 128,
            'img_width': 128
        },
        dag=dag
    )

    # Set the dependencies between tasks
    create_mfcc_dir >> create_source_dir >> generate_mfcc_task


"""
Applies filtering to ticket-indexed data as specified by the user, aggregates ticketing
data at a non-directional OD-level, models KMeans clusters to which observations are
assigned, and runs unit tests.

Usage:
    src/model_pipeline.py [options]
    src/model_pipeline.py (-h | --help)
    src/model_pipeline.py

Options:
    --state=<state>                         State OD filter [default: None].
    --airport=<airport>                     Airport OD filter [default: None].
    --rpcarrier=<rpcarrier>                 RPCarrier filter [default: None].
    --roundtrip=<roundtrip>                 Rountrip ticket filter [default: None].
    --distance_min=<distance_min>           Minimum combined travel distance [default: None].
    --distance_max=<distance_max>           Maximum combined travel distance [default: None].
    --n_clusters=<n_clusters>               Number of clusters, overriding systematic n_cluster selection [default: None].
    --save_dir=<save_dir>                   Path to data directory [default: ./data/].
    --help                                  Prints this help prompt.
    --version                               Prints version.

"""

from filter_aggregate import Filter, Aggregate
from modeling import Clustering, UnitTest
from docopt import docopt

class ModelPipeline():

    def __init__(self, args):
        """
        Assigns arguments passed by docopt to self.

        Parameters
        ----------
        args : docopt
            Arguments passed via docopt.
        """

        self.args = args

    def filter_aggregate(self):
        """
        Applies filtering based on specified users paramters,
        then aggregated ticket-indexed data to non-directional
        OD-indexed data.

        For more detailed documentation, see `filter_aggregate.py`.
        """

        filter = Filter(self.args)
        filter.read_tickets()
        filter.filter_subset()
        filter.save_tickets()

        aggregate = Aggregate(self.args)
        aggregate.read_tickets()
        aggregate.aggregate()
        aggregate.save_od_ndir()

    def model(self):
        """
        Trains a KMeans clustering model using on non-directional
        OD-indexed feature data, and produces per-cluster summary
        statistics and per-observation cluster assignment table data,
        along with the model's Silhouette, Elbow, and Interdistance
        plots. It applies unit tests on the modeling workflow.

        For more detailed documentation, see `modeling.py`.
        """

        model = Clustering(self.args)
        model.train_model()

        test = UnitTest(self.args)
        test.test_scaling()
        test.test_model_output()

if __name__ == "__main__":

    args = docopt(__doc__, version="0.1.0")

    data_pipeline = ModelPipeline(args)
    data_pipeline.filter_aggregate()
    data_pipeline.model()


# -*- coding: utf-8 -*-
import os
import sys
from glob import glob
from os.path import basename, dirname, join, realpath

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from astropy import units as u
from astropy.coordinates import EarthLocation
from astropy.io import fits
from astropy.time import Time
from exoorbit.orbit import Orbit
from pysme.continuum_and_radial_velocity import determine_radial_velocity
from pysme.iliffe_vector import Iliffe_vector
from pysme.sme import SME_Structure
from scipy.constants import speed_of_light
from scipy.ndimage import binary_dilation, gaussian_filter1d, median_filter
from tqdm import tqdm

from exoplanet_transit_snr.petitradtrans import petitRADTRANS
from exoplanet_transit_snr.snr_estimate import (
    cross_correlation_reference,
    kp_vsys_combine,
    run_cross_correlation_ptr,
)
from exoplanet_transit_snr.stats import nanmad
from exoplanet_transit_snr.stellardb import StellarDb
from exoplanet_transit_snr.sysrem import Sysrem

c_light = speed_of_light * 1e-3
path = realpath(join(dirname(__file__), "../plots"))

def load_data(data_dir):
    files_fname = join(data_dir, "cr2res_obs_nodding_extracted[AB].fits")
    files = glob(files_fname)

    flat_data_dir = "/DATA/ESO/CRIRES+/GTO/211028_LTTS1445Ab/??"
    flat_fname = realpath(
        join(flat_data_dir, "../cr2res_util_calib_calibrated_collapsed_extr1D.fits")
    )
    flat = fits.open(flat_fname)

    fluxlist, wavelist, uncslist, times = [], [], [], []
    nodding = []
    for f in tqdm(files):
        hdu = fits.open(f)
        header = hdu[0].header
        # The header of the datafile is unchanged during the extraction
        # i.e. the header info is ONLY the info from the first observation
        # in the sequence. Thus we add the expsoure time to get the center
        # of the four observations

        # Get the actual times of the observations from the original fits files
        sof_fname = join(dirname(f), basename(dirname(f)) + ".sof")
        with open(sof_fname) as sof_file:
            sof = sof_file.readlines()

        sof = [s.split() for s in sof]
        sof = [s[0] for s in sof if s[1] == "OBS_NODDING_OTHER"]
        sof = [join(dirname(data_dir), basename(s)) for s in sof]
        headers = [fits.getheader(s) for s in sof]
        nodpos = [h["ESO SEQ NODPOS"] for h in headers]
        header = [h for i, h in enumerate(headers) if nodpos[i] == f[-6]][0]

        time = Time(header["MJD-OBS"], format="mjd")
        exptime = np.asarray(header["ESO DET SEQ1 EXPTIME"]) << u.s
        time += exptime / 2
        nodding += [f[-6]]
        fluxes, waves, uncses = [], [], []

        for i in [1, 2, 3]:
            chip = f"CHIP{i}.INT1"
            data = hdu[chip].data
            for order in range(1, 10):
                spec_name = f"{order:02}_01_SPEC"
                wave_name = f"{order:02}_01_WL"
                uncs_name = f"{order:02}_01_ERR"
                try:
                    blaze = flat[chip].data[spec_name]
                    waves += [data[wave_name]]
                    fluxes += [data[spec_name] / blaze]
                    uncses += [data[uncs_name] / blaze]
                except KeyError:
                    pass

        nseg = len(fluxes)
        npoints = len(fluxes[0])
        segments = np.arange(0, (nseg + 1) * npoints, npoints)

        flux = np.concatenate(fluxes)
        wave = np.concatenate(waves)
        uncs = np.concatenate(uncses)
        fluxlist += [flux]
        wavelist += [wave]
        uncslist += [uncs]
        times += [time]
        hdu.close()

    fluxlist = np.stack(fluxlist)
    wavelist = np.stack(wavelist)
    uncslist = np.stack(uncslist)
    nodding = np.asarray(nodding)
    # Sort observations in time
    times = Time(times)
    sort = np.argsort(times)

    fluxlist = fluxlist[sort]
    wavelist = wavelist[sort]
    uncslist = uncslist[sort]
    times = times[sort]
    nodding = nodding[sort]

    # Sort the segments by wavelength
    for i in range(len(wavelist)):
        sort = np.argsort(wavelist[i])
        wavelist[i] = wavelist[i][sort]
        fluxlist[i] = fluxlist[i][sort]
        uncslist[i] = uncslist[i][sort]

    # Convert to AA
    wavelist *= u.nm.to(u.AA)

    return wavelist, fluxlist, uncslist, times, segments, header, nodding

def determine_rv_offset(wave_A, spec_A, wave_B, spec_B, segments):
    sme = SME_Structure()
    sme.wave = [wave_A[low:upp] for low, upp in enum(segments)]
    sme.spec = [spec_A[low:upp] for low, upp in enum(segments)]
    syn_wave_B = Iliffe_vector([wave_B[low:upp] for low, upp in enum(segments)])
    syn_spec_B = Iliffe_vector([spec_B[low:upp] for low, upp in enum(segments)])
    sme.vrad_flag = "whole"
    vrad = determine_radial_velocity(sme, syn_wave_B, syn_spec_B, range(18), whole=True)
    return vrad

def remove_outliers(flux, unc=None, copy=False):
    if copy:
        flux = np.copy(flux)
        if unc is not None:
            unc = np.copy(unc)

    model = median_filter(flux, (5, 3))
    resid = flux - model
    std = nanmad(resid)
    flux[flux > model + 5 * std] = np.nan
    flux[flux < model - 10 * std] = np.nan

    spec = np.nanmedian(flux, axis=0)
    std = nanmad(flux - spec, axis=0)
    flux[:, std > np.nanmedian(std) * 5] = np.nan

    mask = np.isfinite(flux.ravel())
    idx = np.where(~mask)[0]
    x = np.arange(flux.size)
    y = flux.ravel()
    flux.ravel()[idx] = np.interp(idx, x[mask], y[mask])
    if unc is not None:
        y = unc.ravel()
        unc.ravel()[idx] = np.interp(idx, x[mask], y[mask])
    return flux, unc

def correct_data(data):
    wave, flux, uncs, times, segments, header, nodding = data

    # Correct for the large scale variations
    for low, upp in tqdm(enum(segments), total=len(segments) - 1, leave=False):
        spec = np.nanmedian(flux[:, low:upp], axis=0)
        mod = flux[:, low:upp] / spec
        mod = np.nan_to_num(mod, nan=1)
        for i in range(flux.shape[0]):
            mod[i] = median_filter(mod[i], 501, mode="constant", cval=1)
            mod[i] = gaussian_filter1d(mod[i], 100)
        flux[:, low:upp] /= mod
        uncs[:, low:upp] /= mod

    # Remove the 20 edge pixels as they are bad
    flux[:, 12186:12269] = np.nan
    for low, upp in enum(segments):
        flux[:, low : low + 20] = np.nan
        flux[:, upp - 20 : upp] = np.nan
        uncs[:, low : low + 20] = np.nan
        uncs[:, upp - 20 : upp] = np.nan

    # divide by the median spectrum
    spec = np.nanmedian(flux, axis=0)
    flux /= spec[None, :]
    uncs /= spec[None, :]

    uncs = np.clip(uncs, 0, None)

    data = wave, flux, uncs, times, segments, header, nodding
    return data

def remove_tellurics(wave, flux, rv_bary=0):
    fname = join(dirname(__file__), "../psg_trn.txt")
    df = pd.read_table(
        fname,
        sep=r"\s+",
        comment="#",
        header=None,
        names=[
            "wave",
            "total",
            "H2O",
            "CO2",
            "O3",
            "N2O",
            "CO",
            "CH4",
            "O2",
            "N2",
            "Rayleigh",
            "CIA",
        ],
    )
    mwave = df["wave"] * u.nm.to(u.AA)
    mflux = df["total"]
    n = wave.shape[0] // 2
    mflux = np.interp(wave[n], mwave * (1 - rv_bary / c_light), mflux, left=1, right=1)
    idx = mflux < 0.90
    idx = binary_dilation(idx, iterations=10)
    flux[:, idx] = np.nan
    return flux

def ptr_spec(wave, star, planet, rv_range, elem=("H2O", "CO", "CO2")):
    wmin, wmax = wave.min() << u.AA, wave.max() << u.AA
    wmin *= 1 - rv_range / c_light
    wmax *= 1 + rv_range / c_light

    mass_fractions = {
        "H2": 0.74,
        "He": 0.24,
        "TiO": 1e-3,
    }
    for el in elem:
        if el not in ["H2", "He", "TiO"]:
            mass_fractions[el] = 1e-3

    ptr = petitRADTRANS(
        wmin,
        wmax,
        line_species=("TiO",),
        mass_fractions=mass_fractions,
    )
    ptr.init_temp_press_profile(star, planet)
    cont_wave, cont_flux = ptr.run()

    ptr = petitRADTRANS(
        wmin,
        wmax,
        line_species=elem,
        mass_fractions=mass_fractions,
    )
    ptr.init_temp_press_profile(star, planet)
    ptr_wave, ptr_flux = ptr.run()

    # Assuming that cont_wave == ptr_wave
    ptr_flux /= cont_flux

    return ptr_wave, ptr_flux

def ptr_ref(wave, ptr_wave, ptr_flux, rv_range, rv_step):
    ref = cross_correlation_reference(
        wave, ptr_wave, ptr_flux, rv_range=rv_range, rv_step=rv_step
    )
    return ref

def run_sysrem(flux, uncs, segments, n2):
    corrected_flux = np.zeros_like(flux)
    for low, upp in enum(segments):
        sysrem = Sysrem(flux[:, low:upp], errors=uncs[:, low:upp])
        corrected_flux[:, low:upp], *_ = sysrem.run(n2)
    return corrected_flux

def enum(segments):
    for i in range(len(segments) - 1):
        yield segments[i], segments[i + 1]

# define the names of the star and planet
# as well as the datasets within the datasets folder
star, planet = "WASP-107", "b"
datasets = "220310_WASP107"

# Load the nominal data for this star and planet from simbad/nasa exoplanet archive
sdb = StellarDb()
# sdb.refresh(star)
star = sdb.get(star)
planet = star.planets[planet]
# Temporary Fix, while the units in the NASA Exoplanet Archive are in the wrong units
planet.transit_duration = planet.transit_duration.to_value(u.day) * u.hour
planet.equilibrium_temperature = lambda x, y: 736 * u.K

# based on alternative in https://iopscience.iop.org/article/10.3847/2041-8213/aabfce/pdf
# planet.intrinsic_temperature = lambda x, y: 500 * u.K

orbit = Orbit(star, planet)
telescope = EarthLocation.of_site("Paranal")

# Define the +- range of the radial velocity points,
# and the density of the sampling
rv_range = 200
rv_step = 0.25

if len(sys.argv) > 1:
    elem = (sys.argv[1],)
    n1 = 0
    n2 = int(sys.argv[2])
else:
    n1, n2 = 0, 15
    elem = ("CO",)
elem_str = "_".join(elem)

# Where to find the data, might need to be adjusted
data_dir = "/DATA/ESO/CRIRES+/GTO/220310_WASP107/1xAB_??"
# load the data from the fits files, returns several objects
data = load_data(data_dir)
wave, flux, uncs, times, segments, header, nodding = data

# Remove obvious excess flux points
flux[flux < 0] = np.nan
flux[flux > np.nanpercentile(flux.ravel(), 99) * 1.5] = np.nan

# Normalize by the 95th percentile of each segment
for low, upp in enum(segments):
    flux[:, low:upp] /= np.nanpercentile(flux[:, low:upp], 95, axis=1)[:, None]

# Split the two nodding positions
wave_A = wave[nodding == "A"][0]
wave_B = wave[nodding == "B"][0]
flux_A = flux[nodding == "A"]
flux_B = flux[nodding == "B"]
uncs_A = uncs[nodding == "A"]
uncs_B = uncs[nodding == "B"]

# Manual masking
flux_A[:, 22956:22960] = np.nan
flux_A[:, 12186:12269] = np.nan
flux_B[:, 12186:12269] = np.nan

# Automatic outlier removal
flux_A, uncs_A = remove_outliers(flux_A, uncs_A)
flux_B, uncs_B = remove_outliers(flux_B, uncs_B)

# Recombine A and B nodding positions
flux[nodding == "A"] = flux_A
flux[nodding == "B"] = flux_B
uncs[nodding == "A"] = uncs_A
uncs[nodding == "B"] = uncs_B

# Determine the radial velocity offset between A and B nodding
spec_A = np.nanmedian(flux_A, axis=0)
spec_B = np.nanmedian(flux_B, axis=0)
vrad = determine_rv_offset(wave_A, spec_A, wave_B, spec_B, segments)

# Barycentric correction
# shift everything into the restframe of the star
rv_bary = -star.coordinates.radial_velocity_correction(
    obstime=times, location=telescope
).to_value("km/s")

# Correct for barycentric and rv offset between nodding positions
# Interpolate to a common wavelength grid
# correct for differences in the wavelength solution between A and B nodding positions

wave_avg = np.nanmean(np.unique(wave, axis=0), axis=0)
for i in range(len(wave)):
    if nodding[i] == "B":
        vel = rv_bary[i] - vrad
    else:
        vel = rv_bary[i]
    vel = 1 - vel / c_light
    flux[i] = np.interp(wave_avg, wave[i] * vel, flux[i])
    wave[i] = wave_avg

# Correct outliers, basic continuum normalization
data = correct_data(data)
wave, flux, uncs, times, segments, header, nodding = data

# Determine telluric lines
# and remove the strongest ones
flux = remove_tellurics(wave, flux, rv_bary=np.mean(rv_bary))

# Calculate airmass
# altaz = star.coordinates.transform_to(AltAz(obstime=times, location=telescope))
# airmass = altaz.secz.value

# Calculate planet transmission spectrum
# with petitradtrans
if elem is None:
    wmin, wmax = wave.min() << u.AA, wave.max() << u.AA
    wmin *= 1 - rv_range / c_light
    wmax *= 1 + rv_range / c_light
    ptr_wave = np.linspace(wmin, wmax, 10_000)
    ptr_flux = np.ones_like(ptr_wave)
else:
    ptr_wave, ptr_flux = ptr_spec(wave, star, planet, rv_range, elem)
    ptr_wave = ptr_wave.to_value(u.AA)

# precompute the reference spectrum at different rv offsets
n = wave.shape[0] // 2
ref = ptr_ref(wave[n], ptr_wave, ptr_flux, rv_range, rv_step)
if elem is not None:
    ref -= 1

# Run Sysrem
# the first number is the number of sysrem iterations accounting for the barycentric velocity
# the second is for regular sysrem iterations
corrected_flux = run_sysrem(flux, uncs, segments, n2)
mad, diff = nanmad(corrected_flux, return_diff=True)
idx = diff > 5 * mad
corrected_flux[idx] = np.nan

# Normalize by the standard deviation in this wavelength column
corrected_flux -= np.nanmedian(corrected_flux, axis=0)
std = nanmad(corrected_flux, axis=0)
std[std == 0] = 1
corrected_flux /= std

# Run the cross correlation between the sysrem residuals and the expected planet spectrum
cc_data, rv_array = run_cross_correlation_ptr(
    corrected_flux,
    ref,
    segments,
    rv_range=rv_range,
    rv_step=rv_step,
)

vsys = star.radial_velocity.to_value("km/s")
kp = Orbit(star, planet).radial_velocity_semiamplitude_planet().to_value("km/s")
kp_range = (-250 - kp, 250 - kp)
vsys_range = (-150 - vsys, 150 - vsys)
for i, (low, upp) in enumerate(enum(segments)):
    selection = "IT"
    kp, vsys, combined = kp_vsys_combine(
        cc_data[i],
        times,
        star,
        planet,
        telescope,
        rv_range,
        rv_step,
        selection=selection,
        vsys_range=vsys_range,
        kp_range=kp_range,
        barycentric_correction=False,
        normalize=False,
    )

    title = ""
    folder = join(
        path,
        f"{star.name}_{planet.name}_{n1}_{n2}_{elem_str}_real_seg{i}_{selection}",
    )

    os.makedirs(folder, exist_ok=True)
    np.savez(
        join(folder, "data.npz"),
        res=combined,
        selection=selection,
        rv_array=rv_array,
        rv_step=rv_step,
        cc_data=cc_data[i],
        corrected_flux=corrected_flux[:, low:upp],
        times=times.mjd,
        kp=kp,
        vsys=vsys,
    )

if len(elem) == 3:
    combined = np.nansum(cc_data[:2], axis=0) + np.nansum(cc_data[3:], axis=0)
elif elem[0] == "H2O":
    combined = np.nansum(cc_data[:2], axis=0) + np.nansum(cc_data[3:], axis=0)
elif elem[0] == "CO2":
    combined = np.nansum(cc_data[:2], axis=0) + np.nansum(cc_data[3:9], axis=0)
elif elem[0] == "CO":
    combined = np.nansum(cc_data[-6:], axis=0)
else:
    combined = np.nansum(cc_data[:2], axis=0) + np.nansum(cc_data[3:], axis=0)

# split into time series?
sections = [0, len(times)]
n_sec = len(sections) - 1

vsys = star.radial_velocity.to_value("km/s")
kp = Orbit(star, planet).radial_velocity_semiamplitude_planet().to_value("km/s")
kp_range = (-250 - kp, 250 - kp)
vsys_range = (-150 - vsys, 150 - vsys)

for i, (low, upp) in enumerate(enum(segments)):
    selection = "IT"
    kp, vsys, kp_vsys = kp_vsys_combine(
        combined[low:upp],
        times[low:upp],
        star,
        planet,
        telescope,
        rv_range,
        rv_step,
        selection=selection,
        vsys_range=vsys_range,
        kp_range=kp_range,
        barycentric_correction=False,
    )

    # Save the results
    folder = join(
        path,
        f"{star.name}_{planet.name}_{n1}_{n2}_{elem_str}_real_{i+1}_{n_sec}_{selection}",
    )

    os.makedirs(folder, exist_ok=True)
    np.savez(
        join(folder, "data.npz"),
        res=kp_vsys,
        kp=kp,
        vsys=vsys,
        selection=selection,
        rv_array=rv_array,
        rv_step=rv_step,
        cc_data=cc_data[:, low:upp],
        combined=combined[low:upp],
        corrected_flux=corrected_flux[low:upp],
        times=times[low:upp].mjd,
    )

    # Run it again, but only for out of transit data
    selection = "OOT"
    kp, vsys, kp_vsys = kp_vsys_combine(
        combined[low:upp],
        times[low:upp],
        star,
        planet,
        telescope,
        rv_range,
        rv_step,
        selection=selection,
        kp_range=kp_range,
        vsys_range=vsys_range,
        barycentric_correction=False,
    )

    # And save to a different directory
    folder = join(
        path,
        f"{star.name}_{planet.name}_{n1}_{n2}_{elem_str}_real_{i+1}_{n_sec}_{selection}",
    )

    os.makedirs(folder, exist_ok=True)
    np.savez(
        join(folder, "data.npz"),
        res=kp_vsys,
        kp=kp,
        vsys=vsys,
        selection=selection,
        rv_array=rv_array,
        rv_step=rv_step,
        cc_data=cc_data[:, low:upp],
        combined=combined[low:upp],
        corrected_flux=corrected_flux[low:upp],
        times=times[low:upp].mjd,
    )

pass


#!/usr/bin/python

import matplotlib.pyplot as plt
from matplotlib import cm
import numpy as np
from mpl_toolkits.mplot3d import Axes3D
import sys

path = ""
if len(sys.argv) > 1:
    path = sys.argv[1]
else:
    print("Missing input file!")
    exit()

file = open(path, "r")

dim = int(file.readline())
if dim == 2:
    label_x = file.readline()

    f = file.readline().split(" ");
    fi = float(f[0])
    ff = float(f[1])
    fd = float(f[2])
    f = np.linspace(fi, ff, int((ff - fi)/fd)+1)

    label_y = file.readline()

    ap = file.readline().split(" ");
    api = float(ap[0])
    apf = float(ap[1])
    apd = float(ap[2])
    ap = np.linspace(api, apf, int((apf - api)/apd)+1)

    A = np.zeros([len(f), len(ap)])
    Sa = np.zeros([len(f), len(ap)])
    for fi in range(len(f)):
        for api in range(len(ap)):
            points = file.readline().split(" ")
            A[fi, api] = float(points[0])
            Sa[fi, api] = float(points[1])

    file.close()

    f, ap = np.meshgrid(f, ap)

    fig1 = plt.figure()
    ax1 = fig1.add_subplot(111, projection='3d')
    ax1.set_xlabel(label_x)
    ax1.set_ylabel(label_y)
    ax1.set_zlabel("A [um^2/um^2]")
    ax1.plot_surface(f, ap, A, cmap=cm.coolwarm,
                           linewidth=0, antialiased=False)

    fig2 = plt.figure()
    ax2 = fig2.add_subplot(111, projection='3d')
    ax2.set_xlabel(label_x)
    ax2.set_ylabel(label_y)
    ax2.set_zlabel("Sa [um]")
    ax2.plot_surface(f, ap, Sa, cmap=cm.coolwarm,
                           linewidth=0, antialiased=False)

    plt.show()

elif dim == 1:

    label_x = file.readline()

    vc = file.readline().split(" ");
    vci = float(vc[0])
    vcf = float(vc[1])
    vcd = float(vc[2])
    vc = np.linspace(vci, vcf, int((vcf - vci)/vcd)+1)

    A = np.zeros([len(vc)])
    Sa = np.zeros([len(vc)])
    for vci in range(len(vc)):
        points = file.readline().split(" ")
        A[vci] = float(points[0])
        Sa[vci] = float(points[1])

    file.close()

    fig1 = plt.figure()
    ax1 = fig1.add_subplot(111)
    ax1.set_xlabel(label_x)
    ax1.set_ylabel("A [um^2/um^2]")
    ax1.plot(vc, A)

    fig2 = plt.figure()
    ax2 = fig2.add_subplot(111)
    ax2.set_xlabel(label_x)
    ax2.set_ylabel("Sa [um]")
    ax2.plot(vc, Sa)

    plt.show()


import pandas as pd
import plotly.express as px
import plotly.graph_objs as go

df = pd.read_csv(r'blkjckhands.csv')
df = df.iloc[:, 1:]

def treemap():
    count_df = df.groupby(['PlayerNo', 'winloss', 'plybustbeat']).size().reset_index(name='count')

    fig = px.treemap(count_df, path=['PlayerNo', 'winloss', 'plybustbeat'], values='count', color='PlayerNo',
                 hover_data={'count': True})

    fig.update_layout(title='Figuur 7: Treemap het resultaat van de gespeelde handen per speler')
    return fig.show()

print(treemap())


from opcua import Client
import time

url = "opc.tcp://10.0.30.103:4840"
client = Client(url)

class SubHandler(object):

    def datachange_notification(self, node, val, data):
        print("New data change event", node, val)

while True:
    client.connect()
    Temp = client.get_node("ns=2; i=2")
    Temperature = Temp.get_value()

    handler = SubHandler()
    sub = client.create_subscription(2000, handler)
    handler = sub.subscribe_data_change(Temp)

    # print(Temperature)
    # time.sleep(2)
    client.disconnect()


import nltk
import string
import os
import collections
import re
import numpy as np
import ast
import pandas as pd
import itertools as it

from sklearn import metrics
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.stem.porter import PorterStemmer
from nltk.stem import LancasterStemmer
from nltk.corpus import PlaintextCorpusReader
from nltk.collocations import BigramCollocationFinder
from nltk.metrics import BigramAssocMeasures
from nltk.collocations import TrigramCollocationFinder
from nltk.metrics import TrigramAssocMeasures
from nltk.corpus import stopwords
from nltk.probability import FreqDist, ConditionalFreqDist
from nltk.classify import MaxentClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn import preprocessing
from sklearn.feature_extraction import DictVectorizer
from sklearn.cross_validation import train_test_split
from sklearn.ensemble import RandomForestClassifier as RF
from sklearn.multiclass import OneVsRestClassifier
from sklearn.metrics import confusion_matrix
from sklearn import tree
from sklearn.externals.six import StringIO
from sklearn.neighbors import NearestNeighbors
"""

def get_equip_equip(inPath,equip_map,outPath):

	use old data to extract the map between equipment and entity-sid and apply to the new data,
	for the ones can not detected in the above method, we then build a list with all possible equipment,
	then check if entity contains the string. If not, then assign the equipment to unknow.

	outFile = open(outPath,'w')
	outFile.write('%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' % (
				'sys','tktNm','crePriority','creWrkList','actWrkList','rpt_chr','createDate','crTime','resolvDate','rsTime',
				'entity','entity_remove','entity_clean','equip','firstCond','lastCond','sID','ne_Lang','channel','source','lastFClrUser','threshType','wfa_TR',
				'msg1','msg2','note','resCode','resText','restTextAdd'))
	counter = 0
	print('equip_map',type(equip_map),len(equip_map))

	equipList = ['lsc','alit','ama','clk','cp','senders bsy','rcan failure','serv loss','congestion','isolation',
	'no paths','cpu','dli','dsf','lgc','fp','ltc','mcc','mdc','mic','mp','power','pwr','rng','rpl','rsu','slc',
	'ss7','tpp','tst','aic','ali','cogn','iptrusta','regen scheduled','backup','fault code','bldg','blk','ccg',
	'fault astn','channel','ne direct','cp flt','digital path','dipflt','diu','dlu','dly stat','dst sw',
	'em fault','emg','ewsd','door','ext alm','hf archive','filesys','sg oper','man svewsd','ost una',
	'group switch fault','isolation','lost','lss','ltg','man busy','man sv','mdd','metallic test errors','mod',
	'nstart','omt','overeng','parse''power','route','rsu','software errors','slct','sp unit flt','lkset','swinv',
	'system operator','tech support','test assist','dpc','x25link','annc','extalm','dtc','rotl tgn','ss','umbcl',
	'unknown message','parse']

	#print equip_map['pcrktxpcdc0']
	#print equip_map['mrcdtxmedc0']
	with open(inPath,'Ur') as inFile:
		for line in inFile:
			if line.startswith('Sys'):
				continue

			sys,tktNm,crePriority,creWrkList,actWrkList,rpt_chr,createDate,crTime,resolvDate,rsTime,entity,firstCond,lastCond,sID,ne_Lang,channel,source,lastFClrUser,threshType,wfa_TR,msg1,msg2, note,resCode,resText,resTextAdd = line.strip().split(',')
			entity = entity.strip()
			entity = re.sub("[^a-zA-Z]"," ", entity)
			sID = re.sub("[^a-zA-Z]"," ", sID)
			msg1 = re.sub("[^a-zA-Z]"," ", msg1)
			print("before",msg1)
			msg1 = msg1.lower()
			print("after", msg1)
			entity = ' '.join(entity.split())
			sID = ' '.join(sID.split())
			msg1 = ' '.join(msg1.split())
			resCode = resCode.lower()
			entity_remove = re.sub(sID," ", entity)
			entity_remove = ' '.join(entity_remove.split())
			entity_clean = ''
			words = entity_remove.strip().split(" ")
			for word in words:
				if 100>len(word)>1:
					entity_clean = entity_clean + ' '+ word
			try:
				equip = equip_map[entity_clean]
				#print entity,equip
			except:
				for substr in equipList:
					if substr in entity_clean:
						equip = substr.upper()
						#print substr
						#print equip
						break
					else:
						equip = 'UNKNOWN'

			#if entity_clean not in equip_map:
			#	print equip

			outFile.write('%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' % (
			sys,tktNm,crePriority,creWrkList,actWrkList,rpt_chr,createDate,crTime,resolvDate,rsTime,
			entity,entity_remove,entity_clean,equip,firstCond,lastCond,sID,ne_Lang,channel,source,lastFClrUser,threshType,wfa_TR,
			msg1,msg2,note,resCode,resText,resTextAdd))

	outFile.close()

def parse_target(inPath,outPath):
	stemmer = LancasterStemmer()
	outFile = open(outPath,'w')
    outFile.write('%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' %
    	('sys','tktNm','crePriority','creWrkList','actWrkList','rpt_chr','createDate','crTime','resolvDate','rsTime',
    	'entity','equip','firstCond','lastCond','sID','ne_Lang','channel','source','lastFClrUser','threshType','wfa_TR',
    	'msg1','msg2','note','resCode','resText','resTextAdd'))
	with open(inPath,'Ur') as inFile:
		for line in inFile:
			equip = ''
			if line.startswith('Sys'):
				continue
			else:
				#items = line.strip().split(',')
				#print len(items)

				sys,tktNm,crePriority,creWrkList,actWrkList,rpt_chr,createDate,crTime,resolvDate,rsTime,entity,firstCond,lastCond,sID,ne_Lang,channel,source,lastFClrUser,threshType,wfa_TR,msg1,msg2, note,resCode,resText,resTextAdd = line.strip().split(',')
				entity = entity.replace("/"," ")
				entity = entity.replace("-"," ")
				entity = entity.replace(":"," ")
				entity = entity.replace("="," ")
				entity = entity.replace("_"," ")
				firstCond = firstCond.replace("/"," ")
				firstCond = firstCond.replace("-"," ")
				firstCond = firstCond.replace(":"," ")
				firstCond = firstCond.replace("="," ")
				firstCond = firstCond.replace("_"," ")
				' '.join(entity.split())
				' '.join(firstCond.split())
				print(entity)
				items = entity.split(' ')
				items = filter(None, items)
				#print items,len(items)
				equip = ''
				for i in range(len(items)):
					k = []
					if len(items[i])>=11 and len(items[i])<=16:
						#office = items[i]
						for j in range(i+1, len(items)):
							if any(char.isdigit() for char in items[j]):
								k.append(j)
								#print k
								equip = items[k[0]-1]
								break
							else:
								equip = equip + ' '+ items[j]
					elif len(items[i])>17:
						#office = items[i][0:17]
						equip = items[i][17:]
				equip =  equip.strip()

			outFile.write('%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' % (
				sys,tktNm,crePriority,creWrkList,actWrkList,rpt_chr,createDate,crTime,resolvDate,rsTime,
				entity,equip,firstCond,lastCond,sID,ne_Lang,channel,source,lastFClrUser,threshType,wfa_TR,
				msg1,msg2,note,resCode,resText,resTextAdd))
    outFile.close()

def parse_target_2(inPath,outPath):
	outFile = open(outPath,'w')
	outFile.write('%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' % ('sys','tktNm','crePriority','creWrkList','actWrkList','rpt_chr','createDate','crTime','resolvDate','rsTime',
				'entity','equip','firstCond','lastCond','sID','ne_Lang','channel','source','lastFClrUser','threshType','wfa_TR',
				'msg1','msg2','note','resCode','resText','resTextAdd'))

	name_dict = {'senders bsy':'cpe','rcan failure':'cpe','serv loss':'cpe','proc':'cpe','dtf':'ds',
	'function not mounted':'mp','pgm mismatch':'mp','cogn':'annc','iptrusta':'audit','regen scheduled':'backup',
	'bkup info':'backup','mts regen':'backup','eqpt':'bldg','blk':'blkd','fault astn':'cdu_flt','ne direct':'channel',
	'door':'ewsd','hf archive':'file','filesys':'file','sg oper':'file','snococ failures':'front end','sn fail':'front end',
	'ost una':'front end','failure witho':'front end','man svewsd':'front end','ost una':'front end',
	'group switch fault':'gs fault','man sv':'manual','metallic test errors':'mlt','software errors':'sftware',
	'sp unit flt':'sp fault','lkset':'ss7','tech support':'tech supt','dpc':'trunking'}

	counter = 0
	counter_line = 0
	equipList = ['lsc','alit','ama','clk','cp','senders bsy','rcan failure','serv loss','congestion','isolation',
	'no paths','cpu','dli','dsf','lgc','fp','ltc','mcc','mdc','mic','mp','power','pwr','rng','rpl','rsu','slc',
	'ss7','tpp','tst','aic','ali','cogn','iptrusta','regen scheduled','backup','fault code','bldg','blk','ccg',
	'fault astn','channel','ne direct','cp flt','digital path','dipflt','diu','dlu','dly stat','dst sw',
	'em fault','emg','ewsd','door','ext alm','hf archive','filesys','sg oper','man svewsd','ost una',
	'group switch fault','isolation','lost','lss','ltg','man busy','man sv','mdd','metallic test errors','mod',
	'nstart','omt','overeng','parse''power','route','rsu','software errors','slct','sp unit flt','lkset','swinv',
	'system operator','tech support','test assist','dpc','x25link']

	with open(inPath,'Ur') as inFile:

		for line in inFile:
			#equip = ''
			counter_line += 1
			if line.startswith('Sys'):
				continue
			else:
				sys,tktNm,crePriority,creWrkList,actWrkList,rpt_chr,createDate,crTime,resolvDate,rsTime,entity,firstCond,lastCond,sID,ne_Lang,channel,source,lastFClrUser,threshType,wfa_TR,msg1,msg2, note,resCode,resText,resTextAdd = line.strip().split(',')
				entity = re.sub("[^a-zA-Z]"," ", entity)
				' '.join(entity.split())
				print(entity)
				for key in name_dict:
					if key in entity:
						entity = re.sub(key,name_dict[key],entity)
				print(entity)

				for substr in equipList:
					if substr in entity:
						equip = substr
						print(equip)
					else:
						equip = 'unknown'

			outFile.write('%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' % (
				sys,tktNm,crePriority,creWrkList,actWrkList,rpt_chr,createDate,crTime,resolvDate,rsTime,
				entity,equip,firstCond,lastCond,sID,ne_Lang,channel,source,lastFClrUser,threshType,wfa_TR,
				msg1,msg2,note,resCode,resText,resTextAdd))

	outFile.close()
"""

def msg_vec_generate(inPath,vec_method="Count",min_frequency=10):
	stemmer = LancasterStemmer()
	msg_list = []
	counter = 0
	with open("ticket_Feb2015_equip.csv",'Ur') as inFile:
		for line in inFile:
			if line.startswith('sys'):
				continue
			sys,tktNm,crePriority,creWrkList,actWrkList,rpt_chr,createDate,crTime,resolvDate,rsTime,entity,entity_remove,entity_clean,equip,firstCond,lastCond,sID,ne_Lang,channel,source,lastFClrUser,threshType,wfa_TR,msg1,msg2,note,resCode,resText,resTextAdd,day = line.strip().split(',')

			msg = ''

			msg1 = re.sub("[^a-zA-Z]"," ", msg1)
			msg1 = msg1.strip().lower()
			msg1 = ' '.join(msg1.split())

			msgTokens = nltk.word_tokenize(msg1)
			stopset = set(stopwords.words('english'))
			for token in msgTokens:
				if len(token) > 2 and token not in stopset and token != 'msg' and token != equip:
					token_stem = stemmer.stem(token)
					msg = msg+" "+token_stem
					msg = msg.strip()
			msg_list.append(msg)

		if vec_method == "Count":
			vectorizer = CountVectorizer(analyzer = "word",tokenizer = None,preprocessor = None, stop_words = None,max_features = 5000,min_df = min_frequency,ngram_range = (1,3))
		elif vec_method == "Tfidf":
			vectorizer = TfidfVectorizer(analyzer = "word",tokenizer = None,preprocessor = None, stop_words = None,max_features = 5000,min_df = min_frequency)
		else:
			print("vec_method")
		msg_vec_features = vectorizer.fit_transform(msg_list)
		msg_vec_features = msg_vec_features.toarray()
		print("msg_vec_feature_shape",msg_vec_features.shape)
		vocab = vectorizer.get_feature_names()
		print('vocabulary starts here ...',vocab)
		print(msg_vec_features,type(msg_vec_features),msg_vec_features.shape)
		return msg_vec_features

def model_selection(inPath,msg_vec_array,outPath):
	outFile =  open(outPath,'w')
	outFile.write('%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' %
		('sys','tktNm','crePriority','creWrkList','actWrkList','rpt_chr','createDate','crTime','resolvDate','rsTime',
		'entity','equip','firstCond','lastCond','sID','ne_Lang','channel','source',
		'lastFClrUser','threshType','wfa_TR','msg1','msg2','note','resCode','resText','resTextAdd'))
	print(msg_vec_array,type(msg_vec_array),msg_vec_array.size)
	print(msg_vec_array[:,0],type(msg_vec_array[:,0]),msg_vec_array[:,0].size)

	equip_list = []
	code_list = []
	cond_list = []
	day_list = []
	creWrkList_list = []
	actWrkList_list = []
	crePriority_list = []
	sys_list = []
	with open(inPath,'Ur') as inFile:
		for line in inFile:
			if line.startswith('sys'):
				continue
			sys,tktNm,crePriority,creWrkList,actWrkList,rpt_chr,createDate,crTime,resolvDate,rsTime,entity,entity_remove,entity_clean,equip,firstCond,lastCond,sID,ne_Lang,channel,source,lastFClrUser,threshType,wfa_TR,msg1,msg2,note,resCode,resText,resTextAdd,day = line.strip().split(',')

			equip = re.sub("[^a-zA-Z]"," ", equip)
			equip = equip.strip().lower()
			equip_list.append(equip)
			#print equip_list

			resCode = re.sub("[^a-zA-Z]"," ", resCode)
			resCode = resCode.strip().lower()
			if resCode != 'almclr' and resCode != 'rts' and resCode !='techsup':
				resCode = "other"
			#if resCode != 'almclr'
			#	resCode = "other"
			outFile.write('%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' %
				(sys,tktNm,crePriority,creWrkList,actWrkList,rpt_chr,createDate,crTime,resolvDate,rsTime,
					entity,equip,firstCond,lastCond,sID,ne_Lang,channel,source,
					lastFClrUser,threshType,wfa_TR,msg1,msg2,note,resCode,resText,resTextAdd,day))

			code_list.append(resCode.strip())

			firstCond = re.sub("[^a-zA-Z]"," ", firstCond)
			firstCond = firstCond.strip().lower()
			cond = ''
			firstCond_list = firstCond.strip().split(" ")
			for i in firstCond_list:
				if len(i)>2:
					cond = cond + ' ' + i
			cond_list.append(cond.strip())

			day = re.sub("[^a-zA-Z]"," ", day)
			day = day.strip().lower()
			day_list.append(day)

			creWrkList = re.sub("[^a-zA-Z]"," ", creWrkList)
			creWrkList = creWrkList.strip().lower()
			creWrkList_list.append(creWrkList)

			actWrkList = re.sub("[^a-zA-Z]"," ", actWrkList)
			actWrkList = actWrkList.strip().lower()
			actWrkList_list.append(actWrkList)

			crePriority = crePriority.strip().lower()
			crePriority_list.append(crePriority)

			sys = re.sub("[^a-zA-Z]"," ", sys)
			sys = sys.strip().lower()
			sys_list.append(sys)

		equip_unique = list(set(equip_list))
		cond_unique = list(set(cond_list))
		resCode_unique = list(set(code_list))
		creWrkList_unique = list(set(creWrkList_list))
		actWrkList_unique = list(set(actWrkList_list))
		sys_unique = list(set(sys_list))
		crePriority_unique = list(set(crePriority_list))
		code_dict = dict([(name,i) for i,name in enumerate(resCode_unique)])
		print(code_dict)

		equip_array = np.array([equip_list])

		cond_array = np.array([cond_list])
		day_array = np.array([day_list])
		creWrkList_array = np.array([creWrkList_list])
		actWrkList_array = np.array([actWrkList_list])
		crePriority_array = np.array([crePriority_list])
		sys_array = np.array([sys_list])
		print("size check",cond_array.shape,creWrkList_array.shape,actWrkList_array.shape)
		features = np.concatenate((equip_array,cond_array,day_array,creWrkList_array,actWrkList_array,crePriority_array,sys_array))

		features_df = pd.DataFrame(features)
		features_df = pd.DataFrame.transpose(features_df)

		vec = DictVectorizer()
		features_vec = vec.fit_transform(features_df.to_dict('records')).toarray()

		#print(features,features.size,features[0].size, features.ndim)
		features_vec_df = pd.DataFrame(features_vec)
		#print(features_df,features_df.size)

		#print(msg_vec_array,msg_vec_array.size,msg_vec_array[0].size, msg_vec_array.ndim)
		msg_vec_df = pd.DataFrame(msg_vec_array)

		#print(msg_vec_df,msg_vec_df.size)
		features_all = pd.concat([features_vec_df,msg_vec_df],axis=1)

		#print("features_all",features_all,features_all.size)

		vec = DictVectorizer()
		features_all_array  = vec.fit_transform(features_all.to_dict('records')).toarray()

		code = [code_dict[x] for x in code_list]
		#print("code_list",code_list)
		#print("code",code)
		code_df = pd.DataFrame(code)
		data = pd.concat([features_all,code_df],axis=1)
		data2 = pd.concat([features_df,msg_vec_df,code_df],axis=1)
		data.to_csv(path_or_buf="text_binary.csv", sep=',')
		data2.to_csv(path_or_buf="text_token.csv", sep=',')
		## Start training

		print("Training the Naive Bayes...")

		x_train, x_test, y_train, y_test = train_test_split(features_all_array, code, test_size=0.33, random_state=42)
		#print("x_train",x_train,"y_train",y_train,"x_test",x_test,"y_test",y_test)

		nb = MultinomialNB(alpha=1,fit_prior=True).fit(x_train, y_train)
		nb_pred = nb.predict(x_test)
		print(metrics.classification_report(y_test, nb_pred,target_names=resCode_unique))
		print(confusion_matrix(y_test, nb_pred))
		#print((nb_pred(x_test) == y_test).mean())
		"""
		print("Training the Random Forest...")

		n_trees = 500
		criteria = ['gini', 'entropy']
		max_feature_params = ["sqrt", "log2"]
		max_depth_params = [10,15,20,25,30,40]
		class_weight_params = ["auto","subsample",None]
		parameter_space = it.product(criteria,max_feature_params, max_depth_params,class_weight_params)

		accuracies = {}
		verbose = 1
		n_jobs = 1
		for criterium, max_feature_param, max_depth_param,class_weight_param in parameter_space:
			rf = RF(n_estimators = n_trees, max_features=max_feature_param,verbose=verbose,
					n_jobs=n_jobs,max_depth=max_depth_param,class_weight=class_weight_param)
			#clf_fit = clf.fit(x, y, sample_weight = balance_weights(y))
			rf.fit(x_train, y_train)
			rf_pred = rf.predict(x_test)

			importances = rf.feature_importances_
			std = np.std([rf.feature_importances_ for tree in rf.estimators_],
             axis=0)
			indices = np.argsort(importances)[::-1]
			print("Feature ranking:")
			for f in range(10):
				print("%d. feature %d (%f)" % (f + 1, indices[f], importances[indices[f]]))

			print([(criterium, max_feature_param, max_depth_param,class_weight_param)])
			print(metrics.classification_report(y_test, rf_pred,target_names=resCode_unique))
			print(confusion_matrix(y_test, rf_pred))
			accuracies[(criterium, max_feature_param, max_depth_param,class_weight_param)] = (rf.predict(x_test) == y_test).mean()
		print(accuracies)

		print("training the decision tree....")
		dt = tree.DecisionTreeClassifier()
		dt.fit(x_train,y_train)
		dt_pred = dt.predict(x_test)
		tree.export_graphviz(dt,out_file='tree.dot')
		print(metrics.classification_report(y_test, dt_pred,target_names=resCode_unique))
		print(confusion_matrix(y_test, dt_pred))
		"""
		print("Training the KNN...")

		x_train, x_test, y_train, y_test = train_test_split(features_all_array, code, test_size=0.33, random_state=42)
		#print("x_train",x_train,"y_train",y_train,"x_test",x_test,"y_test",y_test)

		knn = NearestNeighbors(n_neighbors=2, algorithm='ball_tree').fit(x_train,y_train)
		knn_pred = knn.predict(x_test)
		print(metrics.classification_report(y_test, knn_pred,target_names=resCode_unique))
		print(confusion_matrix(y_test, knn_pred))
		#print((nb_pred(x_test) == y_test).mean())

def main():
	#get_equip_equip("ticket_Feb2015.csv",equip_map_entity_remove_sid,"ticket_Feb2015_equip.csv")
	#get_equip_equip("ticket_Jan2015.csv",equip_map_entity_remove_sid,"ticket_Jan2015_equip.csv")
	msg_vec_array = msg_vec_generate("ticket_Feb2015_equip.csv",vec_method="Tfidf",min_frequency=10)
	model_selection('ticket_Feb2015_equip.csv',msg_vec_array,'ticket_Feb2015_equip_comb.csv')

if __name__ == "__main__":
    main()


import sys
import importlib

def hide_module(name):
    mod = importlib.import_module(name)
    mod_fname = mod.__file__

    ending = f"/{name}/__init__.py"
    assert mod_fname.endswith(ending)
    mod_folder = mod_fname[:-len(ending)]
    sys.path.remove(mod_folder)

#    del mod
    del sys.modules[name]

class HiddenModule:

    def __init__(self, name):
        self.name = name
        self.was_loaded = (name in sys.modules)

        self.mod = mod = importlib.import_module(name)
        mod_fname = mod.__file__

        ending = f"/{name}/__init__.py"
        assert mod_fname.endswith(ending)
        self.mod_folder = mod_folder = mod_fname[:-len(ending)]
        sys.path.remove(mod_folder)

        del sys.modules[name]

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        sys.path.append(self.mod_folder)
        if self.was_loaded:
            sys.modules[self.name] = self.mod
        return (exc_type is None)

if __name__ == "__main__":
    with HiddenModule("jungfrau_utils"):
        try:
            import jungfrau_utils as ju
        except ImportError:
            print("worked")
        else:
            print("failed")

    assert sys.modules.get("jungfrau_utils") is None
    import jungfrau_utils as ju
    assert sys.modules.get("jungfrau_utils")

    with HiddenModule("jungfrau_utils"):
        try:
            import jungfrau_utils as ju
        except ImportError:
            print("worked")
        else:
            print("failed")

    assert sys.modules.get("jungfrau_utils")
    import jungfrau_utils as ju

    #hide_module("jungfrau_utils")

    #import jungfrau_utils as ju


# %%
import re
import requests
from bs4 import BeautifulSoup
import pandas as pd

# %%

url = "https://www.playbasket.it/veneto/league.php?lt=2&lr=VE&lp=VE&lc=PM&season=2022&lf=M&lg=3&mod={mod}"
team = re.compile("arcella", re.IGNORECASE)

# %% Calendario

dfs = pd.read_html(url.format(mod="cl"), match=team)
df = pd.concat(dfs)
df.columns = ["data", "locali", "ospiti",
              "tabellini", "pt-locali", "pt-ospiti", "extra"]
df.drop(columns=["tabellini", "extra"], inplace=True)
mask = df["locali"].apply(lambda x: "arcella" in x.lower()) |\
    df["ospiti"].apply(lambda x: "arcella" in x.lower())
df = df[mask]

df.to_csv("data/calendario.csv", index=False)

# %% Classifica

r = requests.get(url.format(mod="st"))
soup = BeautifulSoup(r.content, "html5lib")
div = soup.find(id="classifica_general")
standing = div.find("table")
df = pd.read_html(str(standing))[0]
df.to_csv("data/classifica.csv", index=False)


from setuptools import setup, find_packages

version = '0.1.0'

setup(name='anachronos',
      version=version,
      description="A turn-based strategy game with time travel.",
      classifiers=[
          #"Development Status :: some_dev_status",
          #"Intended Audience :: if_ya_got_one",
          #"License :: ????",
          "Programming Language :: Python :: 2",
          "Topic :: Games/Entertainment :: Turn Based Strategy",
      ],
      keywords='anachronos game turn-based strategy time',
      author='Samuel Lucidi',
      #author_email='whatever@ya.like',
      url='http://github.com/mansam/anachronos',
      #license='????',
      packages=find_packages(exclude=['ez_setup', 'tests']),
      include_package_data=True,
      zip_safe=False,
      install_requires=["fabulous"],
      #entry_points="""
      #[console_scripts]
      #anachronos = anachronos:???
      #"""
      )


#
# This script is an example of how to interface with an online Zotero library, and
# in particular how to use the resultant tagged data to narrow down the potential
# entries of interest on the basis of the tags assigned to each entry.
#
# The script downloads and caches a tagged data set from Zotero; if a previously
# cached data set exists in the local directory, the download from Zotero is skipped.
#
# The use of cached data prevents us having to hit the Zotero website every time the
# script is run; this could cause problems if the Zotero website is down, or if
# Zotero throttles the number of requests allowed over its REST API. In any case,
# sending out for an additional download from Zotero each time would slow things down!
#
# Requirements:
#
# PyZotero : https://github.com/urschrei/pyzotero (https://pyzotero.readthedocs.io/en/latest/)
#
# E.g.: pip3 install pyzotero
#

import os, json
from pyzotero import zotero

##########################################################################
# CHANGE THIS BEFORE ADDING TO ANY PUBLIC REPOSITORIES!                  #
#                                                                        #
# Temporary hardwiring - don't leave the api key in the code, obviously! #
# These should be environment variables or passed via command line.      #
##########################################################################

# Identify the Zotero library, and specify whether it's a user-specific
# library, or a group library
library_id = "2547147"
library_type = "group"

# This access key is generated from a Zotero account that has access to the
# specified library: https://www.zotero.org/support/dev/web_api/v3/basics
api_key = "YOUR_ZOTERO_ACCESS_KEY_HERE"

# Local output file for cached Zotero results (json format, portable)
cache_path = 'items.json'

#
# Simple utility class for Zotero results to allow mapping of tags into
# Zotero items. We assume items as per "entry" data generated by PyZotero.
#
# For our purposes, the important parts of PyZotero's "entry" data structure are:
#
#   entry['key'] = unique key to identify this entry in the Zotero library
#   entry['data']['tags'] = tagsm in the format [ {'tag': $tag1}, {'tag': $tag2}, ... ]
#
class ZoteroMap:

	def __init__(self, entries=None):
		#
		# This class uses two internal maps:
		#
		# - tag => [key, key, ...]
		# - key => PyZotero entry
		#
		self.tag_to_keys = {}
		self.key_to_entry = {}

		if entries != None: self.add_entries(entries)

	def add_entries(self, entries):
		for entry in entries:
			key, data = entry['key'], entry['data']
			tags = [ x['tag'] for x in data['tags'] ]

			self.key_to_entry[key] = entry

			for tag in tags:
				if tag not in self.tag_to_keys:
					self.tag_to_keys[tag] = [key]
				else:
					self.tag_to_keys[tag].append(key)

	# returns [[key,...],...] : list of keys for each tag in tags[]
	def tags_to_keysets(self, tags):
		t2k = self.tag_to_keys
		return [t2k[tag] for tag in tags if tag in t2k]

	# returns flattened version of [[key,...],...] from tags_to_keysets(), with
	# duplicates removed.
	def tags_to_keyset(self, tags):
		keysets = self.tags_to_keysets(tags)
		return { k: True for k in [k for ks in keysets for k in ks] }.keys()

	# Returns list of keys that have ANY of the specified tags
	def any_tag(self, tags):
		return self.tags_to_keyset(tags)

		keys, x = [k for ks in self.tags_to_keysets(tags) for k in ks], {}
		for key in keys: x[key] = True
		return list(x.keys())

	# Returns list of keys that have ALL of the specified tags
	def all_tags(self, tags):
		keys, x = [k for ks in self.tags_to_keysets(tags) for k in ks], {}
		for key in keys: x[key] = x[key]+1 if (key in x) else 1
		return [key for key in x if (x[key] == len(tags))]

	# ToDo: add progressive filtering by passing (optional) set of keys
	# into the above routines.

	# Add note regarding an NOT operation!

#
# Try to load previously cached data, or fetch from Zotero website if no
# cached data file is present.
# Saving data as "prettified" json produces a file ~30% larger in some cases,
# which currently means approx. 630 kB -> 940 kB, so we likely shouldn't use
# prettification unless we know the cached file is going to be read by humans!
#

if os.path.isfile(cache_path):

	print(f'Reading cache data from "{cache_path}"')
	s = open(cache_path).read()
	items = json.loads(s)

else:

	print(f'Fetching library data from Zotero')
	zot = zotero.Zotero(library_id, library_type, api_key)
	items = zot.everything( zot.top() )
	s = json.dumps(items)

	print(f'Writing cache data to "{cache_path}"')
	with open(cache_path,'w') as f:
		f.write(s)

	print(f'Writing prettified cache data')
	with open('items.pretty.json','w') as f:
		for item in items:
			print(json.dumps(item, sort_keys=True,indent=4), file=f)

#
# Generate a Zotero object from the data we retrieved
# ToDo - note regarding which option we take and why.

zm = ZoteroMap(items)
for tag in zm.tag_to_keys:
	print(tag, zm.tag_to_keys[tag])

tags = list(zm.tag_to_keys.keys())
print(tags)

ksets = zm.tags_to_keysets(tags)
for (i,kset) in enumerate(ksets):
	print(i, tags[i])
	for k in kset:
		print(f'  {k}')
	print()

print()
print()

print('HERE:')
ksets = zm.tags_to_keysets(tags)
blah = []
for kset in ksets:
	for k in kset:
		if k in blah: continue
		blah.append(k)

blah2 = list(zm.tags_to_keyset(tags))

print(len(blah), blah)
print(len(blah2), blah2)
for i,k in enumerate(blah):
	if k != blah2[i]:
		print(f'PROBLEM: {i} : {k} != {blah2[i]}')
		sys.exit(-1)

tags2 = tags[:5] + ['Statistics'] # tags[:5]
print(tags2)

print(f'Any tag: {tags2}')
keys = zm.any_tag(tags2)
for key in keys:
	tags = zm.key_to_entry[key]['data']['tags']
	print(f'  {key} : {[t["tag"] for t in tags]}')

print()
print()

print(f'All tags: {tags2}')
keys = zm.all_tags(tags2)
for key in keys:
	entry = zm.key_to_entry[key]
	print(f'  {key} : {entry["data"]["tags"]}')


#!/usr/bin/env python3
import main
import datetime
import subprocess
import os
import database_ops as db
import speedtest as sp

def get_and_save_devices():
	# Start on home directory
	subprocess.call("cd /home", shell=True)
	# It gets current date and time
	fecha_log = get_date()
	# Message to show on command line that it going to start look for devices
	print(" Start looking for devices connected to the network...")
	# It executes Wiom app to find the devices that are connected to the network and save all of them into
	# a log file
	subprocess.call("sudo wiom -w > ../Logs/logs_devices/last.log", shell=True)

	archivo = open("../Logs/logs_devices/last.log","r")

	# Call procesar_datos in order to obtain Ip, Mac, Name
	listaIp, listaMac, listaName  = process_data(archivo)

	# connect to ddbb
	connection = db.devices_database(listaIp, listaMac, listaName)

	# We rename the last log with todays date
	src = "../Logs/logs_devices/last.log"
	dst = "../Logs/logs_devices/%s.log" % fecha_log
	os.rename(src,dst)

def get_and_save_speeds():
    speed = sp.Speedtest()

    # We modify the round the results in order to see it better
    downspeed = round((round(speed.download()) / 1048576), 2)
    upspeed = round((round(speed.upload()) / 1048576), 2)

    db.speed_database(upspeed, downspeed)

#-----------------------------------------------------------------------------------
def get_date():
	fecha_log = datetime.datetime.now().strftime("%d_%m_%Y_%H:%M")
	return fecha_log

# To obtain data from logs
def process_data(archivo):
    listIp = []
    listMac = []
    listName = []
    lineas = archivo.readlines()
    for i in lineas:
     if ("IP Addres"in i):
        listIp.append(i[12:26])
     if ("Mac Address"in i):
        listMac.append(i[14:-1])
     if ("Device"in i):
        listName.append(i[8:-1])

    return listIp, listMac, listName
#---------------------------------------------------------------------------------

def main():
    get_and_save_devices()
    get_and_save_speeds()

if __name__ == "__main__":
    main()


# mcc_parser.py
# Functions to parse Midi files.

from mido import MidiFile, tempo2bpm

# Mapping of MIDI numeric notes to RTTTL key+octave notes.
# Opposite mapping from RTTTL2MIDI in mcc_waves module.
# Based on this: https://en.wikipedia.org/wiki/Piano_key_frequencies#List
MIDI2RTTTL = {
	24: 'c1', 25: 'c#1', 26: 'd1', 27: 'd#1', 28: 'e1', 29: 'f1', 30: 'f#1', 31: 'g1', 32: 'g#1', 33: 'a1',
	34: 'a#1', 35: 'b1', 36: 'c2', 37: 'c#2', 38: 'd2', 39: 'd#2', 40: 'e2', 41: 'f2', 42: 'f#2', 43: 'g2',
	44: 'g#2', 45: 'a2', 46: 'a#2', 47: 'b2', 48: 'c3', 49: 'c#3', 50: 'd3', 51: 'd#3', 52: 'e3', 53: 'f3',
	54: 'f#3', 55: 'g3', 56: 'g#3', 57: 'a3', 58: 'a#3', 59: 'b3', 60: 'c4', 61: 'c#4', 62: 'd4', 63: 'd#4',
	64: 'e4', 65: 'f4', 66: 'f#4', 67: 'g4', 68: 'g#4', 69: 'a4', 70: 'a#4', 71: 'b4', 72: 'c5', 73: 'c#5',
	74: 'd5', 75: 'd#5', 76: 'e5', 77: 'f5', 78: 'f#5', 79: 'g5', 80: 'g#5', 81: 'a5', 82: 'a#5', 83: 'b5',
	84: 'c6', 85: 'c#6', 86: 'd6', 87: 'd#6', 88: 'e6', 89: 'f6', 90: 'f#6', 91: 'g6', 92: 'g#6', 93: 'a6',
	94: 'a#6', 95: 'b6', 96: 'c7', 97: 'c#7', 98: 'd7', 99: 'd#7', 100: 'e7', 101: 'f7', 102: 'f#7',
	103: 'g7', 104: 'g#7', 105: 'a7', 106: 'a#7', 107: 'b7'
}

INSTRUMENTS = [
	'Acoustic Grand Piano','Bright Acoustic Piano','Electric Grand Piano','Honky-tonk Piano','Electric Piano 1','Electric Piano 2','Harpsichord',
	'Clavi','Celesta','Glockenspiel','Music Box','Vibraphone','Marimba','Xylophone','Tubular Bells','Dulcimer','Drawbar Organ','Percussive Organ',
	'Rock Organ','Church Organ','Reed Organ','Accordion','Harmonica','Tango Accordion','Acoustic Guitar (nylon)','Acoustic Guitar (steel)',
	'Electric Guitar (jazz)','Electric Guitar (clean)','Electric Guitar (muted)','Overdriven Guitar','Distortion Guitar','Guitar harmonics',
	'Acoustic Bass','Electric Bass (finger)','Electric Bass (pick)','Fretless Bass','Slap Bass 1','Slap Bass 2','Synth Bass 1','Synth Bass 2',
	'Violin','Viola','Cello','Contrabass','Tremolo Strings','Pizzicato Strings','Orchestral Harp','Timpani','String Ensemble 1','String Ensemble 2',
	'SynthStrings 1','SynthStrings 2','Choir Aahs','Voice Oohs','Synth Voice','Orchestra Hit','Trumpet','Trombone','Tuba','Muted Trumpet','French Horn',
	'Brass Section','SynthBrass 1','SynthBrass 2','Soprano Sax','Alto Sax','Tenor Sax','Baritone Sax','Oboe','English Horn','Bassoon','Clarinet','Piccolo',
	'Flute','Recorder','Pan Flute','Blown Bottle','Shakuhachi','Whistle','Ocarina','Lead 1 (square)','Lead 2 (sawtooth)','Lead 3 (calliope)','Lead 4 (chiff)',
	'Lead 5 (charang)','Lead 6 (voice)','Lead 7 (fifths)','Lead 8 (bass + lead)','Pad 1 (new age)','Pad 2 (warm)','Pad 3 (polysynth)','Pad 4 (choir)',
	'Pad 5 (bowed)','Pad 6 (metallic)','Pad 7 (halo)','Pad 8 (sweep)','FX 1 (rain)','FX 2 (soundtrack)','FX 3 (crystal)','FX 4 (atmosphere)','FX 5 (brightness)',
	'FX 6 (goblins)','FX 7 (echoes)','FX 8 (sci-fi)','Sitar','Banjo','Shamisen','Koto','Kalimba','Bag pipe','Fiddle','Shanai','Tinkle Bell','Agogo','Steel Drums',
	'Woodblock','Taiko Drum','Melodic Tom','Synth Drum','Reverse Cymbal','Guitar Fret Noise','Breath Noise','Seashore','Bird Tweet','Telephone Ring','Helicopter',
	'Applause','Gunshot'
]

def program_to_instrument(program:int) ->  str:
	"""
	Maps a program number to a specific instrument under
	the INSTRUMENTS lookup.
	"""
	if 1 <= program <= 128:
		return INSTRUMENTS[program - 1]
	else:
		return INSTRUMENTS[0]

def open_midi(filepath:str) -> MidiFile:
	"""
	Return the MidiFile object given its filename.
	"""
	assert ".mid" in filepath, "MCC: Cannot open non-MIDI file."
	return MidiFile(filepath, clip=True)

def get_note_lengths(file_info: list) -> dict:
	bpm = file_info[2][1]
	notes = {"whole note": 240 / bpm, "half note": 120 / bpm, "quarter note": 60 / bpm, "eighth note": 30 / bpm,
			 "sixteenth note": 15 / bpm}
	return notes

"""
currently, this implementation only works assuming the format of the midi files we've seen so far is the usual, ie. the
time signature is first, then the key signature, then the tempo is set. This is why if you pass it the list of extracted info
from the midi file, the tempo should always be the 2nd entry of the 3rd list in the input
"""

def extract_midi_info(meta_messages:list) -> dict:
	"""
	Input the first track of a MIDI file containing meta messages
	with key info such as timestamps for tempo change.
	Return a dictionary of relevant information for a MIDI file.
	"""
	info = {}
	for msg in meta_messages:
		if msg.type == "time_signature":
			info["time_signature"] = (msg.numerator, msg.denominator)
		elif msg.type == "key_signature":
			info["key_signature"] = msg.key
		elif msg.type == 'set_tempo':
			info["tempo"] = (tempo2bpm(msg.tempo), msg.time)
	return info

def extract_midi_tracks(mid_tracks:list) -> list:
	"""
	Iterate over a list of tracks and create a list for each
	track containing the notes in each track, preserving the order.

	Each note is stored as a 4-tuple:
		(
			note: int, the note key on a piano
			velocity: int, the strength of the key where 0 means the note is off, or a rest
			time: int, the wait time between the last and current note or operation, so the
				duration of a note is the sum of time in between the two nearest messages
				containing the same note
			on/off: bool, essentially whether this note is off or not
			instrument: the instrument playing this note
		)
	"""
	notes_tracks = []
	for track in mid_tracks:
		track_notes = []
		for msg in track:
			if msg.type == 'program_change':
				current_instrument = program_to_instrument(msg.program)
			# Only work on messages that describe notes.
			elif msg.type == "note_on" or msg.type == "note_off":
				note = (msg.note, msg.velocity, msg.time, msg.velocity > 0, current_instrument)
				track_notes.append(note)
		if len(track_notes) > 0:
			notes_tracks.append(track_notes)
	return notes_tracks

def assign_note(beat:int, ticks_per_beat:int) -> str:
	"""
	Input the time (in ticks) that can be found in midi tracks, and
	ticks per beat convert time to beats for RTTTL format
	"""
	beat = beat / ticks_per_beat
	if beat == 0:
		return "0"
	elif beat >= 3:
		return "1"
	elif beat >= 1.5:
		return "2"
	elif beat >= 0.75:
		return "4"
	elif beat >= 0.375:
		return "8"
	elif beat >= 0.1875:
		return "16"
	else:
		return "32"

def midi_to_rtttl(midi_tuple_list:list, ticks_per_beat:int) -> str:
	"""
	Input ONE element of the output of extract_midi_tracks function. i.e. just one list should be the input.
	Specify ticks_per_beat of a MidiFile object `mid` as `mid.ticks_per_beat`.

	Returns RTTTL string of the midi note list.
	"""

	rtttlList = ""

	# Tuple: (note:int, velocity:int, time:int, on/off:bool, current_instrument:str)
	for i, tuple in enumerate(midi_tuple_list):
		# Check first MIDI note for offset time from the start. Only add a "rest" if the time value > 0.
		if i == 0 and tuple[2] > 0:
			# Insert RTTTL rest with duration based on initial time.
			rtttlList += "," + assign_note(tuple[2], ticks_per_beat) + "p"

		# skipping the last element, since there's no next tuple's time
		if i == len(midi_tuple_list) - 1:
			break

		# note on
		next_tuple = midi_tuple_list[i + 1]
		beat_in_note = assign_note(next_tuple[2], ticks_per_beat)
		if beat_in_note == "0":
			continue

		if tuple[3] == True:
			newNote = beat_in_note + MIDI2RTTTL.get(tuple[0]) # time of next tuple + note
			rtttlList += "," + newNote

		# note off
		else:
			newNote = beat_in_note + "p"
			if beat_in_note != "0":
				rtttlList += "," + newNote

	return rtttlList[1:]	# removing the first comma


import streamlit as st
import pandas as pd
import numpy as np
import re
import plotly.express as px
from bs4 import BeautifulSoup
from datetime import datetime

from html_scripts import boxkpi,boxnumberpercentage
from datafunctions import inspeccion_callsample,inspeccion_callhistory,inspeccion_seguimiento_ofertas

# px.colors.named_colorscales()
# color_continuous_scale=px.colors.sequential.Viridis / invers: px.colors.sequential.Viridis[::-1]
# color_discrete_sequence=px.colors.qualitative.Vivid

def dashboard_gestion_ofertas(comercial=None,modulo_gestion_lead=True,modulo_seguimiento_ofertas=True,modulo_inconsistencias=True):

    data                  = inspeccion_callsample()
    datacallhistory       = inspeccion_callhistory()
    dataseguimientofertas = inspeccion_seguimiento_ofertas()

    if modulo_gestion_lead:
        st.write('---')
        st.markdown('

Gestión de Inmuebles de Oportunidad

', unsafe_allow_html=True)

        col1, col2, col3, col4, col5 = st.columns(5)
        with col1:
            totalregistros     = len(data['id_inmueble'].unique())
            sinasignacion_call = len(data[data['fecha_asignacion_callagent'].isnull()]['id_inmueble'].unique())
            html               = boxnumberpercentage(totalregistros,f'({sinasignacion_call} sin asignar)','Total oportundiades')
            html_struct        = BeautifulSoup(html, 'html.parser')
            st.markdown(html_struct, unsafe_allow_html=True)

        with col2:
            asignacion_call     = len(data[data['fecha_asignacion_callagent'].notnull()]['id_inmueble'].unique())
            por_asignacion_call = asignacion_call/totalregistros
            por_asignacion_call = "{:.1%}".format(por_asignacion_call)
            html          = boxnumberpercentage(asignacion_call,f'({por_asignacion_call})','Llamada asignada')
            html_struct   = BeautifulSoup(html, 'html.parser')
            st.markdown(html_struct, unsafe_allow_html=True)

        with col3:
            asignacion_visit     = len(data[data['fecha_asignacion_visitagent'].notnull()]['id_inmueble'].unique())
            por_asignacion_visit = asignacion_visit/asignacion_call
            por_asignacion_visit = "{:.1%}".format(por_asignacion_visit)
            html          = boxnumberpercentage(asignacion_visit,f'({por_asignacion_visit})','Visita asignada')
            html_struct   = BeautifulSoup(html, 'html.parser')
            st.markdown(html_struct, unsafe_allow_html=True)

        with col4:
            estado_pendiente     = len(data[data['finalizado'].str.lower()=='pendiente']['id_inmueble'].unique())
            por_estado_pendiente = estado_pendiente/asignacion_visit
            por_estado_pendiente = "{:.1%}".format(por_estado_pendiente)
            html          = boxnumberpercentage(estado_pendiente,f'({por_estado_pendiente})','Pendiente de visita')
            html_struct   = BeautifulSoup(html, 'html.parser')
            st.markdown(html_struct, unsafe_allow_html=True)

        with col5:
            estado_finalizado     = len(data[data['finalizado'].str.lower()=='finalizado']['id_inmueble'].unique())
            por_estado_finalizado = estado_finalizado/asignacion_visit
            por_estado_finalizado = "{:.1%}".format(por_estado_finalizado)
            html          = boxnumberpercentage(estado_finalizado,f'({por_estado_finalizado})','Visita Finalizada')
            html_struct   = BeautifulSoup(html, 'html.parser')
            st.markdown(html_struct, unsafe_allow_html=True)

        datacallhistory                = datacallhistory[datacallhistory['id_inmueble'].notnull()]
        datacallhistory['call_status'] = datacallhistory['call_status'].replace(['NÚMERO EQUIVOCADO','DESISTE COMERCIALIZACIÓN','NO HAY COMUNICACIÓN'],['NUMERO EQUIVOCADO','DESISTE COMERCIALIZACION','NO CONTESTA'])
        datacallhistory['order']       = datacallhistory['call_status'].replace(['EFECTIVA', 'NUMERO EQUIVOCADO', 'NO CUMPLE CON REQUISITOS','DESISTE COMERCIALIZACION','VENDIDO', 'ARRENDADO', 'NO NEGOCIABLE', 'HACER SEGUIMIENTO', 'NO CONTESTA'],[1,2,3,4,5,6,7,8,9])
        datacallhistory                = datacallhistory[datacallhistory['order'].notnull()]
        datacallhistory                = datacallhistory.sort_values(by='order',ascending=True).drop_duplicates(subset='id_inmueble',keep='first')

        col1, col2, col3 = st.columns(3)
        with col1:
            st.markdown('

Llamadas contestadas

', unsafe_allow_html=True)
            df          = datacallhistory[['call_status']]
            df          = df[df['call_status'].notnull()]
            df['count'] = 1
            df          = df.groupby('call_status')['count'].count().reset_index()
            df.columns  = ['call_status','count']
            df          = df.sort_values(by='count',ascending=False)
            df['colors'] = px.colors.sequential.Viridis[0:len(df)]

            if df.empty is False:
                # color_continuous_scale=px.colors.sequential.Viridis / invers: px.colors.sequential.Viridis[::-1]
                # color_discrete_sequence=px.colors.qualitative.Vivid
                fig = px.bar(df, x='call_status', y='count', text='count', color='count',color_continuous_scale=px.colors.sequential.RdBu)
                fig.update_traces(textposition='outside')
                fig.update_layout(
                    plot_bgcolor='rgba(0, 0, 0, 0)',
                    paper_bgcolor='rgba(0, 0, 0, 0)',
                    xaxis_title='',
                    yaxis_title='',
                    legend_title_text=None,
                    autosize=True,
                    showlegend=False,
                    coloraxis_showscale=False,
                    #xaxis={'tickangle': -90},
                    #width=800,
                    #height=500
                )
                st.plotly_chart(fig, theme="streamlit",use_container_width=True)

        with col2:
            st.markdown('

Fuente del Lead

', unsafe_allow_html=True)
            df          = data[['procedencia']]
            df          = df[df['procedencia'].notnull()]
            df['count'] = 1
            df          = df.groupby('procedencia')['count'].count().reset_index()
            df.columns  = ['procedencia','count']
            df          = df.sort_values(by='count',ascending=False)

            if df.empty is False:
                fig = px.bar(df, x='procedencia', y='count', text='count',color='count',color_continuous_scale=px.colors.sequential.RdBu)
                fig.update_traces(textposition='outside')
                fig.update_layout(
                    plot_bgcolor='rgba(0, 0, 0, 0)',
                    paper_bgcolor='rgba(0, 0, 0, 0)',
                    xaxis_title='',
                    yaxis_title='',
                    legend_title_text=None,
                    autosize=True,
                    showlegend=False,
                    coloraxis_showscale=False,
                    #xaxis={'tickangle': -90},
                    #width=800,
                    #height=500
                )
                st.plotly_chart(fig, theme="streamlit",use_container_width=True)

        with col3:
            st.markdown('

Visitas realizadas

', unsafe_allow_html=True)
            df          = data[['fecha_asignacion_visitagent','finalizado','id_inmueble']]
            df          = df[df['finalizado'].str.lower()=='finalizado'].drop_duplicates(subset='id_inmueble',keep='first')
            df['count'] = 1
            df          = df[['fecha_asignacion_visitagent','count']]
            df.columns  = ['fecha','count']
            df          = df.set_index(['fecha'])
            g           = df[['count']].groupby(pd.Grouper(freq='M'))
            g           = pd.concat([g.sum()], axis=1).reset_index()
            g['fecha']  = g['fecha'].apply(lambda x: x.strftime("%b-%y"))

            if g.empty is False:
                fig = px.bar(g, x='fecha', y='count', text='count',  color_discrete_sequence=['#2166ac'])
                fig.update_traces(textposition='outside')
                fig.update_layout(
                    plot_bgcolor='rgba(0, 0, 0, 0)',
                    paper_bgcolor='rgba(0, 0, 0, 0)',
                    xaxis_title='',
                    yaxis_title='',
                    legend_title_text=None,
                    autosize=True,
                    #xaxis={'tickangle': -90},
                    #width=800,
                    #height=500
                )
                st.plotly_chart(fig, theme="streamlit",use_container_width=True)

    if modulo_seguimiento_ofertas:
        st.write('---')
        st.markdown('

Seguimiento de Ofertas Realizadas

', unsafe_allow_html=True)

        col1, col2, col3, col4, col5 = st.columns(5)
        with col1:
            estado_finalizado = len(data[data['finalizado'].str.lower()=='finalizado']['id_inmueble'].unique())
            html              = boxnumberpercentage(estado_finalizado,' ','# Visitas')
            html_struct       = BeautifulSoup(html, 'html.parser')
            st.markdown(html_struct, unsafe_allow_html=True)

        with col2:
            totalofertas       = len(dataseguimientofertas[dataseguimientofertas['estado'].notnull()])
            por_totalofertas   = totalofertas/estado_finalizado
            por_totalofertas   = "{:.1%}".format(por_totalofertas)
            html               = boxnumberpercentage(totalofertas,f'({por_totalofertas})','Total ofertas realizadas')
            html_struct        = BeautifulSoup(html, 'html.parser')
            st.markdown(html_struct, unsafe_allow_html=True)

        with col3:
            rechazadas         = sum(dataseguimientofertas['estado'].str.lower()=='rechazada')
            por_rechazadas     = rechazadas/totalofertas
            por_rechazadas     = "{:.1%}".format(por_rechazadas)
            html               = boxnumberpercentage(rechazadas,f'({por_rechazadas})','Ofertas rechazadas')
            html_struct        = BeautifulSoup(html, 'html.parser')
            st.markdown(html_struct, unsafe_allow_html=True)

        with col4:
            aceptadas          = sum(dataseguimientofertas['estado'].str.lower()=='comprado')
            por_aceptadas      = aceptadas/totalofertas
            por_aceptadas      = "{:.1%}".format(por_aceptadas)
            html               = boxnumberpercentage(aceptadas,f'({por_aceptadas})','Ofertas aceptadas')
            html_struct        = BeautifulSoup(html, 'html.parser')
            st.markdown(html_struct, unsafe_allow_html=True)

        with col5:
            pendientes          = sum(dataseguimientofertas['estado'].str.lower().str.contains('pendiente'))
            por_pendientes      = pendientes/totalofertas
            por_pendientes      = "{:.1%}".format(por_pendientes)
            html               = boxnumberpercentage(pendientes,f'({por_pendientes})','Pendientes de respuesta')
            html_struct        = BeautifulSoup(html, 'html.parser')
            st.markdown(html_struct, unsafe_allow_html=True)

        col1, col2, col3 = st.columns(3)
        with col1:
            st.markdown('

Estado de las ofertas

', unsafe_allow_html=True)
            df          = dataseguimientofertas[['fecha_oferta','estado']]
            df['count'] = 1
            df.columns  = ['fecha','estado','count']
            df          = df.set_index(['fecha'])
            g           = df.groupby([pd.Grouper(freq='M'),'estado'])
            g           = pd.concat([g.sum()], axis=1).reset_index()
            g           = g[g['count']>0]
            g['fecha']  = g['fecha'].apply(lambda x: x.strftime("%b-%y"))

            if g.empty is False:
                fig = px.bar(g, x='fecha', y='count', text='count', barmode='group', color='estado')
                fig.update_traces(textposition='outside')
                fig.update_layout(
                    plot_bgcolor='rgba(0, 0, 0, 0)',
                    paper_bgcolor='rgba(0, 0, 0, 0)',
                    xaxis_title='',
                    yaxis_title='',
                    legend_title_text=None,
                    autosize=True,
                    legend=dict(
                        orientation="h",
                        entrywidth=70,
                        yanchor="bottom",
                        y=1.02,
                        xanchor="right",
                        x=1),
                    #xaxis={'tickangle': -90},
                    #width=800,
                    #height=500
                )
                st.plotly_chart(fig, theme="streamlit",use_container_width=True)

        with col2:
            st.markdown('

Asesor

', unsafe_allow_html=True)
            df          = dataseguimientofertas[['asesor']]
            df          = df[df['asesor'].notnull()]
            df['count'] = 1
            df          = df.groupby('asesor')['count'].count().reset_index()
            df.columns  = ['asesor','count']
            df          = df.sort_values(by='count',ascending=False)

            if df.empty is False:
                fig = px.bar(df, x='asesor', y='count', text='count', color='count',color_continuous_scale=px.colors.sequential.RdBu)
                fig.update_traces(textposition='outside')
                fig.update_layout(
                    plot_bgcolor='rgba(0, 0, 0, 0)',
                    paper_bgcolor='rgba(0, 0, 0, 0)',
                    xaxis_title='',
                    yaxis_title='',
                    legend_title_text=None,
                    showlegend=False,
                    coloraxis_showscale=False,
                    #width=800,
                    #height=500
                )
                st.plotly_chart(fig, theme="streamlit",use_container_width=True)

        with col3:
            st.markdown('

Data de Ofertas Pendientes

', unsafe_allow_html=True)
            datapaso = dataseguimientofertas[dataseguimientofertas['estado'].str.lower().str.contains('pendiente')][['id', 'id_inmueble', 'asesor', 'direccion', 'nombre_conjunto', 'areaconstruida', 'habitaciones', 'banos', 'garajes', 'piso', 'depositos', 'url', 'valoradministracion', 'precio_publicacion', 'precio_maximo', 'precio_oferta', 'fecha_oferta', 'precio_compra', 'observaciones', 'estado']]
            for i in ['areaconstruida', 'habitaciones', 'banos', 'garajes', 'piso', 'depositos']:
                datapaso[i] = pd.to_numeric(datapaso[i],errors='coerce')
                idd = datapaso[i]>=0
                if sum(idd)>0:
                    datapaso.loc[idd,i] = datapaso.loc[idd,i].astype(int).astype(str)
            for i in ['valoradministracion', 'precio_publicacion', 'precio_maximo', 'precio_oferta', 'fecha_oferta', 'precio_compra']:
                datapaso[i] = pd.to_numeric(datapaso[i],errors='coerce')
                idd = datapaso[i]>=0
                if sum(idd)>0:
                    datapaso.loc[idd,i] = datapaso.loc[idd,i].apply(lambda x: f"${x:,.0f}")
            datapaso.index = range(len(datapaso))
            st.dataframe(datapaso)


#!/usr/bin/python3
import gog_extractor as gog
import steamprices_extractor as prices

import csv
import logging
import argparse
import time

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Export stats from GOG Galaxy 2.0 to csv file.')
    parser.add_argument('-d', '--database', help='Path to GOG Galaxy 2.0 database.', default='galaxy-2.0.db')
    parser.add_argument('-o', '--output', help='Path to output csv file.', default='output.csv')
    parser.add_argument(
        '-p',
        '--proxied',
        help=
        'Using proxies to scrape websites faster. Use at your own risk! Without this setting the robots.txt is used to configure the delay between requests.',
        nargs='?',
        const=True,
        default=False)
    parser.add_argument(
        '--skip-prices',
        help=
        'Skips the annotations of price data from steamprices.com. Keep in mind that further analysis might not work because of the missing fields.',
        nargs='?',
        const=True,
        default=False)
    parser.add_argument('-l',
                        '--logging',
                        help='Defines log level.',
                        default='INFO',
                        choices=['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET'])

    args = parser.parse_args()
    logging.basicConfig(level=args.logging, format='%(asctime)s %(levelname)-8s [%(name)s] %(message)s')
    logger = logging.getLogger('main')
    logger.debug('Provided command line args: %s', args)

    start = time.time()
    games = gog.extract(args)
    done = time.time()
    logger.debug('Elapsed time: %.2fs', done - start)

    if not args.skip_prices:
        start = time.time()
        games = prices.extract(args, games)
        done = time.time()
        logger.debug('Elapsed time: %.2fs', done - start)

    logger.info("Writing results to file...")
    headers = set()
    for value in games.values():
        headers.update(value.keys())
    headers = sorted(headers)

    with open(args.output, 'w', encoding='utf8') as f:
        writer = csv.DictWriter(f, fieldnames=headers)
        writer.writeheader()
        for game in games.values():
            writer.writerow(game)
    logger.info("Exporting finished!")


import os
import sys
import shlex
import subprocess
import gettext

import config

from ovirt_engine import configfile
from ovirt_engine import service
from ovirt_engine import util
from ovirt_engine import java

def _(m):
    return gettext.dgettext(message=m, domain='ovirt-engine-dwh')

class Daemon(service.Daemon):

    def __init__(self):
        super(Daemon, self).__init__()
        self._tempDir = None
        self._defaults = os.path.abspath(
            os.path.join(
                os.path.dirname(sys.argv[0]),
                'ovirt-engine-dwhd.conf',
            )
        )

    def _getClasspath(self):
        p = subprocess.Popen(
            args=(
                os.path.join(
                    self._config.get('PKG_DATA_DIR'),
                    'bin',
                    'dwh-classpath.sh',
                ),
                'run',
            ),
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            close_fds=True,
        )
        stdout, stderr = p.communicate()
        stdout = stdout.decode('utf-8', 'replace').splitlines()
        stderr = stderr.decode('utf-8', 'replace').splitlines()
        if p.returncode != 0:
            raise RuntimeError(_('Cannot setup classpath (%s)') % stderr)
        classpath = stdout[0]
        self.logger.debug('classpath: %s', classpath)
        return classpath

    def _checkInstallation(
        self,
        pidfile,
    ):
        # Check the required directories and files:
        self.check(
            os.path.join(
                self._config.get('PKG_DATA_DIR'),
                'services',
            ),
            directory=True,
        )
        self.check(
            self._config.get('PKG_LOG_DIR'),
            directory=True,
            writable=True,
        )
        self.check(
            os.path.join(
                self._config.get('PKG_JAVA_LIB'),
                'historyETL.jar',
            ),
        )
        for log in ('ovirt-engine-dwhd.log',):
            self.check(
                name=os.path.join(
                    self._config.get('PKG_LOG_DIR'),
                    log,
                ),
                mustExist=False,
                writable=True,
            )
        if pidfile is not None:
            self.check(
                name=pidfile,
                writable=True,
                mustExist=False,
            )

    def daemonSetup(self):

        if os.geteuid() == 0:
            raise RuntimeError(
                _('This service cannot be executed as root')
            )

        if not os.path.exists(self._defaults):
            raise RuntimeError(
                _(
                    "The configuration defaults file '{file}' "
                    "required but missing"
                ).format(
                    file=self._defaults,
                )
            )

        self._config = configfile.ConfigFile(
            (
                self._defaults,
                config.DWH_VARS,
            ),
        )

        #
        # the earliest so we can abort early.
        #
        self._executable = os.path.join(
            java.Java().getJavaHome(),
            'bin',
            'java',
        )

        self._checkInstallation(
            pidfile=self.pidfile,
        )

        self._tempDir = service.TempDir()
        self._tempDir.create()

        settings = os.path.join(self._tempDir.directory, 'settings.properties')
        with open(settings, 'w') as f:
            f.write(
                util.processTemplate(
                    os.path.join(
                        self._config.get('PKG_DATA_DIR'),
                        'conf',
                        'settings.properties.in'
                    ),
                    dict(
                        ('@%s@' % k, util.escape(v, ':=\\ ')) for (k, v) in
                        self._config.values.items()
                    ),
                )
            )

        self._serviceArgs = [
            'ovirt-engine-dwhd',
            '-Dorg.ovirt.engine.dwh.settings=%s' % settings,
        ]

        # Add arguments for the java heap size:
        self._serviceArgs.extend([
            '-Xms%s' % self._config.get('DWH_HEAP_MIN'),
            '-Xmx%s' % self._config.get('DWH_HEAP_MAX'),
        ])

        for engineProperty in shlex.split(
            self._config.get('DWH_PROPERTIES')
        ):
            if not engineProperty.startswith('-D'):
                engineProperty = '-D' + engineProperty
            self._serviceArgs.append(engineProperty)

        for arg in shlex.split(self._config.get('DWH_JVM_ARGS')):
            self._serviceArgs.append(arg)

        engineDebugAddress = self._config.get('DWH_DEBUG_ADDRESS')
        if engineDebugAddress:
            self._serviceArgs.append(
                (
                    '-Xrunjdwp:transport=dt_socket,address=%s,'
                    'server=y,suspend=n'
                ) % (
                    engineDebugAddress
                )
            )

        if self._config.getboolean('DWH_VERBOSE_GC'):
            self._serviceArgs.extend([
                '-verbose:gc',
                '-XX:+PrintGCTimeStamps',
                '-XX:+PrintGCDetails',
            ])

        self._serviceArgs.extend([
            '-classpath', '%s:%s' % (
                os.path.join(
                    self._config.get('PKG_JAVA_LIB'),
                    '*',
                ),
                self._getClasspath(),
            ),
            'ovirt_engine_dwh.historyetl_4_5.HistoryETL',
            '--context=Default',
        ])

        self._serviceEnv = os.environ.copy()
        self._serviceEnv.update({
            'PATH': (
                '/usr/local/sbin:/usr/local/bin:'
                '/usr/sbin:/usr/bin:/sbin:/bin'
            ),
            'LANG': 'en_US.UTF-8',
            'LC_ALL': 'en_US.UTF-8',
        })

    def daemonStdHandles(self):
        consoleLog = open(
            os.path.join(
                self._config.get('PKG_LOG_DIR'),
                'ovirt-engine-dwhd.log'
            ),
            'a+',
        )
        return (consoleLog, consoleLog)

    def daemonContext(self):
        self.daemonAsExternalProcess(
            executable=self._executable,
            args=self._serviceArgs,
            env=self._serviceEnv,
            stopTime=self._config.getinteger(
                'DAEMON_STOP_TIME'
            ),
            stopInterval=self._config.getinteger(
                'DAEMON_STOP_INTERVAL'
            ),
        )

    def daemonCleanup(self):
        if self._tempDir:
            self._tempDir.destroy()

if __name__ == '__main__':
    service.setupLogger()
    d = Daemon()
    d.run()

# vim: expandtab tabstop=4 shiftwidth=4


import argparse
import numpy as np
from pathlib import Path
import cv2
from model import get_model
from noise_model import get_noise_model
import os
from PIL import Image
import pytesseract
pytesseract.pytesseract.tesseract_cmd = r'C:/Program Files/Tesseract-OCR/tesseract'

def get_args(): #콘솔로부터 인자를 받아오는 함수
    parser = argparse.ArgumentParser(description="Test trained model",
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--image_dir", type=str, required=True,
                        help="test image dir")
    parser.add_argument("--model", type=str, default="srresnet",
                        help="model architecture ('srresnet' or 'unet')")
    parser.add_argument("--weight_file", type=str, required=True,
                        help="trained weight file")
    parser.add_argument("--test_noise_model", type=str, default="gaussian,25,25",
                        help="noise model for test images")
    parser.add_argument("--output_dir", type=str, default=None,
                        help="if set, save resulting images otherwise show result using imshow")
    args = parser.parse_args()
    return args

def get_image(image):   #이미지를 받아오는 함수
    image = np.clip(image, 0, 255)
    return image.astype(dtype=np.uint8)

def main():
    # get_args() 함수로부터 인자를 받아옴
    args = get_args()
    image_dir = args.image_dir
    weight_file = args.weight_file
    val_noise_model = get_noise_model(args.test_noise_model)
    model = get_model(args.model)
    model.load_weights(weight_file)

    if args.output_dir: #out_path를 입력받았다면
        output_dir = Path(args.output_dir)
        output_dir.mkdir(parents=True, exist_ok=True)

    image_paths = list(Path(image_dir).glob("*.*"))

    for image_path in image_paths:  #image_path 내에 있는 모든 이미지 파일을 순차적으로 작업함
        image = cv2.imread(str(image_path))
        h, w, _ = image.shape
        image = image[:(h // 16) * 16, :(w // 16) * 16]
        h, w, _ = image.shape

        out_image = np.zeros((h, w * 3, 3), dtype=np.uint8)
        noise_image = val_noise_model(image)
        pred = model.predict(np.expand_dims(noise_image, 0))
        denoised_image = get_image(pred[0])

        #out_image에 imgae, noise_image, denoised_image를 넣음.(사용하지 않음)
        execution_path = os.getcwd()
        out_image[:, :w] = image
        out_image[:, w:w * 2] = noise_image
        out_image[:, w * 2:] = denoised_image
        #cv2.imwrite("denoised.jpg", out_image)
        cv2.imwrite("denoised.jpg", denoised_image) #denoised된 이미지를 기본경로에 저장
        #img = cv2.imread(execution_path, "denoised.jpg")
        #경로 내에 존재하는 denoised.jpg파일을 tesseract로 불러옴
        data = pytesseract.image_to_string(Image.open('denoised.jpg'), lang='eng')
        #불러온 image 내의 문자열을 data 변수에 string으로 입력
        f = open("result.txt", 'w', encoding='UTF8')
        f.write(data)   #result.txt파일 안에 해당 string을 저장하고 기본 경로에 저장.
        f.close()

        if args.output_dir: #output_path를 입력받았다면 그 경로에 저장
            cv2.imwrite("denoised.png", out_image)
        #else:
            #cv2.imshow("result", out_image)

if __name__ == '__main__':
    main()


from requests import get
from requests.exceptions import RequestException
from contextlib import closing
from bs4 import BeautifulSoup

import bs4

import Algorithmia

client = Algorithmia.client("simWYZZvrLoNQu0jhCwHWQvuz1J1")

def simple_get(url):
    try:
        with closing(get(url, stream=True)) as resp:
            if is_good_response(resp):
                return resp.content
            else:
                return None

    except RequestException as e:
        log_error('Error during requests to {0} : {1}'.format(url, str(e)))
        return None

def get_links():
    """Gets links from URL"""
    input = "https://twitter.com/rashtrapatibhvn/status/1166247247027113985"
    if input.startswith("http") or input.startswith("https"):
        algo = client.algo('web/GetLinks/0.1.5')
        links = algo.pipe(input).result
        return links
    else:
        print("Please enter a properly formed URL")

def get_content():
    url = 'http://www.fabpedigree.com/james/mathmen.htm'
    response = simple_get(url)

    if response is not None:
        html = BeautifulSoup(response, 'html.parser')
        names = set()
        for li in html.select('li'):
            for name in li.text.split('\n'):
                if len(name) > 0:
                    names.add(name.strip())
        return list(names)