#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Cleanup script to make transparency work for inky hats.

Attribution:
    This script is courtesy of user mikeyp on pimoroni forums.

Notes:
    Fixes issues with GIMP's transparency export by using a green screen method.

    Any pixels that should be transparent are first painted green using
    the fourth color of the colormap.

    Once the image is saved in the assets directory this script can be run to clean
    any images of greenscreen, leaving behind transparency.

"""

from PIL import Image  # type: ignore
import glob
import re
import png  # type: ignore

for sprite in glob.glob("assets/sprites/*.png"):
    pre_chunk_list = list(png.Reader(sprite).chunks())
    img = Image.open(sprite)
    img.save(sprite, transparency=3, optimize=1)
    post_chunk_list = list(png.Reader(sprite).chunks())
    for chunk in pre_chunk_list:
        if re.compile(b"..Xt").match(chunk[0]):
            post_chunk_list.insert(2, chunk)
    with open(sprite, "wb") as out_sprite:
        png.write_chunks(out_sprite, post_chunk_list)

for ui_piece in glob.glob("assets/ui/*.png"):
    pre_chunk_list = list(png.Reader(ui_piece).chunks())
    img = Image.open(ui_piece)
    img.save(ui_piece, transparency=3, optimize=1)
    post_chunk_list = list(png.Reader(ui_piece).chunks())
    for chunk in pre_chunk_list:
        if re.compile(b"..Xt").match(chunk[0]):
            post_chunk_list.insert(2, chunk)
    with open(ui_piece, "wb") as out_ui_piece:
        png.write_chunks(out_ui_piece, post_chunk_list)


#!/usr/bin/python
# -*- coding: utf-8 -*-

import numpy as np
import sys
import os

from matplotlib import pyplot as plt
import matplotlib.cm as cm
import matplotlib.patheffects as PathEffects

import netCDF4 as nc
import cartopy.crs as ccrs
import pandas as pd

from numpy import linalg as LA
from eofs.standard import Eof
from scipy import stats
from scipy import interpolate as itrp
import itertools as itt

from sklearn.cluster import KMeans

from datetime import datetime
import pickle

import climtools_lib as ctl
import climdiags as cd

from copy import deepcopy as cp

##############################################
### constants
# cp = 4185.5 # J kg-1 K-1 QUESTA E L'ACQUA COJOOOO
cp = 1005.0 # specific enthalpy dry air - J kg-1 K-1
cpw = 1840.0 # specific enthalpy water vapor - J kg-1 K-1
# per la moist air sarebbe cpm = cp + q*cpw, ma conta al massimo per un 3 %
L = 2501000.0 # J kg-1
Lsub = 2835000.0 # J kg-1
g = 9.81 # m s-2
Rearth = 6371.0e3 # mean radius
#####################################

#plt.ion()
print('TEST run on February 1988\n\n')

cart_in = '/data-hobbes/fabiano/SPHINX/heat_flux/1988_daily/'

cart_out = '/home/fabiano/Research/lavori/SPHINX_for_lisboa/heat_flux/'
figure_file = cart_out+'heat_flux_calc_vs_ERAref.pdf'

figure_file2 = cart_out+'hfc_ERAref_recalc.pdf'
figures = []

figure_file_cross = cart_out+'hfc_crosssections_ERAref.pdf'
figures_cross = []

fluxes = dict()
fluxnames = ['mshf', 'mpef', 'mlhf']
fluxlongnames = ['Sensible Heat', 'Potential Energy', 'Latent Heat']
factors = [cp/g, 1., L/g]

vars = dict()
varnames = ['hus', 'ta', 'va', 'zg']
fils = ['lcs0_day_1988_{}.nc'.format(varna) for varna in varnames]

for varna, fi in zip(varnames, fils):
    var, level, lat, lon, dates, time_units, var_units, time_cal = ctl.read4Dncfield(cart_in+fi)
    var, okda = ctl.sel_season(var, dates, 'Feb')
    vars[varna] = var

press0, latdad, londsad, datespress, time_units, var_units = ctl.read3Dncfield(cart_in+'lcs0_day_1988_ps.nc')
press0, _ = ctl.sel_season(press0, datespress, 'Feb')
press0 = np.mean(press0, axis = 0)

mshf = factors[0]*vars['va']*vars['ta']
mpef = vars['va']*vars['zg']
mlhf = factors[2]*vars['va']*vars['hus']

mshf = np.mean(mshf, axis = 0)
mpef = np.mean(mpef, axis = 0)
mlhf = np.mean(mlhf, axis = 0)

mshfint = np.zeros(mshf.shape[1:])
mpefint = np.zeros(mpef.shape[1:])
mlhfint = np.zeros(mlhf.shape[1:])
print(mshf.shape)

# for ila in range(len(lat)):
#     for ilo in range(len(lon)):
#         print(lat[ila], lon[ilo])
#         p0 = press0[ila, ilo]
#         exclude_pres = level > p0
#         if np.any(exclude_pres):
#             lev0 = np.argmax(exclude_pres)
#         else:
#             lev0 = None
#         levels_ok = np.append(level[:lev0], p0)
#
#         if not np.all(np.diff(levels_ok) > 0):
#             print(levels_ok)
#             raise ValueError('x not increasing')
#
#         coso = mshf[:, ila, ilo]
#         coso = np.append(coso[:lev0], np.interp(p0, level, coso))
#         mshfint[ila, ilo] = np.trapz(coso, x = levels_ok)
#         print(levels_ok, coso.mean(), coso[-1], mshfint[ila, ilo])
#
#         coso = mpef[:, ila, ilo]
#         coso = np.append(coso[:lev0], np.interp(p0, level, coso))
#         mpefint[ila, ilo] = np.trapz(coso, x = levels_ok)
#
#         coso = mlhf[:, ila, ilo]
#         coso = np.append(coso[:lev0], np.interp(p0, level, coso))
#         mlhfint[ila, ilo] = np.trapz(coso, x = levels_ok)
#
# ctl.save2Dncfield(lat,lon,mshfint,'mshf',cart_out+'mshf.nc')
# ctl.save2Dncfield(lat,lon,mpefint,'mpef',cart_out+'mpef.nc')
# ctl.save2Dncfield(lat,lon,mlhfint,'mlhf',cart_out+'mlhf.nc')

zonal_factor = 2*np.pi*Rearth*np.cos(np.deg2rad(lat))
mshfcross = np.mean(mshf, axis = -1)*zonal_factor
mpefcross = np.mean(mpef, axis = -1)*zonal_factor
mlhfcross = np.mean(mlhf, axis = -1)*zonal_factor

figures_cross.append(ctl.plot_lat_crosssection(mshfcross, lat, level, title = 'SPHINX - SH cross section', cbar_range = (-3.e12, 3.e12), plot_anomalies = True))
figures_cross.append(ctl.plot_lat_crosssection(mpefcross, lat, level, title = 'SPHINX - PE cross section', cbar_range = (-1.e12, 1.e12), plot_anomalies = True))
figures_cross.append(ctl.plot_lat_crosssection(mpefcross, lat, level, title = 'SPHINX - PE cross section', cbar_range = (-1.e12, 1.e12), plot_anomalies = True, set_logscale_levels = True))
figures_cross.append(ctl.plot_lat_crosssection(mlhfcross, lat, level, title = 'SPHINX - LH cross section', cbar_range = (-1.e11, 1.e11), plot_anomalies = True))

# mshfzon = np.mean(mshfint, axis = 1)*zonal_factor
# mpefzon = np.mean(mpefint, axis = 1)*zonal_factor
# mlhfzon = np.mean(mlhfint, axis = 1)*zonal_factor
#
# fig = plt.figure()
# plt.title('Mycoso')
# plt.plot(lat, mshfzon, label = 'SH')
# plt.plot(lat, mpefzon, label = 'PE')
# plt.plot(lat, mlhfzon, label = 'LH')
# plt.plot(lat, mshfzon+mpefzon+mlhfzon, label = 'Total')
# plt.legend()
# figures.append(fig)

pinopress = itrp.RectBivariateSpline(lat, lon, press0)

cart_in = '/data-hobbes/fabiano/OBS/ERA/ERAInterim/'

#leggo gia fatto
fi = 'prova_heatflux_1988_MM.nc'
cpc = nc.Dataset(cart_in+fi)
era_lat = cpc.variables['latitude'][:]
era_lon = cpc.variables['longitude'][:]
era_zonal_factor = 2*np.pi*Rearth*np.cos(np.deg2rad(era_lat))
era_tot_Feb = list(cpc.variables.values())[-1][1, ...]
era_mpef_Feb = list(cpc.variables.values())[-2][1, ...]
era_mshf_Feb = list(cpc.variables.values())[-3][1, ...]

fi = 'northward_water_flux.nc'
cpc = nc.Dataset(cart_in+fi)
era_mlhf_Feb = L*list(cpc.variables.values())[-1][1, ...]

era_tot_Febzon = np.mean(era_tot_Feb, axis = 1)*era_zonal_factor
era_mshf_Febzon = np.mean(era_mshf_Feb, axis = 1)*era_zonal_factor
era_mpef_Febzon = np.mean(era_mpef_Feb, axis = 1)*era_zonal_factor
era_mlhf_Febzon = np.mean(era_mlhf_Feb, axis = 1)*era_zonal_factor

# fig = plt.figure()
# plt.title('ERA ref - February 1988')
# plt.plot(era_lat, era_mshf_Febzon, label = 'SH')
# plt.plot(era_lat, era_mpef_Febzon, label = 'PE')
# plt.plot(era_lat, era_mlhf_Febzon, label = 'LH')
# plt.plot(era_lat, era_mlhf_Febzon+era_mshf_Febzon+era_mpef_Febzon, label = 'Sum')
# plt.plot(era_lat, era_tot_Febzon, label = 'Total')
# plt.legend()
# figures.append(fig)

# calcolo il mio, senza correzione per la surface pressure
cpc = nc.Dataset(cart_in+'all_vtgq_1988_daily.nc')
era_levels = 100.0*cpc.variables['level'][:]
era_zonal_factor = 2*np.pi*Rearth*np.cos(np.deg2rad(era_lat))
v = list(cpc.variables.values())[-1][:]
q = list(cpc.variables.values())[-2][:]
t = list(cpc.variables.values())[-3][:]
z = list(cpc.variables.values())[-4][:]/g

mshf = factors[0]*v*t
mpef = v*z
mlhf = factors[2]*v*q
mshf = np.mean(mshf, axis = 0)
mpef = np.mean(mpef, axis = 0)
mlhf = np.mean(mlhf, axis = 0)

mshfint1 = np.trapz(mshf, x = era_levels, axis = 0).squeeze()
mpefint1 = np.trapz(mpef, x = era_levels, axis = 0).squeeze()
mlhfint1 = np.trapz(mlhf, x = era_levels, axis = 0).squeeze()

# ctl.save2Dncfield(lat,lon,mshfint,'mshf',cart_out+'era_mshf_Feb.nc')
# ctl.save2Dncfield(lat,lon,mpefint,'mpef',cart_out+'era_mpef_Feb.nc')
# ctl.save2Dncfield(lat,lon,mlhfint,'mlhf',cart_out+'era_mlhf_Feb.nc')

mshfzon1 = np.mean(mshfint1, axis = 1)*era_zonal_factor
mpefzon1 = np.mean(mpefint1, axis = 1)*era_zonal_factor
mlhfzon1 = np.mean(mlhfint1, axis = 1)*era_zonal_factor

# plt.figure()
# plt.title('ERA calc - February 1988 - daily')
# plt.plot(era_lat, mshfzon1, label = 'SH')
# plt.plot(era_lat, mpefzon1, label = 'PE')
# plt.plot(era_lat, mlhfzon1, label = 'LH')
# plt.plot(era_lat, mshfzon1+mpefzon1+mlhfzon1, label = 'Total')
# plt.legend()

# calcolo il mio, senza correzione per la surface pressure
cpc = nc.Dataset(cart_in+'all_vtgq_1988_6hrs.nc')
era_levels = 100.0*cpc.variables['level'][:]
era_zonal_factor = 2*np.pi*Rearth*np.cos(np.deg2rad(era_lat))
v = list(cpc.variables.values())[-1][:]
q = list(cpc.variables.values())[-2][:]
t = list(cpc.variables.values())[-3][:]
z = list(cpc.variables.values())[-4][:]/g

mshf00 = factors[0]*v*t
mpef00 = v*z
mlhf00 = factors[2]*v*q
mshf = np.mean(mshf00, axis = 0)
mpef = np.mean(mpef00, axis = 0)
mlhf = np.mean(mlhf00, axis = 0)

mshfint2 = np.trapz(mshf, x = era_levels, axis = 0).squeeze()
mpefint2 = np.trapz(mpef, x = era_levels, axis = 0).squeeze()
mlhfint2 = np.trapz(mlhf, x = era_levels, axis = 0).squeeze()

# ctl.save2Dncfield(lat,lon,mshfint,'mshf',cart_out+'era_mshf_Feb.nc')
# ctl.save2Dncfield(lat,lon,mpefint,'mpef',cart_out+'era_mpef_Feb.nc')
# ctl.save2Dncfield(lat,lon,mlhfint,'mlhf',cart_out+'era_mlhf_Feb.nc')

mshfzon2 = np.mean(mshfint2, axis = 1)*era_zonal_factor
mpefzon2 = np.mean(mpefint2, axis = 1)*era_zonal_factor
mlhfzon2 = np.mean(mlhfint2, axis = 1)*era_zonal_factor

mshfcross_era = np.mean(mshf, axis = -1)*era_zonal_factor
mpefcross_era = np.mean(mpef, axis = -1)*era_zonal_factor
mlhfcross_era = np.mean(mlhf, axis = -1)*era_zonal_factor

figures_cross.append(ctl.plot_lat_crosssection(mshfcross_era, era_lat, era_levels, title = 'Era - SH cross section', cbar_range = (-3.e12, 3.e12), plot_anomalies = True))
figures_cross.append(ctl.plot_lat_crosssection(mpefcross_era, era_lat, era_levels, title = 'Era - PE cross section', cbar_range = (-1.e12, 1.e12), plot_anomalies = True))
figures_cross.append(ctl.plot_lat_crosssection(mpefcross_era, era_lat, era_levels, title = 'Era - PE cross section', cbar_range = (-1.e12, 1.e12), plot_anomalies = True, set_logscale_levels = True))
figures_cross.append(ctl.plot_lat_crosssection(mlhfcross_era, era_lat, era_levels, title = 'Era - LH cross section', cbar_range = (-1.e11, 1.e11), plot_anomalies = True))

pinomshfcross = itrp.RectBivariateSpline(np.log(era_levels), era_lat[::-1], mshfcross_era[:,::-1])
pinompefcross = itrp.RectBivariateSpline(np.log(era_levels), era_lat[::-1], mpefcross_era[:,::-1])
pinomlhfcross = itrp.RectBivariateSpline(np.log(era_levels), era_lat[::-1], mlhfcross_era[:,::-1])

mshfcross_itrp = np.zeros(mshfcross.shape)
mpefcross_itrp = np.zeros(mshfcross.shape)
mlhfcross_itrp = np.zeros(mshfcross.shape)
for ila, la in enumerate(lat):
    for ile, le in enumerate(level):
        mshfcross_itrp[ile,ila] = pinomshfcross(np.log(le), la)
        mpefcross_itrp[ile,ila] = pinompefcross(np.log(le), la)
        mlhfcross_itrp[ile,ila] = pinomlhfcross(np.log(le), la)

figures_cross.append(ctl.plot_lat_crosssection(mshfcross-mshfcross_itrp, lat, level, title = 'SPHINX - Era diff - SH cross section', plot_anomalies = True))
figures_cross.append(ctl.plot_lat_crosssection(mpefcross-mpefcross_itrp, lat, level, title = 'SPHINX - Era - PE cross section', plot_anomalies = True))
figures_cross.append(ctl.plot_lat_crosssection(mpefcross-mpefcross_itrp, lat, level, title = 'SPHINX - Era - PE cross section', plot_anomalies = True, set_logscale_levels = True))
figures_cross.append(ctl.plot_lat_crosssection(mlhfcross-mlhfcross_itrp, lat, level, title = 'SPHINX - Era - LH cross section', plot_anomalies = True))

# Ora calcolo con press0
mshfint2p0 = np.zeros(mshf.shape[1:])
mpefint2p0 = np.zeros(mpef.shape[1:])
mlhfint2p0 = np.zeros(mlhf.shape[1:])

nupress = np.zeros((len(era_lat), len(era_lon)))

for ila in range(len(era_lat)):
    for ilo in range(len(era_lon)):
        print(era_lat[ila], era_lon[ilo])
        #p0 = press0[ila, ilo]
        p0 = pinopress(era_lat[ila], era_lon[ilo])
        nupress[ila,ilo] = p0
        exclude_pres = era_levels > p0
        if np.any(exclude_pres):
            lev0 = np.argmax(exclude_pres)
        else:
            lev0 = None
        levels_ok = np.append(era_levels[:lev0], p0)
        if not np.all(np.diff(levels_ok) > 0):
            print(levels_ok)
            raise ValueError('x not increasing')

        coso = mshf[:, ila, ilo]
        coso = np.append(coso[:lev0], np.interp(p0, era_levels, coso))
        mshfint2p0[ila, ilo] = np.trapz(coso, x = levels_ok)
        #print(levels_ok, coso.mean(), coso[-1], mshfint2p0[ila, ilo])

        coso = mpef[:, ila, ilo]
        coso = np.append(coso[:lev0], np.interp(p0, era_levels, coso))
        mpefint2p0[ila, ilo] = np.trapz(coso, x = levels_ok)

        coso = mlhf[:, ila, ilo]
        coso = np.append(coso[:lev0], np.interp(p0, era_levels, coso))
        mlhfint2p0[ila, ilo] = np.trapz(coso, x = levels_ok)

mshfzon2p0 = np.mean(mshfint2p0, axis = 1)*era_zonal_factor
mpefzon2p0 = np.mean(mpefint2p0, axis = 1)*era_zonal_factor
mlhfzon2p0 = np.mean(mlhfint2p0, axis = 1)*era_zonal_factor

nupress = np.stack([nupress, nupress])
mshfzon2p0_recalc = cd.quant_flux_calc(mshf00, era_lat, era_lon, era_levels, None, nupress, None, seasons = [])['zonal']['year']
mpefzon2p0_recalc = cd.quant_flux_calc(mpef00, era_lat, era_lon, era_levels, None, nupress, None, seasons = [])['zonal']['year']
mlhfzon2p0_recalc = cd.quant_flux_calc(mlhf00, era_lat, era_lon, era_levels, None, nupress, None, seasons = [])['zonal']['year']

# plt.figure()
# plt.title('ERA calc - February 1988 - 6 hrs')
# plt.plot(era_lat, mshfzon2, label = 'SH')
# plt.plot(era_lat, mpefzon2, label = 'PE')
# plt.plot(era_lat, mlhfzon2, label = 'LH')
# plt.plot(era_lat, mshfzon2+mpefzon2+mlhfzon2, label = 'Total')
# plt.legend()

# calcolo il mio, senza correzione per la surface pressure
cpc = nc.Dataset(cart_in+'all_vtgq_1988_daily_morelev.nc')
era_levels = 100.0*cpc.variables['level'][:]
era_zonal_factor = 2*np.pi*Rearth*np.cos(np.deg2rad(era_lat))
v = list(cpc.variables.values())[-1][:]
q = list(cpc.variables.values())[-2][:]
t = list(cpc.variables.values())[-3][:]
z = list(cpc.variables.values())[-4][:]/g

mshf = factors[0]*v*t
mpef = v*z
mlhf = factors[2]*v*q
mshf = np.mean(mshf, axis = 0)
mpef = np.mean(mpef, axis = 0)
mlhf = np.mean(mlhf, axis = 0)

mshfint3 = np.trapz(mshf, x = era_levels, axis = 0).squeeze()
mpefint3 = np.trapz(mpef, x = era_levels, axis = 0).squeeze()
mlhfint3 = np.trapz(mlhf, x = era_levels, axis = 0).squeeze()

# ctl.save2Dncfield(lat,lon,mshfint,'mshf',cart_out+'era_mshf_Feb.nc')
# ctl.save2Dncfield(lat,lon,mpefint,'mpef',cart_out+'era_mpef_Feb.nc')
# ctl.save2Dncfield(lat,lon,mlhfint,'mlhf',cart_out+'era_mlhf_Feb.nc')

mshfzon3 = np.mean(mshfint3, axis = 1)*era_zonal_factor
mpefzon3 = np.mean(mpefint3, axis = 1)*era_zonal_factor
mlhfzon3 = np.mean(mlhfint3, axis = 1)*era_zonal_factor

# plt.figure()
# plt.title('ERA calc - February 1988 - daily more levels')
# plt.plot(era_lat, mshfzon3, label = 'SH')
# plt.plot(era_lat, mpefzon3, label = 'PE')
# plt.plot(era_lat, mlhfzon3, label = 'LH')
# plt.plot(era_lat, mshfzon3+mpefzon3+mlhfzon3, label = 'Total')
# plt.legend()

fig = plt.figure()
plt.title('ERA calc - February 1988 - SH diffs')
#plt.plot(era_lat, mshfzon1, label = 'SH - daily')
plt.plot(era_lat, mshfzon2, label = 'SH - 6hrs')
plt.plot(era_lat, mshfzon2p0, label = 'SH - 6hrs - int_p0')
plt.plot(era_lat, mshfzon2p0_recalc, label = 'SH - hfc recalc')
#plt.plot(era_lat, mshfzon3, label = 'SH - daily more lev')
#plt.plot(lat, mshfzon, label = 'SH - sphinx daily')
plt.plot(era_lat, era_mshf_Febzon, label = 'SH era ref', linestyle = ':')
plt.legend()
figures.append(fig)

fig = plt.figure()
plt.title('ERA calc - February 1988 - PE diffs')
#plt.plot(era_lat, mpefzon1, label = 'PE - daily')
plt.plot(era_lat, mpefzon2, label = 'PE - 6hrs')
plt.plot(era_lat, mpefzon2p0, label = 'PE - 6hrs - int_p0')
plt.plot(era_lat, mpefzon2p0_recalc, label = 'PE - hfc recalc')
#plt.plot(era_lat, mpefzon3, label = 'PE - daily more lev')
#plt.plot(lat, mpefzon, label = 'PE - sphinx daily')
plt.plot(era_lat, era_mpef_Febzon, label = 'PE era ref', linestyle = ':')
plt.legend()
figures.append(fig)

fig = plt.figure()
plt.title('ERA calc - February 1988 - LH diffs')
#plt.plot(era_lat, mlhfzon1, label = 'LH - daily')
plt.plot(era_lat, mlhfzon2, label = 'LH - 6hrs')
plt.plot(era_lat, mlhfzon2p0, label = 'LH - 6hrs - int_p0')
plt.plot(era_lat, mlhfzon2p0_recalc, label = 'LH - hfc recalc')
#plt.plot(era_lat, mlhfzon3, label = 'LH - daily more lev')
#plt.plot(lat, mlhfzon, label = 'LH - sphinx daily')
plt.plot(era_lat, era_mlhf_Febzon, label = 'LH era ref', linestyle = ':')
plt.legend()
figures.append(fig)

# fig = ctl.plot_double_sidebyside(era_mshf_Feb, mshfint, [era_lat, lat], [era_lon, lon], title = 'era_ref vs sphinx', use_different_grids = True)
# figures.append(fig)
# fig = ctl.plot_double_sidebyside(era_mshf_Feb, mshfint2, era_lat, era_lon, title = 'era_ref vs era 6 hrs')
# figures.append(fig)
# fig = ctl.plot_double_sidebyside(mshfint2, mshfint2p0, era_lat, era_lon, title = 'era 6 hrs vs era 6 hrs int p0')
# figures.append(fig)

ctl.plot_pdfpages(figure_file2, figures)
figures_cross = np.array(figures_cross)
figures_cross = figures_cross[[0,4,8,1,5,9,2,6,10,3,7,11]]
ctl.plot_pdfpages(figure_file_cross, figures_cross)


import json
from .ajax import ajax
from .Progress import Progress

class MediaWiki:
	def __init__(self, lang = 'ru'):
		self.__url = 'https://' + lang + '.wikipedia.org/w/api.php'

	def execQuery(self, query, onDone, onFail = (lambda e: None)):
		def onData(data):
			js = json.loads(data.decode('utf-8'))
			if 'error' in js:
				err = js['error']['info']
				onFail(err)
				sublime.error_message(err)
			else:
				onDone(js)

		ajax({
			'url': self.__url,
			'data': query,
			'done': onData,
			'fail': onFail
		})

	def execQueryWithProgress(self, query, msg, onDone):
		pr = Progress(msg)

		def onMyDone(js):
			pr.stop()
			onDone(js)

		def onMyFail(err):
			pr.stop();

		self.execQuery(query, onMyDone, onMyFail)

	def search(self, query, onDone):
		self.execQueryWithProgress({
			'action': 'query',
			'list': 'search',
			'srlimit': 5,
			'srprop': '',
			'srredirects': 1,
			'format': 'json',
			'srsearch': query
		}, 'Searching', lambda js: onDone(list(map(lambda x: x['title'], js['query']['search']))))

	def preview(self, text, onDone):
		self.execQueryWithProgress({
			'action': 'parse',
			'text': text,
			'contentmodel': 'wikitext',
			'format': 'json'
		}, 'Building preview', lambda js: onDone(js['parse']['text']['*']))

	def getInterwiki(self, links, lang, onDone):
		limit = 50
		links = list(set(links)) # uniqify
		result = {}
		pr = Progress('Looking up...')

		def go(index):
			if index >= len(links):
				pr.stop()
				onDone(result)
			else:
				self.execQuery({
					'action': 'query',
					'prop': 'langlinks',
					'lllang': lang,
					'redirects': '',
					'lllimit': limit,
					'titles': '|'.join(links[index : index+limit]),
					'format': 'json',
				}, lambda json: onResult(json['query'], index), lambda err: pr.stop())

		def mapToDic(json, name):
			if (name in json):
				return dict(map(lambda m: (m['to'], m['from']), json[name]))
			else:
				return {}

		def onResult(json, index):
			redirects = dict(mapToDic(json, 'redirects'), **mapToDic(json, 'normalized'))
			for page in json['pages'].values():
				title = page['title']
				if ('langlinks' in page):
					result[redirects.get(title, title)] = page['langlinks'][0]['*']
			go(index + limit)

		go(0)


import requests
import os

#--- https://docs.polygonscan.com/
def get(params: dict):
    x = "&".join([f"{k}={v}" for k,v in params.items()])
    url = f"{os.environ['EndpointURL']}?apikey={os.environ['API_KEY']}&{x}"
    # print(url)

    response = requests.get(url)
    if response.status_code == 200:
        return response.json()
    else:
        error_msg = f"Error fetching data. Status code: {response.status_code}"
        raise requests.exceptions.HTTPError(error_msg, response=response)

# if __name__ == "__main__":
#     params = {
#        "module": "account",
#        "action": "balance"
#     }

#     get(params)


from django.urls import path
from django.views.generic import RedirectView

# views
from .views import (
		PartyDetailView,
		PartyListView,
		PartyCreateView,
		# PartyUpdateView,
		PartyDeleteView,
		FollowingListView,
		StarredListView,
		JoinedListView,
		PartyDuplicateView,
		PartyKickallView,
		PartyKickView,
		PartyCloseView,
		PartyLeaveView,
	)

# app name specifies how we refer to these urls
# for example, the event detail page is "parties:detail"
app_name = 'parties'
# /events routes to this
urlpatterns = [
# also note that <int:> just tells the url what type of input it can accept
	path('', RedirectView.as_view(url='/')), # redirects to home
    path('search/', PartyListView.as_view(), name='list'), # /search/?q=
    path('following/', FollowingListView.as_view(), name='following-list'), # /search/?q=
    path('starred/', StarredListView.as_view(), name='starred-list'),
    path('joined/', JoinedListView.as_view(), name='joined-list'),
    path('<int:pk>', PartyDetailView.as_view(), name='detail'), # /1
    path('create/', PartyCreateView.as_view(), name='create'), # /create
    # path('<int:pk>/update/', PartyUpdateView.as_view(), name='update'), # /1/update
    path('<int:pk>/delete/', PartyDeleteView.as_view(), name='delete'), # /1/delete
    path('<int:pk>/duplicate/', PartyDuplicateView.as_view(), name='duplicate'), # /1/duplicate
    path('<int:pk>/kickall/', PartyKickallView.as_view(), name='kickall'), # /1/duplicate
    path('<int:pk>/kick/<username>', PartyKickView.as_view(), name='kick'),
    path('<int:pk>/close/', PartyCloseView.as_view(), name='close'),
    path('<int:pk>/leavequeue/', PartyLeaveView.as_view(), name='leavequeue'),
]


#    Hitsuki (A telegram bot project)

#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.

#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.

#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <https://www.gnu.org/licenses/>.

import re

from telegram import ParseMode, InlineKeyboardMarkup, InlineKeyboardButton
from telegram.ext import CallbackQueryHandler
from telegram.ext import CommandHandler

from hitsuki import dispatcher
from hitsuki.modules.connection import connected
from hitsuki.modules.helper_funcs.chat_status import user_admin
from hitsuki.modules.sql.locales_sql import switch_to_locale, prev_locale
from hitsuki.modules.tr_engine.list_locale import list_locales
from hitsuki.modules.tr_engine.strings import tld, LANGUAGES

@user_admin
def locale(bot, update, args):
    chat = update.effective_chat
    message = update.effective_message
    if len(args) > 0:
        locale = args[0].lower()
        if locale == 'en-us':
            locale = 'en-US'

        if locale in list_locales:
            if locale in LANGUAGES:
                switch_to_locale(chat.id, locale)
                if chat.type == "private":
                    message.reply_text(
                        tld(chat.id, 'language_switch_success_pm').format(
                            list_locales[locale]))
                else:
                    message.reply_text(
                        tld(chat.id, 'language_switch_success').format(
                            chat.title, list_locales[locale]))
            else:
                text = tld(chat.id, "language_not_supported").format(
                    list_locales[locale])
                text += "\n\n*Currently available languages:*\n"
                for lang in LANGUAGES:
                    locale = list_locales[lang]
                    text += "\n *{}* - `{}`".format(locale, lang)
                message.reply_text(text, parse_mode=ParseMode.MARKDOWN)
        else:
            text = tld(chat.id, "language_code_not_valid")
            text += "\n\n*Currently available languages:*\n"
            for lang in LANGUAGES:
                locale = list_locales[lang]
                text += "\n *{}* - `{}`".format(locale, lang)
            message.reply_text(text, parse_mode=ParseMode.MARKDOWN)
    else:
        LANGUAGE = prev_locale(chat.id)
        if LANGUAGE:
            locale = LANGUAGE.locale_name
            native_lang = list_locales[locale]
            message.reply_text(tld(
                chat.id, "language_current_locale").format(native_lang),
                parse_mode=ParseMode.MARKDOWN)
        else:
            message.reply_text(tld(
                chat.id, "language_current_locale").format("English (US)"),
                parse_mode=ParseMode.MARKDOWN)

@user_admin
def locale_button(bot, update):
    chat = update.effective_chat
    user = update.effective_user
    query = update.callback_query
    lang_match = re.findall(r"en-US|pt", query.data)
    if lang_match:
        if lang_match[0]:
            switch_to_locale(chat.id, lang_match[0])
            query.answer(text=tld(chat.id, 'language_switch_success_pm').
                         format(list_locales[lang_match[0]]))
        else:
            query.answer(text="Error!", show_alert=True)

    try:
        LANGUAGE = prev_locale(chat.id)
        locale = LANGUAGE.locale_name
        curr_lang = list_locales[locale]
    except Exception:
        curr_lang = "English (US)"

    text = tld(chat.id, "language_select_language")
    text += tld(chat.id, "language_user_language").format(curr_lang)

    conn = connected(bot, update, chat, user.id, need_admin=False)

    if conn:
        try:
            chatlng = prev_locale(conn).locale_name
            chatlng = list_locales[chatlng]
            text += tld(chat.id, "language_chat_language").format(chatlng)
        except Exception:
            chatlng = "English (US)"

    text += tld(chat.id, "language_sel_user_lang")

    bot.edit_message_text(
        chat_id=query.message.chat_id,
        message_id=query.message.message_id,
        text=text,
        parse_mode=ParseMode.MARKDOWN,
        reply_markup=InlineKeyboardMarkup([[
            InlineKeyboardButton("English (US) ðŸ‡ºðŸ‡¸",
                                 callback_data="set_lang_en-US"),
            InlineKeyboardButton("Portuguese ðŸ‡§ðŸ‡·",
                                 callback_data="set_lang_pt")
        ]] + [[
            InlineKeyboardButton(f"{tld(chat.id, 'btn_go_back')}",
                                 callback_data="bot_start")
        ]]))

    # query.message.delete()
    bot.answer_callback_query(query.id)

LOCALE_HANDLER = CommandHandler(["set_locale", "locale", "lang", "setlang"],
                                locale,
                                pass_args=True)
locale_handler = CallbackQueryHandler(locale_button, pattern="chng_lang")
set_locale_handler = CallbackQueryHandler(locale_button, pattern=r"set_lang_")

dispatcher.add_handler(LOCALE_HANDLER)
dispatcher.add_handler(locale_handler)
dispatcher.add_handler(set_locale_handler)


import tkinter as tk
from tkinter import filedialog
from modules import mail, logger

def unexpected_error(error):
    error_log = logger.error_logger()
    error_log.exception(error)
    print(f"\n{error}")
    print(f"An unexpected error occurred! Please contact sys.execute@gmail.com\n")
    input("Press Enter to close program...")
    exit(1)

def connection_error(error):
    error_log = logger.error_logger()
    error_log.exception(error)
    print(f"\n{error}! Couldn't connect to SMTP server!\n")
    input('Press Enter to close program...')
    exit(1)

def project_title(title):
    print(80*'=')
    print(title.center(80, '-'))
    print(80*'_' + '\n')

def open_csv():
    root = tk.Tk()
    root.withdraw()
    filepath = filedialog.askopenfilename(title='Select Records',
                                          filetypes=(('CSV (*.csv)', '*.csv'), ))
    root.destroy()

    print(filepath + '\n')
    return filepath

def csv_info(len_records, len_fields):
    print(f"Records: {len_records}\t\tFields: {len_fields}", end='')

def select_email(fields_email):
    length = len(fields_email)
    email_field = None

    if length == 1:
        email_field = fields_email[0]
        print(f"\t\tField of emails: {email_field}\n")
    else:
        print("\n\nEmail addresses detected in:")
        for i in range(length):
            print(f"{i+1}) {fields_email[i]}")

        selected = False
        while not selected:
            try:
                index = int(input(f"Select primary email field(1-{length}): "))
                email_field, selected = fields_email[index - 1], True
                print(f"\n'{email_field}' is selected as primary email field\n")
            except ValueError or IndexError:
                print(f"\nInvalid selection! Please insert an integer from 1 to {length}\n")
            except Exception as err:
                unexpected_error(err)

    return email_field

def open_txt():
    root = tk.Tk()
    root.withdraw()
    filepath = filedialog.askopenfilename(title='Select Contents',
                                          filetypes=(('Text Documents (*.txt)', '*.txt'), ))
    root.destroy()

    print(filepath + '\n')
    return filepath

def show_content(subject, body_template):
    print(f"Subject: {subject}")
    print(body_template)
    print(80*'_')

def invalid_variable(variable):
    print(f"Invalid Variable! %{{{variable}}}% doesn't match with any fields\n")
    input('Press Enter to close program...')
    exit(1)

def variables_matched(len_vars):
    print(f"All variables({len_vars}) matched and replaceable with records\n")

def get_login_info(tried):
    if tried == 0:
        print("Login to your gmail account")
    else:
        print("Please try again!")

    login_info = dict()
    login_info['email'] = input('Email: ')
    login_info['password'] = input('Password: ')
    return login_info

def authentication_error():
    print("\nLogin failed! SMTP authentication error. Possible reasons:")
    print("1) Email/password was incorrect.\n2) Your account doesn't allow access via app.\n")

def connection_lost():
    print("\nConnection lost to server! Trying to reconnect...")
    mail.connect_server()
    print("Reconnected to SMTP server!\n")

def login_success():
    input("\nSuccessfully logged in! Press Enter to continue...\n")

def already_sent(sent, total):
    print(f"\rEmails sent: {sent} out of {total}", end='')

def sent_all(len_records):
    print(f"\nAll emails are sent successfully! (Total: {len_records})\n")
    input("Press Enter to close program...")


from selenium.webdriver.firefox.webdriver import WebDriver

from django.test import LiveServerTestCase

class TitleTest(LiveServerTestCase):

    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.selenium = WebDriver()
        cls.selenium.implicitly_wait(10) # will wait up to 10 seconds for something to happen - code quite possibly running ahead of what browser is doing

    @classmethod
    def tearDownClass(cls):
        cls.selenium.quit()
        super().tearDownClass()

    def test_title_on_homepage(self):
        self.selenium.get(self.live_server_url)
        self.assertIn('Travel Wishlist', self.selenium.title) # think of selenium as the browser in these tests

# to run test: cd to wishlist directory, then 'python manage.py test travel_wishlist.functional_tests'

class AddPlacesTest(LiveServerTestCase):

    fixtures = ['test_places']

    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.selenium = WebDriver()
        cls.selenium.implicitly_wait(10) # will wait up to 10 seconds for something to happen - code quite possibly running ahead of what browser is doing

    @classmethod
    def tearDownClass(cls):
        cls.selenium.quit()
        super().tearDownClass()

    def test_add_new_place(self):

        self.selenium.get(self.live_server_url)
        input_name = self.selenium.find_element_by_id('id_name')
        input_name.send_keys('Denver')

        add_button = self.selenium.find_element_by_id('add-new-place')
        add_button.click()
        # immitates expected browser behavior
        denver = self.selenium.find_element_by_id('place-name-5')
        self.assertEqual('Denver', denver.text)

        self.assertIn('Denver', self.selenium.page_source)
        self.assertIn('New York', self.selenium.page_source)
        self.assertIn('Tokyo', self.selenium.page_source)


import numpy as np
import matplotlib.pyplot as plt
import open3d as o3d
from sklearn.cluster import Birch

pcd = o3d.io.read_point_cloud("Pointclouds/bathroom.ply")
# pcd.estimate_normals(search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.1, max_nn=16), fast_normal_computation=True)
# pcd.paint_uniform_color([0.6, 0.6, 0.6])
# #o3d.visualization.draw_geometries([pcd]) #Works only outside Jupyter/Colab
# # plane_model, inliers = pcd.segment_plane(distance_threshold=0.01,ransac_n=3,num_iterations=1000)
# plane_model, inliers = pcd.segment_plane(distance_threshold=0.02,ransac_n=3,num_iterations=2000)
# [a, b, c, d] = plane_model
# print(f"Plane equation: {a:.2f}x + {b:.2f}y + {c:.2f}z + {d:.2f} = 0")
# inlier_cloud = pcd.select_by_index(inliers)
# outlier_cloud = pcd.select_by_index(inliers, invert=True)
# inlier_cloud.paint_uniform_color([1.0, 0, 0])
# outlier_cloud.paint_uniform_color([0.6, 0.6, 0.6])
# o3d.visualization.draw_geometries([inlier_cloud, outlier_cloud])
# ODSTRANENIE PLANOV
plane_model, inliers = pcd.segment_plane(distance_threshold=0.07, ransac_n=3, num_iterations=30000)
inlier_cloud = pcd.select_by_index(inliers)
outlier_cloud = pcd.select_by_index(inliers, invert=True)
o3d.visualization.draw_geometries([outlier_cloud])

while True:
    plane_model, inliers = outlier_cloud.segment_plane(distance_threshold=0.04, ransac_n=3, num_iterations=30000)
    inlier_cloud = outlier_cloud.select_by_index(inliers)
    outlier_cloud = outlier_cloud.select_by_index(inliers, invert=True)
    o3d.visualization.draw_geometries([outlier_cloud])

    if len(outlier_cloud.points) < 274000:
        break

pcd_without_planes = outlier_cloud
data_points = np.asarray(pcd_without_planes.points)
# Train Birch model on the data points
birch = Birch(threshold=0.8, branching_factor=5, n_clusters=None)
birch.fit(data_points)

# Predikcia zhlukov pre dátové body
labels = birch.predict(data_points)
max_label = labels.max()
colors = plt.get_cmap("tab20")(labels / (max_label if max_label > 0 else 1))
colors[labels < 0] = 0
pcd_without_planes.colors = o3d.utility.Vector3dVector(colors[:, :3])
o3d.visualization.draw_geometries([pcd_without_planes])

# o3d.visualization.draw_geometries([pcd_without_planes])
# -------------------------- SEGMENT
# segment_models, inliers = pcd_without_planes.segment_plane(distance_threshold=0.04, ransac_n=3, num_iterations=3000)
# segments = pcd_without_planes.select_by_index(inliers)

# data_points = np.asarray(segments.points)
# # Inicializujte BIRCH algoritmus
# birch = Birch(threshold=5,branching_factor=50,n_clusters=None)

# # Natrénujte BIRCH model na dátových bodoch
# birch.fit(data_points)

# # Predikcia zhlukov pre dátové body
# labels = birch.predict(data_points)
# max_label = labels.max()
# colors = plt.get_cmap("tab20")(labels / (max_label if max_label > 0 else 1))
# colors[labels < 0] = 0
# pcd_without_planes.colors = o3d.utility.Vector3dVector(colors[:, :3])
# o3d.visualization.draw_geometries([pcd_without_planes])

# labels = np.array(pcd.cluster_dbscan(eps=0.04, min_points=10))
# max_label = labels.max()
# print(f"point cloud has {max_label + 1} clusters")

# colors = plt.get_cmap("tab20")(labels / (max_label if max_label > 0 else 1))
# colors[labels < 0] = 0
# pcd.colors = o3d.utility.Vector3dVector(colors[:, :3])

# segment_models={}
# segments={}
# max_plane_idx=1
# rest=pcd_without_planes
# d_threshold=0.01
# # BIRCH ALGORITMUS
# for i in range(max_plane_idx):
#     colors = plt.get_cmap("tab20")(i)
#     segment_models[i], inliers = rest.segment_plane(distance_threshold=0.04, ransac_n=3, num_iterations=5000)
#     segments[i] = rest.select_by_index(inliers)

#     # Konvertujte dáta na numpy array pre použitie BIRCH algoritmu
#     data_points = np.asarray(segments[i].points)

#     # Inicializujte BIRCH algoritmus
#     birch = Birch(threshold=1,branching_factor=50,n_clusters=None)

#     # Natrénujte BIRCH model na dátových bodoch
#     birch.fit(data_points)

#     # Predikcia zhlukov pre dátové body
#     labels = birch.predict(data_points)

#     candidates = [len(np.where(labels == j)[0]) for j in np.unique(labels)]
#     best_candidate = int(np.unique(labels)[np.argmax(candidates)])
#     print("the best candidate is: ", best_candidate)
#     rest = rest.select_by_index(inliers, invert=True) + segments[i].select_by_index(list(np.where(labels != best_candidate)[0]))
#     segments[i] = segments[i].select_by_index(list(np.where(labels == best_candidate)[0]))
#     segments[i].paint_uniform_color(list(colors[:3]))
#     print("pass", i+1, "/", max_plane_idx, "done.")

# o3d.visualization.draw_geometries([segments[i] for i in range(max_plane_idx)]+[rest])


from django.core import paginator as django_paginator
from django.db import models

from djangae.test import TestCase
from djangae.contrib import sleuth
from djangae.contrib.pagination import (
    paginated_model,
    Paginator,
    PaginationOrderingRequired
)

from .paginator import (
    queryset_identifier,
    _get_marker,
)

@paginated_model(orderings=[
    "first_name",  # single field declared as a string
    ("last_name",),  # single field declared as a tuple
    ("first_name", "last_name"),
    ("first_name", "-last_name"),
    ("created",),
    ("-created",),
    "pk",  # it's possible to order by the model pk
])
class TestUser(models.Model):
    first_name = models.CharField(max_length=200)
    last_name = models.CharField(max_length=200)
    created = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return " ".join([self.first_name, self.last_name])

    class Meta:
        ordering = ("first_name", "last_name")

@paginated_model(orderings=[
    "name",
    "pk",  # it's possible to order by the model pk
])
class SimpleModelWithoutOrdering(models.Model):
    name = models.CharField(max_length=200)

class SimpleModelWithOrdering(SimpleModelWithoutOrdering):
    class Meta:
        ordering = ["name"]

class PaginatedModelTests(TestCase):
    def test_fields_added_correctly(self):
        self.assertIsNotNone(TestUser._meta.get_field("pagination_first_name"))
        self.assertIsNotNone(TestUser._meta.get_field("pagination_last_name"))
        self.assertIsNotNone(TestUser._meta.get_field("pagination_first_name_last_name"))
        self.assertIsNotNone(TestUser._meta.get_field("pagination_first_name_neg_last_name"))
        self.assertIsNotNone(TestUser._meta.get_field("pagination_created"))

    def test_precalculate_field_values(self):
        user = TestUser.objects.create(pk=1, first_name="Luke", last_name="Benstead")

        self.assertEqual(u"Luke\x001", user.pagination_first_name)
        self.assertEqual(u"Benstead\x001", user.pagination_last_name)
        self.assertEqual(u"Luke\x00Benstead\x001", user.pagination_first_name_last_name)
        self.assertEqual("%s\x001" % user.created.isoformat(), user.pagination_created)

        reversed_last_name = "".join([chr(0xffff - ord(x)) for x in "Benstead"])

        self.assertEqual(u"Luke\x00{}\x001".format(reversed_last_name), user.pagination_first_name_neg_last_name)

class DatastorePaginatorTests(TestCase):
    def setUp(self):
        super(DatastorePaginatorTests, self).setUp()

        self.u1 = TestUser.objects.create(id=1, first_name="A", last_name="A")
        self.u2 = TestUser.objects.create(id=2, first_name="A", last_name="B")
        self.u3 = TestUser.objects.create(id=3, first_name="B", last_name="A")
        self.u4 = TestUser.objects.create(id=4, first_name="B", last_name="B")

    def test_default_ordering(self):
        """ Tests that pagination works using just the model's default ordering. """
        paginator = Paginator(TestUser.objects.all(), 1, readahead=2)
        paginator.page(1)
        self.assertEqual([self.u1, self.u2, self.u3, self.u4], list(paginator.object_list))

    def test_ordering(self):
        paginator = Paginator(TestUser.objects.all().order_by("created"), 1, readahead=2)
        paginator.page(1)
        self.assertEqual([self.u1, self.u2, self.u3, self.u4], list(paginator.object_list))

        paginator = Paginator(TestUser.objects.all().order_by("-created"), 1, readahead=2)
        paginator.page(1)
        self.assertEqual([self.u4, self.u3, self.u2, self.u1], list(paginator.object_list))

    def test_count_up_to(self):
        paginator = Paginator(TestUser.objects.all().order_by("first_name"), 1, readahead=2)
        paginator.page(1)
        self.assertEqual(3, paginator.count)

        paginator = Paginator(TestUser.objects.all().order_by("first_name"), 1, readahead=10)
        paginator.page(1)
        self.assertEqual(4, paginator.count)

    def test_count_reads_ahead(self):
        paginator = Paginator(TestUser.objects.all().order_by("first_name"), 1, readahead=2)

        paginator.page(1)
        self.assertEqual(3, paginator.count)

        paginator.page(3)
        self.assertEqual(4, paginator.count)

    def test_page_jump_updates_count_correctly(self):
        paginator = Paginator(TestUser.objects.all().order_by("first_name"), 1, readahead=1)
        paginator.page(1)
        self.assertEqual(2, paginator.count)
        paginator.page(3)
        self.assertEqual(4, paginator.count)

    def test_that_marker_is_read(self):
        paginator = Paginator(TestUser.objects.all().order_by("first_name"), 1, readahead=1)
        paginator.page(2)

        with sleuth.watch("djangae.contrib.pagination.paginator._get_marker") as get_marker:
            paginator.page(4)

            self.assertTrue(get_marker.called)
            self.assertIsNotNone(get_marker.call_returns[0][0])
            self.assertEqual(1, get_marker.call_returns[0][1])

    def test_that_readahead_stores_markers(self):
        paginator = Paginator(TestUser.objects.all().order_by("first_name"), 1, readahead=4)

        expected_markers = [None] + list(
            TestUser.objects.all().order_by("first_name").
            values_list(paginator.field_required, flat=True)
        )[:3]

        paginator.page(1)

        query_id = queryset_identifier(paginator.object_list)

        actual_markers = []
        for i in range(1, 5):
            actual_markers.append(_get_marker(query_id, i)[0])

        self.assertEqual(expected_markers, actual_markers)

        # Now change the per page number
        paginator = Paginator(TestUser.objects.all().order_by("first_name"), 2, readahead=4)

        all_markers = list(TestUser.objects.all().order_by("first_name").values_list(
            paginator.field_required, flat=True)
        )
        expected_markers = [None, all_markers[1]]

        paginator.page(1)

        query_id = queryset_identifier(paginator.object_list)

        actual_markers = []
        for i in range(1, 3):
            actual_markers.append(_get_marker(query_id, i)[0])

        self.assertEqual(expected_markers, actual_markers)

    def test_ordering_required_exception_is_thrown_when_no_order_specified(self):
        # The exception should not be thrown when an order is specified
        query_set = SimpleModelWithoutOrdering.objects.order_by("name")
        Paginator(query_set, 25, readahead=10)

        # The exception should not be thrown when the model has a default order
        query_set = SimpleModelWithOrdering.objects.all()
        Paginator(query_set, 25, readahead=10)

        # The exception should be thrown when no order is specified on the model or in the query set
        query_set = SimpleModelWithoutOrdering.objects.all()
        with self.assertRaises(PaginationOrderingRequired):
            Paginator(query_set, 25, readahead=10)

    def test_pages_correct(self):
        paginator = Paginator(TestUser.objects.all().order_by("first_name"), 1)  # 1 item per page

        self.assertEqual("A", paginator.page(1).object_list[0].first_name)
        self.assertEqual("A", paginator.page(2).object_list[0].first_name)
        self.assertEqual("B", paginator.page(3).object_list[0].first_name)
        self.assertEqual("B", paginator.page(4).object_list[0].first_name)

        paginator = Paginator(TestUser.objects.all().order_by("first_name", "last_name"), 1)  # 1 item per page
        self.assertEqual(self.u1, paginator.page(1).object_list[0])
        self.assertEqual(self.u2, paginator.page(2).object_list[0])
        self.assertEqual(self.u3, paginator.page(3).object_list[0])
        self.assertEqual(self.u4, paginator.page(4).object_list[0])

        paginator = Paginator(TestUser.objects.all().order_by("first_name", "-last_name"), 1)  # 1 item per page
        self.assertEqual(self.u2, paginator.page(1).object_list[0])
        self.assertEqual(self.u1, paginator.page(2).object_list[0])
        self.assertEqual(self.u4, paginator.page(3).object_list[0])
        self.assertEqual(self.u3, paginator.page(4).object_list[0])

        paginator = Paginator(TestUser.objects.all().order_by("-first_name"), 1)  # 1 item per page
        self.assertEqual(self.u4, paginator.page(1).object_list[0])
        self.assertEqual(self.u3, paginator.page(2).object_list[0])
        self.assertEqual(self.u2, paginator.page(3).object_list[0])
        self.assertEqual(self.u1, paginator.page(4).object_list[0])

        with self.assertRaises(PaginationOrderingRequired):
            paginator = Paginator(TestUser.objects.all().order_by("-first_name", "last_name"), 1)  # 1 item per page
            list(paginator.page(1).object_list)

        # Test paging when last page has less than per_page objects
        paginator = Paginator(TestUser.objects.all().order_by("first_name"), 3)  # 3 items per page
        self.assertEqual(
            sorted([self.u1.pk, self.u2.pk, self.u3.pk]),
            sorted([x.pk for x in paginator.page(1).object_list])
        )
        self.assertEqual(
            sorted([self.u4.pk]),
            sorted([x.pk for x in paginator.page(2).object_list])
        )

    def test_empty_page(self):
        paginator = Paginator(TestUser.objects.all().order_by("-first_name"), 1, allow_empty_first_page=False)
        with self.assertRaises(django_paginator.EmptyPage):
            self.assertEqual(paginator.page(5).object_list, [])

        paginator = Paginator(TestUser.objects.all().order_by("-first_name"), 1)
        self.assertEqual(paginator.page(5).object_list, [])


import datetime
import sys
import pylast
import os
import time

# Define the paths to the files
CONFIG_DIR = os.path.expanduser("~/.config/pylast/")
API_KEY_FILE = os.path.join(CONFIG_DIR, "api_key")
API_SECRET_FILE = os.path.join(CONFIG_DIR, "api_secret")
SESSION_KEY_FILE = os.path.join(CONFIG_DIR, ".session_key")

# Dry run, yes or no?
DRY = False

# Make sure the API_KEY and API_SECRET files exist
if not os.path.exists(API_KEY_FILE) or not os.path.exists(API_SECRET_FILE):
    print("API_KEY and/or API_SECRET file(s) not found.")
    sys.exit(1)

# Read API_KEY and API_SECRET from their files
with open(API_KEY_FILE, 'r') as file:
    API_KEY = file.read().strip()

with open(API_SECRET_FILE, 'r') as file:
    API_SECRET = file.read().strip()

# Check if the session key file exists, if not generate and save session key
if not os.path.exists(SESSION_KEY_FILE):
    skg = pylast.SessionKeyGenerator(network)
    url = skg.get_web_auth_url()

    print(f"Please authorize this script to access your account: {url}\n")
    import webbrowser
    webbrowser.open(url)

    while True:
        try:
            SESSION_KEY = skg.get_web_auth_session_key(url)
            with open(SESSION_KEY_FILE, "w") as f:
                f.write(SESSION_KEY)
            break
        except pylast.WSError:
            time.sleep(1)
else:
    with open(SESSION_KEY_FILE, 'r') as file:
        SESSION_KEY = file.read().strip()

network = pylast.LastFMNetwork(api_key=API_KEY, api_secret=API_SECRET, session_key=SESSION_KEY)

def search_albums(artist):
    artist_obj = network.get_artist(artist)
    albums = artist_obj.get_top_albums(limit=10)
    n_albums = len(albums)

    # print albums in reverse order (most popular at the bottom)
    for index, album in enumerate(reversed(albums), 1):
        try:
            track_count = len(album.item.get_tracks())
            album_name = album.item.get_name()
            print(f'#{n_albums-index+1}: {album_name} ({track_count} tracks)')

        except pylast.WSError as e:
            print(f"#{n_albums - index + 1}: error while processing album: {e}")
            continue  # Skip to the next album

    print("#0: to do nothing & exit")
    while True:
        try:
            user_input = int(input("scrobble #> "))
            if user_input == 0:
                print("BYE BYE")
                break
            elif 1 <= user_input <= (n_albums + 1):
                current_time = time.strftime("%I:%M %p", time.localtime(time.time()))
                time_input = str(input(f"time? (blank for {current_time}) > "))
                if (time_input == ''):
                    time_input = current_time
                album_selected_name = albums[user_input - 1].item.get_name()
                print(f'attempting to scrobble "{album_selected_name}" @ {time_input}...')
                scrobble_album(artist, album_selected_name, time_input)
                break
            else:
                print("scrobble #> ")
        except ValueError:
            print("error: you either selected an album that is out of bounds, " \
                    "or you did not follow the correct time format (HH:MM AM)")

def scrobble_album(artist, album, time_str=None):
    # TODO: general error handling -- scrobbling albums with 0 tracks,
    #       searching for non-existant artists, etc
    album_info = network.get_album(artist, album)

    tracks = album_info.get_tracks()

    # TODO: should allow user to write something like 5:55pm/PM etc, uppercase/lowercase
    #       with/without the space
    now = int(time.time())
    if time_str is not None:
        # Convert the input time string to a time object
        time_obj = datetime.datetime.strptime(time_str, "%I:%M %p").time()
        # Get the current date and combine with the input time to form a datetime object
        today_date = datetime.date.today()
        now = int(datetime.datetime.combine(today_date, time_obj).timestamp())

    # Calculate the scrobble times for each track
    scrobble_times = []
    total_duration = sum(track.get_duration() for track in tracks)
    #print(tracks)
    try:
        album_title = tracks[0].get_album().title
    except AttributeError:
        # trouble grabbing album title from first track; defaulting to whatever user input was
        album_title = album
    except IndexError:
        # TODO: maybe just hide 0 track albums then? this error will
        # still be needed for manual 'Artist' 'Album' scrobbles though
        print("sorry! last.fm doesn't think that album has any tracks, so it can't be scrobbled")
        exit()

    for track in reversed(tracks):
        track_duration = track.get_duration()
        total_duration -= track_duration

        # Calculate the timestamp for this track's scrobble time
        timestamp = now - (total_duration) // 1000
        # Add the scrobble time to the list
        scrobble_times.append(timestamp)

    # Scrobble each track at the corresponding scrobble time
    for i, track in enumerate(tracks):
        le_time = time.strftime("%I:%M %p", time.localtime(scrobble_times[i]))
        print(f"Track {i+1} - {track.title} @ {le_time}")
        if (not DRY):
            network.scrobble(
                artist=track.artist.name,
                title=track.title,
                album=album_title,
                timestamp=scrobble_times[i]
            )

if __name__ == "__main__":
    if len(sys.argv) < 2 or len(sys.argv) > 4:
        print("Usage: python scrobble_album.py 'Artist Name' [Album Name] [Time]")
        sys.exit(1)

    artist_name = sys.argv[1]

    # Check if album name is supplied
    if len(sys.argv) > 2:
        album_name = sys.argv[2]
    else:
        album_name = None

    # Check if time_str is supplied
    if len(sys.argv) == 4:
        time_str = sys.argv[3]
    else:
        time_str = None

    # If album name is not provided, search albums instead of scrobbling
    if album_name is None:
        search_albums(artist_name)
    else:
        scrobble_album(artist_name, album_name, time_str)


import numpy as np
from scipy.io import loadmat

def get_X_y(file_name, encoding='binary', normalize=False, k_target=0, n=0):
    """
    Args:
        file_name [string]: path of file from current directory
        encoding [string]: 'binary' or 'value'
        normalize [boolean]: whether to normalize the data using buffer identity functions (NOTE: ONLY active for
            encoding = 'binary')
        k_target [int]: target k value for normalization
        n [int]: length of input bit string
    Returns:
        X [np.array]: fature matrix of shape (n_examples x d)
        y [np.array]: label vector of shape (n_sampels,)
    """
    datafile = loadmat(file_name)
    POS_dataset = NEG_dataset = None
    if encoding == 'binary':
        POS_dataset = datafile['POS_dataset']
        NEG_dataset = datafile['NEG_dataset']
        if normalize:
            POS_dataset, NEG_dataset = normalize_dimension(POS_dataset,NEG_dataset,n,k_target)
    elif encoding == 'value':
        POS_dataset = datafile['POS_storage']
        NEG_dataset = datafile['NEG_storage']
    X = np.concatenate((POS_dataset, NEG_dataset))
    y = np.concatenate((np.ones(POS_dataset.shape[0], ).astype(int), np.zeros(NEG_dataset.shape[0], )))
    return X, y

def get_training_test_data(X, y, n_train_total, n_test_total=-1, random_state=2508):
    """
    Obtain a training and test set with the desired number of examples from the dataset
    Args:
        X: np.array, parent design matrix of size (n_samples * d)
        y: np.array, parent label vector of size (n_samples,)
        n_train_total: int, number of training points
        n_test_total: int, number of test points (Default: -1, all points not chosen for
            training are included)
        random_state: int, seed the random number genenator for reproducability
    Returns:
        X_train: np.array, training data of size (n_train_total * d)
        y_train: np.array, training label of size (n_train_total,)
        X_test: np.array, test data of size (n_test_total * d)
        y_test: np.array, test label of size (n_test_total * d)
    """
    X_train, y_train, X_test, y_test = get_subset_n(X,y,n_train_total,random_state)
    if n_test_total != -1:
        X_test, y_test, _, _ = get_subset_n(X_test, y_test, n_test_total,random_state)
    return X_train, y_train, X_test, y_test

def get_subset_n(X, y, n_total_points, random_state=2508):
    """
    Randomly extract a balanced dataset of size n_total_points from a parent dataset
    Args:
        X: np.array, parent design matrix of size (n_samples * d)
        y: np.array, parent label vector of size (n_samples,)
        n_total_points: int, total points to extract (including both classes)
            requires: (n_total_points / 2) < n_pos_samples, n_neg_samples
        random_state: int, seed the random number genenator for reproducability

    Returns:
        X_chosen: np.array, balanced subset design matrix of size (n_total_points * d)
        y_chosen: np.array, subset label vector of size (n_total_points,)
        X_left: np.array, leftover design matrix of size ((n_samples - n_total_points) * d)
        y_left: np.array, leftover label vector of size ((n_samples - n_total_points),)
    """
    n_class_examples = int(n_total_points / 2)
    np.random.seed(random_state)
    class0_ind = np.random.choice(np.where(y == 0)[0], n_class_examples, replace=False)
    class1_ind = np.random.choice(np.where(y == 1)[0], n_class_examples, replace=False)
    chosen_ind = np.concatenate((class0_ind, class1_ind))
    X_chosen = np.concatenate((X[class1_ind], X[class0_ind]))
    y_chosen = np.concatenate((np.ones(n_class_examples, ).astype(int), np.zeros(n_class_examples, ).astype(int)))
    X_left = np.delete(X,chosen_ind,axis=0)
    y_left = np.delete(y,chosen_ind,axis=0)
    return X_chosen, y_chosen, X_left, y_left

def normalize_dimension(POS_dataset, NEG_dataset, n, k_target):
    """Buffer a dataset with low k to one with higher k with identity functions """
    k_add = int(k_target - np.shape(POS_dataset)[1] / n)
    if k_add == 0:
        return POS_dataset, NEG_dataset
    POS_dataset_ = np.c_[POS_dataset, np.zeros((np.shape(POS_dataset)[0],k_add * n))]
    NEG_dataset_ = np.c_[NEG_dataset, np.zeros((np.shape(NEG_dataset)[0],k_add * n))]
    return POS_dataset_.astype(int), NEG_dataset_.astype(int)


import logging
import os

def level_num(name, default_level):
    """Get level num from name"""
    level_mapping = dict(ERROR=logging.ERROR, INFO=logging.INFO,
                         WARNING=logging.WARNING, DEBUG=logging.DEBUG)
    return level_mapping.get(name.upper(), default_level)

# set up test logger
_log = logging.getLogger(__name__)

level = logging.WARNING
if 'IDAES_TEST_LOG_LEVEL' in os.environ:
    env_level = os.environ['IDAES_TEST_LOG_LEVEL']
    level = level_num(env_level, level)

_h = logging.StreamHandler()
_h.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] '
                                  '%(filename)s:%(lineno)d :: %(message)s'))
_log.addHandler(_h)
_log.setLevel(level)
_log.propagate = False


import json
import re
import sys
from urllib import parse

from html5lib import HTMLParser, treebuilders
from loguru import logger
from lxml import etree
from requests import get

from user_agent import random_ua

magnet_head = 'magnet:?xt=urn:btih:'

def run_crawler(key: str, search_terms: str, page: int, sort: str, proxies):
    rule_name = f'rule/{key}.json'
    logger.info('搜索任务开始执行')
    logger.info(f'文件名称: {rule_name}, 搜索词: {search_terms}, 页数: {page}, 排序规则: {sort}')

    try:
        with open(rule_name, encoding = 'utf-8') as rule_json:
            rule = json.load(rule_json)

            # 排序条件
            # TODO: 各个网站规则不一，暂不实现
            if len(sort) == 0:
                path = rule['path']['default'].format(k = search_terms, p = page)
            else:
                path = ''
            page_url = rule['url'] + path
            logger.info(f'搜索网址: {page_url}')

            # 发送请求获取数据
            headers = {
                'referer':    parse.quote(rule['referer']),
                'user-agent': random_ua()
            }
            content = get(page_url, timeout = 10, headers = headers, proxies = proxies)
            parser = HTMLParser(treebuilders.getTreeBuilder('lxml'), namespaceHTMLElements = False)
            document = parser.parse(content.text)

            # 获取页数
            page_sel = rule['parse']['page']
            page_num = ''
            for sel in page_sel:
                if sel['type'] is None:
                    page_num = '0'
                    break
                elif sel['type'] == 'xpath':
                    page_elem = document.xpath(sel['xpath'])
                    if len(page_elem) == 0:
                        page_num = '0'
                    else:
                        page_num = page_elem[0]
                elif sel['type'] == 'regex':
                    page_num = re.findall(sel['regex'], page_num)
                    if len(page_num) == 0:
                        page_num = '0'
                    else:
                        page_num = page_num[0]
            page_num = int(page_num)

            # 获取数据列表（字符串格式）
            data_result = []
            item_list = []
            if rule['parse']['item']["type"] == "xpath":
                elem_list = document.xpath(rule['parse']['item']['xpath'])
                for elem in elem_list:
                    item_list.append(etree.tostring(elem, encoding = str))
            elif rule['parse']['item']["type"] == "url":
                item_urls = document.xpath(rule['parse']['item']['xpath'])
                start = 0
                end = len(item_urls)
                if page_num == 0:
                    page_num = int(len(item_urls) / 10)
                    if page <= page_num:
                        start = (page - 1) * 10
                        end = start + 10
                for index in range(start, end):
                    url = rule['url'] + item_urls[index]
                    item_content = get(url, timeout = 10, headers = headers, proxies = proxies)
                    item_list.append(item_content.text)
            logger.info(f'结果列表: {item_list}')

            # 解析数据列表
            for index in range(rule['parse']['item']['startIndex'], len(item_list)):
                # 按照规则解析数据
                item_data = {
                    'magnet': 'magnet',
                    'name':   'name',
                    'hot':    'hot',
                    'size':   'size',
                    'time':   'time',
                }
                for key in item_data:
                    value = ''

                    # 遍历规则处理数据
                    for sel in rule['parse'][item_data[key]]:
                        # xpath匹配规则
                        if sel['type'] == 'xpath':
                            value = etree.HTML(item_list[index]).xpath(sel['xpath'])
                            # xpath匹配结果都是列表
                            if len(value) == 0:
                                value = ''
                            else:
                                value = value[0]

                        # xpath列表匹配规则
                        if sel['type'] == 'xpathList':
                            values = etree.HTML(item_list[index]).xpath(sel['xpath'])
                            # xpath匹配结果是列表
                            if len(values) == 0:
                                value = ''
                            else:
                                value = ''
                                for text in values:
                                    value += text

                        # 字符串截取规则
                        elif sel['type'] == 'subIndex':
                            # 起始和结束不设置时
                            # 需要为null值
                            start = 0 if sel['start'] is None else sel['start']
                            end = len(value) if sel['end'] is None else sel['end']
                            value = value[start:end]

                        # 正则表达式匹配规则
                        elif sel['type'] == 'regex':
                            value = re.findall(sel['regex'], value)
                            # 正则匹配结果是列表
                            if len(value) == 0:
                                value = ''
                            else:
                                value = value[0]

                        # 没有规则，取默认值
                        elif sel['type'] is None:
                            value = sel['default']
                            break

                    # fix: 某些链接提取后不完整
                    if key == 'magnet' and magnet_head not in value:
                        value = magnet_head + value

                    # fix: 移除一些错误字符
                    if key == 'name':
                        # &nbsp
                        value = value.replace(u'\xa0', u'')

                    # 保存解析后数据
                    item_data[key] = value

                data_result.append(item_data)
    except Exception as e:
        # 监听到异常
        logger.error(f'出现异常, {e.with_traceback(sys.exc_info()[2])}')
        return None
    else:
        # 没有异常返回正确数据
        logger.success(f'解析后结果列表: {data_result}')
        return {
            "page": page_num,
            "list": data_result
        }
    finally:
        # 有无异常都会最终执行的操作
        logger.info('搜索任务执行完毕')


from swatchM8.core import Swatch
from swatchM8.core import Image
from swatchM8.core.util.ExeptionHandeling import InvalidInputExeption

import logging
logger = logging.getLogger(__name__)

class Texture:

    def __init__(self, name, filePath, size, amountOfSwatchesOnRow, availableCoordinates=None):
        # type: (str,str,int,int,list) -> None

        self.name = name
        self.filePath = filePath
        self.size = size
        self.amountOfSwatchesOnRow = amountOfSwatchesOnRow

        self.swatchSize = self.calculateSwatchSize(amountOfSwatchesOnRow, size)
        self.swatches = []  # lijst van alle swatch objecten

        self.availableCoordinates = availableCoordinates  # lijst met booleans
        if availableCoordinates is None:
            self.availableCoordinates = self.generateAvailableCoordinates()

        logger.info('done Initializing TextureObject with Name: {}, filePath: {}, size: {}, amountOnRow: {}'.format(name, filePath, size, amountOfSwatchesOnRow))

    def saveImage(self):
        # type: () -> None

        image = Image.Image(self)
        image.save()

    def createSwatch(self, swatchName, color, coordinate=None):
        # type: (str, list, object) -> object

        try:
            if self.checkIfSwatchExists(swatchName) is not None:
                raise ("Swatch with name: {} already exists".format(swatchName))

        except InvalidInputExeption as e:
            logger.exception(e)
            raise

        if coordinate is None:
            coordinate = self.requestCoordinate()

        swatch = Swatch.Swatch(swatchName, coordinate, self.swatchSize, color)
        self.swatches.append(swatch)

        if coordinate is not None:
            self.setCoordinateUnavailable(swatch)

        logger.info("swatch created with name: {} on Coordinate: {} using Color: {}".format(swatchName, coordinate, color))
        return swatch

    def deleteSwatch(self, swatchName):
        # type: (str) -> list

        swatch = self.getSwatchByName(swatchName)

        self.setCoordinateAvailable(swatch)

        self.swatches.remove(swatch)

        logger.info("Swatch with name {} successfully deleted".format(swatchName))
        return self.swatches

    def getSwatchByName(self, swatchName):
        # type: (str) -> object

        if self.checkIfSwatchExists(swatchName):
            for i in self.swatches:
                if i.name == swatchName:
                    logger.debug('requested and returned Swatch with name: {}'.format(swatchName))
                    return i
        else:
            logger.debug('requested Swatch with name: {}, returned None'.format(swatchName))
            return None

    def generateAvailableCoordinates(self):
        # type: () -> None

        possibleCoordinates = []

        for i in range(self.amountOfSwatchesOnRow):
            col = []
            for j in range(self.amountOfSwatchesOnRow):
                col.append(True)

            possibleCoordinates.append(col)

        for i in self.swatches:
            possibleCoordinates[i.coordinate[0]][i.coordinate[1]] = False

        availableCoordinates = possibleCoordinates

        logger.debug('generated list with availible Coordinates: {}'.format(availableCoordinates))
        return availableCoordinates

    def requestCoordinate(self):
        # type: () -> List

        for i in range(self.amountOfSwatchesOnRow):
            for j in range(self.amountOfSwatchesOnRow):

                if self.availableCoordinates[i][j] == True:
                    logger.debug('requested coordinated returned: {}'.format([i, j]))
                    return [i, j]

        try:
            raise ValueError('All the coordinates are used!')

        except ValueError as e:
            logger.exception(e)
            raise
        # return None

    def setCoordinateAvailable(self, swatch):
        # type: (object) -> None

        self.availableCoordinates[(swatch.coordinate)[0]][(swatch.coordinate)[1]] = True
        logger.debug('Coordinate {} back availible'.format([(swatch.coordinate)[0], (swatch.coordinate)[1]]))

    def setCoordinateUnavailable(self, swatch):
        # type: (object) -> None

        self.availableCoordinates[(swatch.coordinate)[0]][(swatch.coordinate)[1]] = False
        logger.debug('Coordinate {} now unavailible'.format([(swatch.coordinate)[0], (swatch.coordinate)[1]]))

    def calculateSwatchSize(self, amountSwatchesOnRow, size):
        # type: (int, int) -> int

        return int(size / amountSwatchesOnRow)

    def checkIfSwatchExists(self, swatchName):
        # type: (str) -> bool

        for i in self.swatches:

            if i.name.lower() == swatchName.lower():
                logger.debug('Swatch with name: {} ALREADY exist'.format(swatchName))

                return i

        return None


from fastapi import FastAPI
from fastapi.responses import StreamingResponse
from huggingface_hub import InferenceClient

from langchain.llms import HuggingFaceHub

from langchain.document_loaders import PyPDFLoader
from langchain.document_loaders import PyPDFDirectoryLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import HuggingFaceHubEmbeddings
from langchain.vectorstores import Chroma

from langchain.retrievers import ContextualCompressionRetriever
from langchain.retrievers.document_compressors import LLMChainExtractor
from langchain.retrievers.multi_query import MultiQueryRetriever

from langchain.chains import RetrievalQA

def prepare_vector_database():
    if 'qa' in globals():
        return

    loader = PyPDFDirectoryLoader('files')
    documents = loader.load()
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=850, chunk_overlap=200)
    docs = text_splitter.split_documents(documents)

    embeddings = HuggingFaceHubEmbeddings()
    db = Chroma.from_documents(docs, embeddings)

    llm = HuggingFaceHub(
        repo_id="mistralai/Mistral-7B-Instruct-v0.1",
        model_kwargs={"temperature":0.1, "max_new_tokens":300}
    )
    #compressor = LLMChainExtractor.from_llm(llm)
    #compression_retriever = ContextualCompressionRetriever(
    #    base_compressor=compressor,
    #    base_retriever=db.as_retriever(
    #        search_type = "mmr", search_kwargs={
    #            "k":3,
    #            "score_threshold": .5
    #        })
    #    )

    #retriever_from_llm = MultiQueryRetriever.from_llm(
    #    retriever=db.as_retriever(), llm=llm
    #)

    retriever = db.as_retriever(
        search_kwargs={
            "k":4,
            "score_threshold": .95
        }
    )
    global qa
    qa = RetrievalQA.from_chain_type(
        llm=llm,
        chain_type="stuff",
        retriever=retriever,
        return_source_documents=True
    )
    print('VectorDB ready!')
    return "Ready!"

prompts = [
    "Summarize Thiago's career in a 250 words",
    "Tell me in bullet points Thiago's education, citing only the intitution names, degree, and period",
    "Tell me in bullet points Thiago's work experiences, citing only the companies' names, title and period",
    "Tell me about Thiago's role at Youper Inc",
    "Tell me about Thiago's role at Softplan",
    "Tell me about Thiago's coding skills",
    "Tell me about Thiago's management skills",
]

async def stream_inference():
    for p in prompts:
        result = qa({"query": p})
        yield result["query"]
        yield "\n"
        yield result["result"]
        yield "\n\n"

app = FastAPI()

@app.get("/")
async def root():
    prepare_vector_database()
    return StreamingResponse(stream_inference(), media_type="text/plain")

@app.get("/{input}")
async def inference(input):
    prepare_vector_database()
    return qa({"query": input})