import datetime
import os
import pickle
import shutil
import subprocess
from io import BytesIO
from pathlib import Path
from typing import List

import boto3
import pandas as pd
from botocore.exceptions import ClientError

from data import constants
from data.constants import (
    READABLE_DATESTRING_FORMAT,
    S3_ACCESS_KEY,
    S3_SECRET_KEY,
    S3_BUCKET_NAME,
    S3_DISEASE_DATA_OBJ_NAME,
    DISEASE_DATA_GITHUB_REPO,
    REPO_DIRPATH,
    DAILY_REPORTS_DIRPATH,
    DEMOGRAPHIC_DATA,
    BED_DATA,
)

def execute_shell_command(command: List[str]):
    return subprocess.run(command, stdout=subprocess.PIPE).stdout.decode("utf-8")

def get_full_and_latest_dataframes_from_csv(csv_filepaths: List[Path]):
    """
    Returns two dataframes - full table (all dates), table of the latest date
    """
    total_df = None

    for fpath in csv_filepaths:
        stem_split = fpath.stem.split("-")
        month = int(stem_split[0])
        day = int(stem_split[1])
        year = int(stem_split[2])

        curr_df = pd.read_csv(fpath)
        if "Country_Region" in curr_df.columns:
            curr_df.rename({"Country_Region": "Country/Region"}, axis=1, inplace=True)
        country_stats_df = curr_df.groupby("Country/Region").agg(
            {"Confirmed": "sum", "Deaths": "sum", "Recovered": "sum"}
        )

        country_stats_df.reset_index(inplace=True)
        country_stats_df["Date"] = datetime.datetime(year=year, month=month, day=day)

        if total_df is None:
            total_df = country_stats_df
        else:
            total_df = total_df.append(country_stats_df, ignore_index=True)

    ## Clean data
    # replacing Mainland china with just China
    total_df["Country/Region"] = total_df["Country/Region"].replace(
        "Mainland China", "China"
    )
    total_df["Country/Region"] = total_df["Country/Region"].replace(
        "US", "United States"
    )

    # sort by date, then country name
    total_df = total_df.sort_values(["Date", "Country/Region"])

    # Latest date table
    latest_date_df = total_df[total_df["Date"] == max(total_df["Date"])]

    # Set country as index
    latest_date_df = latest_date_df.set_index("Country/Region")
    total_df = total_df.set_index("Country/Region")

    return total_df, latest_date_df

def _get_data_from_repo(path):
    # Go to daily reports directory and fetch all CSV files
    csv_filepaths = list(Path(path).glob("*.csv"))

    # Get the full and latest table
    full_df, latest_df = get_full_and_latest_dataframes_from_csv(csv_filepaths)
    data_object = {"full_table": full_df, "latest_table": latest_df}

    return data_object

def download_data(cleanup=True):
    """
     Clone the JHU COVID GitHub repo (takes about a minute) and return paths to CSVs.
    """
    cmd = ["git", "clone", DISEASE_DATA_GITHUB_REPO]
    execute_shell_command(cmd)

    data_object = _get_data_from_repo(path=DAILY_REPORTS_DIRPATH)

    if cleanup:
        # Remove GitHub repo directory
        shutil.rmtree(REPO_DIRPATH)
        print("Cleaned up.")

    print("Data fetch complete")

    return data_object

def pull_latest_data(path=REPO_DIRPATH):
    print("Updating the local data storage")

    current_dir = os.path.curdir

    # For some reason when I used subprocess I lost the STDOUT
    os.system(f"cd {path} && git pull")

    os.system(f"cd {current_dir}")

    data_object = _get_data_from_repo(path=DAILY_REPORTS_DIRPATH)

    return data_object

def get_data_locally_or_download(data_path_locally=constants.DATA_DIR):
    """A helper function for users running locally"""
    if "COVID-19" not in [p.stem for p in data_path_locally.parent.iterdir()]:
        # Don't delete the repo we're cloning if running locally
        data_object = download_data(cleanup=False)
    else:
        # We need to make sure the data is up to date
        data_object = pull_latest_data(REPO_DIRPATH)

    return data_object

def _configure_s3_client():
    # Upload the file
    s3_client = boto3.client(
        "s3", aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY
    )
    return s3_client

def upload_data_to_s3(data: bytes, object_name: str = S3_DISEASE_DATA_OBJ_NAME):
    """
    Upload a file to an S3 bucket
    :param data: Bytes to upload.
    :param object_name: S3 object name.
    :return: True if file was uploaded, else False
    """
    buf = BytesIO(data)
    s3_client = _configure_s3_client()
    try:
        s3_client.put_object(Body=buf, Bucket=S3_BUCKET_NAME, Key=object_name)
    except ClientError as e:
        print(e)
        return False
    return True

def download_data_from_s3(object_name: str = S3_DISEASE_DATA_OBJ_NAME):
    """
    Download a file from S3 bucket.
    :param object_name: Name of object to download.
    :return: Object bytes and date last modified.
    """
    s3_client = _configure_s3_client()
    try:
        download = s3_client.get_object(Key=object_name, Bucket=S3_BUCKET_NAME)
    except ClientError:
        return None
    content = download["Body"].read()

    # e.g. Sunday 30 November 2014
    last_modified = download["LastModified"].strftime(READABLE_DATESTRING_FORMAT)
    return content, last_modified

def build_country_data(demographic_data=DEMOGRAPHIC_DATA, bed_data=BED_DATA):
    # Try to download from S3, else download from JHU
    objects = download_data_from_s3()
    if objects is None:
        data_dict = get_data_locally_or_download()
        last_modified = datetime.datetime.now().strftime(READABLE_DATESTRING_FORMAT)
    else:
        data_dict_pkl_bytes, last_modified = objects
        data_dict = pickle.loads(data_dict_pkl_bytes)

    full_disease_data, latest_disease_data = (
        data_dict["full_table"],
        data_dict["latest_table"],
    )

    # Rename name "US" to "United States" in demographics data to match bed data
    demographic_data = demographic_data.rename(index={"US": "United States"})
    demographic_data = demographic_data.merge(bed_data, on="Country/Region")
    demographic_data["Num Hospital Beds"] = (
        demographic_data["Latest Bed Estimate"] * demographic_data["Population"]
    )

    country_data = latest_disease_data.merge(demographic_data, on="Country/Region")
    country_data = country_data.loc[
        :, ["Confirmed", "Deaths", "Recovered", "Population", "Num Hospital Beds"]
    ]

    # Check that all of the countries in our selectable dropdown are also present in the full data
    assert set(latest_disease_data.index.unique()).issubset(
        set(full_disease_data.index.unique())
    )

    return country_data.to_dict(orient="index"), last_modified, full_disease_data

def check_if_aws_credentials_present():
    if len(constants.S3_ACCESS_KEY) is 0:
        print(
            "No S3 credentials present, using local file storage. "
            "This make some time the first time you run. We will clone "
            "the John Hopkins data repo (COVID-19) into this one."
        )
    else:
        print("S3 credentials found, using AWS.")


#!/usr/bin/env python
# coding: utf-8

#
COVID-19: ULAs

#
#

#

# Rodrigo Carrillo-Larco, Manuel Castillo-Cara

# Universidad Peruana Cayetano Heredia

# Universidad Nacional de Ingeniería
#
#

# ---
#
#
Índice

#
#
# * [1. Introducción](#section1)
# * [2. Análisis Exploratorio de Datos](#section2)
#     * [2.1. Limpieza de datos](#section21)
#     * [2.2. Boxplots](#section22)
#     * [2.3. Histogramas](#section23)
#     * [2.4. Matriz de correlación](#section24)
#     * [2.5. Matriz de dispersión](#section25)
# * [3. Transformaciones categórico a dummy](#section3)
# * [4. PCA + k-means](#section4)
# * [5. Análisis clustering](#section5)
#     * [5.1. Tamaño/Densidad de clústers](#section51)
#     * [5.2. Centroides de clústers (Formas)](#section52)

# ---
#
# #  1. Introducción

# In[2]:

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pylab as pl
import pandas as pd
import os

# ### Lectura de datos

# In[3]:

df_original = pd.read_csv('dataset/data_2020-03-22.csv', parse_dates=[2])
df_original.head(1)

# In[5]:

df = pd.read_csv('dataset/data_2020-03-22.csv', parse_dates=[2])
df.drop(['pop_men'], axis=1, inplace=True)
df.drop(['pop_women'], axis=1, inplace=True)
#df.drop(['p.db'], axis=1, inplace=True)
#df.drop(['p.copd'], axis=1, inplace=True)
#df.drop(['p.hiv'], axis=1, inplace=True)
#df.drop(['p.tbc'], axis=1, inplace=True)
df.drop(['ylds.db'], axis=1, inplace=True)
df.drop(['ylds.copd'], axis=1, inplace=True)
df.drop(['ylds.hiv'], axis=1, inplace=True)
df.drop(['ylds.tbc'], axis=1, inplace=True)
df.drop(['Country'], axis=1, inplace=True)
df.drop(['Speed.Number'], axis=1, inplace=True)
df.drop(['first_date'], axis=1, inplace=True)
df.drop(['order'], axis=1, inplace=True)
df.drop(['total_deaths'], axis=1, inplace=True)

# ---
#

#
#

# ---
#
# #  2. Análisis Exploratorio de Datos

# In[6]:

df.dtypes

# In[7]:

df.describe().transpose()

# In[8]:

df.shape

# In[9]:

list_columns = df.columns

# ---
#
# ##  2.1. Data Cleaning: Handling missing data

# In[10]:

df.isnull().any().any(), df.shape

# In[11]:

df.isnull().sum(axis=0)

# ---
#
# ##  2.2. Boxplots

# In[15]:

pl.figure(1)
plt.subplots(figsize=(20,20))
#pl.subplot(431)
#sns.boxplot(df["Speed.Number"])
#pl.grid(True)
pl.subplot(432)
sns.boxplot(df["p.db"])
pl.grid(True)
#pl.subplot(433)
#sns.boxplot(df["ylds.db"])
#pl.grid(True)
pl.subplot(434)
sns.boxplot(df["p.copd"])
pl.grid(True)
#pl.subplot(435)
#sns.boxplot(df["ylds.copd"])
#pl.grid(True)
pl.subplot(436)
sns.boxplot(df["p.hiv"])
pl.grid(True)
#pl.subplot(437)
#sns.boxplot(df["ylds.hiv"])
#pl.grid(True)
pl.subplot(438)
sns.boxplot(df["p.tbc"])
pl.grid(True)
pl.subplot(439)
sns.boxplot(df["GDP_2017"])
pl.grid(True)
#=======================
pl.figure(2)
plt.subplots(figsize=(20,20))
#pl.subplot(431)
#sns.boxplot(df["pop_men"])
pl.grid(True)
#pl.subplot(432)
#sns.boxplot(df["pop_women"])
#pl.grid(True)
pl.subplot(433)
sns.boxplot(df["prop_men"])
pl.grid(True)
pl.subplot(434)
sns.boxplot(df["pm2.5"])
pl.grid(True)
pl.subplot(435)
sns.boxplot(df["UHC_index_2017"])
pl.grid(True)

pl.show()

# ---
#
# ##  2.3. Histogramas

# In[17]:

get_ipython().run_line_magic('matplotlib', 'inline')
# Univariate Histograms
f, axes = plt.subplots(4, 4, figsize=(14, 14))
#sns.distplot(df["Speed.Number"], rug=True, ax=axes[0, 0])
sns.distplot(df["p.db"], rug=True, ax=axes[0, 1])
#sns.distplot(df["ylds.db"], rug=True, ax=axes[0, 2])
sns.distplot(df["p.copd"], rug=True, ax=axes[0, 3])
#sns.distplot(df["ylds.copd"], rug=True, ax=axes[1, 0])
sns.distplot(df["p.hiv"], rug=True, ax=axes[1, 1])
#sns.distplot(df["ylds.hiv"], rug=True, ax=axes[1, 2])
sns.distplot(df["p.tbc"], rug=True, ax=axes[2, 0])
sns.distplot(df["GDP_2017"], rug=True, ax=axes[2, 1])
#sns.distplot(df["pop_men"], rug=True, ax=axes[2, 2])
#sns.distplot(df["pop_women"], rug=True, ax=axes[2, 3])
sns.distplot(df["prop_men"], rug=True, ax=axes[3, 0])
sns.distplot(df["pm2.5"], rug=True, ax=axes[3, 1])
sns.distplot(df["UHC_index_2017"], ax=axes[3, 2])

# ---
#
# ##  2.4. Matriz de correlación

# In[18]:

correlation = df.corr()
plt.figure(figsize=(10,10))
ax = sns.heatmap(correlation, vmax=1, square=True, annot = True, cmap = 'viridis')
# Esto se ponde debido al bug de Matplotlib 3.1.1 (quitarlo en versiones diferentes)
bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)
# ----------
plt.title('Correlación entre variables')
plt.show()

# ---
#
# ##  2.5. Matriz de dispersión

# Al igual que el diagrama de matriz de correlación anterior, la matriz de diagrama de dispersión es simétrica. Esto es útil para mirar las relaciones por pares desde diferentes perspectivas. Debido a que no tiene mucho sentido dibujar un diagrama de dispersión de cada variable consigo mismo, la diagonal muestra histogramas de cada atributo.
#
# **Aquí es importante ver el comportamiento entre variables, es decir, si tienen un comportamiento clusterizado entre ambas o tiene un comportamiento lineal**

# In[19]:

sns.pairplot(df);

# ---
#

#
#

# ---
#
# # 3. Transformaciones categórico a dummy

# In[12]:

cat_features = ['Country', 'first_date']

#num_features = ['Speed.Number', 'order', 'total_deaths', 'p.db', 'ylds.db', 'p.copd', 'ylds.copd', 'p.hiv', 'ylds.hiv', 'p.tbc', 'ylds.tbc', 'GDP_2017', 'prop_men', 'pm2.5', 'UHC_index_2017']
num_features = ['Speed.Number', 'order', 'total_deaths', 'p.db', 'p.copd', 'p.hiv', 'p.tbc', 'GDP_2017', 'prop_men', 'pm2.5', 'UHC_index_2017']
#num_features = ['Speed.Number', 'order', 'total_deaths', 'ylds.db', 'ylds.copd', 'ylds.hiv', 'ylds.tbc', 'GDP_2017', 'prop_men', 'pm2.5', 'UHC_index_2017']

def generateonlyDummies2(X):
    outd_X = pd.DataFrame()
    categories = {}
    for c in cat_features:
        categories[c] = X[c].dropna().unique().tolist()

    for c in cat_features:
            # Cambiamos las variables object a categorical para que las categorias tengan siempre el mismo orden
            c_df = X[c].astype(pd.api.types.CategoricalDtype(categories = categories[c]))
            #c_df.reset_index(drop=True, inplace=True)
            # One-hot encoding. Utilizamos get_dummies de pandas!
            c_dummies = pd.get_dummies(c_df, prefix=c)
            outd_X = pd.concat([outd_X, c_dummies], axis=1)
    outc_X = pd.DataFrame(X[num_features],columns=num_features)

    for c in num_features:
        outd_X[c] = outc_X[c].values
    return outd_X

# ---
#

#
#

#
# ---

#
# # 4.PCA + k-Means

# Se generan las variables dummies correspondientes y con ello ya se tiene listo el dataset para aplicar k-means.
# El código en comentario es dependiendo las variables a analizar, también si hay que pasar a dummy o no.
#
# Cambiar los componentes de PCA
#
# O si no se utiliza PCA comentar el bloque correspondiente

# In[20]:

df_original = pd.read_csv('dataset/data_2020-03-22.csv', parse_dates=[2])
df_original.head(1)

# In[21]:

df = pd.read_csv('dataset/data_2020-03-22.csv', parse_dates=[2])
df.drop(['pop_men'], axis=1, inplace=True)
df.drop(['pop_women'], axis=1, inplace=True)
#df.drop(['p.db'], axis=1, inplace=True)
#df.drop(['p.copd'], axis=1, inplace=True)
#df.drop(['p.hiv'], axis=1, inplace=True)
#df.drop(['p.tbc'], axis=1, inplace=True)
df.drop(['ylds.db'], axis=1, inplace=True)
df.drop(['ylds.copd'], axis=1, inplace=True)
df.drop(['ylds.hiv'], axis=1, inplace=True)
df.drop(['ylds.tbc'], axis=1, inplace=True)
df.drop(['Country'], axis=1, inplace=True)
df.drop(['Speed.Number'], axis=1, inplace=True)
df.drop(['first_date'], axis=1, inplace=True)
df.drop(['order'], axis=1, inplace=True)
df.drop(['total_deaths'], axis=1, inplace=True)

# In[22]:

df.head(1)

# In[23]:

df_dummy = df.copy()
#df_dummy= generateonlyDummies2(df)

# In[24]:

df_dummy.head(1)

# In[25]:

from sklearn.decomposition import PCA
n_components = 3 # reducir de 67 a 40 variables
pca = PCA(n_components=n_components, iterated_power='auto', svd_solver='auto', whiten=True).fit(df_dummy)

# In[26]:

pca_trans = pca.transform(df_dummy)

# In[27]:

total_variance = np.cumsum(pca.explained_variance_ratio_)
total_variance[:5] # Cinco primeros...

# In[28]:

df_pca = pd.DataFrame(pca_trans)

# In[29]:

df_dummyo = df_pca.copy()
#df_dummyo = df_dummy.copy()

# In[30]:

df_dummyo.head(1)

# In[31]:

from sklearn.metrics import pairwise_distances

def cost(data, k, centroids, clusters):
    cost = 0.0
    for c in range(k):
        points_cluster = data[clusters==c, :]
        if points_cluster.shape[0] > 0:
            distances = pairwise_distances(points_cluster, [centroids[c]], metric='euclidean')
            cost += np.sum(distances**2)
    return cost/len(clusters)

# In[32]:

from sklearn.cluster import MiniBatchKMeans

# Arrat que guardará los costes
costsD = np.zeros(10,)
# Valores de k de 1 a 64
ks = 1+np.arange(10)
# Guarda los costes
#costs = np.zeros(20,)
# Valores de k de 5 a 125
#ks = np.linspace(1,20,20).astype(int)

# Ejecuta k-means para cada valor de k, y guarda el coste asociado
for i,k in enumerate(ks):
    kmeans = MiniBatchKMeans(n_clusters=k, init='k-means++', max_iter=500)
    kmeans.fit(df_dummyo.values)
    centroidsD = kmeans.cluster_centers_
    clustersD = kmeans.labels_
    costsD[i] = cost(df_dummyo.values, k, centroidsD, clustersD)
    print(i,end=',')

plt.xlabel('No. Clusters')
plt.ylabel('Cost')
plt.title('Cost Function')
plt.plot(ks,costsD, 'bo-');
plt.savefig('coste.eps', format='eps', dpi=400)

# In[33]:

for i in range(10):
    print(costsD[i])

# In[34]:

np.mean(costsD)

# La gráfica muestra claramente que existe un punto a partir del cual la disminución del coste es significativamente más lenta. En concreto, hasta 2-4 clusters, la disminución del coste es considerable. A partir de ese punto, es mucho más lenta.

# ### Con 2 clústers

# In[35]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_2 = KMeans(n_clusters=2, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_2.fit(df_dummy)
df_dummy_2 = df_dummy.copy()
df_dummy_2['label'] = km_2.labels_
df_dummy_2['Country'] = df_original['Country']
df_dummy_2.to_csv(r'Dataset_2_clusters.csv', index = False)
#df_dummy.drop(['Country'], axis=1, inplace=True)
#df_dummy.drop(['label'], axis=1, inplace=True)

# In[36]:

df_dummy_2.head(1)

# ### Con 3 clústers

# In[37]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_3 = KMeans(n_clusters=3, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_3.fit(df_dummy)
df_dummy_3 = df_dummy.copy()
df_dummy_3['label'] = km_3.labels_
df_dummy_3['Country'] = df_original['Country']
df_dummy_3.to_csv(r'Dataset_3_clusters.csv', index = False)
#df_dummy.drop(['Country'], axis=1, inplace=True)
#df_dummy.drop(['label'], axis=1, inplace=True)

# ### Con 4 clústers

# In[38]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_4 = KMeans(n_clusters=4, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_4.fit(df_dummy)
df_dummy_4 = df_dummy.copy()
df_dummy_4['label'] = km_4.labels_
df_dummy_4['Country'] = df_original['Country']
df_dummy_4.to_csv(r'Dataset_4_clusters.csv', index = False)

# ### Con 5 clústers

# In[39]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_5 = KMeans(n_clusters=5, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_5.fit(df_dummy)
df_dummy_5 = df_dummy.copy()
df_dummy_5['label'] = km_5.labels_
df_dummy_5['Country'] = df_original['Country']
df_dummy_5.to_csv(r'Dataset_5_clusters.csv', index = False)

# ### Con 6 clústers

# In[40]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_6 = KMeans(n_clusters=6, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_6.fit(df_dummy)
df_dummy_6 = df_dummy.copy()
df_dummy_6['label'] = km_6.labels_
df_dummy_6['Country'] = df_original['Country']
df_dummy_6.to_csv(r'Dataset_6_clusters.csv', index = False)

# ### Con 7 clústers

# In[41]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_7 = KMeans(n_clusters=7, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_7.fit(df_dummy)
df_dummy_7 = df_dummy.copy()
df_dummy_7['label'] = km_7.labels_
df_dummy_7['Country'] = df_original['Country']
df_dummy_7.to_csv(r'Dataset_7_clusters.csv', index = False)

# ### Con 8 clústers

# In[42]:

from sklearn.cluster import KMeans
# Elegimos con 2 clusters
km_8 = KMeans(n_clusters=8, init='k-means++', max_iter=500, random_state=0)
df_dummy = df_dummyo.copy()
km_8.fit(df_dummy)
df_dummy_8 = df_dummy.copy()
df_dummy_8['label'] = km_8.labels_
df_dummy_8['Country'] = df_original['Country']
df_dummy_8.to_csv(r'Dataset_8_clusters.csv', index = False)

# In[59]:

df_dummy

# ---
#

#
#

#
# ---

#
# # 5. Análisis clustering

#
# ## 5.1. Tamaño/Densidad de clústers

# In[46]:

print(90*'_')
print("\nNumero de paises en cada cluster 2")
print(90*'_')

pd.value_counts(km_2.labels_, sort=False)

# In[47]:

print(90*'_')
print("\nNumero de paises en cada cluster 3")
print(90*'_')

pd.value_counts(km_3.labels_, sort=False)

# In[48]:

print(90*'_')
print("\nNumero de paises en cada cluster 4")
print(90*'_')

pd.value_counts(km_4.labels_, sort=False)

# In[49]:

print(90*'_')
print("\nNumero de paises en cada cluster 5")
print(90*'_')

pd.value_counts(km_5.labels_, sort=False)

# In[50]:

print(90*'_')
print("\nNumero de paises en cada cluster 6")
print(90*'_')

pd.value_counts(km_6.labels_, sort=False)

# In[51]:

print(90*'_')
print("\nNumero de paises en cada cluster 7")
print(90*'_')

pd.value_counts(km_7.labels_, sort=False)

# In[52]:

print(90*'_')
print("\nNumero de paises en cada cluster 8")
print(90*'_')

pd.value_counts(km_8.labels_, sort=False)

#
# ## 5.2. Centroides de clústers (Formas)

# In[54]:

def pd_centers(featuresUsed, centers):
	from itertools import cycle, islice
	from pandas.plotting import parallel_coordinates
	import matplotlib.pyplot as plt
	import pandas as pd
	import numpy as np

	colNames = list(featuresUsed)
	colNames.append('prediction')

	# Zip with a column called 'prediction' (index)
	Z = [np.append(A, index) for index, A in enumerate(centers)]

	# Convert to pandas for plotting
	P = pd.DataFrame(Z, columns=colNames)
	P['prediction'] = P['prediction'].astype(int)
	return P

def parallel_plot(data):
	from itertools import cycle, islice
	from pandas.plotting import parallel_coordinates
	import matplotlib.pyplot as plt

	my_colors = list(islice(cycle(['b', 'r', 'g', 'y', 'k']), None, len(data)))
	plt.figure(figsize=(15,8)).gca().axes.set_ylim([-2.5,+2.5])
	parallel_coordinates(data, 'prediction', color = my_colors, marker='o')

#
# First, let us create some utility functions for Plotting
#

def pd_centers(featuresUsed, centers):
	from itertools import cycle, islice
	from pandas.plotting import parallel_coordinates
	import matplotlib.pyplot as plt
	import pandas as pd
	import numpy as np

	colNames = list(featuresUsed)
	colNames.append('prediction')

	# Zip with a column called 'prediction' (index)
	Z = [np.append(A, index) for index, A in enumerate(centers)]

	# Convert to pandas for plotting
	P = pd.DataFrame(Z, columns=colNames)
	P['prediction'] = P['prediction'].astype(int)
	return P

def parallel_plot(data):
	from itertools import cycle, islice
	from pandas.plotting import parallel_coordinates
	import matplotlib.pyplot as plt

	my_colors = list(islice(cycle(['b', 'r', 'g', 'y', 'k']), None, len(data)))
	plt.figure(figsize=(15,8)).gca().axes.set_ylim([-2.5,+2.5])
	parallel_coordinates(data, 'prediction', color = my_colors, marker='o')

# ### 2 Clusters

# In[60]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_2.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ### 3 Clusters

# In[61]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_3.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ### 4 Clusters

# In[62]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_4.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ### 5 Clusters

# In[63]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_5.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ### 6 Clusters

# In[64]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_6.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ### 7 Clusters

# In[65]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_7.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ### 8 Clusters

# In[66]:

get_ipython().run_line_magic('matplotlib', 'inline')
P1 = pd_centers(featuresUsed=df_dummy, centers=km_8.cluster_centers_)
P1.head(3)
parallel_plot(P1)

# ---
#

#
#

#
# ---


from Persona import persona
from conexion import Conexion
from beautifultable import BeautifulTable

class personaDAO:
    def __init__(self) -> None:
        pass
    def eliminarPersona(self, id_rut):

        Conexion.cursor.execute("delete from PERSONA where id_rut=:Prut", [id_rut])
        Conexion.connection.commit()
        return "Se ha eliminado correctamente"

    def buscarPersona(self, rut)->persona:
        Conexion.cursor.execute("select * from PERSONA where rut=:1", [rut])
        row=Conexion.cursor.fetchone()
        if row is None:
            return  None
        else:
            return persona (row[0], row[1], row[2], row[3], row[4], row[5], row[6])

    def insertarPersona(self, Persona):

        Conexion.cursor.execute("""insert into persona(id_rut, nombre, apellido, ciudad, direccion, telefono, id_tipo) values(:Prut,:pnom,:pap,:pci,:pdi,:pte,:idt)""",[Persona.id_rut, Persona.Nombre, Persona.Apellido, Persona.Ciudad, Persona.Direccion, Persona.Telefono, Persona.ID_TipoPersona])
        Conexion.connection.commit()
        return "Los datos fueron ingresados de forma correcta"

    def obtenerPersona(self):
        tabla=BeautifulTable()
        tabla.columns.header=["Rut", "Nombre", "Apellido", "Ciudad", "Direccion", "Telefono", "Tipo Persona"]
        for row in Conexion.cursor.execute('SELECT ID_RUT, PERSONA.NOMBRE, APELLIDO, CIUDAD, DIRECCION, TELEFONO, TIPO_PERSONA.NOMBRE as "tipo de persona" FROM PERSONA left join TIPO_PERSONA ON TIPO_PERSONA.ID_PERSONA = PERSONA.ID_TIPO'):
            tabla.rows.append(row)
        if len(tabla.rows)>0:
            print(tabla)
        else:
            print("Persona no ingresada")


#!/usr/bin/python3
import random
import warnings

import pandas as pd
from prettytable import PrettyTable
from sklearn import linear_model, svm, neighbors, naive_bayes, tree, ensemble
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, precision_recall_curve
from xgboost import XGBClassifier

random.seed(123)
warnings.filterwarnings('ignore')

#加载数据
def load_data(sample_file, label_file):
    """
    从文件中读取数据
    :param sample_file: 数据信息
    :param label_file: 标签信息
    :return: xs,ys
    """
    xs = pd.read_csv(sample_file, header=None)
    ys = pd.read_csv(label_file, header=None)
    return xs, ys

#计算评价指标
def get_metrics(y_pred, y_true):
    """
    计算评价指标
    :param y_pred: 预测值
    :param y_true: 真实值
    :return: acc, f1, p, recall, p_r
    """
    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred)
    p = precision_score(y_true, y_pred)
    recall = recall_score(y_true, y_pred)
    p_r = precision_recall_curve(y_true, y_pred)
    return acc, f1, p, recall, p_r

#模型训练与预测
def get_result(name, model, x_train, y_train, x_test, y_test, table):
    """
    模型训练与预测
    :param name: 模型名
    :param model: 模型
    :param x_train: 训练集数据x
    :param y_train: 训练集标签y
    :param x_test: 测试集数据x
    :param y_test: 测试集标签y
    :param table: 结果记录表格
    """
    model.fit(x_train, y_train)
    y_pred = model.predict(x_test)
    acc, f1, p, recall, p_r = get_metrics(y_pred, y_test)
    table.add_row([name, f'{acc * 100:.4f}', f'{f1 * 100:.4f}', f'{p * 100:.4f}', f'{recall * 100:.4f}'])

def split_data(xs, ys, ratio=0.7):
    """
    划分数据集
    :param xs: 原始数据
    :param ys: 原始标签
    :param ratio: 训练集比例
    :return: x_train, y_train, x_test, y_test
    """
    # 建立数据索引
    indexes_1, indexes_0 = list(range(int(len(xs) * 0.5))), list(range(int(len(xs) * 0.5), len(xs)))
    # 打乱顺序
    random.shuffle(indexes_1), random.shuffle(indexes_0)
    # 数据划分为train和test
    indexes_train = indexes_1[:int(len(indexes_1) * ratio)] + indexes_0[:int(len(indexes_0) * ratio)]
    indexes_test = indexes_1[int(len(indexes_1) * ratio):] + indexes_0[int(len(indexes_0) * ratio):]
    x_train, y_train = xs.loc[indexes_train, :], ys.loc[indexes_train, :]
    x_test, y_test = xs.loc[indexes_test, :], ys.loc[indexes_test, :]
    return x_train, y_train, x_test, y_test

def main():
    """
    机器学习方法对比
    """
    # 读取数据
    #xs, ys = load_data('data/Sample15_1.csv', 'data/label.csv')
    #xs, ys = load_data('data/Sample_4_15_1.csv', 'data/label_4.csv')
    xs, ys = load_data('E:\\G-1149\\trafficCongestion\\训练数据\\4小时文件\\trainData\\res\\resSam.csv',
                             'E:\\G-1149\\trafficCongestion\\训练数据\\4小时文件\\trainData\\res\\resLabel.csv')
    x_train, y_train, x_test, y_test = split_data(xs, ys, ratio=0.7)

    # 测试不同方法
    # 评价指标
    table = PrettyTable(['Methods', 'Accuracy', 'F1', 'Precision', 'Recall'])
    methods = {}
    # 1.Logistic Regression
    lr = linear_model.LogisticRegression(solver='lbfgs')
    methods['Logistic Regression'] = lr

    # 2. Support Vector Machines
    svc = svm.SVC()
    methods['Support Vector Machines'] = svc

    # 3. Stochastic Gradient Descent
    # sgd = linear_model.SGDClassifier()
    # methods['Stochastic Gradient Descent'] = sgd

    # 4. Nearest Neighbors
    knn = neighbors.KNeighborsClassifier(n_neighbors=5, weights='uniform')
    methods['Nearest Neighbors'] = knn

    # 5. Naive Bayes
    gnb = naive_bayes.GaussianNB()
    methods['Naive Bayes'] = gnb

    # 6.Decision Trees
    dct = tree.DecisionTreeClassifier()
    methods['Decision Trees'] = dct

    # 7. Boost Methods
    # 7.1 AdaBoost
    ada = ensemble.AdaBoostClassifier()
    methods['AdaBoost'] = ada

    # 7.2 Bagging
    # bag = ensemble.BaggingClassifier()
    # methods['Bagging'] = bag

    # 7.3 Random Forest
    rf = ensemble.RandomForestClassifier()
    methods['Random Forest'] = rf

    # 7.4 Gradient Tree Boosting
    gbdt = ensemble.GradientBoostingClassifier()
    methods['Gradient Tree Boosting'] = gbdt

    # 7.5 XGBoosting
    # xg = XGBClassifier()
    # methods['XGBoosting'] = xg
    for name, model in methods.items():
        get_result(name, model, x_train, y_train, x_test, y_test, table)
    print(table)

if __name__ == '__main__':
    main()


from setuptools import setup, find_packages
from os import path
import sys
from io import open

here = path.abspath(path.dirname(__file__))
sys.path.append(here)
import properties

# Get the long description from the README file
with open(path.join(path.dirname(here), 'README.md'), encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='syncmanagerapi',
    version=properties.__version__,
    description='Provides Server for managing multiple synchronizations via unison, git, ...',
    long_description=long_description,
    long_description_content_type='text/markdown',
    url='https://github.com/Grid-LTS/syncmanager',
    author='Gerd Friemel',
    author_email='gerd.friemel@gmail.com',
    license = 'MIT',
    # For a list of valid classifiers, see https://pypi.org/classifiers/
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'Topic :: Software Development :: Build Tools',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
    ],
    keywords='git unison',
    packages=find_packages(exclude=['contrib', 'docs', 'tests']),
    package_data={'syncmanagerapi': ['swagger.yaml']},
    include_package_data=True,
    install_requires=['configparser','connexion','flask','flask-marshmallow','flask-sqlalchemy','flask-basicauth',
                      'gitpython','marshmallow>=3.0.0','marshmallow-sqlalchemy', 'mysqlclient', 'python-dotenv'],
    python_requires='>=3',

    extras_require={
        'dev': [
            'setuptools>=35.0.1'
            'wheel>=0.29.0'
                ],
    },

    # executes the function `main` from this package when invoked:
    entry_points={
        'console_scripts': [
            'syncmanagerapi=syncmanagerapi:main',
        ],
    }

)


import os
import sqlite3
import json
import string
from functools import lru_cache
from typing import Dict
from shutil import copyfile
from pathlib import Path
import copy
import sqlite3

class CopyStyle(object):
    conn = None
    COLUMNS_NAMES = ["CLASS", "CATEGORY", "NAME", "TAGS", "CONTENT",
                     "KEY"]  # "ID" is the first column, should be an autonumber
    INSERT_SQL = f"INSERT INTO ITEMS({','.join(COLUMNS_NAMES)}) VALUES(?,?,?,?,?,?)"

    def __init__(self, stylex: str):
        # create a database connection
        self.style = stylex

    @staticmethod
    def _create_connection(db_file):
        """ create a database connection to the SQLite database
          specified by the db_file
            :param db_file: database file
            :return: Connection object or None
        """
        try:
            conn = sqlite3.connect(db_file)
            return conn
        except Exception as e:
            print(e)

        return None

    @lru_cache()
    def _select_all_from_table(self, conn, table_name: str):
        """
        Query all rows in the defined table
        :param conn: the Connection object
        :return:
        """
        cur = conn.cursor()
        cur.execute(f"SELECT * FROM {table_name}")

        return cur.fetchall()

    def copy_style(self, symbol_key: str, copy_count: int, out_file: str):
        copyfile(self.style, out_file)
        conn = self._create_connection(out_file)
        query_sql = f"SELECT * FROM ITEMS WHERE KEY = '{symbol_key}'"
        cur = conn.cursor()
        cur.execute(query_sql)
        exiting_symbol = cur.fetchone()
        if not exiting_symbol:
            return
        sym_cim = json.loads(exiting_symbol[5].strip('\x00'), parse_float=str)
        insert_rows = []
        for i in range(copy_count):
            new_sym = copy.deepcopy(sym_cim)
            new_sym['symbolLayers'][0]['primitiveName'] = f"{new_sym['symbolLayers'][0]['primitiveName']}_{i}"
            insert_rows.append(dict(
                CLASS=exiting_symbol[1],
                CATEGORY=exiting_symbol[2],
                NAME=exiting_symbol[3],
                TAGS=exiting_symbol[4],
                CONTENT=sqlite3.Binary(json.dumps(new_sym).encode()),
                KEY=f"{exiting_symbol[6]}_{i}"))
        insert_rows = [tuple(row.values()) for row in insert_rows]
        cur.executemany(self.INSERT_SQL, insert_rows)
        cur.close()
        conn.commit()
        conn.close()

if __name__ == "__main__":
    key_to_copy = 'ConduitPoint'
    source_style = r"C:\_MyFiles\github\arcade-expressions\dictionary_renderer\Conduit\base_symbol.stylx"
    output_style = r"C:\_MyFiles\github\arcade-expressions\dictionary_renderer\Conduit\ConduitKnockouts.stylx"
    num_copies = 1000
    CopyStyle(source_style).copy_style(key_to_copy, num_copies, output_style)


from lxml import etree
from datetime import datetime

class BotEmailParser:

	def __init__(self, filename):
		self.filename = filename

	def parse_currency(self, currency):
		if currency == 'GBP':
			return '£'
		elif currency == 'EUR':
			return '€'
		elif currency == 'USD' or currency == 'AUD':
			return '$'
		else:
			return None

	def construct_email(self, origin, currency, advenchas, api_key_16):
		currency_symbol = self.parse_currency(currency)

		parser = etree.HTMLParser()
		tree = etree.parse(self.filename, parser)
		#main-title
		find = etree.XPath("//h1[@id = 'main-title']")
		main_title = find(tree)[0]
		main_title.text = "Deals From %s" % origin

		#details of each offer
		count = 1
		for advencha in advenchas:
			if count == 4:
				break
			xp = "//strong[@id = 'location-%s-title']" % str(count)
			find = etree.XPath(xp)
			location_title = find(tree)[0]
			location_title.text = advencha.return_place
			xp = "//p[@id = 'location-%s-text']" % str(count)
			find = etree.XPath(xp)
			location_text = find(tree)[0]

			dept_date = datetime.strptime(advencha.departure_date, "%Y-%m-%dT%H:%M:%S")
			dept_string = dept_date.strftime("%A %d-%b-%y")
			ret_date = datetime.strptime(advencha.return_date, "%Y-%m-%dT%H:%M:%S")
			ret_string = ret_date.strftime("%A %d-%b-%y")

			location_text.text = "%s to %s" % (dept_string, ret_string)
			xp = "//a[@id = 'location-%s-button']" % str(count)
			find = etree.XPath(xp)
			location_button = find(tree)[0]
			url = "http://partners.api.skyscanner.net/apiservices/referral/v1.0/GB/%s/en-GB/%s/%s/%s/%s?apiKey=%s" % (currency, advencha.departure_iata, advencha.return_iata, str(advencha.departure_date.split('T')[0]), str(advencha.return_date.split('T')[0]), api_key_16)
			location_button.attrib['href'] = url
			location_button.text = "%s%s" % (currency_symbol, str(advencha.price))
			count = count + 1

		#hide the unused tiles from the email
		count2 = 3
		while count2 > count - 1:
			xp = "//table[@id = 'location-%s-table']" % str(count2)
			find = etree.XPath(xp)
			location_table = find(tree)[0]
			hidden_style = "display: none; visibility: hidden;";
			location_table.attrib['style'] = hidden_style
			count2 = count2 - 1

		return etree.tostring(tree, encoding="ascii", pretty_print=True, method="html")


import numpy as np
import os

import torch
import torch.nn as nn
import torch.nn.functional as F

from models.reid.inception import Inception_v1
from models.reid.cbp import CompactBilinearPooling as pooling

class Body(nn.Module):
    def __init__(self, depth_dim, num_features_part=128, use_relu=False, dilation=1, initialize=True):
        super(Body, self).__init__()

        self.depth_dim = depth_dim
        self.use_relu = use_relu
        self.dilation = dilation

        self.conv1_1 = nn.Conv2d(3, out_channels=64, kernel_size=3, stride=1, padding=1)
        self.conv1_2 = nn.Conv2d(64, out_channels=64, kernel_size=3, stride=1, padding=1)
        self.pool1_stage1 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0, ceil_mode=True)

        self.conv2_1 = nn.Conv2d(64, out_channels=128, kernel_size=3, stride=1, padding=1)
        self.conv2_2 = nn.Conv2d(128, out_channels=128, kernel_size=3, stride=1, padding=1)
        self.pool2_stage1 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0, ceil_mode=True)

        self.conv3_1 = nn.Conv2d(128, out_channels=256, kernel_size=3, stride=1, padding=1)
        self.conv3_2 = nn.Conv2d(256, out_channels=256, kernel_size=3, stride=1, padding=1)
        self.conv3_3 = nn.Conv2d(256, out_channels=256, kernel_size=3, stride=1, padding=1)
        self.conv3_4 = nn.Conv2d(256, out_channels=256, kernel_size=3, stride=1, padding=1)
        self.pool3_stage1 = nn.MaxPool2d(kernel_size=2, stride=2, padding=0, ceil_mode=True)

        self.conv4_1 = nn.Conv2d(256, out_channels=512, kernel_size=3, stride=1, padding=1)
        self.conv4_2 = nn.Conv2d(512, out_channels=512, kernel_size=3, stride=1, padding=1)
        self.conv4_3_CPM = nn.Conv2d(512, out_channels=256, kernel_size=3, stride=1, padding=1)
        self.conv4_4_CPM = nn.Conv2d(256, out_channels=128, kernel_size=3, stride=1, padding=1)

        # Stage1
        # limbs
        self.conv5_1_CPM_L1 = nn.Conv2d(128, out_channels=128, kernel_size=3, stride=1, padding=1)
        self.conv5_2_CPM_L1 = nn.Conv2d(128, out_channels=128, kernel_size=3, stride=1, padding=1)
        self.conv5_3_CPM_L1 = nn.Conv2d(128, out_channels=128, kernel_size=3, stride=1, padding=1)
        self.conv5_4_CPM_L1 = nn.Conv2d(128, out_channels=512, kernel_size=1, stride=1, padding=0)
        self.conv5_5_CPM_L1 = nn.Conv2d(512, out_channels=38, kernel_size=1, stride=1, padding=0)
        # joints
        self.conv5_1_CPM_L2 = nn.Conv2d(128, out_channels=128, kernel_size=3, stride=1, padding=1)
        self.conv5_2_CPM_L2 = nn.Conv2d(128, out_channels=128, kernel_size=3, stride=1, padding=1)
        self.conv5_3_CPM_L2 = nn.Conv2d(128, out_channels=128, kernel_size=3, stride=1, padding=1)
        self.conv5_4_CPM_L2 = nn.Conv2d(128, out_channels=512, kernel_size=1, stride=1, padding=0)
        self.conv5_5_CPM_L2 = nn.Conv2d(512, out_channels=19, kernel_size=1, stride=1, padding=0)

        if self.dilation == 1:
            self.concat_stage3_pool = nn.MaxPool2d(kernel_size=2, stride=2, padding=0, ceil_mode=True)

        # Stage2
        # limbs
        self.Mconv1_stage2_L1 = nn.Conv2d(185, out_channels=128, kernel_size=7, stride=1, padding=3)
        self.Mconv2_stage2_L1 = nn.Conv2d(128, out_channels=128, kernel_size=7, stride=1, padding=3)
        self.Mconv3_stage2_L1 = nn.Conv2d(128, out_channels=128, kernel_size=7, stride=1, padding=3)
        self.Mconv4_stage2_L1 = nn.Conv2d(128, out_channels=128, kernel_size=7, stride=1, padding=3)
        self.Mconv5_stage2_L1 = nn.Conv2d(128, out_channels=128, kernel_size=7, stride=1, padding=3)
        self.Mconv6_stage2_L1 = nn.Conv2d(128, out_channels=128, kernel_size=1, stride=1, padding=0)
        self.Mconv7_stage2_L1 = nn.Conv2d(128, out_channels=38, kernel_size=1, stride=1, padding=0)
        # joints
        self.Mconv1_stage2_L2 = nn.Conv2d(185, out_channels=128, kernel_size=7, stride=1, padding=3)
        self.Mconv2_stage2_L2 = nn.Conv2d(128, out_channels=128, kernel_size=7, stride=1, padding=3)
        self.Mconv3_stage2_L2 = nn.Conv2d(128, out_channels=128, kernel_size=7, stride=1, padding=3)
        self.Mconv4_stage2_L2 = nn.Conv2d(128, out_channels=128, kernel_size=7, stride=1, padding=3)
        self.Mconv5_stage2_L2 = nn.Conv2d(128, out_channels=128, kernel_size=7, stride=1, padding=3)
        self.Mconv6_stage2_L2 = nn.Conv2d(128, out_channels=128, kernel_size=1, stride=1, padding=0)
        self.Mconv7_stage2_L2 = nn.Conv2d(128, out_channels=19, kernel_size=1, stride=1, padding=0)

        self.pose1 = nn.Conv2d(185, out_channels=num_features_part, kernel_size=3, stride=1, padding=1)
        self.bn = nn.BatchNorm2d(num_features_part, affine=False)

        if initialize:
            self.init_pretrained()

    def forward(self, inputs):
        ## similar to resnet structure and pose extractor from CMU-openpose
        output1 = 0.0039 * inputs

        output1 = F.relu(self.conv1_1(output1))
        output1 = F.relu(self.conv1_2(output1))
        output1 = self.pool1_stage1(output1)

        output1 = F.relu(self.conv2_1(output1))
        output1 = F.relu(self.conv2_2(output1))
        output1 = self.pool2_stage1(output1)

        output1 = F.relu(self.conv3_1(output1))
        output1 = F.relu(self.conv3_2(output1))
        output1 = F.relu(self.conv3_3(output1))
        output1 = F.relu(self.conv3_4(output1))
        output1 = self.pool3_stage1(output1)

        output1 = F.relu(self.conv4_1(output1))
        output1 = F.relu(self.conv4_2(output1))
        output1 = F.relu(self.conv4_3_CPM(output1))
        output1 = F.relu(self.conv4_4_CPM(output1))

        output2_1 = F.relu(self.conv5_1_CPM_L1(output1))
        output2_1 = F.relu(self.conv5_2_CPM_L1(output2_1))
        output2_1 = F.relu(self.conv5_3_CPM_L1(output2_1))
        output2_1 = F.relu(self.conv5_4_CPM_L1(output2_1))
        output2_1 = self.conv5_5_CPM_L1(output2_1)

        output2_2 = F.relu(self.conv5_1_CPM_L2(output1))
        output2_2 = F.relu(self.conv5_2_CPM_L2(output2_2))
        output2_2 = F.relu(self.conv5_3_CPM_L2(output2_2))
        output2_2 = F.relu(self.conv5_4_CPM_L2(output2_2))
        output2_2 = self.conv5_5_CPM_L2(output2_2)

        output2 = torch.cat([output2_1, output2_2, output1], dim=self.depth_dim)

        output3_1 = F.relu(self.Mconv1_stage2_L1(output2))
        output3_1 = F.relu(self.Mconv2_stage2_L1(output3_1))
        output3_1 = F.relu(self.Mconv3_stage2_L1(output3_1))
        output3_1 = F.relu(self.Mconv4_stage2_L1(output3_1))
        output3_1 = F.relu(self.Mconv5_stage2_L1(output3_1))
        output3_1 = F.relu(self.Mconv6_stage2_L1(output3_1))
        output3_1 = self.Mconv7_stage2_L1(output3_1)

        output3_2 = F.relu(self.Mconv1_stage2_L2(output2))
        output3_2 = F.relu(self.Mconv2_stage2_L2(output3_2))
        output3_2 = F.relu(self.Mconv3_stage2_L2(output3_2))
        output3_2 = F.relu(self.Mconv4_stage2_L2(output3_2))
        output3_2 = F.relu(self.Mconv5_stage2_L2(output3_2))
        output3_2 = F.relu(self.Mconv6_stage2_L2(output3_2))
        output3_2 = self.Mconv7_stage2_L2(output3_2)

        output3 = torch.cat([output3_1, output3_2, output1], dim=self.depth_dim)
        if self.dilation == 1:
            output3 = self.concat_stage3_pool(output3)
        output3 = self.pose1(output3)
        output3 = self.bn(output3)
        if self.use_relu:
            output3 = F.relu(output3)

        return output3

    def init_pretrained(self):
        # state_dict = torch.load(self.pretrained)
        state_dict = torch.load('/home/SharedData/swapnil/tracker/re-id/reid/pretrained/pose_iter_440000.caffemodel.pth')
        state_dict['pose1.weight'] = nn.init.xavier_uniform_(self.pose1.weight).detach()
        state_dict['pose1.bias'] = torch.zeros(self.pose1.bias.size())
        #TODO
        state_dict['bn.running_mean'] = self.bn.running_mean
        state_dict['bn.running_var'] = self.bn.running_var

        model_dict = {}
        for k,v in state_dict.items():
            for l,p in self.state_dict().items():
                if k in l:
                    model_dict[l] = torch.from_numpy(np.array(v)).view_as(p)

        self.load_state_dict(model_dict, strict=True)
        print('appearance_body pretrained model loaded!')

class Bilinear_Pooling(nn.Module):

    def __init__(self, num_feat1=512, num_feat2=128, num_feat_out=512):
        super(Bilinear_Pooling, self).__init__()
        self.num_feat1 = num_feat1
        self.num_feat2 = num_feat2
        self.num_feat_out = num_feat_out

        self.cbp = pooling(self.num_feat1, self.num_feat2, self.num_feat_out, sum_pool=True)

    def forward(self, input1, input2):
        assert(self.num_feat1 == input1.shape[1])
        assert(self.num_feat2 == input2.shape[1])
        output = self.cbp(input1, input2)
        output = F.normalize(output, dim=1)
        #output = output.squeeze()

        return output

class Appearance_Body(nn.Module):
    def __init__(self, num_features=512, use_bn=True, use_relu=False, dilation=1, initialize=True):
        super(Appearance_Body, self).__init__()
        num_features_app, num_features_part = (512, 128)
        self.app_feat_extractor = Inception_v1(num_features=num_features_app, dilation=dilation, initialize=initialize)
        self.part_feat_extractor = Body(1, num_features_part=num_features_part, dilation=dilation, initialize=initialize)
        self.pooling = Bilinear_Pooling(num_feat1=num_features_app, num_feat2=num_features_part, num_feat_out=num_features)

    def forward(self, inputs):
        output_app = self.app_feat_extractor(inputs)
        output_part = self.part_feat_extractor(inputs)
        outp = self.pooling(output_app, output_part)
        #print( outp.shape,outp)
        return outp

    def save_dict(self):
        state_dict = {'app_state_dict': self.app_feat_extractor.state_dict(),
                      'part_state_dict': self.part_feat_extractor.state_dict()}
        return state_dict

    def load(self, checkpoint):
        self.app_feat_extractor.load_state_dict(checkpoint['app_state_dict'])
        self.part_feat_extractor.load_state_dict(checkpoint['part_state_dict'])

    def init_pretrained(self):
        self.app_feat_extractor.init_pretrained()
        self.part_feat_extractor.init_pretrained()

def appearance_body_map(features=512, use_relu=False, dilation=1, initialize=True):
    model = Appearance_Body(num_features=features, use_relu=use_relu, dilation=dilation, initialize=initialize)

    return model


import requests
import json
import time
from config import (
    MATRIX_URL,
    VERSION,
    USERNAME)

def request(method: str, path: str, payload="{}"):
    url = f"/_matrix/client/{VERSION}/{path}"
    headers = {
        'Authorization': f'Bearer {TOKEN}',
        'Content-Type': 'application/json'
    }
    response = requests.request(method, url, headers=headers, data=payload)
    if not response.status_code == 429:
        return response
    else:
        time.sleep(3)
        return request(method, path, payload=payload)

def create_dm(user: str):
    payload = json.dumps({
        "is_direct": True,
        "invite": [
            user
        ],
        "preset": "trusted_private_chat"
    })

    response = request("POST", "createRoom", payload=payload)
    return response.json()["room_id"]

def get_user_room(user):
    response = request("GET", f"user/{USERNAME}/account_data/m.direct")
    if user in response.json():
        return response.json()[user][0]
    else:
        return create_dm(user)

def invite_user(user, roomid):
    payload = json.dumps({
        "reason": "You joined to working group using website",
        "user_id": user
    })
    response = request("POST", F"rooms/{roomid}/invite", payload=payload)

def send_message(roomid, message):
    payload = json.dumps({
        "msgtype": "m.text",
        "body": message,
        "mimetype": "text/html"
    })

    headers = {
        'Content-Type': 'application/json'
    }

    request("POST", f"rooms/{roomid}/send/m.room.message", payload=payload)


import serial
import time

# 数据保存地址
save_file_path = "xxx"

try:
    # 通讯串口配置
    port_name_sen = "com7"     # 传感器
    port_name_emd = "com3"     # 驱动
    sen = serial.Serial(port=port_name_sen, baudrate=115200, timeout=0.01, parity=serial.PARITY_NONE, bytesize=8)
    emd = serial.Serial(port=port_name_emd, baudrate=115200, timeout=0.01, parity=serial.PARITY_NONE, bytesize=8)
    if sen.isOpen():
        print(sen)
    if emd.isOpen():
        print(emd)

    # 启动
    emd.write(b'Stable_1st\r\n')

    # 接收数据并通讯
    while True:
        time.sleep(0.1)
        if emd.in_waiting:
            message = emd.readline(emd.in_waiting).decode('gbk')
            if message == "Stable_1st_Finish\r\n":     # 已核对
                sen.write(b'Read_1st\r\n')

            if message == "Stable_2nd_Finish\r\n":     # 已核对
                sen.write(b'Read_2nd\r\n')

            if message == "Stable_3rd_Finish\r\n":     # 已核对
                sen.write(b'Read_3rd\r\n')

            # if message == "MagneticEndFinish\r\n":     # 给机械臂发送指令，移动到下一个指定位置

        time.sleep(0.1)
        if sen.in_waiting:
            message = sen.readline(sen.in_waiting).decode('gbk')
            if message == "Read_1st_Finish\r\n":
                emd.write(b'Stable_2nd\r\n')

            if message == "Read_2nd_Finish\r\n":
                emd.write(b'Stable_3rd\r\n')

            if message == "Read_3rd_Finish\r\n":
                emd.write(b'Stable_1st')

            if message[9:0] == "FinishRead":
                emd.write(b'MagneticEnd\r\n')
                # 保存数据
                file_handle = open(save_file_path, mode='a+')
                file_handle.write(message)
                file_handle.close()

except Exception as e:
    print('error', e)

# w 只能操作写入  r 只能读取   a 向文件追加
# w+ 可读可写   r+可读可写    a+可读可追加
# wb+写入进制数据
# w模式打开文件，如果而文件中有数据，再次写入内容，会把原来的覆盖掉


from aiogram.types import ReplyKeyboardMarkup
from data.config import BASE_URL
import requests

rs = requests.get(url=f"{BASE_URL}main/schedule/create/")
data = rs.json()

available_time = ReplyKeyboardMarkup(row_width=3, resize_keyboard=True)
for i in data:
    if i['user_id'] is None:
        available_time.add(f"{i['month']} {i['day']} {i['time']}")
available_time.add('Bekor qilish')


from flask_restful import Resource
from flask import request
from model.observation import Observation
from database import db
import datetime

class ObservationsEndpoint(Resource):
    def get(self):
        DEFAULT_START = str(datetime.datetime.min)  # format: YYYY-MM-DD hh:mm:ss.mmmmmm
        DEFAULT_END = str(datetime.datetime.max)    # format: YYYY-MM-DD hh:mm:ss.mmmmmm

        start = request.args.get("start", default=DEFAULT_START, type=str)
        end = request.args.get("end", default=DEFAULT_END, type=str)
        sensor_id = request.args.get("sensor_id", default=None, type=int)

        query = db.select(Observation)
        query = query.filter(Observation.timestamp >= start)
        query = query.filter(Observation.timestamp <= end)

        if sensor_id is not None:
            query = query.filter(Observation.sensor_id == sensor_id)

        observations_list = db.session.execute(query).scalars()
        return [o.serialize() for o in observations_list], 200

    def post(self):
        data = request.json
        observation = Observation(
            sensor_id=data["sensor_id"],
            timestamp=data["timestamp"],
            value=data["value"]
        )
        db.session.add(observation)
        db.session.commit()
        return observation.serialize(), 201


from Gaia_knock_off.on_road import *
import requests
from osgeo import gdal, ogr, osr
from fiona.ogrext import Iterator, ItemsIterator, KeysIterator
from geopandas import GeoDataFrame
import fiona
import os
import pyrosm
import matplotlib
from pathlib import Path

'''
set up roads API infastructure

find patch of area defined by, nearest road being 10km away from nearest road within
distance start_distance to end distance radius given by the user

find list of roads that border the area,

find road closet to starting point
find road closet to ending point
get 2 on road directions

link to the via patch

off road instructions

'''

BASE_DIR = Path(__file__).resolve().parent.parent

class nearest_roads_api(object):

    def __init__(self, pointers, payload = {}, headers = {}, api_key = None):
        '''
        :cvar
        pointers: [('lat','lng'),...]
        payload: dictionary
        headers: dictionary
        api_key: google directions api key

        :return
        '''
        self.pointers = pointers
        self.payload = payload
        self.headers = headers
        self.api_key = api_key

    def pointers_string(self):

        pointers = self.pointers
        pointer_string = "points="
        for lat, lng in pointers:

            pointer_string = pointer_string + "{},{}|".format(lat,lng)

        pointer_string = pointer_string[:-1]

        pointer_string.replace(",", "%2C")
        pointer_string.replace("|", "%7C")

        return pointer_string

    def url(self):

        base_url = "https://roads.googleapis.com/v1/nearestRoads?"
        parameters = self.pointers_string()

        string = "{}{}&key={}".format(base_url, parameters, self.api_key)

        return string

    def response(self):

        url = self.url()
        response = requests.request("GET", url, headers=self.headers, data=self.payload)
        return response

    def text(self):
        response = self.response()

        return response.text

class OSM_PBF_file_generator(object):

    def __init__(self,place_name, save_location,update=True):

        self.place_name = place_name
        self.save_location = save_location
        self.update = update

    def generate(self):

        dir_path = os.path.join(BASE_DIR, self.save_location)
        file_name = "{}.osm.pbf".format(self.place_name)
        full_file_path = os.path.join(dir_path,file_name)

        if not self.update:
            if os.path.exists(full_file_path):
                print("exists")
                return full_file_path

        file_path = pyrosm.get_data(self.place_name, directory=dir_path, update=self.update)
        return file_path

'''
PSUEDO ALGO FOR OFF ROAD DIRECTIONS ARE IN THE FOLLOWING

based on start and end destination, check which geographic location
based on lattitude and longitude

the and try to box it in on the osm.pbf file and feed it into

from offroad_routing import Geometry

filename = "../maps/user_area.osm.pbf"
bbox = [34, 59, 34.2, 59.1]
geom = Geometry.parse(filename=filename, bbox=bbox)

https://denikozub.github.io/Offroad-routing-engine/#installation

From here an off road route can be generated

You may use stopping points based on longitude and latitude and feed into the on_road API built

that will give you the instructions via text format
'''


import datetime
import os
from pathlib import Path

import numpy as np
import gdal

__author__ = "J Gomez-Dans"
__copyright__ = "Copyright 2017, 2018 J Gomez-Dans"
__license__ = "GPLv3"
__email__ = "j.gomez-dans@ucl.ac.uk"

GDAL2NUMPY = {gdal.GDT_Byte:   np.uint8,
              gdal.GDT_UInt16:   np.uint16,
              gdal.GDT_Int16:   np.int16,
              gdal.GDT_UInt32:   np.uint32,
              gdal.GDT_Int32:   np.int32,
              gdal.GDT_Float32:   np.float32,
              gdal.GDT_Float64:   np.float64,
              gdal.GDT_CInt16:   np.complex64,
              gdal.GDT_CInt32:   np.complex64,
              gdal.GDT_CFloat32:   np.complex64,
              gdal.GDT_CFloat64:   np.complex128
              }

def find_granules(dire, tile, product, start_time, end_time):
    """Find MCD43 granules based on folder, tile and product type (A1
    or A2). Returns a dictionary of datetimes of the products and
    granules, or raises an IOError exception if not files found."""
    times = []
    fnames = []
    path = Path(dire)
    start_year = start_time.year
    end_year = end_time.year
    granules_start = path.rglob(f"**/MCD43{product:s}.A{start_year:4d}*.{tile:s}.*.hdf")
    granules = [f  for f in granules_start]
    if end_year != start_year:
        granules_end = path.rglob(f"**/MCD43{product:s}.A{end_year:4d}*.{tile:s}.*.hdf")
        granules = granules + [f for f in granules_end]
    granules = list(set(granules))
    if len(granules) == 0:
        raise IOError("Couldn't find any MCD43%s files in %s" % (product, dire))
    for granule in granules:
        fich = os.path.basename (granule)
        timex = datetime.datetime.strptime(fich.split(".")[1][1:], "%Y%j")
        if timex >= start_time and \
            (end_time is None or timex <= end_time ):
            times.append (timex)
            fnames.append(granule.as_posix())
    return dict(list(zip(times, fnames)))

def process_time_input(timestamp):
    """Processes a timestamp given either as (i) a string in
    "%Y-%m-%d" format, (ii) a string in "%Y%j" format or
    (iii) a datetime.datetime object. Returns a datetime.datetime
    ojbect, and raises ValueError if none of the options fits."""
    if type(timestamp) == datetime.datetime:
        output_time = timestamp
    elif type(timestamp) == str:
        try:
            output_time = datetime.datetime.strptime(timestamp,
                                                     "%Y-%m-%d")
        except ValueError:
            try:
                output_time = datetime.datetime.strptime(timestamp,
                                                         "%Y%j")
            except ValueError:
                raise ValueError("The passed timestamp wasn't either " +
                                 'a "%Y-%m-%d" string, a "%Y%j" string')
    else:
        raise ValueError("You can only use a string or a datetime object")
    return output_time

def open_gdal_dataset(fname, roi=None):
    g = gdal.Open(fname)
    if g is None:
        raise IOError("Can't open %s" % fname)
    if roi is None:
        data = g.ReadAsArray()
    else:
        ulx, uly, lrx, lry = roi
        xoff = ulx
        yoff = uly
        xcount = lrx - ulx
        ycount = lry - uly
        data = g.ReadAsArray(xoff, yoff, xcount, ycount).astype(
                             GDAL2NUMPY[g.GetRasterBand(1).DataType])
    return data

def process_masked_kernels(band_no, a1_granule, a2_granule,
                           band_transfer=None, roi=None):
    if band_transfer is not None:
        band_no = band_transfer[band_no]

    fname_a1 = 'HDF4_EOS:EOS_GRID:"%s":MOD_Grid_BRDF:' % (a1_granule)
    fname_a2 = 'HDF4_EOS:EOS_GRID:"%s":MOD_Grid_BRDF:' % (a2_granule)
    try:
        fdata = fname_a1 + 'BRDF_Albedo_Parameters_Band%d' % (band_no)
    except TypeError:
        fdata = fname_a1 + 'BRDF_Albedo_Parameters_%s' % (band_no)

    fsnow = fname_a2 + 'Snow_BRDF_Albedo'
    fland = fname_a2 + 'BRDF_Albedo_LandWaterType'
    func = fname_a2 + 'BRDF_Albedo_Uncertainty'   # % a2_granule
    try:
        fqa = fname_a2 + 'BRDF_Albedo_Band_Quality_Band%d' % band_no
    except TypeError:
        fqa = fname_a1 + 'BRDF_Albedo_Band_Mandatory_Quality_%s' % band_no

    for fname in [fdata, fsnow, fland, fqa]:
        data = open_gdal_dataset(fname, roi)
        if fname.find("Albedo_Parameters") >= 0:
            # Read kernels, post process
            kernels = process_kernels(data)
        elif fname.find("Snow") >= 0:
            # Read snow mask... post process
            snow = process_snow(data)
        elif fname.find("XXXXXLandWaterType") >= 0:
            shp = data.shape
            land = np.in1d(data, [1, 3, 4, 5])    # data == 1 # Only land
            land = land.reshape(shp)

        # elif fname.find("BRDF_Albedo_Uncertainty") >= 0:
        #    unc = process_unc (data)
        elif fname.find("BRDF_Albedo_Band_Quality") >= 0 or \
                fname.find("BRDF_Albedo_Band_Mandatory_Quality") >= 0:
            qa = np.where(data <= 1, True, False)   # Best & good
            qa_val = data*1

    # Create mask:
    # 1. Ignore snow
    # 2. Only land
    # 3. Only good and best
    mask = snow * qa   # *land * qa
    qa_val = np.where(mask, qa_val, np.nan)
    return kernels, mask, qa_val

def process_unc(unc):
    """Process uncertainty. Fuck know what it means..."""
    unc = np.where(unc == 32767, np.nan, unc/1000.)

def process_snow(snow):
    """Returns True if snow free albedo retrieval"""
    return np.where(snow == 0, True, False)

def process_kernels(kernels):
    """Scales the kernels, maybe does other things"""
    kernels = np.where(kernels == 32767, np.nan, kernels/1000.)
    return kernels

class RetrieveBRDFDescriptors(object):
    """Retrieving BRDF descriptors."""

    def __init__(self, tile, mcd43a1_dir, start_time, end_time=None,
                 mcd43a2_dir=None, roi=None):
        """The class needs to locate the data granules. We assume that
        these are available somewhere in the filesystem and that we can
        index them by location (MODIS tile name e.g. "h19v10") and
        time. The user can give a folder for the MCD43A1 and A2 granules,
        and if the second is ignored, it will be assumed that they are
        in the same folder. We also need a starting date (either a
        datetime object, or a string in "%Y-%m-%d" or "%Y%j" format. If
        the end time is not specified, it will be set to the date of the
        latest granule found."""

        self.tile = tile
        self.start_time = process_time_input(start_time)
        if end_time is not None:
            self.end_time = process_time_input(end_time)
        else:
            self.end_time = None
        if os.path.exists(mcd43a1_dir):
            self.mcd43a1_dir = mcd43a1_dir
        else:
            raise IOError("mcd43a1_dir does not exist!")
        self.a1_granules = find_granules(self.mcd43a1_dir, tile, "A1",
                                         self.start_time, self.end_time)
        if mcd43a2_dir is None:
            self.mcd43a2_dir = mcd43a1_dir
        else:
            if os.path.exists(mcd43a2_dir):
                self.mcd43a2_dir = mcd43a2_dir
            else:
                raise IOError("mcd43a2_dir does not exist!")
        self.a2_granules = find_granules(self.mcd43a2_dir, tile, "A2",
                                         self.start_time, self.end_time)
        a1_dates = set(self.a1_granules.keys())
        a2_dates = set(self.a2_granules.keys())
        if a1_dates != a2_dates:
            raise ValueError("A1 and A2 product files do not overlap!")

        self.band_transfer = None

        if roi is not None:
            assert len(roi) == 4,\
                        f"ROI box needs 4 elements! It has only {len(roi)}!"
            ulx, uly, lrx, lry = roi
            assert(ulx < lrx), f" ulx{ulx} !< lrx{lrx}"
            assert(uly < lry), f" uly{uly} !< lry{lry}"
            self.roi = roi
        else:
            self.roi = None
    def get_brdf_descriptors(self, band_no, date):
        #        if not (1 <= band_no <= 7) :
        #            raise ValueError ("Bands can only go from 1 to 7!")

        the_date = process_time_input(date)
        try:
            a1_granule = self.a1_granules[the_date]
        except KeyError:
            return None
        a2_granule = self.a2_granules[the_date]
        kernels, mask, qa = process_masked_kernels(band_no, a1_granule,
                                                   a2_granule,
                                                   band_transfer=self.band_transfer,
                                                   roi=self.roi)
        return kernels, mask, qa

if __name__ == "__main__":
    mcd43a1_dir = "/group_workspaces/cems2/qa4ecv/vol2/modis.c6.brdf/ladsweb.nascom.nasa.gov/allData/6/MCD43A1/2015/"
    mcd43a2_dir = "/group_workspaces/cems2/qa4ecv/vol2/modis.c6.brdf/ladsweb.nascom.nasa.gov/allData/6/MCD43A2/2015/"
    rr = RetrieveBRDFDescriptors("h17v05",
                                 mcd43a1_dir,
                                 "2015-01-01", mcd43a2_dir=mcd43a2_dir,
                                 end_time="2015-12-31")
    roi=[1100, 640, 1400,740]
    rr_chunk = RetrieveBRDFDescriptors("h17v05",
                                 mcd43a1_dir,
                                 "2015-01-01",
                                 mcd43a2_dir=mcd43a2_dir,
                                 end_time="2015-12-31",
                                    roi=roi)


#-*- coding: utf-8 -*-
import requests
import json
import configparser
import xlrd
from vendoasg.vendoasg import Vendo
config = configparser.ConfigParser()
config.read('vendo.ini')

# połączenie z bazą vendo
vendoApi = Vendo(config.get('vendo','vendo_API_port'))
vendoApi.logInApi(config.get('vendo','logInApi_user'),config.get('vendo','logInApi_pass'))
vendoApi.loginUser(config.get('vendo','loginUser_user'),config.get('vendo','loginUser_pass'))

###################################################################
def addTag(numerID,nazwa_wd,nowa_lista):
    response_data = vendoApi.getJson ('/json/reply/Magazyn_Towary_Aktualizuj', {"Token":vendoApi.USER_TOKEN,"Model":{
        "ID":numerID,
        "PolaUzytkownika":{
            "NazwaWewnetrzna":nazwa_wd ,
            "Wartosc":nowa_lista
            }
        }})
###################################################################
def checkWD():
    plik = xlrd.open_workbook("tagi.xls")
    strona = plik.sheet_by_index(0)
    total_cols = strona.ncols
    total_rows = strona.nrows
    jezyki = ['pl','de','fr','en','cz']
    for jezyk in jezyki:
        i = 1
        nazwa_wd = f'tagi_{jezyk}'
        for e in range(total_rows-1):
            kod_towaru = strona.cell(i,0).value
            nowy_tag = str(strona.cell(i,1).value)
            if kod_towaru != "":
                if isinstance(kod_towaru, float):
                    kod_towaru = str(int(kod_towaru))
                if len(kod_towaru) == 4:
                    kod_towaru = "0" + kod_towaru
                print(kod_towaru)
                try:
                    kod_query = vendoApi.getJson ('/Magazyn/Towary/TowarRozszerzony', {"Token":vendoApi.USER_TOKEN,"Model":{
                        "ZwrocWartosciDowolne":True,
                        "Towar":{"Kod":kod_towaru}
                        }})
                    kod_query = kod_query["Wynik"]

                    numerID = kod_query["KartaTowaru"]["Towar"]["ID"]
                    wd = kod_query["WartosciDowolne"]
                    lista_wd = []
                    for w in wd:
                        lista_wd.append(w['Nazwa'])

                        if w['Nazwa'] == nazwa_wd:
                            lista_tagow = w['Wartosci'][0].replace(" ", "").split(",")
                            dodac = True
                            for tag in lista_tagow:
                                if nowy_tag in tag:
                                    dodac = False
                                    break
                            if dodac == True:
                                lista_tagow = (";").join(lista_tagow)

                                lista_tagow = lista_tagow +";"+ nowy_tag

                                addTag(numerID,nazwa_wd,lista_tagow)
                            else:
                                print("TAG już jest")
                    if nazwa_wd not in lista_wd:
                        addTag(numerID,nazwa_wd,nowy_tag)
                        print("BYŁO PUSTE")
                    i +=1
                except:
                    i +=1
                    continue

if __name__ == '__main__':
    checkWD()
# zakończenie pracy z bazą vendo
    vendoApi.logOutApi()


# Process_5_CompileData.py

__author__ = "Scott Matsumura"
__last_update__ = "4.15.2022"

import os
import pickle
import bz2
import pandas as pd
from ete3 import NCBITaxa

PATH_FASTA = '/PyCharm/DNAEntropy/fasta/'
LIST_KINGDOM = ['bacteria', 'fungi', 'metazoa', 'plants', 'protozoa', 'viridae']

def compile_to_csv(list_kingdom, path_fasta):
    """ Convert data into a single csv graph to be used to run statistical analysis on.

    :param list_kingdom: List of kingdom names to be processed.
    :param path_fasta: Location of fasta directory from the home directory, which will also be used as save directory.
    """

    ''' Nested functions '''
    def load_bz2(filename):
        """ Loads binary bzip2 file to array list

        :param filename: file name
        :return: file
        """
        with bz2.BZ2File(filename=filename, mode='rb') as f:
            return_file = pickle.load(f)
        f.close()
        return return_file

    def load_text_to_list(filename):
        """ Loads text file to array list

        :param filename: file name
        :return: array list
        """
        with open(file=filename, mode='rt') as f:
            return_array = f.read().splitlines()
        f.close()
        return return_array

    ''' Start of "def process_species '''

    # Directory and file names
    dir_process_species = 'ProcessSpecies'
    dir_fileindex = 'FileIndex'
    dir_csvdata = 'zz_CSVdata'
    file_csvdata = 'entropy_data.csv'
    file_index_correct_txt = 'file_index_correct.txt'

    path_home = os.getenv('HOME')
    path_save = path_home + path_fasta + '/' + dir_csvdata + '/'

    if not os.path.exists(path_save):
        os.makedirs(path_save)

    ncbi = NCBITaxa()

    # Column names of CSV file
    list_columns = ['NCBI_id',                  # 0
                    'OrthoDB_seq',              # 1
                    'species_name',             # 2
                    'amino_entropy',            # 3
                    'protein_entropy',          # 4
                    'constructive_entropy',     # 5
                    'divergence_time',          # 6
                    'amino_edge_E(G)',          # 7
                    'amino_total_N',            # 8
                    'protein_edge_E(G)',        # 9
                    'protein_total_N',          # 10
                    'superkingdom',             # 11
                    'superkingdom_name',        # 12
                    'kingdom',                  # 13
                    'kingdom_name',             # 14
                    'subkingdom',               # 15
                    'subkingdom_name',          # 16
                    'phylum',                   # 17
                    'phylum_name',              # 18
                    'subphylum',                # 19
                    'subphylum_name',           # 20
                    'superclass',               # 21
                    'superclass_name',          # 22
                    'class',                    # 23
                    'class_name',               # 24
                    'subclass',                 # 25
                    'subclass_name',            # 26
                    'order',                    # 27
                    'order_name',               # 28
                    'family',                   # 29
                    'family_name',              # 30
                    'genus',                    # 31
                    'genus_name']               # 32
    df_data_compile = pd.DataFrame(columns=list_columns)

    print("Processing Compile to CSV:")
    for str_kingdom in list_kingdom:
        path_index = path_home + path_fasta + str_kingdom + '/' + dir_fileindex + '/'
        path_load = path_home + path_fasta + str_kingdom + '/' + dir_process_species + '/'
        list_index_corrected = load_text_to_list(path_index + file_index_correct_txt)

        size_index_corrected = len(list_index_corrected)
        for int_index, str_file in enumerate(list_index_corrected, 1):
            path_load_species_bz2 = path_load + str_file
            dict_species_data = load_bz2(path_load_species_bz2)
            dict_data = dict_species_data['data']
            list_lineage = dict_data['lineage_id']
            dict_rank = ncbi.get_rank(list_lineage)
            dict_rank = {value: key for key, value in dict_rank.items()}
            list_lineage_id = [dict_rank.get('superkingdom'),
                               dict_rank.get('kingdom'),
                               dict_rank.get('subkingdom'),
                               dict_rank.get('phylum'),
                               dict_rank.get('subphylum'),
                               dict_rank.get('superclass'),
                               dict_rank.get('class'),
                               dict_rank.get('subclass'),
                               dict_rank.get('order'),
                               dict_rank.get('family'),
                               dict_rank.get('genus')]
            list_lineage_id = [1 if x is None else x for x in list_lineage_id]
            dict_lineage_name = ncbi.get_taxid_translator(list_lineage_id)
            list_lineage_name = [dict_lineage_name[list_lineage_id[0]],
                                 dict_lineage_name[list_lineage_id[1]],
                                 dict_lineage_name[list_lineage_id[2]],
                                 dict_lineage_name[list_lineage_id[3]],
                                 dict_lineage_name[list_lineage_id[4]],
                                 dict_lineage_name[list_lineage_id[5]],
                                 dict_lineage_name[list_lineage_id[6]],
                                 dict_lineage_name[list_lineage_id[7]],
                                 dict_lineage_name[list_lineage_id[8]],
                                 dict_lineage_name[list_lineage_id[9]],
                                 dict_lineage_name[list_lineage_id[10]]]
            list_lineage_id = [x if x != 1 else None for x in list_lineage_id]
            list_lineage_name = [x if x != 'root' else None for x in list_lineage_name]

            # dict_line = {list_columns[0]: dict_data['species_id'],
            #              list_columns[1]: dict_data['sequence_num'],
            #              list_columns[2]: dict_data['species_name'],
            #              list_columns[3]: dict_data['amino_entropy'],
            #              list_columns[4]: dict_data['protein_entropy'],
            #              list_columns[5]: dict_data['constructive_entropy'],
            #              list_columns[6]: dict_data['divergence_time'],
            #              list_columns[7]: dict_data['amino_edge_count'],
            #              list_columns[8]: dict_data['amino_count'],
            #              list_columns[9]: dict_data['protein_edge_count'],
            #              list_columns[10]: dict_data['protein_count'],
            #              list_columns[11]: list_lineage_id[0],
            #              list_columns[12]: list_lineage_name[0],
            #              list_columns[13]: list_lineage_id[1],
            #              list_columns[14]: list_lineage_name[1],
            #              list_columns[15]: list_lineage_id[2],
            #              list_columns[16]: list_lineage_name[2],
            #              list_columns[17]: list_lineage_id[3],
            #              list_columns[18]: list_lineage_name[3],
            #              list_columns[19]: list_lineage_id[4],
            #              list_columns[20]: list_lineage_name[4],
            #              list_columns[21]: list_lineage_id[5],
            #              list_columns[22]: list_lineage_name[5],
            #              list_columns[23]: list_lineage_id[6],
            #              list_columns[24]: list_lineage_name[6],
            #              list_columns[25]: list_lineage_id[7],
            #              list_columns[26]: list_lineage_name[7],
            #              list_columns[27]: list_lineage_id[8],
            #              list_columns[28]: list_lineage_name[8],
            #              list_columns[29]: list_lineage_id[9],
            #              list_columns[30]: list_lineage_name[9],
            #              list_columns[31]: list_lineage_id[10],
            #              list_columns[32]: list_lineage_name[10]}
            # df_data_compile = df_data_compile.append(dict_line, ignore_index=True)

            dict_line = {list_columns[0]: [dict_data['species_id']],
                         list_columns[1]: [dict_data['sequence_num']],
                         list_columns[2]: [dict_data['species_name']],
                         list_columns[3]: [dict_data['amino_entropy']],
                         list_columns[4]: [dict_data['protein_entropy']],
                         list_columns[5]: [dict_data['constructive_entropy']],
                         list_columns[6]: [dict_data['divergence_time']],
                         list_columns[7]: [dict_data['amino_edge_count']],
                         list_columns[8]: [dict_data['amino_count']],
                         list_columns[9]: [dict_data['protein_edge_count']],
                         list_columns[10]: [dict_data['protein_count']],
                         list_columns[11]: [list_lineage_id[0]],
                         list_columns[12]: [list_lineage_name[0]],
                         list_columns[13]: [list_lineage_id[1]],
                         list_columns[14]: [list_lineage_name[1]],
                         list_columns[15]: [list_lineage_id[2]],
                         list_columns[16]: [list_lineage_name[2]],
                         list_columns[17]: [list_lineage_id[3]],
                         list_columns[18]: [list_lineage_name[3]],
                         list_columns[19]: [list_lineage_id[4]],
                         list_columns[20]: [list_lineage_name[4]],
                         list_columns[21]: [list_lineage_id[5]],
                         list_columns[22]: [list_lineage_name[5]],
                         list_columns[23]: [list_lineage_id[6]],
                         list_columns[24]: [list_lineage_name[6]],
                         list_columns[25]: [list_lineage_id[7]],
                         list_columns[26]: [list_lineage_name[7]],
                         list_columns[27]: [list_lineage_id[8]],
                         list_columns[28]: [list_lineage_name[8]],
                         list_columns[29]: [list_lineage_id[9]],
                         list_columns[30]: [list_lineage_name[9]],
                         list_columns[31]: [list_lineage_id[10]],
                         list_columns[32]: [list_lineage_name[10]]}
            df_data_compile = pd.concat([df_data_compile, pd.DataFrame.from_dict(dict_line)], ignore_index=True)

            print(str_file + ' : ' + str(int_index) + ' of ' + str(size_index_corrected) + ' ' + str_kingdom)

        path_save_compile = path_save + file_csvdata
        df_data_compile.to_csv(path_save_compile, index=False)

    print("Done.")

compile_to_csv(list_kingdom=LIST_KINGDOM, path_fasta=PATH_FASTA)


from flask import request, Response
from panoptes_client import Subject
import json
import ast
import tqdm
import io
import datetime
import numpy as np
from astropy.io import ascii
from astropy import Table
from .app import app
from .utils import NpEncoder

data = None

@app.route('/backend/get-subject-info/<subject_id>', methods=['GET', 'POST'])
def get_subject_info(subject_id):
    '''
        Retrieve the metadata information for a given subject
    '''
    try:
        # check if the subject exists in the subject set
        subject = Subject(int(subject_id))
    except Exception as e:
        print(e)
        return json.dumps({"error": "No such subject found"})

    # get the relevant metadata (including the image for the subject)
    url = subject.raw['locations'][0]['image/png']
    lat = float(subject.metadata['latitude'])
    PJ = int(subject.metadata['perijove'])
    lon = float(subject.metadata['longitude'])

    # send the keys back as a response
    response = Response(json.dumps({'subject_id': subject_id,
                                    'subject_url': url,
                                    'latitude': lat,
                                    'longitude': lon,
                                    'PJ': PJ}))
    return response

@app.route('/backend/get-exploration-data/', methods=['GET'])
def get_exploration_data(dsub=None):
    if dsub is None:
        dsub = data
    names = dsub.colnames
    subject_data = [dict(zip(names, row)) for row in dsub]
    variables = [n for n in names if n not in ['url', 'subject_ID']]
    dtypes = dict(zip(variables, [dsub[v].dtype.name for v in variables]))

    return json.dumps({'subject_data': subject_data,
                       'variables': variables,
                       'dtypes': dtypes}, cls=NpEncoder)

@app.route('/backend/create-export/', methods=['POST'])
def create_export():
    '''
        Generate a subject export based on the requested
        `subject_IDs` from the POST request
    '''
    if request.method == 'POST':
        # get the set of subject IDs
        subjects = request.json['subject_IDs']
    else:
        return json.dumps({'error': 'error! request failed'})

    # open the subject data CSV
    subject_data = ascii.read('jvh_subjects.csv', format='csv')

    # create a clone of the table that we can fill in
    export_table = Table(names=('subject_id', 'latitude', 'longitude',
                                'perijove', 'location',
                                'classification_count', 'retired_at',
                                'retirement_reason'),
                         dtype=('i8', 'f8', 'f8', 'i8', 'U40', 'i8',
                                'U40', 'U40'))

    # for each requested subject, add the metadata from the CSV file
    for subject_id in subjects:
        datai = subject_data[(subject_data['subject_id'] == subject_id) & (
            subject_data['subject_set_id'] != 105808)]
        for row in datai:
            meta = ast.literal_eval(row[4])

            latitude = float(meta['latitude'])
            longitude = float(meta['longitude'])
            perijove = int(meta['perijove'])

            location = ast.literal_eval(row[5])["0"]

            export_table.add_row([row[0], latitude, longitude, perijove,
                                  location, row[6], row[7], row[8]])

    # save it out to CSV-like string so we can send it to the frontend
    # to package it and submit for download
    outfile = io.StringIO()
    export_table.write(outfile, format='csv')
    output = outfile.getvalue()

    return json.dumps({'filedata': output})

@app.route('/backend/refresh-vortex-list/', methods=['GET'])
def update_vortex_list():
    # prepare the subject data
    subject_data = ascii.read('jvh_subjects.csv', format='csv')

    subject_data = subject_data[subject_data['subject_set_id'] == 105808]

    subject_ID_list = np.asarray(subject_data['subject_id'])
    subject_IDs = np.unique(subject_ID_list)

    for subject in tqdm.tqdm(subject_IDs, desc='Updating vortex info'):
        data['is_vortex'][data['subject_ID'] == subject] = 1

    return f"Done at {datetime.datetime.now()}"

@app.route('/backend/upload-umap/', methods=['POST'])
def upload_umap():
    if request.method == 'POST':
        umap_file = request.files.get('umap')
    else:
        return Response(status=500, response="Please upload a file!")

    with io.BytesIO(umap_file.read()) as umap_file_data:
        umap_file_data.seek(0)
        umap_data = ascii.read(umap_file_data, format='csv')

    if 'subject_ID' not in umap_data.colnames:
        return Response(status=500, response="UMAP file must have a `subject_ID` column")

    umap_subject_id = np.asarray(umap_data['subject_ID'][:])
    subject_ids = np.asarray(data['subject_ID'][:])

    n_umaps = len(umap_data.colnames) - 1
    dsub = data.copy()

    umap_coords = np.zeros((len(subject_ids), n_umaps))

    for i, subject in enumerate(tqdm.tqdm(subject_ids, desc='Loading UMAP data')):
        umap_idx = np.where(subject == umap_subject_id)[0]

        if len(umap_idx) < 1:
            return Response(status=500, response=f"{subject} not found in UMAP file!")

        umap_coord_i = umap_data[umap_idx[0]]

        umap_coords[i, :] = np.asarray([umap_coord_i[key] for key in umap_data.colnames if key not in ['subject_ID']])

    for j in range(n_umaps):
        dsub.add_column(umap_coords[:, j], name=f'UMAP{j+1}')

    return get_exploration_data(dsub=dsub)


import NormalizingFlows as nf
import torch
from torch.distributions import Independent, Uniform
from Simulation import *
import numpy

if torch.cuda.is_available():
    device = torch.device("cuda")
else:
    device = torch.device("cpu")

D = 5

a = torch.tensor([1,2,3,1,2]).to(device)

exact_l = 1339/1440

def target_weight(x):
    x = x*a
    x1 = x[...,0]
    x2 = x[...,1]
    x3 = x[...,2]
    x4 = x[...,3]
    x5 = x[...,4]
    h = torch.stack((x1+x4,x1+x3+x5,x2+x3+x4,x2+x5), dim = -1).min(-1).values
    return h

d = Independent(Uniform(torch.zeros(D).to(device), torch.ones(D).to(device)),1)

perm = torch.cat([torch.arange(D//2,D),torch.arange(D//2)])
flows = []
for i in range(5):
    flows.append(nf.CouplingFlow(D, normalise=True, N=1))
    flows.append(nf.PermutationFlow(perm))

net = nf.NormFlows(flows, d).to(device)

trainer = Trainer(net, device = device, bs = 10000)
results = Results(net, plot_ss = 10000, eval_ss = 10000)

train_net(
    net, trainer, results, 300000, target_weight,
    weight_log = False,
    criterion = "KL1",
    penalty = None,
    lr = 0.0001,
    wd = 0.0001,
    print_interval = 10,
    last_only = False,
    save_dir = "experiments/bridge_network")

z = net.sample(10000)
logpy, y = net.logpy(z = z, return_zy = True)
path = target_weight(y)
print(target_weight(z).std(True).item())
print((path/logpy.exp()).std(True).item())
print(target_weight(z).mean().item())
print((path/logpy.exp()).mean().item())

#Estimate KL divergence
target_logp = (path/exact_l).log()
log_rat = target_logp - logpy
KL_samples = log_rat.exp()*log_rat
print(KL_samples.mean().item(),KL_samples.std(unbiased = True).item()/numpy.sqrt(len(KL_samples)))


"""Fourth programming assignment of the course 'Using Databases with Python'."""

import json
import sqlite3

# Create a Connection object:
conn = sqlite3.connect('rosterdb.sqlite')

# Create a Cursor object:
cur = conn.cursor()

# Make sure we always start fresh:
cur.executescript(
    """
    DROP TABLE IF EXISTS User;
    DROP TABLE IF EXISTS Course;
    DROP TABLE IF EXISTS Member
    """
)

# Create a table for storing the data related to the USERS:
cur.execute(
    """
    CREATE TABLE User (
        id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
        name TEXT UNIQUE
    )
    """
)

# Create a table for storing the data related to the COURSES:
cur.execute(
    """
    CREATE TABLE Course (
        id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
        title TEXT UNIQUE
    )
    """
)

# Create a table for storing the data related to MEMBERSHIP:
cur.execute(
    """
    CREATE TABLE Member (
        user_id INTEGER,
        course_id INTEGER,
        role INTEGER,
        PRIMARY KEY (user_id, course_id)
    )
    """
)

# Read the JSON file:
with open('roster_data.json', 'r') as json_file:
    str_data = json_file.read()
json_data = json.loads(str_data)

# Loop over the entries in the JSON file:
for entry in json_data:

    # Get the NAME of the USER:
    name = entry[0]
    # Get the TITLE of the COURSE:
    title = entry[1]
    # Get the ROLE of that user in that course:
    role = entry[2]

    # Insert the data on the USER into the database.
    # Use 'OR IGNORE' due to the uniqueness constraint on 'name'.
    cur.execute('INSERT OR IGNORE INTO User (name) VALUES (?)', (name,))

    # Get the 'id' of the user we just inserted into the database:
    cur.execute('SELECT id FROM User WHERE name = ?', (name,))
    user_id = cur.fetchone()[0]

    # Insert the data on the COURSE into the database.
    # Use 'OR IGNORE' due to the uniqueness constraint on 'title'.
    cur.execute('INSERT OR IGNORE INTO Course (title) VALUES (?)', (title,))

    # Get the 'id' of the course we just inserted into the database:
    cur.execute('SELECT id FROM Course WHERE title = ?', (title,))
    course_id = cur.fetchone()[0]

    # Insert the MEMBERSHIP data into the database:
    cur.execute(
        """
        INSERT OR REPLACE INTO Member (user_id, course_id, role)
        VALUES (?, ?, ?)
        """,
        (user_id, course_id, role)
    )

    # Commit the changes:
    conn.commit()

# Close the connection:
conn.close()


"""
Server so nltk doesn't initialize for every meme request.

Author: Jack Romo <sharrackor@gmail.com>
"""

from flask import Flask, request
from main import DogeGen

class DogeServer(object):
    """
    Server to generate Doge images.
    """

    app = Flask(__name__)
    dogegen = None

    @staticmethod
    def run_server(resources, host_name='0.0.0.0', port=5000):
        """
        Run the internal server.

        Args:
            resources (str): Resources directory, eg. "./resources".
            host_name (str): Host name of server.
            port (int): Port of server.
        """
        if not DogeServer.dogegen:
            DogeServer.dogegen = DogeGen(resources)
        DogeServer.app.run(debug=True, host=host_name, port=port)

    @staticmethod
    @app.route('/makememe')
    def makememe():
        """
        Make a meme upon a server GET request.

        Returns:
            str: Directory of image.

        """
        inptext = request.args.get('inptext')
        outdir = request.args.get('outdir')
        maxphrases = int(request.args.get('maxphrases'))
        DogeServer.dogegen.make_meme(inptext, outdir, maxphrases)
        return outdir

    @staticmethod
    @app.route('/shutdown', methods=['POST'])
    def shutdown():
        """
        Shut down server internally.

        Returns:
            str: Status string.
        """
        func = request.environ.get('werkzeug.server.shutdown')
        if func is None:
            raise RuntimeError('Not running with the Werkzeug Server')
        func()
        return "Shutting down."