import tensorflow as tf
import pathlib
import os
import cv2
import numpy as np
import tqdm
import argparse

class TFRecordsSeg:
    def __init__(self,
                 image_dir="/datasets/custom/cityscapes",
                 label_dir="/datasets/custom/cityscapes",
                 tfrecord_path="data.tfrecords",
                 classes=34,
                 img_pattern="*.png",
                 label_pattern="*.png"):
        """
        :param data_dir: the path to iam directory containing the subdirectories of xml and lines from iam dataset
        :param tfrecord_path:
        """
        # self.data_dir = data_dir
        # self.labels_dir = os.path.join(data_dir, "gtFine/{}".format(split))
        # self.image_dir = os.path.join(data_dir, "leftImg8bit/{}".format(split))
        self.image_dir = image_dir
        self.labels_dir = label_dir
        self.tfrecord_path = tfrecord_path
        self.labels = []
        self.classes = classes
        self.img_pattern = img_pattern
        self.label_pattern = label_pattern
        self.image_feature_description = \
            {
                'label': tf.io.FixedLenFeature([], tf.string),
                'image': tf.io.FixedLenFeature([], tf.string)
            }

    @staticmethod
    def _bytes_feature(value):
        """Returns a bytes_list from a string / byte."""
        if isinstance(value, type(tf.constant(0))):
            value = value.numpy()  # BytesList won't unpack a string from an EagerTensor.
        return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

    @staticmethod
    def _float_feature(value):
        """Returns a float_list from a float / double."""
        return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))

    @staticmethod
    def _int64_feature(value):
        """Returns an int64_list from a bool / enum / int / uint."""
        return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

    def _parse_example_function(self, example_proto):
        # Parse the input tf.Example proto using the dictionary above.
        return tf.io.parse_example(example_proto, self.image_feature_description)

    def image_example(self, image_string, label):
        feature = {
            'label': self._bytes_feature(label),
            'image': self._bytes_feature(image_string)
        }

        return tf.train.Example(features=tf.train.Features(feature=feature))

    def return_inst_cnts(self, inst_ex):
        inst_cnt = np.zeros(inst_ex.shape)
        for unique_class in np.unique(inst_ex):
            inst_img = (inst_ex == unique_class) / 1
            cnts, _ = cv2.findContours(inst_img.astype("uint8"), cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
            inst_cnt = cv2.drawContours(inst_cnt, cnts, -1, (1., 1., 1.), thickness=1)
        return inst_cnt

    def write_tfrecords(self, training=False, dataset_name=""):
        img_paths = sorted(pathlib.Path(self.image_dir).rglob(self.img_pattern))
        label_paths = sorted(pathlib.Path(self.labels_dir).rglob(self.label_pattern))
        with tf.io.TFRecordWriter(self.tfrecord_path) as writer:
            for img_path, label_path in tqdm.tqdm(zip(img_paths, label_paths)):
                img_string = open(str(img_path), 'rb').read()
                label_string = open(str(label_path), 'rb').read()
                tf_example = self.image_example(img_string, label_string)
                writer.write(tf_example.SerializeToString())
            if training:
                import json
                if os.path.exists('{}/data_samples.json'.format(os.path.dirname(self.tfrecord_path))):
                    with open('{}/data_samples.json'.format(os.path.dirname(self.tfrecord_path))) as f:
                        data = json.load(f)
                    if dataset_name in list(data.keys()):
                        print("Dataset {} value was already present but value was updated".format(dataset_name))
                else:
                    data = {}
                data[dataset_name] = len(img_paths)
                with open('{}/data_samples.json'.format(os.path.dirname(self.tfrecord_path)), 'w') as json_file:
                    json.dump(data, json_file)

    def decode_strings(self, record):
        images = tf.io.decode_jpeg(record['image'], 3)
        labels = tf.io.decode_jpeg(record['label'], 3)
        return images, labels

    def read_tfrecords(self):
        """
        Read iam tfrecords
        :return: Returns a tuple of images and their label (images, labels)
        """
        raw_dataset = tf.data.TFRecordDataset(self.tfrecord_path)
        parsed_dataset = raw_dataset.map(self._parse_example_function)
        decoded_dataset = parsed_dataset.map(self.decode_strings)
        return decoded_dataset

if __name__ == "__main__":
    classes = 150
    dataset_name = "ade20k1"
    train = TFRecordsSeg(image_dir="/volumes2/datasets/ADEChallengeData2016/images/training",
                         label_dir="/volumes2/datasets/ADEChallengeData2016/annotations/training",
                         tfrecord_path="/data/input/datasets/tf2_segmentation_tfrecords/{}_train.tfrecords".format(dataset_name),
                         classes=classes, img_pattern="*.jpg",
                         label_pattern="*.png")
    # train = TFRecordsSeg(data_dir="/data/input/datasets/cityscape_processed", tfrecord_path="/volumes1/train.tfrecords", split='train')
    val = TFRecordsSeg(image_dir="/volumes2/datasets/ADEChallengeData2016/images/validation",
                       label_dir="/volumes2/datasets/ADEChallengeData2016/annotations/validation",
                       tfrecord_path="/data/input/datasets/tf2_segmentation_tfrecords/{}_val.tfrecords".format(dataset_name),
                       classes=classes, img_pattern="*.jpg",
                       label_pattern="*.png")
    train.write_tfrecords(training=True, dataset_name=dataset_name)
    val.write_tfrecords()
    # example = train
    # image_dataset = example.read_tfrecords().repeat(10)
    # cv2.namedWindow("img", 0)
    # cv2.namedWindow("label", 0)
    # for image_features in image_dataset:
    #     img = image_features[0][..., ::-1]
    #     label = image_features[1]
    #     print(np.unique(label.numpy()))
    #     insts = image_features[2]
    #     cv2.imshow("img", img.numpy())
    #     cv2.imshow("label", label.numpy()/classes)
    #     cv2.waitKey()

    #     print(image_features[0].shape, image_features[1].shape, image_features[2].shape)
    # example.write_tfrecords()
    # image_dataset = example.read_tfrecords().shuffle(10000)
    #
    # for image_features in image_dataset.take(10):
    #     print(image_features[0].shape, image_features[1].numpy())


#!/usr/bin/python3
"""Starts a basic flask web application"""

from flask import Flask, render_template
from markupsafe import escape

from models import storage
from models.state import State
from models.city import City
app = Flask(__name__)

@app.teardown_appcontext
def teardown(self):
    """procedure to run after request"""
    storage.close()

@app.route("/states_list", strict_slashes=False)
def states_list():
    """Function to run when '/states_list' is accessed"""
    states = [state for state in storage.all(State).values()]
    states.sort(reverse=False, key=lambda state: state.name)
    return (render_template('7-states_list.html', states=states))

@app.route("/cities_by_states", strict_slashes=False)
def cities_by_statesb():
    """Function to run when '/cities_by_states' is accessed"""
    states = storage.all(State).values()
    return (render_template('8-cities_by_states.html', states=states))

if (__name__ == '__main__'):
    app.run(host='0.0.0.0', port=5000, debug=False)


import sys
sys.path.append('..')

from common.wrapped_input import wrapped_input
from common.clean_screen import clean_screen

__TERMINATE_MARKS__ = ['***', '****']

class Reader:
    def __init__(self, args):
        self.loop = True

    def run(self, parser):
        print("""
 _ __                   __,
( /  )         o       (
 /--< _   __, ,  _ _    `.   ,_   __  ,
/___// (_(_/(_(_/ / /_(___)_/|_)_/ (_/_
                            /|      /
Interactive shell          (/      '

==========================================
        """)

        last_input = ''

        while last_input not in __TERMINATE_MARKS__:
            last_input = wrapped_input()

            if last_input in __TERMINATE_MARKS__:
                print('[INFO] Querying, please wait...')
                return last_input

            parser.add(last_input)


import os, sys
import subprocess

# os.environ['DISPLAY'] = ':99.0'
# os.environ['PYVISTA_OFF_SCREEN'] = 'true'
# os.environ['PYVISTA_USE_IPYVTK'] = 'true'
# bashCommand ="Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & sleep 3"
# process = subprocess.Popen(bashCommand, stdout=subprocess.PIPE, shell=True)
# process.wait()
sys.path.insert(0, os.path.abspath("../../../.."))
from copy import deepcopy
import numpy as np
import torch
import pyvista as pv
import matplotlib.pyplot as plt
from shapmagn.global_variable import Shape, shape_type
from shapmagn.datasets.data_utils import read_json_into_list, get_obj, get_file_name
from shapmagn.shape.shape_pair_utils import create_shape_pair
from shapmagn.utils.obj_factory import obj_factory
from shapmagn.utils.visualizer import (
    visualize_point_fea,
    visualize_point_pair,
    visualize_multi_point,
)
from shapmagn.utils.local_feature_extractor import *

def get_pair(source_path, target_path, expand_bch_dim=True, return_tensor=True):

    get_obj_func = get_obj(
        reader_obj,
        normalizer_obj,
        sampler_obj,
        device,
        expand_bch_dim=expand_bch_dim,
        return_tensor=return_tensor,
    )
    source_obj, source_interval = get_obj_func(source_path)
    target_obj, target_interval = get_obj_func(target_path)
    return source_obj, target_obj

def plot_pair_weight_distribution(
    source_weight, target_weight, use_log=False, title="", show=True, save_path=None
):
    plt.style.use("bmh")
    fig, ax = plt.subplots()
    source_weight = np.log(source_weight) if use_log else source_weight
    target_weight = np.log(target_weight) if use_log else target_weight
    ax.hist(source_weight, bins=1000, density=0, histtype="stepfilled", alpha=0.7)
    ax.hist(target_weight, bins=1000, density=0, histtype="stepfilled", alpha=0.5)
    title += "weight" if not use_log else "log_weight"
    ax.set_title(title)
    if show:
        plt.show()
    if save_path:
        plt.savefig(save_path, dpi=300)
    plt.clf()

def plot_pair_weight_distribution_before_and_after_radius_matching(
    source_weight1,
    target_weight1,
    source_weight2,
    target_weight2,
    use_log=False,
    title="",
    show=True,
    save_path=None,
):
    plt.style.use("bmh")

    fig, axes = plt.subplots(nrows=2, ncols=2)
    ax0, ax1, ax2, ax3 = axes.flatten()
    source_weight_matched1 = matching_np_radius(source_weight1, target_weight1)
    smw_sum1, sw_sum1, tp_sum1 = (
        source_weight_matched1.sum(),
        source_weight1.sum(),
        target_weight1.sum(),
    )

    source_weight1 = np.log(source_weight1) if use_log else source_weight1
    target_weight1 = np.log(target_weight1) if use_log else target_weight1
    ax0.hist(source_weight1, bins=1000, density=0, histtype="stepfilled", alpha=0.7)
    ax0.hist(target_weight1, bins=1000, density=0, histtype="stepfilled", alpha=0.5)
    ax0.set_title("sw_sum: {:.3f}, tp_sum:{:.3f}".format(sw_sum1, tp_sum1), fontsize=10)
    source_weight_matched1_norm = (
        np.log(source_weight_matched1) if use_log else source_weight_matched1
    )
    ax1.hist(
        source_weight_matched1_norm,
        bins=1000,
        density=0,
        histtype="stepfilled",
        alpha=0.7,
    )
    ax1.hist(target_weight1, bins=1000, density=0, histtype="stepfilled", alpha=0.5)
    ax1.set_title(
        "smw_sum: {:.3f}, tp_sum:{:.3f}".format(smw_sum1, tp_sum1), fontsize=10
    )

    source_weight_matched2 = matching_np_radius(source_weight2, target_weight2)
    smw_sum2, sw_sum2, tp_sum2 = (
        source_weight_matched2.sum(),
        source_weight2.sum(),
        target_weight2.sum(),
    )
    source_weight2 = np.log(source_weight2) if use_log else source_weight2
    target_weight2 = np.log(target_weight2) if use_log else target_weight2
    ax2.hist(source_weight2, bins=1000, density=0, histtype="stepfilled", alpha=0.7)
    ax2.hist(target_weight2, bins=1000, density=0, histtype="stepfilled", alpha=0.5)
    ax2.set_title("sw_sum: {:.3f}, tp_sum:{:.3f}".format(sw_sum2, tp_sum2), fontsize=10)
    source_weight_matched2_norm = (
        np.log(source_weight_matched2) if use_log else source_weight_matched2
    )
    ax3.hist(
        source_weight_matched2_norm,
        bins=1000,
        density=0,
        histtype="stepfilled",
        alpha=0.7,
    )
    ax3.hist(target_weight2, bins=1000, density=0, histtype="stepfilled", alpha=0.5)
    ax3.set_title(
        "smw_sum: {:.3f}, tp_sum:{:.3f}".format(smw_sum2, tp_sum2), fontsize=10
    )

    fig.subplots_adjust(hspace=0.3)
    fig.suptitle(title)
    if show:
        plt.show()
    if save_path:
        plt.savefig(save_path, dpi=300)
    plt.clf()
    return source_weight_matched1, source_weight_matched2

def get_half_lung(lung, normalize_weight=False):
    weights = lung.weights.detach()
    points = lung.points.detach()
    pos_filter = points[..., 0] < 0
    points = points[pos_filter][None]
    weights = weights[pos_filter][None]
    weights = weights
    weights = weights / weights.sum() if normalize_weight else weights
    half_lung = Shape()
    half_lung.set_data(points=points, weights=weights)
    return half_lung

def get_key_vessel(lung, thre=2e-05):
    weights = lung.weights.detach()
    points = lung.points.detach()
    mask = (lung.weights > thre)[..., 0]
    weights = weights[mask][None]
    points = points[mask][None]
    key_lung = Shape()
    key_lung.set_data(points=points, weights=weights)
    return key_lung

def sampled_via_radius(source, target):
    min_npoints = min(source.npoints, target.npoints)
    tw = target.weights.squeeze()
    sw = source.weights.squeeze()
    t_sorted, t_indices = torch.sort(tw, descending=True)
    s_sorted, s_indices = torch.sort(sw, descending=True)
    t_sampled_indices = t_indices[:min_npoints]
    s_sampled_indices = s_indices[:min_npoints]
    tp_sampled = target.points[:, t_sampled_indices]
    sp_sampled = source.points[:, s_sampled_indices]
    tw_sampled = target.weights[:, t_sampled_indices]
    sw_sampled = source.weights[:, s_sampled_indices]
    target_sampled, source_sampled = Shape(), Shape()
    target_sampled.set_data(points=tp_sampled, weights=tw_sampled)
    source_sampled.set_data(points=sp_sampled, weights=sw_sampled)
    return source_sampled, target_sampled

def hist_match(source, template):
    """
    Adjust the pixel values of a grayscale image such that its histogram
    matches that of a target image.
    Code adapted from
    http://stackoverflow.com/questions/32655686/histogram-matching-of-two-images-in-python-2-x
    Arguments:
    -----------
        source: np.ndarray
            Image to transform; the histogram is computed over the flattened
            array
        template: np.ndarray
            Template image; can have different dimensions to source
    Returns:
    -----------
        matched: np.ndarray
            The transformed output image
    """

    oldshape = source.shape
    source = source.ravel()
    template = template.ravel()

    # get the set of unique pixel values and their corresponding indices and
    # counts
    s_values, bin_idx, s_counts = np.unique(
        source, return_inverse=True, return_counts=True
    )
    t_values, t_counts = np.unique(template, return_counts=True)

    # take the cumsum of the counts and normalize by the number of pixels to
    # get the empirical cumulative distribution functions for the source and
    # template images (maps pixel value --> quantile)
    s_quantiles = np.cumsum(s_counts).astype(np.float64)
    s_quantiles /= s_quantiles[-1]
    t_quantiles = np.cumsum(t_counts).astype(np.float64)
    t_quantiles /= t_quantiles[-1]

    # interpolate linearly to find the pixel values in the template image
    # that correspond most closely to the quantiles in the source image
    interp_t_values = np.interp(s_quantiles, t_quantiles, t_values)

    return interp_t_values[bin_idx].reshape(oldshape)

def matching_np_radius(source_weights, target_weights):
    """

    :param source_weights: Nx1
    :param target_weights: Mx1
    :param matched_weights: Nx1
    :return:
    """

    ns = source_weights.shape[0]
    sw = source_weights.squeeze()
    tw = target_weights.squeeze()
    range = [min(sw.min(), tw.min()), max(sw.max(), tw.max())]
    resol = 10000
    interp = (range[1] - range[0]) / resol
    bins = np.linspace(range[0] - 2 * interp, range[1] + 2 * interp, resol)
    sw_indice = np.digitize(sw, bins, right=False)
    tw_indice = np.digitize(tw, bins, right=False)
    sw_digitize = bins[sw_indice]
    tw_digitize = bins[tw_indice]
    sw_transformed = hist_match(sw_digitize, tw_digitize)
    return sw_transformed.reshape(ns, 1).astype(np.float32)

def matching_shape_radius(source, target, sampled_by_radius=False, show=True):
    if sampled_by_radius:
        source, target = sampled_via_radius(source, target)
    device = source.points.device
    sn = source.npoints
    tn = target.npoints
    sw = source.weights.squeeze().cpu().numpy()
    tw = target.weights.squeeze().cpu().numpy()
    range = [min(sw.min(), tw.min()), max(sw.max(), tw.max())]
    resol = 10000
    interp = (range[1] - range[0]) / resol
    bins = np.linspace(range[0] - 2 * interp, range[1] + 2 * interp, resol)
    sw_indice = np.digitize(sw, bins, right=False)
    tw_indice = np.digitize(tw, bins, right=False)
    sw_digitize = bins[sw_indice]
    tw_digitize = bins[tw_indice]
    sw_transformed = hist_match(sw_digitize, tw_digitize)
    if show:
        plot_pair_weight_distribution(sw_digitize, tw_digitize, use_log=True)
        plot_pair_weight_distribution(sw_transformed, tw_digitize, use_log=True)
        visualize_point_pair(
            source.points,
            target.points,
            source.weights,
            target.weights,
            title1="source(before)",
            title2="target(before)",
        )

        visualize_point_pair(
            source.points,
            target.points,
            sw_transformed,
            tw_digitize,
            title1="source(after)",
            title2="target(after)",
        )
    source.weights = (
        torch.tensor(sw_transformed.astype(np.float32)).to(device).view(1, sn, 1)
    )
    target.weights = (
        torch.tensor(tw_digitize.astype(np.float32)).to(device).view(1, tn, 1)
    )
    return source, target

def source_weight_transform(weights, compute_on_half_lung=False):
    weights = weights * 1
    weights_cp = deepcopy(weights)
    thre = 1.9e-05
    thre = thre  # if not compute_on_half_lung else thre*2
    weights[weights_cp < thre] = 1e-7
    return weights

def flowed_weight_transform(weights, compute_on_half_lung=False):
    weights = weights * 1
    weights_cp = deepcopy(weights)
    thre = 1.9e-05
    thre = thre  # if not compute_on_half_lung else thre * 2
    weights[weights_cp < thre] = 1e-7
    return weights

def target_weight_transform(weights, compute_on_half_lung=False):
    weights = weights * 1
    weights_cp = deepcopy(weights)
    thre = 1.9e-05
    thre = thre  # if not compute_on_half_lung else thre * 2
    weights[weights_cp < thre] = 1e-7
    # weights[weights_cp > 1.1e-05] = 1e-7
    return weights

def pair_shape_transformer(init_thres=2.9e-5, nstep=5):
    # todo the next step of the transformer is to return a smoothed mask to constrain the movement of the lung
    def transform(source, target, cur_step):
        min_weights = min(torch.min(source.weights), torch.min(target.weights))
        max_weights = min(torch.max(source.weights), torch.max(target.weights))
        max_weights = max_weights.item()
        cur_step = cur_step.item()
        assert init_thres > min_weights
        thres = init_thres - (init_thres - min_weights) / nstep * cur_step
        s_weights = source.weights.clone()
        t_weights = target.weights.clone()
        s_weights[source.weights < thres] = 1e-7
        t_weights[target.weights < thres] = 1e-7
        s_transformed, t_transformed = Shape(), Shape()
        s_transformed.set_data(
            points=source.points, weights=s_weights, pointfea=source.pointfea
        )
        t_transformed.set_data(
            points=target.points, weights=t_weights, pointfea=target.pointfea
        )
        print("the weight of the lung pair are updated")
        return s_transformed, t_transformed

    return transform

def capture_plotter(save_source=False):
    from shapmagn.utils.visualizer import visualize_point_pair_overlap

    inner_count = 0

    def save(record_path, name_suffix, shape_pair):
        nonlocal inner_count
        source, flowed, target = shape_pair.source, shape_pair.flowed, shape_pair.target
        for sp, fp, tp, sw, fw, tw, pair_name in zip(
            source.points,
            flowed.points,
            target.points,
            source.weights,
            flowed.weights,
            target.weights,
            pair_name_list,
        ):
            if inner_count == 0 or save_source:
                path = os.path.join(
                    record_path, "source_target" + "_" + name_suffix + ".png"
                )
                visualize_point_pair_overlap(
                    sp,
                    tp,
                    flowed_weight_transform(fw, True),
                    target_weight_transform(tw, True),
                    title1="source",
                    title2="target",
                    rgb_on=False,
                    saving_capture_path=path,
                    show=False,
                )
            path_1 = os.path.join(
                record_path,
                pair_name + "_flowed_target" + "_main_" + name_suffix + ".png",
            )
            path_2 = os.path.join(
                record_path,
                pair_name + "_flowed_target" + "_whole_" + name_suffix + ".png",
            )
            visualize_point_pair_overlap(
                fp,
                tp,
                flowed_weight_transform(fw, True),
                target_weight_transform(tw, True),
                title1="flowed",
                title2="target",
                rgb_on=False,
                saving_capture_path=path_1,
                show=False,
            )
            visualize_point_pair_overlap(
                fp,
                tp,
                fw,
                tw,
                title1="flowed",
                title2="target",
                rgb_on=False,
                saving_capture_path=path_2,
                show=False,
            )
        inner_count += 1

    return save

def lung_isolated_leaf_clean_up(
    lung, radius=0.032, principle_weight=None, normalize_weights=True
):

    points = lung.points.detach()
    weights = lung.weights.detach()
    mass, dev, cov = compute_local_moments(points, radius=radius)
    eigenvector_main = compute_local_fea_from_moments(
        "eigenvector_main", weights, mass, dev, cov
    )
    filter = mass[..., 0].squeeze() > 2
    to_remove = ~filter
    print(
        "In the first step, num of points are removed {}, {}".format(
            torch.sum(to_remove), torch.sum(to_remove) / len(filter)
        )
    )
    points_toremove = points[:, to_remove]
    mass_toremove = mass[:, to_remove]
    mass = mass[:, filter]
    points = points[:, filter]
    weights = weights[:, filter]
    eigenvector_main = eigenvector_main[:, filter]
    visualize_point_fea_with_arrow(points, mass, eigenvector_main * 0.01, rgb_on=False)
    visualize_point_overlap(
        points,
        points_toremove,
        mass,
        mass_toremove,
        title="cleaned points",
        point_size=(10, 20),
        rgb_on=False,
        opacity=("linear", 1.0),
    )

    Gamma = compute_anisotropic_gamma_from_points(
        points,
        cov_sigma_scale=radius,
        aniso_kernel_scale=radius,
        principle_weight=principle_weight,
    )
    mass, dev, cov = compute_aniso_local_moments(points, Gamma)
    eigenvector_main = compute_local_fea_from_moments(
        "eigenvector_main", weights, mass, dev, cov
    )
    filter = mass[..., 0].squeeze() > 2.5
    to_remove = ~filter
    print(
        "In the second step, num of points are removed {}, {}".format(
            torch.sum(to_remove), torch.sum(to_remove) / len(filter)
        )
    )
    points_toremove = points[:, to_remove]
    mass_toremove = mass[:, to_remove]
    mass = mass[:, filter]
    points = points[:, filter]
    weights = weights[:, filter]
    eigenvector_main = eigenvector_main[:, filter]
    visualize_point_fea_with_arrow(points, mass, eigenvector_main * 0.01, rgb_on=False)
    visualize_point_overlap(
        points,
        points_toremove,
        mass,
        mass_toremove,
        title="cleaned points",
        point_size=(10, 20),
        rgb_on=False,
        opacity=("linear", 1.0),
    )

    Gamma = compute_anisotropic_gamma_from_points(
        points,
        cov_sigma_scale=radius,
        aniso_kernel_scale=radius,
        principle_weight=principle_weight,
    )
    mass, dev, cov = compute_aniso_local_moments(points, Gamma)
    eigenvector_main = compute_local_fea_from_moments(
        "eigenvector_main", weights, mass, dev, cov
    )
    filter = mass[..., 0].squeeze() > 3
    to_remove = ~filter
    print(
        "In the third step, num of points are removed {}, {}".format(
            torch.sum(to_remove), torch.sum(to_remove) / len(filter)
        )
    )
    points_toremove = points[:, to_remove]
    mass_toremove = mass[:, to_remove]
    mass = mass[:, filter]
    points = points[:, filter]
    weights = weights[:, filter]
    eigenvector_main = eigenvector_main[:, filter]
    visualize_point_fea_with_arrow(points, mass, eigenvector_main * 0.01, rgb_on=False)
    visualize_point_overlap(
        points,
        points_toremove,
        mass,
        mass_toremove,
        title="cleaned points",
        point_size=(10, 20),
        rgb_on=False,
        opacity=("linear", 1.0),
    )

    cleaned_lung = Shape()
    cleaned_lung.points, cleaned_lung.weights = (
        points,
        weights / torch.sum(weights) if normalize_weights else weights,
    )
    return cleaned_lung

def analysis_large_vessel(
    source,
    target,
    source_weight_transform=source_weight_transform,
    target_weight_transform=target_weight_transform,
    title1="source",
    title2="target",
):
    source_points, source_weights, = (
        source.points.detach().cpu(),
        source.weights.detach().cpu(),
    )
    target_points, target_weights, = (
        target.points.detach().cpu(),
        target.weights.detach().cpu(),
    )
    plot_pair_weight_distribution(
        source_weight_transform(source_weights).squeeze().numpy(),
        target_weight_transform(target_weights).squeeze().numpy(),
        use_log=True,
    )
    visualize_point_pair(
        source_points,
        target_points,
        source_weight_transform(source_weights),
        target_weight_transform(target_weights),
        title1=title1,
        title2=title2,
    )

def compute_atlas(weight_list):
    atlas_weight = np.concatenate(weight_list)
    return atlas_weight

def transfer_radius_and_save_sample(
    cur_obj, atlas_distri, radius_transfered_saing_path
):
    cur_obj["weights"] = matching_np_radius(cur_obj["weights"], atlas_distri)
    data = pv.PolyData(cur_obj["points"])
    for key, item in cur_obj.items():
        if key not in ["points"]:
            data.point_arrays[key] = item
    data.save(radius_transfered_saing_path)
    return cur_obj

if __name__ == "__main__":
    assert (
        shape_type == "pointcloud"
    ), "set shape_type = 'pointcloud'  in global_variable.py"
    device = torch.device("cpu")  # cuda:0  cpu
    reader_obj = "lung_dataloader_utils.lung_reader()"
    normalizer_obj = (
        "lung_dataloader_utils.lung_normalizer(weight_scale=60000,scale=[100,100,100])"
    )
    phase = "train"

    use_local_mount = False
    remote_mount_transfer = lambda x: x.replace(
        "/playpen-raid1", "/home/zyshen/remote/llr11_mount"
    )
    path_transfer = (
        (lambda x: remote_mount_transfer(x)) if use_local_mount else (lambda x: x)
    )

    dataset_json_path = (
        "/playpen-raid1/zyshen/data/lung_expri/{}/pair_data.json".format(phase)
    )
    dataset_json_path = path_transfer(dataset_json_path)

    sampler_obj = "lung_dataloader_utils.lung_sampler( method='voxelgrid',scale=0.0003)"
    get_obj_func = get_obj(
        reader_obj,
        normalizer_obj,
        sampler_obj,
        device,
        expand_bch_dim=False,
        return_tensor=False,
    )
    altas_path = "/playpen-raid1/Data/UNC_vesselParticles/10067M_INSP_STD_MSM_COPD_wholeLungVesselParticles.vtk"
    altas_path = path_transfer(altas_path)
    atlas, _ = get_obj_func(altas_path)
    sampler_obj = "lung_dataloader_utils.lung_sampler( method='combined',scale=0.0003,num_sample=30000,sampled_by_weight=True)"
    get_obj_func = get_obj(
        reader_obj,
        normalizer_obj,
        sampler_obj,
        device,
        expand_bch_dim=False,
        return_tensor=False,
    )
    sampled_atlas, _ = get_obj_func(altas_path)

    radius_transfered_saing_path = "/playpen-raid1/zyshen/data/lung_atlas/{}".format(
        phase
    )
    radius_transfered_saing_path = path_transfer(radius_transfered_saing_path)
    os.makedirs(radius_transfered_saing_path, exist_ok=True)

    pair_name_list, pair_info_list = read_json_into_list(dataset_json_path)
    pair_path_list = [
        [pair_info["source"]["data_path"], pair_info["target"]["data_path"]]
        for pair_info in pair_info_list
    ]
    pair_id = 3
    output_path = "/playpen-raid1/zyshen/data/lung_data_analysis/val"
    for pair_id in range(len(pair_name_list)):
        pair_path = pair_path_list[pair_id]
        pair_path = [path_transfer(path) for path in pair_path]
        sampler_obj = (
            "lung_dataloader_utils.lung_sampler( method='voxelgrid',scale=0.0003)"
        )

        ########################
        plot_saving_path = os.path.join(radius_transfered_saing_path, "origin_plots")
        os.makedirs(plot_saving_path, exist_ok=True)

        source_path, target_path = pair_path_list[pair_id]
        source, target = get_pair(
            source_path, target_path, expand_bch_dim=False, return_tensor=False
        )

        saving_path = os.path.join(plot_saving_path, pair_name_list[pair_id] + ".png")
        camera_pos = [
            (-4.924379645467042, 2.17374925796456, 1.5003730890759344),
            (0.0, 0.0, 0.0),
            (0.40133888001174545, 0.31574165540339943, 0.8597873634998591),
        ]
        visualize_point_pair(
            source["points"],
            target["points"],
            source["weights"],
            target["weights"],
            title1="source",
            title2="target",
            saving_capture_path=saving_path,
            camera_pos=camera_pos,
            show=False,
        )
        plot_saving_path = os.path.join(radius_transfered_saing_path, "plots")
        os.makedirs(plot_saving_path, exist_ok=True)

        # vtk_saving_path = os.path.join(radius_transfered_saing_path,"data")
        # os.makedirs(vtk_saving_path,exist_ok=True)
        # saving_path = os.path.join(vtk_saving_path,get_file_name(source_path)+".vtk")
        # mapped_source = transfer_radius_and_save_sample(source, atlas["weights"], saving_path)
        # saving_path = os.path.join(vtk_saving_path,get_file_name(target_path)+".vtk")
        # mapped_target = transfer_radius_and_save_sample(target, atlas["weights"], saving_path)
        # plot_saving_path = os.path.join(radius_transfered_saing_path, "plots")
        # source_vg_weight, target_vg_weight = source["weights"], target["weights"]
        # sampler_obj ="lung_dataloader_utils.lung_sampler( method='combined',scale=0.0003,num_sample=30000,sampled_by_weight=True)"
        # source, target = get_pair(source_path, target_path, expand_bch_dim=False, return_tensor=False)
        # source_combined_weight, target_combined_weight = source["weights"], target["weights"]
        # os.makedirs(plot_saving_path,exist_ok=True)
        # saving_file_path = os.path.join(plot_saving_path,pair_info_list[pair_id]["source"]["name"]+"_weights_distribution.png")
        # title = pair_info_list[pair_id]["source"]["name"] + "_" +"n_sp:{} ".format(len(source_vg_weight))+"n_tp:{}".format(len(atlas["weights"]))
        # _,source_combined_mapped_weight =plot_pair_weight_distribution_before_and_after_radius_matching(source_vg_weight, atlas["weights"],source_combined_weight,sampled_atlas["weights"], use_log=True,title=title,show=False,save_path=saving_file_path)
        # saving_file_path = os.path.join(plot_saving_path,    pair_info_list[pair_id]["target"]["name"] + "_weights_distribution.png")
        # title = pair_info_list[pair_id]["target"]["name"] + "_" + "n_sp:{} ".format(len(target_vg_weight)) + "n_tp:{}".format(len(atlas["weights"]))
        # _,target_combined_mapped_weight =plot_pair_weight_distribution_before_and_after_radius_matching(target_vg_weight, atlas["weights"], target_combined_weight, sampled_atlas["weights"],use_log=True, title=title, show=False,save_path=saving_file_path)

        # saving_path = os.path.join(plot_saving_path, pair_name_list[pair_id]+"_mapped.png")
        # camera_pos = [(-4.924379645467042, 2.17374925796456, 1.5003730890759344), (0.0, 0.0, 0.0),
        #               (0.40133888001174545, 0.31574165540339943, 0.8597873634998591)]
        # visualize_point_pair(source["points"], target["points"],
        #                      source_combined_mapped_weight,
        #                      target_combined_mapped_weight,
        #                      title1="source", title2="target", rgb_on=False,saving_capture_path=saving_path,camera_pos=camera_pos,show=False )

    # source, target = get_pair(*pair_path)
    # source_vg_weight, target_vg_weight = source["weights"], target["weights"]
    # title = pair_name_list[pair_id] + "_" +"n_sp:{} ".format(len(source_vg_weight))+"n_tp:{}".format(len(target_vg_weight))
    # sampler_obj ="lung_dataloader_utils.lung_sampler( method='combined',scale=0.0003,num_sample=30000,sampled_by_weight=True)"
    # source, target = get_pair(source_path, target_path, expand_bch_dim=False, return_tensor=False)
    # source_combined_weight, target_combined_weight = source["weights"], target["weights"]
    # plot_saving_path = os.path.join(radius_transfered_saing_path,"plots")
    # saving_folder_path = os.path.join(output_path,pair_name_list[pair_id])
    # os.makedirs(saving_folder_path,exist_ok=True)
    # saving_file_path = os.path.join(saving_folder_path,pair_name_list[pair_id]+"_weights_distribution.png")
    # plot_pair_weight_distribution_before_and_after_radius_matching(source_vg_weight, target_vg_weight,source_combined_weight,target_combined_weight, use_log=True,title=title,show=False,save_path=saving_file_path)
    #
    # visualize_point_pair(source["points"], target["points"],
    #                 source["weights"],
    #                  target["weights"],
    #                  title1="source", title2="target", rgb_on=False)

    #
    #
    # shape_pair = create_shape_pair(source, target)
    # source_half = get_half_lung(source)
    # target_half = get_half_lung(target)
    # cleaned_source_half = lung_isolated_leaf_clean_up(source_half,radius=0.02, principle_weight=[2,1,1], normalize_weights=False)
    # # visualize_point_pair(source_half.points, cleaned_source_half.points,
    # #                      source_weight_transform(source_half.weights),
    # #                      source_weight_transform(cleaned_source_half.weights),
    # #                      title1="source", title2="cleaned_source", rgb_on=False)
    # #
    # # plot_pair_weight_distribution(source_weight_transform(source_half.weights).cpu().squeeze().numpy(),
    # #                               target_weight_transform(target_half.weights).cpu().squeeze().numpy(),
    # #                               use_log=True)
    #
    # visualize_point_pair(source_half.points, target_half.points,
    #                      source_weight_transform(source_half.weights),
    #                      target_weight_transform(target_half.weights),
    #                      title1="source", title2="target", rgb_on=False)


import sys
import click
import logging
from pprint import pprint
from ftmstore import get_dataset
from servicelayer.cache import get_redis, get_fakeredis
from servicelayer.logs import configure_logging
from servicelayer.jobs import Job, Dataset
from servicelayer import settings as sl_settings
from servicelayer.archive.util import ensure_path

from ingestors import settings
from ingestors.manager import Manager
from ingestors.directory import DirectoryIngestor
from ingestors.analysis import Analyzer
from ingestors.worker import IngestWorker, OP_ANALYZE, OP_INGEST

log = logging.getLogger(__name__)
STAGES = [OP_ANALYZE, OP_INGEST]

@click.group()
def cli():
    configure_logging(level=logging.DEBUG)

@cli.command()
@click.option("-s", "--sync", is_flag=True, default=False, help="Run without threads")
def process(sync):
    """Start the queue and process tasks as they come. Blocks while waiting"""
    num_threads = None if sync else sl_settings.WORKER_THREADS
    worker = IngestWorker(stages=STAGES, num_threads=num_threads)
    code = worker.run()
    sys.exit(code)

@cli.command()
@click.argument("dataset")
def cancel(dataset):
    """Delete scheduled tasks for given dataset"""
    conn = get_redis()
    Dataset(conn, dataset).cancel()

@cli.command()
def killthekitten():
    """Completely kill redis contents."""
    conn = get_redis()
    conn.flushall()

def _ingest_path(db, conn, dataset, path, languages=[]):
    context = {"languages": languages}
    job = Job.create(conn, dataset)
    stage = job.get_stage(OP_INGEST)
    manager = Manager(db, stage, context)
    path = ensure_path(path)
    if path is not None:
        if path.is_file():
            entity = manager.make_entity("Document")
            checksum = manager.store(path)
            entity.set("contentHash", checksum)
            entity.make_id(checksum)
            entity.set("fileName", path.name)
            log.info("Queue: %r", entity.to_dict())
            manager.queue_entity(entity)
        if path.is_dir():
            DirectoryIngestor.crawl(manager, path)
    manager.close()

@cli.command()
@click.option("--languages", multiple=True, help="3-letter language code (ISO 639)")
@click.option("--dataset", required=True, help="Name of the dataset")
@click.argument("path", type=click.Path(exists=True))
def ingest(path, dataset, languages=None):
    """Queue a set of files for ingest."""
    conn = get_redis()
    db = get_dataset(dataset, OP_INGEST)
    _ingest_path(db, conn, dataset, path, languages=languages)

@cli.command()
@click.option("--dataset", required=True, help="Name of the dataset")
def analyze(dataset):
    db = get_dataset(dataset, OP_ANALYZE)
    analyzer = None
    for entity in db.partials():
        if analyzer is None or analyzer.entity.id != entity.id:
            if analyzer is not None:
                analyzer.flush()
            # log.debug("Analyze: %r", entity)
            analyzer = Analyzer(db, entity, {})
        analyzer.feed(entity)
    if analyzer is not None:
        analyzer.flush()

@cli.command()
@click.option("--languages", multiple=True, help="3-letter language code (ISO 639)")
@click.argument("path", type=click.Path(exists=True))
def debug(path, languages=None):
    """Debug the ingest for the given path."""
    conn = get_fakeredis()
    settings.fts.DATABASE_URI = "sqlite:////tmp/debug.sqlite3"
    db = get_dataset("debug", origin=OP_INGEST, database_uri=settings.fts.DATABASE_URI)
    db.delete()
    _ingest_path(db, conn, "debug", path, languages=languages)
    worker = IngestWorker(conn=conn, stages=STAGES)
    worker.sync()
    for entity in db.iterate():
        pprint(entity.to_dict())

if __name__ == "__main__":
    cli()


from netaddr import IPNetwork, IPAddress
import logging
from pymongo import MongoClient

logger = logging.getLogger( "ucn_logger" )

class VPNResolve(object):

	def __init__( self, cidr, dbcfg):
		self.logscollection = dbcfg['logscollection']
		self.devicecollection = dbcfg['devicecollection']
		self.db = dbcfg['db']
		self.cidr = cidr
		self.mc = MongoClient(dbcfg['host'], dbcfg['port'])

	def clientip(self, request):

		if len(request.access_route) > 1:
			host = request.access_route[-1]
		else:
			host = request.access_route[0]

		logger.debug("seen a client ip %s" % host)

		if IPAddress(host) not in IPNetwork(self.cidr):
			logger.debug("is not local, looking up in openvpn status")
			return self.findlocal(host)
		else:
			return host

	def findlocal(self, host):
		db = self.mc[self.db]
		devices = db[self.logscollection].find({"untrusted_client_ip": host}).sort("ts", -1).limit(1)
		devicename = None
		protocol = None
		for device in devices:
			devicename = device['common_name']
			protocol = device['proto']

		#now lookup device name in the devices collection
		device = db[self.devicecollection].find_one({"login":devicename})

		if device is not None:
			if protocol is not None:
				if protocol == "udp":
					if 'vpn_udp_ip' in device:
						logger.debug("retreived udp ip %s" %  device['vpn_udp_ip'])
						return device['vpn_udp_ip']
				elif protocol == "tcp":
					if 'vpn_tcp_ip' in device:
						logger.debug("retreived tcp ip %s" %  device['vpn_tcp_ip'])
						return device['vpn_tcp_ip']

		logger.debug("no corresponding ip for %s in db" % host)
		return None


import numpy as np
import glob
import os
import pandas as pd

from tqdm import tqdm
import nltk
import string
from nltk.tokenize import word_tokenize
import random
import pickle
from nltk.corpus import stopwords

from autocorrect import Speller
import re
from nltk.corpus import wordnet
from nltk.stem.wordnet import WordNetLemmatizer

from hyperopt import fmin, tpe, hp

# load a document
def load(filename):
    file = open(filename, encoding='utf-8')
    text = file.read()
    file.close()
    return text

# split a document into news story and highlights
def split(doc):
    # find first highlight
    index = doc.find('@highlight')
    # split into story and highlights
    story, highlights = doc[:index], doc[index:].split('@highlight')
    # strip extra white space around each highlight
    highlights = [h.strip() for h in highlights if len(h) > 0]
    return story, highlights

# load all stories from a directory
def load_stories(directory):
    stories = []
    for name in os.listdir(directory):
        filename = directory + '/' + name
        # load document
        doc = load(filename)
        # split into story and highlights
        story, highlights = split(doc)
        # store
        stories.append({'story':story, 'highlights':highlights})
    return stories

directory = r'C:\Users\ymaha\Desktop\cnn\stories'
stories = load_stories(directory)
print('Loaded Stories %d' % len(stories))

def preprocesing(lines):

    # function to convert nltk tag to wordnet tag
    def nltk_tag_to_wordnet_tag(nltk_tag):
        if nltk_tag.startswith('J'):
            return wordnet.ADJ
        elif nltk_tag.startswith('V'):
            return wordnet.VERB
        elif nltk_tag.startswith('N'):
            return wordnet.NOUN
        elif nltk_tag.startswith('R'):
            return wordnet.ADV
        else:
            return None

    def lemmatize_sentence(sentence):
        #tokenize the sentence and find the POS tag for each token
        nltk_tagged = nltk.pos_tag(nltk.word_tokenize(sentence))
        #tuple of (token, wordnet_tag)
        wordnet_tagged = map(lambda x: (x[0], nltk_tag_to_wordnet_tag(x[1])), nltk_tagged)
    #     print(wordnet_tagged)
        lemmatized_sentence = []
        for word, tag in wordnet_tagged:
            if tag is None:
                #if there is no available tag, append the token as is
                lemmatized_sentence.append(word)
            else:
                #else use the tag to lemmatize the token
                lemmatized_sentence.append(lemmatizer.lemmatize(word, tag))
    #         if tag is not None:
    #             lemmatized_sentence.append(lemmatizer.lemmatize(word, tag))

        return " ".join(lemmatized_sentence)

    temp = []
    for line in lines:
        # strip source cnn
        index = line.find('(CNN)')
        if index > -1:
            line = line[index+len('(CNN)'):]

        # tokenize on white space
        line = line.split()
        # convert to lower case
        line = [word.lower() for word in line]
        # remove punctuation and special characters from each token
        line = [w.replace('[<>!#@$:.,%\?-_]+', ' ') for w in line]
        # remove non ascii characters
        line = [w.replace('[^\x00-\x7f]', ' ') for w in line]
        # remove tokens with numbers in them
        line = [word for word in line if word.isalpha()]
#         # removing stop words
#         line = [word for word in line if word not in stop_list]
        # removing words of length 1
        line = [word for word in line if len(word) > 1]
#         # Lemmatizing the words and combing them into a line
#         temp.append(lemmatize_sentence(' '.join(line)))
        # Combining the words into a line
        temp.append(' '.join(line))
    # remove empty strings
    temp = [c for c in temp if len(c) > 0]
    return temp

stop_list = stopwords.words('english')
lemmatizer = WordNetLemmatizer()
stemmer = nltk.stem.PorterStemmer()

for i in tqdm(range(len(stories))):
# for example in stories:
    stories[i]['story'] = preprocesing(stories[i]['story'].split('\n'))
    stories[i]['highlights'] = preprocesing(stories[i]['highlights'])

# save to file
from pickle import dump
dump(stories, open('processed_cnn_data.pkl', 'wb'))


"""Unsupervised Model scheleton."""

from __future__ import division
from __future__ import print_function

import tensorflow as tf

from yadlt.core.model import Model
from yadlt.utils import tf_utils

class UnsupervisedModel(Model):
    """Unsupervised Model scheleton class.

    The interface of the class is sklearn-like.

    Methods
    -------

    * fit(): model training procedure.
    * transform(): model inference procedure.
    * reconstruct(): model reconstruction procedure (autoencoders).
    * score(): model scoring procedure (mean error).
    """

    def __init__(self, name):
        """Constructor."""
        Model.__init__(self, name)

    def fit(self, train_X, train_Y=None, val_X=None, val_Y=None, graph=None):
        """Fit the model to the data.

        Parameters
        ----------

        train_X : array_like, shape (n_samples, n_features)
            Training data.

        train_Y : array_like, shape (n_samples, n_features)
            Training reference data.

        val_X : array_like, shape (N, n_features) optional, (default = None).
            Validation data.

        val_Y : array_like, shape (N, n_features) optional, (default = None).
            Validation reference data.

        graph : tf.Graph, optional (default = None)
            Tensorflow Graph object.

        Returns
        -------
        """
        g = graph if graph is not None else self.tf_graph

        with g.as_default():
            # Build model
            self.build_model(train_X.shape[1])
            with tf.Session() as self.tf_session:
                # Initialize tf stuff
                summary_objs = tf_utils.init_tf_ops(self.tf_session)
                self.tf_merged_summaries = summary_objs[0]
                self.tf_summary_writer = summary_objs[1]
                self.tf_saver = summary_objs[2]
                # Train model
                self._train_model(train_X, train_Y, val_X, val_Y)
                # Save model
                self.tf_saver.save(self.tf_session, self.model_path)

    def transform(self, data, graph=None):
        """Transform data according to the model.

        Parameters
        ----------

        data : array_like, shape (n_samples, n_features)
            Data to transform.

        graph : tf.Graph, optional (default = None)
            Tensorflow Graph object

        Returns
        -------
        array_like, transformed data
        """
        g = graph if graph is not None else self.tf_graph

        with g.as_default():
            with tf.Session() as self.tf_session:
                self.tf_saver.restore(self.tf_session, self.model_path)
                feed = {self.input_data: data, self.keep_prob: 1}
                return self.encode.eval(feed)

    def reconstruct(self, data, graph=None):
        """Reconstruct data according to the model.

        Parameters
        ----------

        data : array_like, shape (n_samples, n_features)
            Data to transform.

        graph : tf.Graph, optional (default = None)
            Tensorflow Graph object

        Returns
        -------
        array_like, transformed data
        """
        g = graph if graph is not None else self.tf_graph

        with g.as_default():
            with tf.Session() as self.tf_session:
                self.tf_saver.restore(self.tf_session, self.model_path)
                feed = {self.input_data: data, self.keep_prob: 1}
                return self.reconstruction.eval(feed)

    def score(self, data, data_ref, graph=None):
        """Compute the reconstruction loss over the test set.

        Parameters
        ----------

        data : array_like
            Data to reconstruct.

        data_ref : array_like
            Reference data.

        Returns
        -------

        float: Mean error.
        """
        g = graph if graph is not None else self.tf_graph

        with g.as_default():
            with tf.Session() as self.tf_session:
                self.tf_saver.restore(self.tf_session, self.model_path)
                feed = {
                    self.input_data: data,
                    self.input_labels: data_ref,
                    self.keep_prob: 1
                }
                return self.cost.eval(feed)


import modules.processing_turn as m_turn
import modules.data_base as m_data

def click_cell(x,y):
    # Умова першого рядка таблиці
    if y < 100 and y > 0:
        # Умова першої комірки по х
        if x > -100 and x < 0 and m_data.list_cells[0] == 0:
            m_turn.who_turn(-100, 100, 0)
        # Умова другої комірки по х
        elif x < 100 and x > 0  and m_data.list_cells[1] == 0:
            m_turn.who_turn(0, 100, 1)
        # Умова третьої комірки по х
        elif x > 100 and x < 200  and m_data.list_cells[2] == 0:
            m_turn.who_turn(100, 100, 2)
    # Умова другого рядка таблиці
    elif y < 0 and y > -100:
        # Умова четвертої комірки по х
        if x > -100 and x < 0  and m_data.list_cells[3] == 0:
            m_turn.who_turn(-100, 0, 3)
        # Умова п'ятої комірки по х
        elif x < 100 and x > 0  and m_data.list_cells[4] == 0:
            m_turn.who_turn(0, 0, 4)
        # Умова шостої комірки по х
        elif x > 100 and x < 200  and m_data.list_cells[5] == 0:
            m_turn.who_turn(100, 0, 5)
    # Умова третього рядка таблиці
    elif y < -100 and y > -200:
        if x > -100 and x < 0 and m_data.list_cells[6] == 0:
            m_turn.who_turn(-100,-100,6)
        elif x < 100 and  x > 0 and m_data.list_cells[7] == 0:
            m_turn.who_turn(0, -100, 7)
        elif x > 100 and x < 200 and m_data.list_cells[8] == 0:
            m_turn.who_turn(100, -100, 8)


import tensorflow as tf
import re
import time, datetime
import os
import data_helper
TOWER_NAME = 'tower'

class CNNClassify(object):
    """CNN图像分类
    """
    def __init__(self, batch_size, num_classes, num_train_examples, initial_lr=0.1, lr_decay_factor=0.1,
                 moving_average_decay=0.9999, num_epochs_per_decay=300, log_frequency=10,
                 max_steps=200000, checkpoint_every=5000, num_gpus=4, session_conf=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True,
                                           gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.3))):
        self.batch_size = batch_size
        self.num_classes = num_classes
        self.moving_average_decay = moving_average_decay  # 用于移动平均的衰减
        self.initial_lr = initial_lr    # 最初的学习速率
        self.lr_decay_factor = lr_decay_factor  # 学习速率衰减因子
        self.num_epochs_per_decay = num_epochs_per_decay  # 多少轮衰减一次
        self.num_train_examples = num_train_examples  # 训练样本数量
        self.log_frequency = log_frequency  # 多少步控制台打印一次结果
        self.max_steps = max_steps
        self.checkpoint_every = checkpoint_every  # 多少步之后保存一次模型
        self.num_checkpoints = 5
        self.num_gpus = num_gpus
        self.session_conf = session_conf

    def _variable_on_cpu(self, name, shape, initializer):
        """帮助创建存储在CPU内存上的变量。"""
        with tf.device('/cpu:0'):
            dtype = tf.float32
            var = tf.get_variable(name, shape, initializer=initializer, dtype=dtype)
        return var

    def _variable_with_weight_decay(self, name, shape, stddev, wd):
        """初始化权重变量
        Args:
          name: name of the variable
          shape: list of ints
          stddev: 高斯函数标准差
          wd: 添加L2范数损失权重衰减系数。如果没有，该变量不添加重量衰减。
        Returns:权重变量
        """
        dtype = tf.float32
        var = self._variable_on_cpu(name, shape, tf.truncated_normal_initializer(stddev=stddev, dtype=dtype))
        if wd is not None:
            weight_decay = tf.multiply(tf.nn.l2_loss(var), wd, name='weight_loss')
            tf.add_to_collection('losses', weight_decay)
        return var

    def _activation_summary(self, x):
        """创建tensorboard摘要 好可视化查看
        """
        tensor_name = re.sub('%s_[0-9]*/' % TOWER_NAME, '', x.op.name)
        tf.summary.histogram(tensor_name + '/activations', x)
        tf.summary.scalar(tensor_name + '/sparsity', tf.nn.zero_fraction(x))

    def average_gradients(self, tower_grads):
        """计算所有tower上所有变量的平均梯度
        """
        average_grads = []
        for grad_and_vars in zip(*tower_grads):
            # 每个梯度和变量类似这样:
            #   ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN))
            grads = []
            for g, _ in grad_and_vars:
                # 添加一个0维度来代表tower  [grad0_gpuN]
                expanded_g = tf.expand_dims(g, 0)

                # [[grad0_gpu1],...,[grad0_gpuN]]
                grads.append(expanded_g)

            # 在tower上进行平均  (上面加维度那部分没理解 加了又合 不是白操作吗？,后续再研究一下)
            grad = tf.concat(axis=0, values=grads)  # [grad0_gpu1,..., grad0_gpuN]
            grad = tf.reduce_mean(grad, 0)          # 平均梯度

            # 把变量拼接回去
            v = grad_and_vars[0][1]
            grad_and_var = (grad, v)
            average_grads.append(grad_and_var)
        return average_grads

    def inference(self, images):
        """向前传播
        """
        # 第一层卷积
        with tf.variable_scope('conv1') as scope:
            kernel = self._variable_with_weight_decay('weights', shape=[5, 5, 3, 64], stddev=5e-2, wd=0.0)  # 权值矩阵
            # 二维卷积
            conv = tf.nn.conv2d(images, kernel, [1, 1, 1, 1], padding='SAME')   #  周围补0 保持形状不变
            biases = self._variable_on_cpu('biases', [64], tf.constant_initializer(0.0))
            pre_activation = tf.nn.bias_add(conv, biases)
            conv1 = tf.nn.relu(pre_activation, name=scope.name)  # relu激活

            self._activation_summary(conv1)

        # pool1 最大池化
        pool1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool1')
        # norm1 增加一个LRN处理,可以增强模型的泛化能力
        norm1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm1')

        # 第二层卷积
        with tf.variable_scope('conv2') as scope:
            kernel = self._variable_with_weight_decay('weights', shape=[5, 5, 64, 64], stddev=5e-2, wd=0.0)
            conv = tf.nn.conv2d(norm1, kernel, [1, 1, 1, 1], padding='SAME')
            biases = self._variable_on_cpu('biases', [64], tf.constant_initializer(0.1))
            pre_activation = tf.nn.bias_add(conv, biases)
            conv2 = tf.nn.relu(pre_activation, name=scope.name)

            self._activation_summary(conv2)

        # 这次先进行LRN处理
        norm2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm2')
        # 最大池化
        pool2 = tf.nn.max_pool(norm2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool2')

        # 全连接隐层 映射到384维向量
        with tf.variable_scope('local3') as scope:
            # 将前面的最大池化输出扁平化成一个单一矩阵 好做全连接
            reshape = tf.reshape(pool2, [self.batch_size, -1])
            dim = reshape.get_shape()[1].value
            weights = self._variable_with_weight_decay('weights', shape=[dim, 384], stddev=0.04, wd=0.004)
            biases = self._variable_on_cpu('biases', [384], tf.constant_initializer(0.1))
            local3 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name)

            self._activation_summary(local3)

        # 再接一个全连接层 映射到192维向量
        with tf.variable_scope('local4') as scope:
            weights = self._variable_with_weight_decay('weights', shape=[384, 192], stddev=0.04, wd=0.004)
            biases = self._variable_on_cpu('biases', [192], tf.constant_initializer(0.1))
            local4 = tf.nn.relu(tf.matmul(local3, weights) + biases, name=scope.name)

            self._activation_summary(local4)

        # 线性输出层 这里不做softmax  因为在损失函数内部执行了，那样效率更高
        with tf.variable_scope('softmax_linear') as scope:
            weights = self._variable_with_weight_decay('weights', [192, self.num_classes], stddev=1 / 192.0, wd=0.0)
            biases = self._variable_on_cpu('biases', [self.num_classes], tf.constant_initializer(0.0))
            softmax_linear = tf.add(tf.matmul(local4, weights), biases, name=scope.name)

            self._activation_summary(softmax_linear)

        return softmax_linear

    def loss(self, logits, labels):
        """损失函数
        """
        # Calculate the average cross entropy loss across the batch.
        labels = tf.cast(labels, tf.int64)
        cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
            labels=labels, logits=logits, name='cross_entropy_per_example')
        cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy')
        tf.add_to_collection('losses', cross_entropy_mean)

        # The total loss is defined as the cross entropy loss plus all of the weight
        # decay terms (L2 loss).
        return tf.add_n(tf.get_collection('losses'), name='total_loss')

    def tower_loss(self, scope, logits, labels):
        _ = self.loss(logits, labels)
        # 把所有损失都集中到当前tower上
        losses = tf.get_collection('losses', scope)
        total_loss = tf.add_n(losses, name='total_loss')
        for l in losses + [total_loss]:
            # 去掉变量名前缀 tower_[0-9],变成和单GPU的时候一样
            loss_name = re.sub('%s_[0-9]*/' % TOWER_NAME, '', l.op.name)
            tf.summary.scalar(loss_name, l)

        return total_loss

    def evaluation(self, logits, labels, k=1):
        """评估函数
        :param logits: 预测
        :param labels: 标签
        """
        correct = tf.nn.in_top_k(logits, labels, k=k)
        # correct = tf.equal(self.predictions, tf.argmax(labels, 1))
        accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
        tf.add_to_collection('accuracy', accuracy)
        return tf.add_n(tf.get_collection('accuracy'), name='accuracy')

    def tower_evaluation(self, scope, logits, labels, k=1):
        """多gpu的评估函数
        """
        _ = self.evaluation(logits, labels, k)
        accuracy = tf.get_collection('accuracy', scope)
        total_accuracy = tf.reduce_mean(accuracy, axis=0, name='total_accuracy')
        return total_accuracy

    def _add_loss_summaries(self, total_loss):
        """增加损失摘要
        """
        # Compute the moving average of all individual losses and the total loss.
        loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg')
        losses = tf.get_collection('losses')
        loss_averages_op = loss_averages.apply(losses + [total_loss])

        # Attach a scalar summary to all individual losses and the total loss; do the
        # same for the averaged version of the losses.
        for l in losses + [total_loss]:
            # Name each loss as '(raw)' and name the moving average version of the loss
            # as the original loss name.
            tf.summary.scalar(l.op.name + ' (raw)', l)
            tf.summary.scalar(l.op.name, loss_averages.average(l))

        return loss_averages_op

    def train_operation(self, total_loss, global_step):
        """训练操作
        """
        num_batches_per_epoch = self.num_train_examples / self.batch_size   # 每轮的批次数
        decay_steps = int(num_batches_per_epoch * self.num_epochs_per_decay)   # 多少步衰减

        # 基于步数，以指数方式衰减学习率。
        lr = tf.train.exponential_decay(self.initial_lr, global_step, decay_steps, self.lr_decay_factor, staircase=True)
        tf.summary.scalar('learning_rate', lr)
        # 损失移动平均
        loss_averages_op = self._add_loss_summaries(total_loss)

        with tf.control_dependencies([loss_averages_op]):
            opt = tf.train.GradientDescentOptimizer(lr)  # 优化器
            grads = opt.compute_gradients(total_loss)    # 梯度

        # 应用梯度
        apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)   # 训练操作

        # 为可训练的变量添加直方图
        for var in tf.trainable_variables():
            tf.summary.histogram(var.op.name, var)

        # 为梯度添加直方图
        for grad, var in grads:
            if grad is not None:
                tf.summary.histogram(var.op.name + '/gradients', grad)

        # 跟踪所有可训练变量的移动平均线
        variable_averages = tf.train.ExponentialMovingAverage(self.moving_average_decay, global_step)
        variables_averages_op = variable_averages.apply(tf.trainable_variables())

        with tf.control_dependencies([apply_gradient_op, variables_averages_op]):
            train_op = tf.no_op(name='train')

        return train_op

    def train_step(self, sess, summary_writer):
        """单步训练
        """
        _, step, cur_loss, cur_acc = sess.run([self.train_op, self.global_step, self._loss, self.accuracy])
        time_str = datetime.datetime.now().isoformat()
        print("{}: step {}, loss {:g}, acc {:g}".format(time_str, step, cur_loss, cur_acc))
        # 存储摘要
        if step % 100 == 0:
            summary_str = sess.run(self.summary)
            summary_writer.add_summary(summary_str, step)
            summary_writer.flush()

    def train(self, filename, out_dir):
        """训练
        """
        with tf.Graph().as_default():
            sess = tf.Session(config=self.session_conf)
            with sess.as_default():

                self.global_step = tf.contrib.framework.get_or_create_global_step()

                with tf.device('/cpu:0'):
                    images, labels = data_helper.distorted_inputs(filename, self.batch_size)
                logits = self.inference(images)
                self._loss = self.loss(logits, labels)
                self.train_op = self.train_operation(self._loss, self.global_step)
                self.accuracy = self.evaluation(logits, labels)
                self.summary = tf.summary.merge_all()

                # 保存点设置
                checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
                checkpoint_prefix = os.path.join(checkpoint_dir, "model")  # 模型存储前缀
                if not os.path.exists(checkpoint_dir):
                    os.makedirs(checkpoint_dir)
                saver = tf.train.Saver(tf.global_variables(), max_to_keep=self.num_checkpoints)
                summary_writer = tf.summary.FileWriter(out_dir + "/summary", sess.graph)

                # 初始化所有变量
                ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
                if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
                    saver.restore(sess, ckpt.model_checkpoint_path)
                else:
                    sess.run(tf.global_variables_initializer())

                tf.train.start_queue_runners(sess=sess)

                for step in range(self.max_steps):
                    self.train_step(sess, summary_writer)  # 训练
                    cur_step = tf.train.global_step(sess, self.global_step)
                    # checkpoint_every 次迭代之后 保存模型
                    if cur_step % self.checkpoint_every == 0 and cur_step != 0:
                        path = saver.save(sess, checkpoint_prefix, global_step=cur_step)
                        print("Saved model checkpoint to {}\n".format(path))

    def multi_gpu_train(self, filename, out_dir):
        with tf.Graph().as_default(), tf.device('/cpu:0'):
            sess = tf.Session(config=self.session_conf)
            with sess.as_default():
                # Create a variable to count the number of train() calls. This equals the
                # number of batches processed * FLAGS.num_gpus.
                self.global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)

                # 学习速率衰减设置
                num_batches_per_epoch = self.num_train_examples / self.batch_size
                decay_steps = int(num_batches_per_epoch * self.num_epochs_per_decay)

                # 根据步数衰减学习速率
                lr = tf.train.exponential_decay(self.initial_lr, self.global_step, decay_steps, self.lr_decay_factor,
                                                staircase=True)
                # 执行梯度下降的优化器
                opt = tf.train.GradientDescentOptimizer(lr)

                images, labels = data_helper.distorted_inputs(filename, self.batch_size)  # 取出数据
                # 批次队列 这个函数不是很懂
                batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue([images, labels], capacity=2 * self.num_gpus)

                tower_grads = []
                summaries = None
                with tf.variable_scope(tf.get_variable_scope()):
                    for i in range(self.num_gpus):
                        with tf.device('/gpu:{}'.format(i)):
                            with tf.name_scope('{}_{}'.format(TOWER_NAME, i)) as scope:
                                # 为gpu列出一个批次
                                image_batch, label_batch = batch_queue.dequeue()
                                # 计算一个tower的损失. 并且每个tower共享权重变量
                                logits = self.inference(image_batch)
                                self._loss = self.tower_loss(scope, logits, label_batch)
                                self.accuracy = self.tower_evaluation(scope, logits, label_batch)
                                # 下一个tower复用变量
                                tf.get_variable_scope().reuse_variables()

                                # 保存最终tower的摘要
                                summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope)

                                # 计算梯度
                                grads = opt.compute_gradients(self._loss)

                                # 跟踪所有tower的梯度
                                tower_grads.append(grads)

                grads = self.average_gradients(tower_grads)  # 平均梯度

                # 添加学习速率的摘要
                summaries.append(tf.summary.scalar('learning_rate', lr))

                # 添加梯度直方图
                for grad, var in grads:
                    if grad is not None:
                        summaries.append(tf.summary.histogram(var.op.name + '/gradients', grad))

                # 应用梯度来调整共享变量
                apply_gradient_op = opt.apply_gradients(grads, global_step=self.global_step)

                # 所有可训练变量添加直方图
                for var in tf.trainable_variables():
                    summaries.append(tf.summary.histogram(var.op.name, var))

                # 跟踪所有可训练变量的移动平均线
                variable_averages = tf.train.ExponentialMovingAverage(self.moving_average_decay, self.global_step)
                variables_averages_op = variable_averages.apply(tf.trainable_variables())

                # 将所有更新集中到一个训练操作
                self.train_op = tf.group(apply_gradient_op, variables_averages_op)

                # 从最后的tower总结摘要
                self.summary = tf.summary.merge(summaries)

                # 保存点设置
                checkpoint_dir = os.path.abspath(os.path.join(out_dir, "checkpoints"))
                checkpoint_prefix = os.path.join(checkpoint_dir, "model")  # 模型存储前缀
                if not os.path.exists(checkpoint_dir):
                    os.makedirs(checkpoint_dir)
                saver = tf.train.Saver(tf.global_variables(), max_to_keep=self.num_checkpoints)
                summary_writer = tf.summary.FileWriter(out_dir + "/summary", sess.graph)

                # 初始化所有变量
                ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
                if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
                    saver.restore(sess, ckpt.model_checkpoint_path)
                else:
                    sess.run(tf.global_variables_initializer())

                # 启动队列
                tf.train.start_queue_runners(sess=sess)

                for step in range(self.max_steps):
                    self.train_step(sess, summary_writer)  # 训练
                    cur_step = tf.train.global_step(sess, self.global_step)
                    # checkpoint_every 次迭代之后 保存模型
                    if cur_step % self.checkpoint_every == 0 and cur_step != 0:
                        path = saver.save(sess, checkpoint_prefix, global_step=cur_step)
                        print("Saved model checkpoint to {}\n".format(path))


from src.utils.all_utils import read_yaml, create_directory
import argparse
import os
import shutil
from tqdm import tqdm
import logging

log_string = "[%(asctime)s: %(levelname)s: %(module)s]: %(message)s"
logs_dir = "Logs"
os.makedirs(logs_dir,exist_ok=True)
logging.basicConfig(filename=os.path.join(logs_dir,"Running_Logs.log"),level=logging.INFO,format=log_string,filemode='a')

def copy_file(source_download_dir,local_data_dir):
    source_files = os.listdir(source_download_dir)
    N = len(source_files)
    for file in tqdm(source_files,total=N,desc= f"Copying File from {source_download_dir} to {local_data_dir}", colour="green"):
        src = os.path.join(source_download_dir,file)
        dst = os.path.join(local_data_dir,file)
        shutil.copy(src, dst)

def get_data(config_path):
    config = read_yaml(config_path)

    source_download_dirs = config["source_download_dirs"]
    local_data_dirs = config["local_data_dirs"]

    for source_download_dir,local_data_dir in tqdm(zip(source_download_dirs,local_data_dirs),total=2,desc= "List of Folders", colour="cyan"):
        create_directory([local_data_dir])
        copy_file(source_download_dir,local_data_dir)

if __name__ == '__main__':
    args = argparse.ArgumentParser()

    args.add_argument("--config", "-c", default="config/config.yaml")

    parsed_args = args.parse_args()

    try:
        logging.info(">>>>>Stage-01 Started...")
        get_data(config_path=parsed_args.config)
        logging.info("Stage-01 Completed , Data saved into local Directory <<<<<<\n")
    except Exception as e:
        raise e


import pandas as pd
import pickle

def buildDataSet():
	#Import Ingredients DF
	print('Loaded Products...')
	ewg_ing_df = pd.read_json('ingredients_products_keys_fixed/ewg_ingredients.json', orient = 'index')

	#Build mapping between Ingredient ID and ingredient Name
	ing_map = {}
	for i in range(len(ewg_ing_df)):
	    ID = ewg_ing_df.iloc[i]['ingredient_id']
	    name = ewg_ing_df.iloc[i]['ingredient_name']
	    ing_map[ID] = name

	#Read in Product Data and Initialize Acne Score
	ewg_prd_df = pd.read_json('ingredients_products_keys_fixed/ewg_products.json', orient = 'index')
	ewg_prd_df['Acne_Score'] = 0
	print('Loaded ingredients')

	#Build Lists of ingredients to modify original DataFrame and Initialize Dataset for Model
	from collections import Counter
	n = len(ewg_prd_df)
	ing_lists = []
	ing_cnts = Counter()
	string_lists = []
	for i in range(n):
	    try:
	        new_list = []
	        strings = ''
	        ing_list = ewg_prd_df.iloc[i]['ingredient_list']
	        for ID in ing_list:
	            new_list.append(ing_map[ID])
	            ing_cnts[ing_map[ID]] += 1
	            #strings = strings + ' ' + ing_map[ID]

	        #print(new_list)
	        ing_lists.append(new_list)
	        string_lists.append(str(new_list))
	    except:
	        ing_lists.append([''])
	        string_lists.append('')
	        print('Failed on',i, 'no ingredient list.')
	print('Finished matching ingredients to keys.')

	ewg_prd_df['New_List'] = ing_lists

	#Build Synonym Dictionary
	synonym_dict = {}
	for i in range(ewg_ing_df.shape[0]):
	    row = ewg_ing_df.iloc[i]
	    syns = row['synonym_list']
	    if type(syns) == list:
	        for syn in syns:
	            synonym_dict[syn.strip()] = row['ingredient_name']
	        synonym_dict[row['ingredient_name']] = row['ingredient_name']
	    else:
	        synonym_dict[row['ingredient_name']] = row['ingredient_name']
	print('Build Synonyms')

	#Initialize Ingredient Score
	ewg_ing_df['Acne_Score'] = 0.0

	#Extract Comodegenic Scores

	comodegenic = []

	with open('comodegenic.csv','r') as f:
	    for line in f:
	        if line[0] != ',':
	            words = line.strip().split(',')
	            if words[1] != '':
	                comodegenic.append(( words[0], words[1], words[2]))
	cd_df = pd.DataFrame(comodegenic)

	#Match Comodegeic Ingredients to EWG
	from fuzzywuzzy import fuzz
	from fuzzywuzzy import process
	matches = []
	print('Matching Comodegenic to EWG...')
	for i in range(cd_df.shape[0]):
	    cur_ingredient = cd_df.iloc[i][0].upper()
	    matches.append(process.extract(cur_ingredient, synonym_dict.keys(),limit=1, scorer=fuzz.token_sort_ratio))

	#Match Comodegenic Ingredients to EWG
	cd_ranks = []
	stop
	for i in range(cd_df.shape[0]):
	    match_score = int(matches[i][0][1])
	    match_name = matches[i][0][0]
	    cd_name = cd_df.iloc[i][0].upper()
	    cd_ranks.append(match_score)

	    if match_score >= 90:
	        ewg_name = synonym_dict[match_name]
	        #print(temp_score, '\t', match_name, '\t', cd_name, '\t', synonym_dict[match_name])
	        #print(cd_df.iloc[i][1],cd_df.iloc[i][0])
	        row= ewg_ing_df[ewg_ing_df['ingredient_name']==ewg_name].index
	        ewg_ing_df.loc[row,'Acne_Score'] = cd_df.iloc[i][1]
	        #print(ewg_ing_df.loc[row]['ingredient_name'], ewg_ing_df.loc[row]['Acne_Score'])
	        #print(ewg_ing_df[ewg_ing_df['ingredient_name']==ewg_name])
	print('Updated EWG with Acne Scores')
	#Update Product Acne Score
	acne_score_list = []
	for i in range(ewg_prd_df.shape[0]):
	    row = ewg_prd_df.iloc[i]
	    total_acne = 0
	    for ing in row['New_List']:
	        try:
	            acne_score = float(ewg_ing_df[ewg_ing_df['ingredient_name']==ing]['Acne_Score'])
	            #print(ing, acne_score)
	            total_acne += acne_score
	        except:
	            None
	    acne_score_list.append(total_acne)
	#print(acne_score_list)
	ewg_prd_df['Acne_Score'] = acne_score_list

	#Save Final Acne Matrix
	pickle_out = open("ewg_prd_df.pickle","wb")
	pickle.dump(ewg_prd_df, pickle_out)
	pickle_out.close()
	print('Saved dataset to "ewg_prd_df.pickle"')

try:

	pickle.load(open("ewg_prd_df.pickle","rb"))
	print('Loaded from Pickle')
	ewg_prd_df = pickle.load(open("ewg_prd_df.pickle","rb"))
except:
	print("Building Dataset from Files...")
	buildDataSet()
	ewg_prd_df = pickle.load(open("ewg_prd_df.pickle","rb"))

#try:
#	X = pickle.load(open("X.pickle","rb"))
#except:
#Need to change to a real function...code block simple
print('Building Dataset...')
#print(ewg_prd_df)
from collections import Counter
n = ewg_prd_df.shape[0]

print(n)
ing_lists = []
ing_cnts = Counter()
string_lists = []

for i in range(n):
	ings = ewg_prd_df.iloc[i]['New_List']
	str_list = ''
	if type(ings) == list:
		#print(type(ings), i)
		for ing in ings:
			if type(ing) == str:
				str_list = str_list + '|' + ing
		string_lists.append(str_list)
	else:
		print('Failed',i)
		string_lists.append('')

#Build TD-IDF Matrix
from sklearn.feature_extraction.text import TfidfVectorizer
def ing_tokenizer(word):
    return word.split('|')

#print(ewg_prd_df['New_List'].tolist())
vectorizer = TfidfVectorizer(tokenizer = ing_tokenizer, lowercase = False, stop_words = ['WATER','GLYCERIN','',
	'TITANIUM DIOXIDE', 'IRON OXIDES','BEESWAX','METHYLPARABEN', 'PROPYLPARABEN', 'PROPYLENE GLYCOL', 'PANTHENOL', 'MICA'] )
X = vectorizer.fit_transform(string_lists)
#print(vectorizer.vocabulary_)
pickle_out = open("X.pickle","wb")
pickle.dump(X, pickle_out)
pickle_out.close()

#print(X)

print('Running Optimization...')

from sklearn.metrics import confusion_matrix
for thresh in [0]:

	for test_size in [.001,.05,.01,.1]:

		for alph in [.001]:
			best_alpha = 0
			best_test_size = 0
			best_thresh_hold = 0
			best_test_score = 0
			best_train_score = 0
			best_model = None

			#Initialize Acne Score by Product
			Y = []
			for i in ewg_prd_df['Acne_Score']:
			    if i > 0 and i < 3:
			        Y.append(1)
			    elif i > 2:
			    	Y.append(2)
			    else:
			        Y.append(0)

			#Split Training and Test Data by 1/3 to 2/3
			from sklearn.model_selection import train_test_split
			X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=test_size, random_state=42)

			#Build NB Model
			from sklearn.naive_bayes import  MultinomialNB
			gnb =  MultinomialNB(alpha = alph)
			gnb_fit = gnb.fit(X_train,y_train)

			y_pred = gnb_fit.predict(X_test)
			#y_pred_tr = gnb_fit.predict(X_train)
			test_score = confusion_matrix(y_test, y_pred)
			#train_score = confusion_matrix(y_train, y_pred_tr)

			#if test_score:
			best_test_score = test_score
			best_alpha = alph
			best_test_size = test_size
			best_thresh_hold = thresh
			best_model = gnb_fit

			print('Best Test Score:',gnb_fit.score(X_test,y_test), '\n', test_score) #,'\t', train_score)
			print('Alpha:\t', best_alpha)
			print('Test_size:\t',test_size)
			print('Thresh:\t', thresh,'\n')

			#print('Thresh:',thresh, 'TestSize\t',test_size,'\n' ,'\tTraining Error:', )
			#print('\tTesting Error', )

pickle_out = open("nb.pickle","wb")
pickle.dump(gnb_fit, pickle_out)
pickle_out.close()

ingredient_weights = {}
i = 0
print(len(gnb.coef_), best_model.coef_, type(best_model.coef_[0]))
for i in range(gnb_fit.coef_[0].shape[0]):
	#print( gnb.coef_[0][i], vectorizer.get_feature_names()[i])
    ingredient_weights[vectorizer.get_feature_names()[i]] =(gnb.coef_[0][i])
    #print(, gnb.coef_[i])

import operator
sorted_weights = sorted(ingredient_weights.items(), key=operator.itemgetter(1))
for i in range(1,20):
    print(sorted_weights[-i])

score = best_model.predict_proba(X_train)
pred = best_model.predict(X_train)

for i in range(100):
	print(ewg_prd_df.iloc[i]['Acne_Score'], score[i], pred[i])

import matplotlib.pyplot as plt
import matplotlib.patches as mpatches

#%matplotlib inline
ewg_prd_df['Acne_Score'].hist(bins=40)
plt.show()
#for i in range(gnb_fit.coef_
#print(gnb_fit.coef_)
#out = gnb_fit.predict_proba(X_test)
#for i in range(len(out)):
#	print(out[i])
#print(gnb_fit.class_log_prior_)
#print(gnb_fit.feature_count_)
#print(gnb_fit.class_count_)
#print(gnb_fit.get_params())


import os
from glob import glob

import torch_em
from . import util

URL = "https://zenodo.org/record/6546550/files/MouseEmbryos.zip?download=1"
CHECKSUM = "bf24df25e5f919489ce9e674876ff27e06af84445c48cf2900f1ab590a042622"

def _require_embryo_data(path, download):
    if os.path.exists(path):
        return
    os.makedirs(path, exist_ok=True)
    tmp_path = os.path.join(path, "mouse_embryo.zip")
    util.download_source(tmp_path, URL, download, CHECKSUM)
    util.unzip(tmp_path, path, remove=True)
    # remove empty volume
    os.remove(os.path.join(path, "Membrane", "train", "fused_paral_stack0_chan2_tp00073_raw_crop_bg_noise.h5"))

def get_mouse_embryo_dataset(
    path,
    name,
    split,
    patch_shape,
    download=False,
    offsets=None,
    boundaries=False,
    binary=False,
    **kwargs,
):
    """Dataset for the segmentation of nuclei in confocal microscopy.

    This dataset is stored on zenodo: https://zenodo.org/record/6546550.
    """
    assert name in ("membrane", "nuclei")
    assert split in ("train", "val")
    assert len(patch_shape) == 3
    _require_embryo_data(path, download)

    # the naming of the data is inconsistent: membrane has val, nuclei has test;
    # we treat nuclei:test as val
    split_ = "test" if name == "nuclei" and split == "val" else split
    file_paths = glob(os.path.join(path, name.capitalize(), split_, "*.h5"))
    file_paths.sort()

    kwargs, _ = util.add_instance_label_transform(
        kwargs, add_binary_target=binary, binary=binary, boundaries=boundaries,
        offsets=offsets, binary_is_exclusive=False
    )

    raw_key, label_key = "raw", "label"
    return torch_em.default_segmentation_dataset(file_paths, raw_key, file_paths, label_key, patch_shape, **kwargs)

def get_mouse_embryo_loader(
    path,
    name,
    split,
    patch_shape,
    batch_size,
    download=False,
    offsets=None,
    boundaries=False,
    binary=False,
    **kwargs,
):
    """Dataloader for the segmentation of nuclei in confocal microscopy. See 'get_mouse_embryo_dataset' for details."""
    ds_kwargs, loader_kwargs = util.split_kwargs(
        torch_em.default_segmentation_dataset, **kwargs
    )
    dataset = get_mouse_embryo_dataset(
        path, name, split, patch_shape,
        download=download, offsets=offsets, boundaries=boundaries, binary=binary,
        **ds_kwargs
    )
    loader = torch_em.get_data_loader(dataset, batch_size, **loader_kwargs)
    return loader


import pandas as pd
from sklearn.model_selection import train_test_split
from transformers import BertTokenizer, BertModel, AutoTokenizer, AutoModelForMaskedLM
from torch import nn
import numpy as np
from sklearn.model_selection import train_test_split, KFold, StratifiedKFold
from torch.optim import Adam
from tqdm import tqdm
import torch
import os
import logging

# **************读取数据和模型************
data = pd.read_csv("../dataset/train.csv")
data_part = data.sample(n=60000, random_state=42, replace=True)
data_shuffled = data_part.sample(frac=1, random_state=42)  # 随机打乱数据
train_data, test_data = train_test_split(
    data_shuffled, test_size=0.3, random_state=42
)  # 分割成训练集和测试集

K_FOLDS = 6  # K折训练
# K折训练的模型
kf = StratifiedKFold(n_splits=K_FOLDS, shuffle=True, random_state=42)

# ***************下载模型*****************
if 1:# 下载模型
    print("下载模型中...")
    tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
    tokenizer.save_pretrained("../model/Tokenizer")
    bert = AutoModelForMaskedLM.from_pretrained("bert-base-cased")
    bert.save_pretrained("../model/BERT_ROW")
    bert_basic = BertModel.from_pretrained("bert-base-cased")
    bert_basic.save_pretrained("../model/BERT_BASIC")
    print("!模型下载结束")
if 0:#
    print("模型加载中...")
    tokenizer = AutoTokenizer.from_pretrained("../model/Tokenizer")
    bert = AutoModelForMaskedLM.from_pretrained("../model/BERT_ROW")
    bert_basic = BertModel.from_pretrained("../model/BERT_BASIC")
    print("模型加载完毕...")
# ***************常量和定义的类与函数************
LABELS = {
    "Literature & Fiction": 0,
    "Animals": 1,
    "Growing Up & Facts of Life": 2,
    "Humor": 3,
    "Cars, Trains & Things That Go": 4,
    "Fairy Tales, Folk Tales & Myths": 5,
    "Activities, Crafts & Games": 6,
    "Science Fiction & Fantasy": 7,
    "Classics": 8,
    "Mysteries & Detectives": 9,
    "Action & Adventure": 10,
    "Geography & Cultures": 11,
    "Education & Reference": 12,
    "Arts, Music & Photography": 13,
    "Holidays & Celebrations": 14,
    "Science, Nature & How It Works": 15,
    "Early Learning": 16,
    "Biographies": 17,
    "History": 18,
    "Children's Cookbooks": 19,
    "Religions": 20,
    "Sports & Outdoors": 21,
    "Comics & Graphic Novels": 22,
    "Computers & Technology": 23,
}

# 日志文件输出目录
logging.basicConfig(filename="../log/train.log", level=logging.INFO)

# ***  封装类 方便数据类型转换 ***************
class Dataset(torch.utils.data.Dataset):
    def __init__(self, df):
        self.labels = [LABELS[label] for label in df["category"]]
        self.texts = [
            tokenizer(
                text,
                padding="max_length",
                max_length=512,
                truncation=True,
                return_tensors="pt",
            )
            for text in df["text"]
        ]

    def classes(self):
        return self.labels

    def __len__(self):
        return len(self.labels)

    def get_batch_labels(self, idx):
        # Fetch a batch of labels
        return np.array(self.labels[idx])

    def get_batch_texts(self, idx):
        # Fetch a batch of inputs
        return self.texts[idx]

    def __getitem__(self, idx):
        batch_texts = self.get_batch_texts(idx)
        batch_y = self.get_batch_labels(idx)
        return batch_texts, batch_y

class BertClassifier(nn.Module):
    def __init__(self, dropout=0.5):
        super(BertClassifier, self).__init__()
        self.bert = bert_basic
        self.dropout = nn.Dropout(dropout)
        self.linear = nn.Linear(768, 24)
        self.relu = nn.ReLU()

    def forward(self, input_id, mask):
        _, pooled_output = self.bert(
            input_ids=input_id, attention_mask=mask, return_dict=False
        )
        dropout_output = self.dropout(pooled_output)
        linear_output = self.linear(dropout_output)
        final_layer = self.relu(linear_output)
        return final_layer

def train(model, train_data, val_data, learning_rate, epochs):
    # 判断是否使用GPU
    use_cuda = torch.cuda.is_available()
    device = torch.device("cuda" if use_cuda else "cpu")

    # 通过Dataset类获取训练和验证集
    train, val = Dataset(train_data), Dataset(val_data)
    # DataLoader根据batch_size获取数据，训练时选择打乱样本
    train_dataloader = torch.utils.data.DataLoader(train, batch_size=8, shuffle=True)
    val_dataloader = torch.utils.data.DataLoader(val, batch_size=8)

    # 定义损失函数和优化器
    criterion = nn.CrossEntropyLoss()
    optimizer = Adam(model.parameters(), lr=learning_rate)

    if use_cuda:
        print("使用gpu")
        model = model.to(device)
        criterion = criterion.to(device)

    # 开始进入训练循环
    for epoch_num in range(epochs):
        # 定义两个变量，用于存储训练集的准确率和损失
        total_acc_train = 0
        total_loss_train = 0

        for train_input, train_label in tqdm(train_dataloader):
            train_label = train_label.to(device)
            train_label = train_label.to(torch.long)
            mask = train_input["attention_mask"].to(device)
            input_id = train_input["input_ids"].squeeze(1).to(device)

            # 通过模型得到输出
            output = model(input_id, mask)
            # 计算损失
            batch_loss = criterion(output, train_label)
            total_loss_train += batch_loss.item()
            # 计算精度
            acc = (output.argmax(dim=1) == train_label).sum().item()
            total_acc_train += acc

            # 模型更新
            model.zero_grad()
            batch_loss.backward()
            optimizer.step()

        # ------ 验证模型 -----------
        # 定义两个变量，用于存储验证集的准确率和损失
        total_acc_val = 0
        total_loss_val = 0

        # 不需要计算梯度
        with torch.no_grad():
            # 循环获取数据集，并用训练好的模型进行验证
            for val_input, val_label in val_dataloader:
                val_label = val_label.to(device)
                val_label = val_label.to(torch.long)
                mask = val_input["attention_mask"].to(device)
                input_id = val_input["input_ids"].squeeze(1).to(device)

                output = model(input_id, mask)

                batch_loss = criterion(output, val_label)
                total_loss_val += batch_loss.item()

                acc = (output.argmax(dim=1) == val_label).sum().item()
                total_acc_val += acc

        logging.info(
            "\n| Epochs: %d \n| Train Loss: %.3f \n| Train Accuracy: %.3f \n| Val Loss: %.3f \n| Val Accuracy: %.3f \n",
            epoch_num + 1,
            total_loss_train / len(train_data),
            total_acc_train / len(train_data),
            total_loss_val / len(val_data),
            total_acc_val / len(val_data),
        )

# ************** 运行部分 ********************

model = BertClassifier()
model.load_state_dict(torch.load("../model/BERT-1"))

learning_rate = 5e-6  # 设置学习率
epochs = 1  # 设置训练轮数

train(model, train_data, test_data, learning_rate, epochs)

torch.save(model.state_dict(), "../model/BERT-1")


import sqlite3 as lite
import sys
from bs4 import BeautifulSoup
import requests
import re

def site_parsing():

    max_page = 10
    pages = []

    id_n = 0
    id_n_price = 0

    for x in range(1, max_page + 1):
        pages.append(requests.get('https://moto.drom.ru/sale/+/Harley-Davidson+Softail/'))

    for n in pages:
        soup = BeautifulSoup(n.text, 'html.parser')

        moto_name = soup.find_all('a', class_="bulletinLink bull-item__self-link auto-shy")

        for rev in moto_name:
            id_n += 1
            a = str(rev.text)
            moto = re.split(r',', a)
            moto_name_s = str(moto[0])
            moto_year = re.sub(r'[ ]', '', moto[1])
            moto_year_s = int(moto_year)
            cur.execute("INSERT INTO moto VALUES(?,?,?)", (id_n, moto_name_s, moto_year_s))

        price = soup.find_all('span', class_='price-block__price')
        pattern = r'(\d{1}\s\d{3}\s\d{3})|(\d{3}\s\d{3})'
        for rev in price:
            id_n_price += 1
            price_str = re.findall(pattern, rev.text)
            price_str = str(price_str)
            price_str = price_str.replace('\\xa0', '')
            price_str = re.sub(r"[\]['(),\s]", '', price_str)
            price_int = int(price_str)
            cur.execute("INSERT INTO moto_price VALUES(?,?)", (id_n_price, price_int))

connect = None

try:
    connect = lite.connect('motos.db')
    cur = connect.cursor()
    cur.execute("CREATE TABLE moto(id INT, moto TEXT, year INT)")
    cur.execute("CREATE TABLE moto_price(id INT, price INT)")

    site_parsing()

except lite.Error as e:
    print(f"Error {e.args[0]}:")
    sys.exit()

with connect:
    cur = connect.cursor()

    rows_join = f'SELECT * FROM moto JOIN moto_price ON moto.id = moto_price.id'

    cur.execute(rows_join)
    rows = cur.fetchall()
    for row in rows:
        print(row)

connect.close()


# num = 100
#
# while num > 0:
#     print(num)
#     num = num + 1
# num = 1
# while num <= 100:
#     if num % 2 == 0:
#         print(num)
#     num += 1
#
# num = 1
# son = 0
# while num <= 100:
#     if num % 4 == 0:
#         son += 1
#     num += 1
#
# print(son)

import random

num = 1
nums = []

while num <= 10:
    random_number = random.randint(1, 50)
    if random_number % 2 == 1:
        nums.append(random_number)
        num += 1
print(nums)


import torch.nn as nn
import torch_geometric.nn as pyg_nn

class iVGAE_Encoder(nn.Module):
    def __init__(self, in_channels, hidden_channels, out_channels):
        super().__init__()
        self.conv0 = pyg_nn.GCNConv(in_channels, hidden_channels)
        self.conv1 = pyg_nn.GCNConv(hidden_channels, hidden_channels)
        self.lin_mean = nn.Linear(hidden_channels, out_channels)
        self.lin_logstd = nn.Linear(hidden_channels, out_channels)

    def forward(self, x, edge_index):
        h = self.conv0(x, edge_index)
        h = nn.ReLU()(h)
        h = self.conv1(h, edge_index)
        h = nn.ReLU()(h)
        mean = self.lin_mean(h)
        logstd = self.lin_logstd(h)
        return mean, logstd

class iVGAE_Decoder(nn.Module):
    def __init__(self, in_channels, hidden_channels, out_channels):
        super().__init__()
        self.conv0 = pyg_nn.GCNConv(in_channels, hidden_channels)
        self.conv1 = pyg_nn.GCNConv(hidden_channels, hidden_channels)
        self.linear = nn.Linear(hidden_channels, out_channels)

    def forward(self, z, edge_index, sigmoid=True):
        h = self.conv0(z, edge_index)
        h = nn.ReLU()(h)
        h = self.conv1(h, edge_index)
        h = nn.ReLU()(h)
        out = self.linear(h)
        if sigmoid:
            out = nn.Sigmoid()(out)
        return out

class iVGAE(pyg_nn.VGAE):
    def __init__(self, encoder, decoder):
        super().__init__(encoder, decoder)

    def decode(self, z, pos_edge_index):
        x_gen = self.decoder(z, pos_edge_index)
        return x_gen

    def forward(self, x, pos_edge_index):
        z = self.encode(x, pos_edge_index)
        x_gen = self.decode(z, pos_edge_index)
        return x_gen, z