from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import argparse
import h5py
import json
import os
import scipy.misc
import sys

import cityscapesscripts.evaluation.instances2dict as cs

import detectron.utils.segms as segms_util
import detectron.utils.boxes as bboxs_util
import numpy as np
import cv2
from pycocotools import mask

def parse_args():
    parser = argparse.ArgumentParser(description='Convert dataset')
    parser.add_argument(
        '--dataset', help="cocostuff, cityscapes", default=None, type=str)
    parser.add_argument(
        '--outdir', help="output dir for json files", default=None, type=str)
    parser.add_argument(
        '--datadir', help="data dir for annotations to be converted",
        default=None, type=str)
    # if len(sys.argv) == 1:
    #     parser.print_help()
    #     sys.exit(1)
    return parser.parse_args()

def convert_coco_stuff_mat(data_dir, out_dir):
    """Convert to png and save json with path. This currently only contains
    the segmentation labels for objects+stuff in cocostuff - if we need to
    combine with other labels from original COCO that will be a TODO."""
    sets = ['train', 'val']
    categories = []
    json_name = 'coco_stuff_%s.json'
    ann_dict = {}
    for data_set in sets:
        file_list = os.path.join(data_dir, '%s.txt')
        images = []
        with open(file_list % data_set) as f:
            for img_id, img_name in enumerate(f):
                img_name = img_name.replace('coco', 'COCO').strip('\n')
                image = {}
                mat_file = os.path.join(
                    data_dir, 'annotations/%s.mat' % img_name)
                data = h5py.File(mat_file, 'r')
                labelMap = data.get('S')
                if len(categories) == 0:
                    labelNames = data.get('names')
                    for idx, n in enumerate(labelNames):
                        categories.append(
                            {"id": idx, "name": ''.join(chr(i) for i in data[
                                n[0]])})
                    ann_dict['categories'] = categories
                scipy.misc.imsave(
                    os.path.join(data_dir, img_name + '.png'), labelMap)
                image['width'] = labelMap.shape[0]
                image['height'] = labelMap.shape[1]
                image['file_name'] = img_name
                image['seg_file_name'] = img_name
                image['id'] = img_id
                images.append(image)
        ann_dict['images'] = images
        print("Num images: %s" % len(images))
        with open(os.path.join(out_dir, json_name % data_set), 'wb') as outfile:
            outfile.write(json.dumps(ann_dict))

# for Cityscapes
def getLabelID(self, instID):
    if (instID < 1000):
        return instID
    else:
        return int(instID / 1000)

def getAreaByPoint(points, h, w, category_id):
    line_type = 1  # cv2.CV_AA
    color = category_id
    seg = []
    for j in range(len(points)):
        coordx = points[j][0]
        coordy = points[j][1]
        point = []
        point.append(int(coordx))
        point.append(int(coordy))
        seg.append(point)

    labelMask = np.zeros((h, w))
    cv2.fillPoly(labelMask, np.array([seg], dtype=np.int32), color, line_type)

    mask_new, contours, hierarchy = cv2.findContours((labelMask).astype(np.uint8), cv2.RETR_TREE,
                                                     cv2.CHAIN_APPROX_SIMPLE)

    ##----------------------------------------------
    polygons = []
    # In practice, only one element.
    for contour in contours:
        contour = contour.flatten().tolist()
        polygons.append(contour)
    labelMask[:, :] = labelMask == color

    labelMask = np.expand_dims(labelMask, axis=2)
    labelMask = labelMask.astype('uint8')
    labelMask = np.asfortranarray(labelMask)

    Rs = mask.encode(labelMask)
    return float(mask.area(Rs))

def convert_cityscapes_instance_only(
        data_dir, out_dir):
    """Convert from cityscapes format to COCO instance seg format - polygons"""
    sets = [
        'gtFine_train',
        'gtFine_val',
        # 'gtFine_test',

        # 'gtCoarse_train',
        # 'gtCoarse_val',
        # 'gtCoarse_train_extra'
    ]
    ann_dirs = [
        'labels_train',
        'labels_val',
        # 'gtFine_trainvaltest/gtFine/test',

        # 'gtCoarse/train',
        # 'gtCoarse/train_extra',
        # 'gtCoarse/val'
    ]
    json_name = 'instancesonly_filtered_%s.json'
    ends_in = '.json'
    img_id = 0
    ann_id = 0
    cat_id = 1
    category_dict = {}

    category_instancesonly = ['__background__',
        'person',
        'rider',
        'car',
        'truck',
        'bus',
        'motorcycle',
        'bicycle',
        'ground',
        'road',
        'sky'
    ]

    for data_set, ann_dir in zip(sets, ann_dirs):
        print('Starting %s' % data_set)
        ann_dict = {}
        images = []
        annotations = []
        ann_dir = os.path.join(data_dir, ann_dir)
        for root, _, files in os.walk(ann_dir):
            for filename in files:
                if filename.endswith(ends_in):
                    if len(images) % 50 == 0:
                        print("Processed %s images, %s annotations" % (
                            len(images), len(annotations)))
                    json_ann = json.load(open(os.path.join(root, filename)))
                    image = {}
                    image['id'] = img_id
                    img_id += 1

                    image['width'] = json_ann['imgWidth']
                    image['height'] = json_ann['imgHeight']
                    sub_name = filename.split('_')[0:3]
                    image['file_name'] = sub_name[0] + "_" + sub_name[1] + "_" + sub_name[2] + "_" + 'leftImg8bit.png'
                    images.append(image)

                    objects = json_ann["objects"]

                    for obj in objects:
                        if obj["label"] not in category_instancesonly:
                            continue  # skip non-instance categories

                        index = category_instancesonly.index(obj["label"])# + 184

                        ann = {}
                        ann['id'] = ann_id
                        ann_id += 1
                        ann['image_id'] = image['id']
                        ann['segmentation'] = [sum(obj['polygon'], [])]

                        ann['category_id'] = index
                        ann['iscrowd'] = 0

                        seg_points = obj["polygon"]
                        ann['area'] = getAreaByPoint(seg_points, image['height'], image['width'], ann['category_id'])
                        ann['bbox'] = bboxs_util.xyxy_to_xywh(
                            segms_util.polys_to_boxes(
                                [ann['segmentation']])).tolist()[0]

                        annotations.append(ann)
                    # break
        ann_dict['images'] = images
        categories = []
        for index, value in enumerate(category_instancesonly):
            categories.append({"id": index, "name": value})
        categories = categories[1:]
        ann_dict['categories'] = categories
        ann_dict['annotations'] = annotations
        print("Num categories: %s" % len(categories))
        print("Num images: %s" % len(images))
        print("Num annotations: %s" % len(annotations))
        with open(os.path.join(out_dir, json_name % data_set), 'wb') as outfile:
            outfile.write(json.dumps(ann_dict))

if __name__ == '__main__':
    args = parse_args()
    args.datadir = "/media/administrator/deeplearning/dataset/test_cityscape"
    args.outdir = "/media/administrator/deeplearning/dataset/test_cityscape/output"
    convert_cityscapes_instance_only(args.datadir, args.outdir)


import os
import re

import yaml

import hou

from metatools.imports import load_entrypoint

def dispatch(entrypoint, args=(), kwargs={}, reload=None):
    func = load_entrypoint(entrypoint, reload=reload)
    return func(*args, **kwargs)

def load(path=None):

    if path is None:
        path = os.environ.get('HOUTOOLS_SHELVES')
        if path is None:
            return
        path = path.split(':')

    for root in path:
        for dir_path, dir_names, file_names in os.walk(root):
            for file_name in file_names:
                if file_name.endswith('.yml') and not file_name.startswith('.'):
                    load_one(os.path.join(dir_path, file_name))

def load_one(path):

    label = os.path.basename(os.path.splitext(path)[0])
    name = 'houtools_' + re.sub(r'\W+', '_', label).lower()

    # Make the shelf.
    shelves = hou.shelves.shelves()
    shelf = shelves.get(name)
    if not shelf:
        shelf = hou.shelves.newShelf(name=name, label=label)

    # Add the shelf to the current desktop.
    desktop = hou.ui.curDesktop()
    dock = desktop.shelfDock()
    shelfSets = dock.shelfSets()
    shelfSet = shelfSets[0]
    if shelf not in shelfSet.shelves():
        shelfSet.setShelves(shelfSet.shelves() + (shelf, ))

    shelf.setTools(())

    # Load the buttons.
    serialized = open(path).read()
    buttons = yaml.load_all(serialized)
    for button in buttons:
        load_button(shelf, button)

def load_button(shelf, spec):

    label = spec['label']
    name = spec.get('name') or ('%s_%s' % (shelf.name(), re.sub(r'\W+', '_', label))).lower()

    # Make sure the tool exists.
    tools = hou.shelves.tools()
    tool = tools.get(name)
    if not tool:
        tool = hou.shelves.newTool(name=name)

    # Make sure the tools is on the shelf.
    shelf.setTools(shelf.tools() + (tool, ))

    # Set up the tool.
    tool.setLabel(label)
    if 'python' in spec:
        tool.setScript(spec['python'])
    elif 'entrypoint' in spec:
        tool.setScript('from houtools.shelf import dispatch; dispatch(%r)' % spec['entrypoint'])


import numpy as np
import glob
import itertools
# import h5py
# import h5py_cache as h5c
from threading import Lock, Thread
from queue import Queue
import time

TRAIN_TEST_SPLIT=0.95

class DataLoaderBase:
    def __init__(self):
        self.train_x = []
        self.train_y = []
        self.test_x = []
        self.test_y = []

class NextItemsDataLoader(DataLoaderBase):

    def __init__(self):
        self.train_x_filenames = sorted(glob.glob('training_data/next_items/processed/*_train_x*.npz'))
        self.test_x_filenames = sorted(glob.glob('training_data/next_items/processed/*_test_x*.npz'))
        self.train_y_filenames = sorted(glob.glob('training_data/next_items/processed/*_train_y*.npz'))
        self.test_y_filenames = sorted(glob.glob('training_data/next_items/processed/*_test_y*.npz'))
        # h5f = h5py.File('training_data/next_items/hdf5/data.h5', 'r')
        # self.train_x_hdf5 = h5f['train_x']
        # self.train_y_hdf5 = h5f['train_y']
        # self.test_x_hdf5 = h5f['test_x']
        # self.test_y_hdf5 = h5f['test_y']

        super().__init__()
        self.readFromNumpyFiles()

    def _generate_train_test(self, train_test_x, train_test_y):
        x = np.concatenate((np.tile(np.arange(5),len(train_test_x))[:,np.newaxis], np.repeat(train_test_x, 5, axis=0)), axis=1)
        y = np.ravel(train_test_y)
        valid_ind = y != 0
        result_y = y[valid_ind]
        result_x = x[valid_ind]
        return result_x, result_y

    def build_HDF5(self):
        self.readFromNumpyFiles()
        print("Files loaded")
        print("Generating training data")
        self.train_x, self.train_y = self._generate_train_test(self.train_x, self.train_y)
        self.test_x, self.test_y = self._generate_train_test(self.test_x, self.test_y)
        print("Training data complete")

        # train_x_y = np.concatenate([self.train_x, self.train_y[:,np.newaxis]], axis = 1)
        # self.train_x, self.train_y = [], []
        # print("Now shuffling")
        # np.random.shuffle(train_x_y)
        # print("Shuffling complete")
        # shuffled = []

        # for i in range(0, len(train_x_y), 10 ** 6):
        #     print("Shuffling {:.2%} complete".format(i / len(train_x_y)))
        #     seg = train_x_y[i:i + 10**6]
        #     np.random.shuffle(seg)
        #     shuffled.extend(seg)

        # print("get arrays back in shape")
        # self.train_x = train_x_y[:, :-1]
        # self.train_y = train_x_y[:, -1]

        print("writing files")
        # chunk_shape_x = (self.train_x.shape[-1],1)
        # chunk_shape_y = (1,)
        h5f = h5c.File('training_data/next_items/hdf5/data.h5', 'w')
        print("0 complete")
        h5f.create_dataset('train_x', data=self.train_x, shape=self.train_x.shape)
        print("1 complete")
        h5f.create_dataset('train_y', data=self.train_y, shape=self.train_y.shape)
        print("2 complete")
        h5f.create_dataset('test_x', data=self.test_x, shape=self.test_x.shape)
        print("3 complete")
        h5f.create_dataset('test_y', data=self.test_y, shape=self.test_y.shape)
        print("4 complete")
        h5f.close()

    class GenerateTrainingData(Thread):
        def __init__(self, queue, gen_x_y):
            Thread.__init__(self)
            self.queue = queue
            self.generate_x_y = gen_x_y

        def run(self):
            while True:
                x, y = self.generate_x_y()
                self.queue.put((x, y))

    def _get_next_examples(self):
        start_index = self.randints.pop()
        end_index = min(start_index+self.batchsize, self.train_len)
        return self.train_x_hdf5[start_index: end_index], self.train_y_hdf5[start_index: end_index]

    def get_next_train_subepoch(self, num_examples):
        if not self.threads:
            # self.
            for _ in range(1):
                t = self.GenerateTrainingData(self.queue, self._get_next_examples)
                self.threads.append(t)
                t.start()
        X,Y = [], []
        for _ in range(num_examples):
            (x,y) = self.queue.get()
            X.append(x)
            Y.append(y)
        return (X,Y)

    def get_test_data(self):
        # X,Y = [],[]
        # for i in range(self.test_x_hdf5.shape[0]):
        #     X.append(self.test_x_hdf5[i])
        #     Y.append(self.test_y_hdf5[i])

        # return self.test_x_hdf5, self.test_y_hdf5
        return self._generate_train_test(self.test_x, self.test_y)
    def get_train_data(self):
        # return self.train_x_hdf5, self.train_y_hdf5
        return self._generate_train_test(self.train_x, self.train_y)

    # def get_train_data(self, pos):
    #     self.readFromNumpyFiles()
    #     train_x, train_y = [], []
    #     for x, y in zip(self.train_x, self.train_y):
    #         y_summ = y[pos]
    #         if y_summ == 0:
    #             continue
    #         else:
    #             train_x += [x]
    #             train_y += [y_summ]
    #     self.train_x = np.array(train_x)
    #     self.train_y = np.array(train_y)
    #     return self.train_x, self.train_y
    #
    # def get_test_data(self, pos):
    #     test_x, test_y = [], []
    #     for x, y in zip(self.test_x, self.test_y):
    #         y_summ = y[pos]
    #         if y_summ == 0:
    #             continue
    #         else:
    #             test_x += [x]
    #             test_y += [y_summ]
    #     self.test_x = np.array(test_x)
    #     self.test_y = np.array(test_y)
    #     return self.test_x, self.test_y

    def readFromNumpyFiles(self):

        if not self.train_x_filenames or not self.train_y_filenames or not self.test_x_filenames or not self.test_y_filenames:
            raise FileNotFoundError("No train or test numpy files in that location")

        for i in self.train_x_filenames:
            data = np.load(i)['arr_0']
            self.train_x += list(data)
        for i in self.train_y_filenames:
            data = np.load(i)['arr_0']
            self.train_y += list(data)
        for i in self.test_x_filenames:
            data = np.load(i)['arr_0']
            self.test_x += list(data)
        for i in self.test_y_filenames:
            data = np.load(i)['arr_0']
            self.test_y += list(data)

class PositionsDataLoader(DataLoaderBase):

    def __init__(self):
        self.train_x_filenames = sorted(glob.glob('training_data/positions/processed/*_train_x*.npz'))
        self.test_x_filenames = sorted(glob.glob('training_data/positions/processed/*_test_x*.npz'))
        super().__init__()

    def get_train_data(self):
        result_x, result_y = [], []
        order = [[1,0,0,0,0],[0,1,0,0,0],[0,0,1,0,0],[0,0,0,1,0],[0,0,0,0,1]]
        for x in self.train_x:

            comp_order = ((comp,pos) for comp, pos in zip(x, order))
            for champ_position in itertools.permutations(comp_order, 5):
                champ_position = np.array(champ_position)
                team_comp = np.stack(champ_position[:, 0])
                positions = np.stack(champ_position[:, 1])
                result_x.append(
                    np.concatenate((np.ravel(team_comp[:, 0]), np.ravel(team_comp[:, 1:3]), np.ravel(team_comp[:, 3:])),
                                   axis=0))
                result_y.append(np.ravel(positions))
        return np.array(result_x), np.array(result_y)

    def get_test_data(self):
        result_x, result_y = [], []
        order = [[1,0,0,0,0],[0,1,0,0,0],[0,0,1,0,0],[0,0,0,1,0],[0,0,0,0,1]]
        for x in self.test_x:

            comp_order = ((comp,pos) for comp, pos in zip(x, order))
            for champ_position in itertools.permutations(comp_order, 5):
                champ_position = np.array(champ_position)
                team_comp = np.stack(champ_position[:, 0])
                positions = np.stack(champ_position[:, 1])
                result_x.append(
                    np.concatenate((np.ravel(team_comp[:, 0]), np.ravel(team_comp[:, 1:3]), np.ravel(team_comp[:, 3:])),
                                   axis=0))
                result_y.append(np.ravel(positions))
        return np.array(result_x), np.array(result_y)

    def readFromNumpyFiles(self):

        if not self.train_x_filenames or not self.test_x_filenames:
            raise FileNotFoundError("No train or test numpy files in that location")

        for i in self.train_x_filenames:
            data = np.load(i)['arr_0']
            self.train_x += list(data)
        for i in self.test_x_filenames:
            data = np.load(i)['arr_0']
            self.test_x += list(data)

if __name__ == "__main__":
    d = NextItemsDataLoader()
    d.build_HDF5()
    # h5f = h5py.File('training_data/next_items/hdf5/data.h5', 'r')
    # train_x = h5f['train_x']
    # print(train_x.shape)
    # rando = np.random.randint(24000000, size=128*100).reshape((100,128))
    # for i in rando:
    #     t1 = time.time()
    #     tmp = train_x[sorted(list(i))]
    #     t2 = time.time()-t1
    #     print(t2)


"""
A module consisting of sinks that the processed events will end up at.
"""

from abc import abstractmethod, ABC
from collections import deque, namedtuple
from datetime import datetime
import logging
import time
import csv

from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

class Sink(ABC):
    """
    An abstract base class for pipeline sinks.
    """

    @abstractmethod
    def handle(self, events):
        """
        Handle each of the given stream of events.
        """
        pass

    @staticmethod
    def parallel(*sinks):
        return ParallelSink(sinks)

class ParallelSink(Sink):
    def __init__(self, sinks):
        self.sinks = sinks

    def handle(self, events):
        for sink in self.sinks:
            sink.handle(events)

class CalculatesAverage(Sink):
    """
    Averaging_period and expiry_time are in seconds. Will calculate a moving average.
    """

    def __init__(self, averaging_period, expiry_time):
        self.averaging_period_ms = 1000 * averaging_period
        self.expiry_time_ms = 1000 * expiry_time

        # Creates millisecond-size bins for calculating moving averages
        current_time_ms = 1000 * time.time()
        self.bins = deque(
            [AverageBin(start=(current_time_ms - self.expiry_time_ms), end=(current_time_ms - self.expiry_time_ms),
                        values=[])])

    def handle(self, events):
        for event in events:
            self.add_to_bin(event)
            expired_bin = self.maybe_expire_first_bin_and_get_average()

            if expired_bin is not None:
                average = expired_bin.average
                logger.info(f'Average value for {average.start} to {average.end} is {average.value}')

    def add_to_bin(self, event):
        # Checks if the event is old and should be ignored (i.e. happened in the past)
        if self.bins[0].start > event.timestamp:
            logger.debug(f'Not averaging old event at timestamp {event.timestamp}')
            return

        # Checks if new bins are needed to process the event
        while self.bins[-1].end < event.timestamp:
            current_last_start, current_last_end = self.bins[-1].start, self.bins[-1].end
            logger.debug(
                f'Adding new bin to deal with event at timestamp {event.timestamp} (Current last bucket is {current_last_start} to {current_last_end})')
            self.bins.append(
                AverageBin(start=current_last_end, end=(current_last_end + self.averaging_period_ms), values=[]))

        # Find the correct bin for an event and add the event's value
        bin_index = int(event.timestamp - self.bins[0].start) // self.averaging_period_ms
        self.bins[bin_index].values.append(event.value)
        self.write_bins_to_file()

    def write_bins_to_file(self):
        # Write bin values to a csv file so overall intensity trends can be identified
        with open('Gas_Averages.csv', mode='w') as csv_file:
            fieldnames = ['Bin Start', 'Bin End', 'Average Value']
            writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
            writer.writeheader()
            for i in range(len(self.bins)):
                if len(self.bins[i].values) != 0:
                    writer.writerow(
                        {'Bin Start': self.bins[i].average.start,
                         'Bin End': self.bins[i].average.end,
                         'Average Value': self.bins[i].average.value})

    def maybe_expire_first_bin_and_get_average(self):
        current_time_ms = 1000 * time.time()
        if current_time_ms - self.expiry_time_ms > self.bins[0].end:
            return self.bins.popleft()

class AverageBin(namedtuple('AverageBucket', 'start end values')):

    @property
    def average(self):
        start_datetime = datetime.fromtimestamp(self.start / 1000)
        end_datetime = datetime.fromtimestamp(self.end / 1000)
        average_value = (sum(self.values) / len(self.values)) if self.values else 0
        return Average(start=start_datetime, end=end_datetime, value=average_value)

class Average(namedtuple('Average', 'start end value')):
    """
    A record of the average value of events between two timestamps.
    """

class EventLocation(namedtuple('EventLocation', 'x y value')):
    """
    An event location, consisting of x and y coordinates the intensity of gas at that event.
    """

class LocationAverage(Sink):
    """
    Calculates the weighted average of each location with an event registered there.
    """

    def __init__(self, locations):
        self.locations = locations

    def handle(self, events):
        self.find_event_locations(events)

    def find_event_locations(self, events):
        event_locations = []
        for event in events:
            for location in self.locations:
                if location.id == event.location_id:
                    event_locations.append(EventLocation(
                        x=location.x, y=location.y, value=event.value
                    ))

        self.plot_results(event_locations)

    @staticmethod
    def plot_results(event_locations):
        x_points = []
        y_points = []
        values = []
        for i in range(len(event_locations)):
            x_points.append(event_locations[i].x)
            y_points.append(event_locations[i].y)
            values.append(event_locations[i].value)

        fig = plt.figure()
        ax = fig.gca(projection='3d')

        surf = ax.plot_trisurf(x_points, y_points, values, cmap='viridis', linewidth=0, antialiased=True)

        fig.colorbar(surf)
        ax.set_xlabel(r'x')
        ax.set_ylabel(r'y')
        ax.set_zlabel(r'Value')
        ax.set_title('GasMon')

        plt.show()


# -*- coding: utf-8 -*-
"""
Created on Tue Apr 07 14:25:34 2020

@author: Timmitim (Tim Laeufer)
"""

import discord
import sqlite3

class db_handler:
    STR_SERVER = 'server'
    STR_USER = 'user'
    STR_CHANNEL = 'channel'
    STR_CATEGORY = 'category'
    STR_MESSAGE = 'message'
    STR_EDITS = 'edits'
    STR_DELETES = 'deletes'

    def log(self, rel_path, message):
        conn = sqlite3.connect('logs/messages.db')
        c = conn.cursor()
        """
        Flow:

        Check if DM
            True
                Check if user exists
                    True
                        user_info = ...
                    False
                        create user
                        user_info = ...
                Check if channel exists
                    True
                        channel_info = ...
                    False
                        create channel
                        channel_info = ...
                server_info = None/0
                category_info = None/0
            False
                Check if server exists
                    True
                        server_info = ...
                    False
                        create_server
                        server_info = ...
                Check if user exists
                    True
                        user_info = ...
                    False
                        create user
                        user_info = ...
                Check if category_exists
                    True
                        category_info = ...
                    False
                        create category
                        category_info = ...
                Check if channel exists
                    True
                        channel_info = ...
                    False
                        create_channel
                        channel_info = ...

        create_message
        """
        ins = False
        allowed = [679614550286663721] #mh2, lonesome town
        #Check if DM
        try:
            cat = message.channel.category
            is_dm = False
        except AttributeError:
            is_dm = True

        if(is_dm):
            self.insert_user(c, message.author)
            self.insert_channel(c, message.channel, is_dm = True)
        else:
            #is not a dm:
            self.insert_user(c, message.author)
            self.insert_server(c, message.guild)
            try:
                self.insert_category(c, message.channel.category)
            except:
                pass
            for el in allowed:
                if(message.guild.id == el):
                    ins = True
            self.insert_channel(c, message.channel)

        if(ins):
            self.insert_message(cursor = c, message_obj = message)
        conn.commit()
        conn.close()

    def log_join(self, member_obj, amount_users):
        conn = sqlite3.connect('logs/messages.db')
        c = conn.cursor()

        query = 'INSERT INTO join (join_id,amount_now) '
        query += 'VALUES ({mem_id},{amount});'

        c.execute(query.format(
            mem_id = member_obj.id,
            amount = amount_users))

        conn.commit()
        conn.close()

    def log_edit(self, rel_path, message_before, message):
        self.log_msg_edit(rel_path, message_before, message)

    def log_msg_edit(self, rel_path, message_before, message_obj):
        conn = sqlite3.connect(rel_path)
        c = conn.cursor()

        try: #add message, if not in db
            self.insert_message(c, message_before)
        except:
            pass

        try:
            cat = message_obj.channel.category.name
            cat_id = message_obj.channel.category.id
        except AttributeError:
            cat = 'None'
            cat_id = 0
        try:
            auth_name = message_obj.author.nick
        except AttributeError:
            auth_name = message_obj.author.name
        query = 'INSERT INTO {table} '
        query += '(message_id,channel_id,user_id,user_nick,time,old_content,new_content,server_id,category_id,channel_name,category_name) '
        query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{old_content}","{content}",{s_id},{cat_id},"{channel_name}","{category_name}");'
        query = query.format(
            table = self.STR_EDITS,
            msg_id = message_obj.id,
            ch_id = message_obj.channel.id,
            u_id = message_obj.author.id,
            u_nick = auth_name,
            time = str(message_obj.created_at) + 'UTC',
            old_content = message_before.content.replace('"', '/'),
            content = message_obj.clean_content.replace('"', '/'),
            s_id = message_obj.guild.id,
            cat_id = cat_id,
            channel_name = message_obj.channel.name,
            category_name = cat)
        c.execute(query)
        conn.commit()
        conn.close()

    def log_delete(self, rel_path, message):
        self.log_msg_delete(rel_path, message)

    def log_msg_delete(self, rel_path, message_obj):
        conn = sqlite3.connect(rel_path)
        cursor = conn.cursor()

        try:
            nick = message_obj.author.nick
        except AttributeError:
            nick = message_obj.author.name

        try: #on server, with category
            query = 'INSERT INTO {table} '
            query += '(message_id,channel_id,user_id,user_nick,time,content,server_id,category_id,channel_name,category_name) '
            query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{content}", {s_id},{cat_id},"{ch_name}","{cat_name}");'
            query = query.format(
                table = self.STR_DELETES,
                msg_id = message_obj.id,
                ch_id = message_obj.channel.id,
                u_id = message_obj.author.id,
                u_nick = nick,
                time = str(message_obj.created_at) + 'UTC',
                content = message_obj.clean_content.replace('"', '/'),
                s_id = message_obj.guild.id,
                cat_id = message_obj.channel.category.id,
                ch_name = message_obj.channel.name,
                cat_name = message_obj.channel.category.name)
        except AttributeError:
            try:#on server, no category
                query = 'INSERT INTO {table} '
                query += '(message_id,channel_id,user_id,user_nick,time,content,server_id,channel_name) '
                query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{content}", {s_id},"{ch_name}");'
                query = query.format(
                    table = self.STR_DELETES,
                    msg_id = message_obj.id,
                    ch_id = message_obj.channel.id,
                    u_id = message_obj.author.id,
                    u_nick = nick,
                    time = str(message_obj.created_at) + 'UTC',
                    content = message_obj.clean_content.replace('"', '/'),
                    s_id = message_obj.guild.id,
                    ch_name = message_obj.channel.name)
            except AttributeError: #not on server
                query = 'INSERT INTO {table} '
                query += '(message_id,channel_id,user_id,user_nick,time,content) '
                query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{content}");'
                query = query.format(
                    table = self.STR_DELETES,
                    msg_id = message_obj.id,
                    ch_id = message_obj.channel.id,
                    u_id = message_obj.author.id,
                    u_nick = nick,
                    time = str(message_obj.created_at) + 'UTC',
                    content = message_obj.clean_content.replace('"', '/'))
        cursor.execute(query)
        conn.commit()
        conn.close()

    def exists_in_db(self, cursor, table, element_id):
        # SELECT EXISTS(SELECT 1 FROM myTbl WHERE u_tag="tag");
        c = cursor

        query = 'SELECT EXISTS(SELECT 1 FROM {table} WHERE {table}_id = {row_id});'
        query = query.format(
            table = table,
            row_id = element_id)
        c.execute(query)

        if(c.fetchone()[0] == 0):
            return False
        else:
            return True

    def insert_user(self, cursor, author_obj):

        if(self.exists_in_db(cursor, self.STR_USER, author_obj.id)):
            return cursor

        query = 'INSERT INTO {table} (user_id,user_name) VALUES ({u_id},"{u_name}");'
        query = query.format(
            table = self.STR_USER,
            u_id = author_obj.id,
            u_name = author_obj.name)
        cursor.execute(query)
        return cursor

    def insert_server(self, cursor, server_obj):

        if(self.exists_in_db(cursor, self.STR_SERVER, server_obj.id)):
            return cursor

        query = 'INSERT INTO {table} (server_id,server_name) VALUES ({s_id},"{s_name}");'
        query = query.format(
            table = self.STR_SERVER,
            s_id = server_obj.id,
            s_name = server_obj.name)
        cursor.execute(query)
        return cursor

    def insert_category(self, cursor, category_obj):

        if(self.exists_in_db(cursor, self.STR_CATEGORY, category_obj.id)):
            return cursor

        query = 'INSERT INTO {table} (category_id,category_name,server_id) VALUES ({c_id},"{c_name}",{s_id});'
        query = query.format(
            table = self.STR_CATEGORY,
            c_id = category_obj.id,
            c_name = category_obj.name,
            s_id = category_obj.guild.id)
        cursor.execute(query)
        return cursor

    def insert_channel(self, cursor, channel_obj, is_dm = False):

        if(self.exists_in_db(cursor, self.STR_CHANNEL, channel_obj.id)):
            return cursor

        try:
            cat = channel_obj.category
            has_category = True
        except AttributeError:
            has_category = False

        if(is_dm):
            pass
        else:
            if(has_category):
                query = 'INSERT INTO {table} (channel_id,channel_name,category_id,server_id) VALUES '
                query += '({c_id},"{c_name}",{cat_id},{s_id});'
                query = query.format(
                    table = self.STR_CHANNEL,
                    c_id = channel_obj.id,
                    c_name = channel_obj.name,
                    cat_id = channel_obj.category.id,
                    s_id = channel_obj.guild.id)
            else:
                query = 'INSERT INTO {table} (channel_id,channel_name,category_id,server_id) VALUES '
                query += '({c_id},"{c_name}",{cat_id},{s_id});'
                query = query.format(
                    table = self.STR_CHANNEL,
                    c_id = channel_obj.id,
                    c_name = channel_obj.name,
                    cat_id = 0,
                    s_id = channel_obj.guild.id)
            cursor.execute(query)
            return cursor

    def insert_message(self, cursor, message_obj):

        try:
            nick = message_obj.author.nick
        except AttributeError:
            nick = message_obj.author.name

        try: #on server, with category
            query = 'INSERT INTO {table} '
            query += '(message_id,channel_id,user_id,user_nick,time,content,server_id,category_id,channel_name,category_name) '
            query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{content}", {s_id},{cat_id},"{ch_name}","{cat_name}");'
            query = query.format(
                table = self.STR_MESSAGE,
                msg_id = message_obj.id,
                ch_id = message_obj.channel.id,
                u_id = message_obj.author.id,
                u_nick = nick,
                time = str(message_obj.created_at) + 'UTC',
                content = message_obj.clean_content.replace('"', '/'),
                s_id = message_obj.guild.id,
                cat_id = message_obj.channel.category.id,
                ch_name = message_obj.channel.name,
                cat_name = message_obj.channel.category.name)
        except AttributeError:
            try:#on server, no category
                query = 'INSERT INTO {table} '
                query += '(message_id,channel_id,user_id,user_nick,time,content,server_id,channel_name) '
                query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{content}", {s_id},"{ch_name}");'
                query = query.format(
                    table = self.STR_MESSAGE,
                    msg_id = message_obj.id,
                    ch_id = message_obj.channel.id,
                    u_id = message_obj.author.id,
                    u_nick = nick,
                    time = str(message_obj.created_at) + 'UTC',
                    content = message_obj.clean_content.replace('"', '/'),
                    s_id = message_obj.guild.id,
                    ch_name = message_obj.channel.name)
            except AttributeError: #not on server
                query = 'INSERT INTO {table} '
                query += '(message_id,channel_id,user_id,user_nick,time,content) '
                query += 'VALUES ({msg_id},{ch_id},{u_id},"{u_nick}","{time}","{content}");'
                query = query.format(
                    table = self.STR_MESSAGE,
                    msg_id = message_obj.id,
                    ch_id = message_obj.channel.id,
                    u_id = message_obj.author.id,
                    u_nick = nick,
                    time = str(message_obj.created_at) + 'UTC',
                    content = message_obj.clean_content.replace('"', '/'))
        cursor.execute(query)
        return cursor

    def fetch_sql_rows(self, db, sql):
        conn = sqlite3.connect(db)
        conn.row_factory = sqlite3.Row
        cursor = conn.cursor()
        lis = cursor.execute(sql).fetchall()
        conn.close()
        return lis


from collections import defaultdict

try:
    from itertools import izip_longest
except ImportError:
    from itertools import zip_longest

from io import StringIO
from os import stat

import pandas as pd
import numpy as np

import pytest

from rpy2.robjects import r, pandas2ri
from triform.create_bed import create_bed

@pytest.fixture
def chip():
    bed_file = "tests/test_data/iranges_to_bigwig.bed"
    df = r["read.table"](bed_file, sep=" ")

    treatment = {bed_file: {
        "chrY":
        r("function(df) {IRanges(start=df$start, width=df$width)}")(df),
        "chr1": r("function(df) {IRanges(start=df$start, width=df$width)}")(df)
    }}

    return treatment

@pytest.fixture
def input():
    bed_file = "tests/test_data/input_iranges_to_bigwig.bed"
    df = r["read.table"](bed_file, sep=" ", header=1)

    control = {bed_file: {
        "chrY":
        r("function(df) {IRanges(start=df$start, width=df$width)}")(df),
        "chr1": r("function(df) {IRanges(start=df$start, width=df$width)}")(df)
    }}

    return control

@pytest.fixture
def enriched_regions():

    result = u"""CHROMOSOME START END QVAL NLQ NLP MAX.NLP LOC WIDTH CVG SURL SURR FORM
chrY 27096723 27096953 0.0030190321570660633 2.520132261086883 3.984008558280453 5.638949274037698 27096861 231 19.0 2.0 24.0 2
chr1 2709562 2719659 0.12753262199776602 0.8943787113752741 1.9970405839422427 4.269576380346016 2709612 98 12.0 3.0 9.0 2"""

    return pd.read_table(StringIO(result), index_col=[0, 1, 2], sep=" ", header=0)

@pytest.fixture
def expected_result():
    result = u"""Chromosome      Start        End      Qval  logFC      Loc
chr1  2709562.0  2719659.0  0.127533    0.0  2709612"""

    return pd.read_table(StringIO(result), header=0, index_col=[0, 1, 2], sep="\s+")

@pytest.mark.current
def test_create_bed(chip, input, enriched_regions, expected_result, args):
    print(chip, "chip " * 3)
    print(input, "input " * 3)
    print(expected_result)

    result = create_bed(chip, input, enriched_regions, args)
    print(result, "result")
    print(expected_result, "expected")

    assert np.allclose(result, expected_result)
# /local/home/endrebak/anaconda3/lib/python3.5/site-packages/rpy2/robjects/functions.py:106: UserWarning: Error in .Call2("Rle_constructor", values, lengths, check, 0L, PACKAGE = "S4Vectors") :
#   integer overflow while summing elements in 'lengths'

#   res = super(Function, self).__call__(*new_args, **new_kwargs)
# /local/home/endrebak/anaconda3/lib/python3.5/site-packages/rpy2/robjects/functions.py:106: UserWarning: In addition:
#   res = super(Function, self).__call__(*new_args, **new_kwargs)
# /local/home/endrebak/anaconda3/lib/python3.5/site-packages/rpy2/robjects/functions.py:106: UserWarning: There were 24 warnings (use warnings() to see them)
#   res = super(Function, self).__call__(*new_args, **new_kwargs)
# /local/home/endrebak/anaconda3/lib/python3.5/site-packages/rpy2/robjects/functions.py:106: UserWarning:

#   res = super(Function, self).__call__(*new_args, **new_kwargs)
# Traceback (most recent call last):
#   File "/local/home/endrebak/anaconda3/bin//triform2", line 4, in <module>
#     __import__('pkg_resources').run_script('triform2==0.0.5', 'triform2')
#   File "/local/home/endrebak/anaconda3/lib/python3.5/site-packages/setuptools-20.7.0-py3.5.egg/pkg_resources/__init__.py", line 719, in run_script
#   File "/local/home/endrebak/anaconda3/lib/python3.5/site-packages/setuptools-20.7.0-py3.5.egg/pkg_resources/__init__.py", line 1504, in run_script
#   File "/local/home/endrebak/anaconda3/lib/python3.5/site-packages/triform2-0.0.5-py3.5.egg/EGG-INFO/scripts/triform2", line 157, in <module>
#     run_triform(args)
#   File "/local/home/endrebak/anaconda3/lib/python3.5/site-packages/triform2-0.0.5-py3.5.egg/triform/run_triform.py", line 96, in run_triform
#     create_bed(treatment_iranges, args.bed, args)
#   File "/local/home/endrebak/anaconda3/lib/python3.5/site-packages/triform2-0.0.5-py3.5.egg/triform/create_bed.py", line 20, in create_bed
#     _create_bed(iranges, outpath, args)
#   File "/local/home/endrebak/anaconda3/lib/python3.5/site-packages/triform2-0.0.5-py3.5.egg/triform/create_bed.py", line 37, in _create_bed
#     genome = r["RleList"](genome_coverage_rpkm)
#   File "/local/home/endrebak/anaconda3/lib/python3.5/site-packages/rpy2/robjects/functions.py", line 178, in __call__
#     return super(SignatureTranslatedFunction, self).__call__(*args, **kwargs)
#   File "/local/home/endrebak/anaconda3/lib/python3.5/site-packages/rpy2/robjects/functions.py", line 106, in __call__
#     res = super(Function, self).__call__(*new_args, **new_kwargs)
# rpy2.rinterface.RRuntimeError: Error in .Call2("Rle_constructor", values, lengths, check, 0L, PACKAGE = "S4Vectors") :
#   integer overflow while summing elements in 'lengths'


from flask import Flask,Blueprint,render_template,abort,session, request
from Classes.oauth_config import oauth
from Classes.login_manager_config import login_required, current_user
from Models.Identity import db, User, Client
from pprint import pprint

oauth_endpoint = Blueprint('oauth_endpoint',__name__,template_folder='templates')

@oauth_endpoint.route('/oauth/authorize/', methods=['GET', 'POST'])
@login_required
@oauth.authorize_handler
def authorize(*args, **kwargs):
    if request.method == 'GET':
        client_id = kwargs.get('client_id')
        client = Client.query.filter_by(client_id=client_id).first()
        kwargs['client'] = client
        return render_template('/oauth/oauthorize.html', user= current_user, **kwargs)

    confirm = request.form.get('confirm', 'no')
    return confirm == 'yes'

@oauth_endpoint.route('/oauth/token/', methods=["POST"])
@oauth.token_handler
def access_token():
    return None


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Glances - An eye on your system
#
# Copyright (C) 2017 Nicolargo <nicolas@nicolargo.com>
#
# Glances is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Glances is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

"""Glances unitary tests suite."""

import time
import unittest

from glances.main import GlancesMain
from glances.stats import GlancesStats
from glances import __version__
from glances.globals import WINDOWS, LINUX
from glances.outputs.glances_bars import Bar
from glances.thresholds import GlancesThresholdOk
from glances.thresholds import GlancesThresholdCareful
from glances.thresholds import GlancesThresholdWarning
from glances.thresholds import GlancesThresholdCritical
from glances.thresholds import GlancesThresholds
from glances.plugins.glances_plugin import GlancesPlugin
from glances.compat import subsample

# Global variables
# =================

# Init Glances core
core = GlancesMain()

# Init Glances stats
stats = GlancesStats(config=core.get_config(),
                     args=core.get_args())

# Unitest class
# ==============
print('Unitary tests for Glances %s' % __version__)

class TestGlances(unittest.TestCase):
    """Test Glances class."""

    def setUp(self):
        """The function is called *every time* before test_*."""
        print('\n' + '=' * 78)

    def test_000_update(self):
        """Update stats (mandatory step for all the stats).

        The update is made twice (for rate computation).
        """
        print('INFO: [TEST_000] Test the stats update function')
        try:
            stats.update()
        except Exception as e:
            print('ERROR: Stats update failed: %s' % e)
            self.assertTrue(False)
        time.sleep(1)
        try:
            stats.update()
        except Exception as e:
            print('ERROR: Stats update failed: %s' % e)
            self.assertTrue(False)

        self.assertTrue(True)

    def test_001_plugins(self):
        """Check mandatory plugins."""
        plugins_to_check = ['system', 'cpu', 'load', 'mem', 'memswap', 'network', 'diskio', 'fs', 'irq']
        print('INFO: [TEST_001] Check the mandatory plugins list: %s' % ', '.join(plugins_to_check))
        plugins_list = stats.getPluginsList()
        for plugin in plugins_to_check:
            self.assertTrue(plugin in plugins_list)

    def test_002_system(self):
        """Check SYSTEM plugin."""
        stats_to_check = ['hostname', 'os_name']
        print('INFO: [TEST_002] Check SYSTEM stats: %s' % ', '.join(stats_to_check))
        stats_grab = stats.get_plugin('system').get_raw()
        for stat in stats_to_check:
            # Check that the key exist
            self.assertTrue(stat in stats_grab, msg='Cannot find key: %s' % stat)
        print('INFO: SYSTEM stats: %s' % stats_grab)

    def test_003_cpu(self):
        """Check CPU plugin."""
        stats_to_check = ['system', 'user', 'idle']
        print('INFO: [TEST_003] Check mandatory CPU stats: %s' % ', '.join(stats_to_check))
        stats_grab = stats.get_plugin('cpu').get_raw()
        for stat in stats_to_check:
            # Check that the key exist
            self.assertTrue(stat in stats_grab, msg='Cannot find key: %s' % stat)
            # Check that % is > 0 and < 100
            self.assertGreaterEqual(stats_grab[stat], 0)
            self.assertLessEqual(stats_grab[stat], 100)
        print('INFO: CPU stats: %s' % stats_grab)

    @unittest.skipIf(WINDOWS, "Load average not available on Windows")
    def test_004_load(self):
        """Check LOAD plugin."""
        stats_to_check = ['cpucore', 'min1', 'min5', 'min15']
        print('INFO: [TEST_004] Check LOAD stats: %s' % ', '.join(stats_to_check))
        stats_grab = stats.get_plugin('load').get_raw()
        for stat in stats_to_check:
            # Check that the key exist
            self.assertTrue(stat in stats_grab, msg='Cannot find key: %s' % stat)
            # Check that % is > 0
            self.assertGreaterEqual(stats_grab[stat], 0)
        print('INFO: LOAD stats: %s' % stats_grab)

    def test_005_mem(self):
        """Check MEM plugin."""
        stats_to_check = ['available', 'used', 'free', 'total']
        print('INFO: [TEST_005] Check MEM stats: %s' % ', '.join(stats_to_check))
        stats_grab = stats.get_plugin('mem').get_raw()
        for stat in stats_to_check:
            # Check that the key exist
            self.assertTrue(stat in stats_grab, msg='Cannot find key: %s' % stat)
            # Check that % is > 0
            self.assertGreaterEqual(stats_grab[stat], 0)
        print('INFO: MEM stats: %s' % stats_grab)

    def test_006_swap(self):
        """Check MEMSWAP plugin."""
        stats_to_check = ['used', 'free', 'total']
        print('INFO: [TEST_006] Check SWAP stats: %s' % ', '.join(stats_to_check))
        stats_grab = stats.get_plugin('memswap').get_raw()
        for stat in stats_to_check:
            # Check that the key exist
            self.assertTrue(stat in stats_grab, msg='Cannot find key: %s' % stat)
            # Check that % is > 0
            self.assertGreaterEqual(stats_grab[stat], 0)
        print('INFO: SWAP stats: %s' % stats_grab)

    def test_007_network(self):
        """Check NETWORK plugin."""
        print('INFO: [TEST_007] Check NETWORK stats')
        stats_grab = stats.get_plugin('network').get_raw()
        self.assertTrue(type(stats_grab) is list, msg='Network stats is not a list')
        print('INFO: NETWORK stats: %s' % stats_grab)

    def test_008_diskio(self):
        """Check DISKIO plugin."""
        print('INFO: [TEST_008] Check DISKIO stats')
        stats_grab = stats.get_plugin('diskio').get_raw()
        self.assertTrue(type(stats_grab) is list, msg='DiskIO stats is not a list')
        print('INFO: diskio stats: %s' % stats_grab)

    def test_009_fs(self):
        """Check File System plugin."""
        # stats_to_check = [ ]
        print('INFO: [TEST_009] Check FS stats')
        stats_grab = stats.get_plugin('fs').get_raw()
        self.assertTrue(type(stats_grab) is list, msg='FileSystem stats is not a list')
        print('INFO: FS stats: %s' % stats_grab)

    def test_010_processes(self):
        """Check Process plugin."""
        # stats_to_check = [ ]
        print('INFO: [TEST_010] Check PROCESS stats')
        stats_grab = stats.get_plugin('processcount').get_raw()
        # total = stats_grab['total']
        self.assertTrue(type(stats_grab) is dict, msg='Process count stats is not a dict')
        print('INFO: PROCESS count stats: %s' % stats_grab)
        stats_grab = stats.get_plugin('processlist').get_raw()
        self.assertTrue(type(stats_grab) is list, msg='Process count stats is not a list')
        print('INFO: PROCESS list stats: %s items in the list' % len(stats_grab))
        # Check if number of processes in the list equal counter
        # self.assertEqual(total, len(stats_grab))

    def test_011_folders(self):
        """Check File System plugin."""
        # stats_to_check = [ ]
        print('INFO: [TEST_011] Check FOLDER stats')
        stats_grab = stats.get_plugin('folders').get_raw()
        self.assertTrue(type(stats_grab) is list, msg='Folders stats is not a list')
        print('INFO: Folders stats: %s' % stats_grab)

    def test_012_ip(self):
        """Check IP plugin."""
        print('INFO: [TEST_012] Check IP stats')
        stats_grab = stats.get_plugin('ip').get_raw()
        self.assertTrue(type(stats_grab) is dict, msg='IP stats is not a dict')
        print('INFO: IP stats: %s' % stats_grab)

    @unittest.skipIf(not LINUX, "IRQs available only on Linux")
    def test_013_irq(self):
        """Check IRQ plugin."""
        print('INFO: [TEST_013] Check IRQ stats')
        stats_grab = stats.get_plugin('irq').get_raw()
        self.assertTrue(type(stats_grab) is list, msg='IRQ stats is not a list')
        print('INFO: IRQ stats: %s' % stats_grab)

    @unittest.skipIf(not LINUX, "GPU available only on Linux")
    def test_013_gpu(self):
        """Check GPU plugin."""
        print('INFO: [TEST_014] Check GPU stats')
        stats_grab = stats.get_plugin('gpu').get_raw()
        self.assertTrue(type(stats_grab) is list, msg='GPU stats is not a list')
        print('INFO: GPU stats: %s' % stats_grab)

    def test_014_sorted_stats(self):
        """Check sorted stats method."""
        print('INFO: [TEST_015] Check sorted stats method')
        aliases = {
            "key2": "alias11",
            "key5": "alias2",
        }
        unsorted_stats = [
            {"key": "key4"},
            {"key": "key2"},
            {"key": "key5"},
            {"key": "key21"},
            {"key": "key3"},
        ]

        gp = GlancesPlugin()
        gp.get_key = lambda: "key"
        gp.has_alias = aliases.get
        gp.stats = unsorted_stats

        sorted_stats = gp.sorted_stats()
        self.assertEqual(len(sorted_stats), 5)
        self.assertEqual(sorted_stats[0]["key"], "key5")
        self.assertEqual(sorted_stats[1]["key"], "key2")
        self.assertEqual(sorted_stats[2]["key"], "key3")
        self.assertEqual(sorted_stats[3]["key"], "key4")
        self.assertEqual(sorted_stats[4]["key"], "key21")

    def test_015_subsample(self):
        """Test subsampling function."""
        print('INFO: [TEST_015] Subsampling')
        for l in [([1, 2, 3], 4),
                  ([1, 2, 3, 4], 4),
                  ([1, 2, 3, 4, 5, 6, 7], 4),
                  ([1, 2, 3, 4, 5, 6, 7, 8], 4),
                  (list(xrange(1, 800)), 4),
                  (list(xrange(1, 8000)), 800)]:
            l_subsample = subsample(l[0], l[1])
            self.assertLessEqual(len(l_subsample), l[1])

    def test_094_thresholds(self):
        """Test thresholds classes"""
        print('INFO: [TEST_094] Thresholds')
        ok = GlancesThresholdOk()
        careful = GlancesThresholdCareful()
        warning = GlancesThresholdWarning()
        critical = GlancesThresholdCritical()
        self.assertTrue(ok < careful)
        self.assertTrue(careful < warning)
        self.assertTrue(warning < critical)
        self.assertFalse(ok > careful)
        self.assertEqual(ok, ok)
        self.assertEqual(str(ok), 'OK')
        thresholds = GlancesThresholds()
        thresholds.add('cpu_percent', 'OK')
        self.assertEqual(thresholds.get(stat_name='cpu_percent').description(), 'OK')

    def test_095_methods(self):
        """Test mandatories methods"""
        print('INFO: [TEST_095] Mandatories methods')
        mandatories_methods = ['reset', 'update']
        plugins_list = stats.getPluginsList()
        for plugin in plugins_list:
            for method in mandatories_methods:
                self.assertTrue(hasattr(stats.get_plugin(plugin), method),
                                msg='{} has no method {}()'.format(plugin, method))

    def test_096_views(self):
        """Test get_views method"""
        print('INFO: [TEST_096] Test views')
        plugins_list = stats.getPluginsList()
        for plugin in plugins_list:
            stats_grab = stats.get_plugin(plugin).get_raw()
            views_grab = stats.get_plugin(plugin).get_views()
            self.assertTrue(type(views_grab) is dict,
                            msg='{} view is not a dict'.format(plugin))

    def test_097_attribute(self):
        """Test GlancesAttribute classe"""
        print('INFO: [TEST_097] Test attribute')
        # GlancesAttribute
        from glances.attribute import GlancesAttribute
        a = GlancesAttribute('a', description='ad', history_max_size=3)
        self.assertEqual(a.name, 'a')
        self.assertEqual(a.description, 'ad')
        a.description = 'adn'
        self.assertEqual(a.description, 'adn')
        a.value = 1
        a.value = 2
        self.assertEqual(len(a.history), 2)
        a.value = 3
        self.assertEqual(len(a.history), 3)
        a.value = 4
        # Check if history_max_size=3 is OK
        self.assertEqual(len(a.history), 3)
        self.assertEqual(a.history_size(), 3)
        self.assertEqual(a.history_len(), 3)
        self.assertEqual(a.history_value()[1], 4)
        self.assertEqual(a.history_mean(nb=3), 4.5)

    def test_098_history(self):
        """Test GlancesHistory classe"""
        print('INFO: [TEST_098] Test history')
        # GlancesHistory
        from glances.history import GlancesHistory
        h = GlancesHistory()
        h.add('a', 1)
        h.add('a', 2)
        h.add('a', 3)
        h.add('b', 10)
        h.add('b', 20)
        h.add('b', 30)
        self.assertEqual(len(h.get()), 2)
        self.assertEqual(len(h.get()['a']), 3)
        h.reset()
        self.assertEqual(len(h.get()), 2)
        self.assertEqual(len(h.get()['a']), 0)

    def test_099_output_bars_must_be_between_0_and_100_percent(self):
        """Test quick look plugin.

        > bar.min_value
        0
        > bar.max_value
        100
        > bar.percent = -1
        > bar.percent
        0
        > bar.percent = 101
        > bar.percent
        100
        """
        print('INFO: [TEST_099] Test progress bar')
        bar = Bar(size=1)
        bar.percent = -1
        self.assertLessEqual(bar.percent, bar.min_value)
        bar.percent = 101
        self.assertGreaterEqual(bar.percent, bar.max_value)

    def test_999_the_end(self):
        """Free all the stats"""
        print('INFO: [TEST_999] Free the stats')
        stats.end()
        self.assertTrue(True)

if __name__ == '__main__':
    unittest.main()


# -*- coding: utf-8 -*-

import numpy as np
from scipy import sparse
from . import Graph

class Grid2d(Graph):
    r"""
    Create a 2 dimensional grid graph.

    Parameters
    ----------
    Nv : int
        Number of vertices along the first dimension (default is 16)
    Mv : int
        Number of vertices along the second dimension (default is Nv)

    Examples
    --------
    >>> from pygsp import graphs
    >>> G = graphs.Grid2d(Nv=32)

    """

    def __init__(self, Nv=16, Mv=None, **kwargs):
        if not Mv:
            Mv = Nv

        # Create weighted adjacency matrix
        K = 2*(Nv - 1)
        J = 2*(Mv - 1)

        i_inds = np.zeros((K*Mv + J*Nv), dtype=float)
        j_inds = np.zeros((K*Mv + J*Nv), dtype=float)

        tmpK = np.arange(K, dtype=int)
        tmpNv1 = np.arange(Nv - 1)
        for i in range(Mv):
            i_inds[i*K + tmpK] = i*Nv + \
                np.concatenate((tmpNv1, tmpNv1 + 1))
            j_inds[i*K + tmpK] = i*Nv + \
                np.concatenate((tmpNv1 + 1, tmpNv1))

        tmp2Nv = np.arange(2*Nv, dtype=int)
        tmpNv = np.arange(Nv)
        for i in range(Mv-1):
            i_inds[(K*Mv) + i*2*Nv + tmp2Nv] = \
                np.concatenate((i*Nv + tmpNv, (i + 1)*Nv + tmpNv))

            j_inds[(K*Mv) + i*2*Nv + tmp2Nv] = \
                np.concatenate(((i + 1)*Nv + tmpNv, i*Nv + tmpNv))

        W = sparse.csc_matrix((np.ones((K*Mv + J*Nv)), (i_inds, j_inds)),
                              shape=(Mv*Nv, Mv*Nv))

        xtmp = np.kron(np.ones((Mv, 1)), (np.arange(Nv)/float(Nv)).reshape(Nv,
                                                                           1))
        ytmp = np.sort(np.kron(np.ones((Nv, 1)),
                               np.arange(Mv)/float(Mv)).reshape(Mv*Nv, 1),
                       axis=0)

        coords = np.concatenate((xtmp, ytmp), axis=1)

        self.Nv = Nv
        self.Mv = Mv
        plotting = {"vertex_size": 30,
                    "limits": np.array([-1./self.Nv, 1 + 1./self.Nv,
                                        1./self.Mv, 1 + 1./self.Mv])}

        super(Grid2d, self).__init__(W=W, gtype='2d-grid', coords=coords,
                                     plotting=plotting, **kwargs)


from flask import Flask, render_template, request
from prometheus_flask_exporter import PrometheusMetrics
from prometheus_flask_exporter.multiprocess import GunicornInternalPrometheusMetrics
import os

app = Flask(__name__)
is_gunicorn = "gunicorn" in os.environ.get("SERVER_SOFTWARE", "")
if is_gunicorn:
    metrics = GunicornInternalPrometheusMetrics(app)
else:
    metrics = PrometheusMetrics(app)

@app.route('/')
def homepage():
    return render_template("main.html")

if __name__ == "__main__":
    app.run(debug=False)


import logging
from pathlib import Path
import time

def timeLogger(func):
    currDate = time.strftime("%Y-%m-%d", time.localtime())
    currTime = time.strftime("%H-%M-%S", time.localtime())
    Path(f"/var/log/ownlog/python-{currDate}/").mkdir(parents=True, exist_ok=True)
    outputPath = f"/var/log/ownlog/python-{currDate}/{currTime}.log"

    logging.basicConfig(filename=outputPath,
                        level=logging.DEBUG,
                        datefmt="%Y-%m-%d %H:%M:%S",
                        format="time function called by %(message)s at %(asctime)s"
                        )
    logging.info(f'{func} {func.__name__}')
    return func

@timeLogger
def func():
    print('done')

if __name__ == '__main__':
    func()


import numpy as np
from pymoo.optimize import minimize
from pymoo.util.nds.non_dominated_sorting import NonDominatedSorting
from pymoo.operators.sampling.random_sampling import FloatRandomSampling
from pymoo.operators.sampling.latin_hypercube_sampling import LatinHypercubeSampling
from algorithm.external import lhs

class Solver:
    '''
    Multi-objective solver.
    '''
    def __init__(self, n_gen, pop_init_method, batch_size, algo, seed=None, **kwargs):
        '''
        Initialize a solver.

        Parameters
        ----------
        n_gen: int
            Number of generations to solve.
        pop_init_method: str
            Method to initialize population.
        batch_size: int
            Batch size, i.e. population size.
        algo: pymoo.model.algorithm.Algorithm
            Class of multi-objective algorithm to use.
        seed: int, default=None
            Random seed.
        kwargs: dict
            Other keyword arguments for algorithm to initialize.
        '''
        self.n_gen = n_gen
        self.pop_init_method = pop_init_method
        self.batch_size = batch_size
        self.seed = seed
        self.algo_type = algo
        self.algo_kwargs = kwargs
        self.solution = None

    def solve(self, problem, X, Y):
        '''
        Solve the multi-objective problem.

        Parameters
        ----------
        problem: mobo.surrogate_problem.SurrogateProblem
            The surrogate problem to be solved.
        X: np.array
            Current design variables.
        Y: np.array
            Current performance values.

        Returns
        -------
        solution: dict
            A dictionary containing information of the solution.\n
            - solution['x']: Proposed design samples.
            - solution['y']: Performance of proposed design samples.
            - solution['algo']: The algoritm of the solver, containing some history information of the solving process.
        '''
        # initialize population
        sampling = self._get_sampling(X, Y)

        # setup algorithm
        algo = self.algo_type(sampling=sampling, **self.algo_kwargs)

        # optimization
        res = minimize(problem, algo, ('n_gen', self.n_gen), seed=self.seed)

        # construct solution
        self.solution = {'x': res.pop.get('X'), 'y': res.pop.get('F'), 'algo': res.algorithm}

        # fill the solution in case less than batch size
        pop_size = len(self.solution['x'])
        if pop_size < self.batch_size:
            indices = np.concatenate([np.arange(pop_size), np.random.choice(np.arange(pop_size), self.batch_size - pop_size)])
            self.solution['x'] = np.array(self.solution['x'])[indices]
            self.solution['y'] = np.array(self.solution['y'])[indices]

        return self.solution

    def _get_sampling(self, X, Y):
        '''
        Initialize population from data.

        Parameters
        ----------
        X: np.array
            Design variables.
        Y: np.array
            Performance values.

        Returns
        -------
        sampling: np.array or pymoo.model.sampling.Sampling
            Initial population or a sampling method for generating initial population.
        '''
        if self.pop_init_method == 'lhs':
            sampling = LatinHypercubeSampling()
        elif self.pop_init_method == 'nds':
            sorted_indices = NonDominatedSorting().do(Y)
            pop_size = self.algo_kwargs['pop_size']
            sampling = X[np.concatenate(sorted_indices)][:pop_size]
            # NOTE: use lhs if current samples are not enough
            if len(sampling) < pop_size:
                rest_sampling = lhs(X.shape[1], pop_size - len(sampling))
                sampling = np.vstack([sampling, rest_sampling])
        elif self.pop_init_method == 'random':
            sampling = FloatRandomSampling()
        else:
            raise NotImplementedError

        return sampling


import os
import sys
currentPath = os.path.dirname(os.path.realpath(__file__))
sys.path.append(currentPath.replace("\manualreview", ""))
from manualreview import rawscriptsmenu
from PyQt5 import QtWidgets
from manualreview import client
import sys
import atexit
import datetime
import configparser
from manualreview import settings
from datetime import timedelta
from manualreview import videoscriptcore
currentPath = os.path.dirname(os.path.realpath(__file__))

def loadVideoScripts():
    pass

class App():
    def __init__(self):
        application = QtWidgets.QApplication(sys.argv)
        application.processEvents()
        loginWindow = client.LoginWindow()
        loginWindow.show()

        #new = rawscriptsmenu.ScriptsMenu()
        #new.show()
        sys.exit(application.exec_())
        client.safeDisconnect()

sys._excepthook = sys.excepthook
def exception_hook(exctype, value, traceback):
    sys._excepthook(exctype, value, traceback)
    print("loooooooooooool")
    sys.exit(1)
sys.excepthook = exception_hook

def init():

    os.chdir(currentPath)

    if not os.path.exists(settings.thumbnailpath):
        os.mkdir(settings.thumbnailpath)

    if not os.path.exists(settings.scriptsaves):
        os.mkdir(settings.scriptsaves)

    app = App()

def exit_handler():
    print("Safe Exit")
    client.sock.close()

if __name__ == "__main__":
    atexit.register(exit_handler)
    settings.generateConfigFile()
    init()


# coding: utf-8

# In[40]:

################################JP 29th august 2015
################################This code takes and runs stuff on the cluster!!!!
import os
import re
import json
from time import ctime
import string
import shutil
maxSplaunchRun=30
currFolder = os.getcwd() + '/'
targetFolder = 'data/proxDist/'
outputFolder = currFolder+'outputReports'
if (os.path.exists(outputFolder) != 1): ###Here we check that the folder exists.
        os.mkdir(outputFolder)

# In[ ]:

# In[41]:

#################################################READ JSON FILE
jsonString = open('clusterRegRunnerConfig.json').read()
parsedJson = json.loads(jsonString)
parsedJson.keys()

##################################################CONTROL OUTPUT FILE:
fileToSaveOutputRoot = 'outputFileTransfer' ####Create a better name
time= ctime()
time = time.translate(None, string.punctuation)
time = time.replace(" ", "")
fileToSaveOutputTime = time
fileToSaveOutputSuffix = '_OUT.txt'
fileToSaveOutput= fileToSaveOutputRoot + fileToSaveOutputTime + fileToSaveOutputSuffix

fileToSaveOutputFullPath = currFolder + fileToSaveOutput

if (os.path.isfile(fileToSaveOutputFullPath) == True): ###Here we check that the folder exists.
    os.remove(fileToSaveOutputFullPath)

####Need to delete it if it already exists

outFile = open(fileToSaveOutputFullPath,'a')

outFile.write('This is a test')

# In[42]:

foldersToSearch = os.listdir(targetFolder)
##Now we filter the files so that they are only files that we are interested in:
matchPattern = '[A-Z]{1,4}[0-9]{1,4}'  ##the outline for acceptable files  [A-D]{1-4}_
foldersToSearch = [elem for elem in foldersToSearch if re.search(matchPattern, elem) != None]

#print number summary
outputMessage = '-----There are the following number of mice: ' + str(len(foldersToSearch)) + '\n'

outFile.write(outputMessage)
#####Here we print output.
outputMessage = '-----Mice being searched: ' + str(foldersToSearch) + '\n' #new line is critical.
outFile.write(outputMessage)
miceToSearch = foldersToSearch

for mouse in range(len(miceToSearch)):
    print(foldersToSearch[mouse])
    sessionFolder = targetFolder+miceToSearch[mouse]+'/'
    sessionsToSearch = os.listdir(sessionFolder)
    ##Now we filter the files so that they are only files that we are interested in:
    matchPattern = '[A-Z]{1,4}[0-9]{1,4}_S[0-9]{1,3}_F[0-9]{1,3}'  ##the outline for acceptable files  [A-D]{1-4}_
    sessionsToSearch = [elem for elem in sessionsToSearch if re.search(matchPattern, elem) != None]

    #print number summary
    outputMessage = '----------There are the following number of sessions: ' + str(len(sessionsToSearch)) + '\n'

    outFile.write(outputMessage)
    #####Here we print output.
    outputMessage = '----------Sessions being searched: ' + str(sessionsToSearch) + '\n' #new line is critical.
    outFile.write(outputMessage)

    print(sessionsToSearch)

    ############################################Now we iterate through each individual sessions registeredDirectory....

    for session in range(len(sessionsToSearch)):

        regFolder = sessionFolder + sessionsToSearch[session] + '/registeredDirectory/'
        filesToSearch = os.listdir(regFolder)
        ##Now we filter the files so that they are only files that we are interested in:
        matchPattern = 'OUTPUTREPORT'  ##the outline for acceptable files  [A-D]{1-4}_
        filesToSearch = [elem for elem in filesToSearch if re.search(matchPattern, elem) != None]

        #print tnumber summary
        outputMessage = '----------There are the following number of output reports: ' + str(len(filesToSearch)) + '\n'
        print(outputMessage)
        outFile.write(outputMessage)
        #####Here we print output.
        outputMessage = '----------Sessions being searched: ' + str(filesToSearch) + '\n' #new line is critical.
        outFile.write(outputMessage)

        #######################Now we move these things to the report
        for fileId in range(len(filesToSearch)):
            fileToMove = regFolder+filesToSearch[fileId]

            shutil.copy(fileToMove, outputFolder)

# In[43]:

outFile.close() ####Close the text output file.


from cassandra.cluster import Cluster
import sys

def lecture(numtest, numprocess):

	rows = session.execute("""
		SELECT registre
		FROM mrc_system.mrc
	""")

	total = 0

	for row in rows:
		total += row[0]

	print (total)

cluster = Cluster()
session = cluster.connect()

try:
	numprocess = int(sys.argv[1])
	numtest = int(sys.argv[2])
	lecture(numtest,numprocess)
except IndexError:
	print ("Erreurs dans la notation des arguments")

cluster.shutdown()


import datetime

def print_header():
    print('-----------------------')
    print('BIRTHDAY CALCULATOR APP')
    print('-----------------------')
    print()

def get_birthday_from_user():
    print("When were you born")
    year = int(input("Year: "))
    month = int(input ("month: "))
    day = int(input("day: "))

    return datetime.date(year, month, day)

def compute_days_between_dates(original_date, target_date):
    current_year = datetime.date(target_date.year, original_date.month, original_date.day)
    days = current_year - target_date
    return days.days

def print_birthday_information(days):
    if days < 0:
        print("Your birthday was {} days ago".format(-days))
    elif days > 0:
        print("Your birthday is {} days from now".format(days))
    else:
        print("Happy Birthday!")

def main():
    print_header()
    birthday = get_birthday_from_user()
    current_date = datetime.date.today()
    number_of_days = compute_days_between_dates(current_date, birthday)
    print_birthday_information(number_of_days)

main()


# -*- coding: utf-8 -*-

from datetime import datetime
try:
    from collections import OrderedDict
except ImportError:
    from ordereddict import OrderedDict
from flask import (
    abort,
    redirect,
    render_template,
    request,
    Response,
    url_for,
    g
    )

from .. import app, lastuser, redis_store
from ..models import db, webmail_domains, JobCategory, JobPost, JobType, POSTSTATUS, newlimit, JobLocation
from ..search import do_search
from ..views.helper import getposts, getallposts, location_geodata
from ..uploads import uploaded_logos

@app.route('/', subdomain='<subdomain>')
@app.route('/')
def index(basequery=None, type=None, category=None, md5sum=None, domain=None,
        location=None, title=None, showall=True, statuses=None):
    now = datetime.utcnow()
    if basequery is None and not (g.user or g.kiosk or (g.board and not g.board.require_login)):
        showall = False
    posts = list(getposts(basequery, pinned=True, showall=showall, statuses=statuses))

    # Cache viewcounts (admin view or not)
    redis_pipe = redis_store.connection.pipeline()
    viewcounts_keys = [p.viewcounts_key for p in posts]
    for key in viewcounts_keys:
        redis_pipe.hgetall(key)
    viewcounts_values = redis_pipe.execute()
    g.viewcounts = dict(zip(viewcounts_keys, viewcounts_values))

    if posts:
        employer_name = posts[0].company_name
    else:
        employer_name = u'a single employer'

    # Make lookup slightly faster in the loop below since 'g' is a proxy
    board = g.board

    if basequery is None and posts and not g.kiosk:
        # Group posts by email_domain on index page only, when not in kiosk mode
        grouped = OrderedDict()
        for post in posts:
            pinned = post.pinned
            if board is not None:
                blink = post.link_to_board(board)
                if blink is not None:
                    pinned = blink.pinned
            if pinned:
                # Make pinned posts appear in a group of one
                grouped.setdefault(('s', post.hashid), []).append((pinned, post))
            elif post.status == POSTSTATUS.ANNOUNCEMENT:
                # Make announcements also appear in a group of one
                grouped.setdefault(('a', post.hashid), []).append((pinned, post))
            elif post.email_domain in webmail_domains:
                grouped.setdefault(('ne', post.md5sum), []).append((pinned, post))
            else:
                grouped.setdefault(('nd', post.email_domain), []).append((pinned, post))
        pinsandposts = None
    else:
        grouped = None
        if g.board:
            pinsandposts = []
            for post in posts:
                pinned = post.pinned
                if board is not None:
                    blink = post.link_to_board(board)
                    if blink is not None:
                        pinned = blink.pinned
                pinsandposts.append((pinned, post))
        else:
            pinsandposts = [(post.pinned, post) for post in posts]

    return render_template('index.html', pinsandposts=pinsandposts, grouped=grouped, now=now,
                           newlimit=newlimit, jobtype=type, jobcategory=category, title=title,
                           md5sum=md5sum, domain=domain, employer_name=employer_name,
                           location=location, showall=showall,
                           is_siteadmin=lastuser.has_permission('siteadmin'))

@app.route('/drafts', subdomain='<subdomain>')
@app.route('/drafts')
@lastuser.requires_login
def browse_drafts():
    basequery = JobPost.query.filter_by(user=g.user)
    return index(basequery=basequery, statuses=[POSTSTATUS.DRAFT, POSTSTATUS.PENDING])

@app.route('/type/<name>', subdomain='<subdomain>')
@app.route('/type/<name>')
def browse_by_type(name):
    if name == 'all':
        return redirect(url_for('index'))
    ob = JobType.query.filter_by(name=name).first_or_404()
    basequery = JobPost.query.filter_by(type_id=ob.id)
    return index(basequery=basequery, type=ob, title=ob.title)

@app.route('/at/<domain>', subdomain='<subdomain>')
@app.route('/at/<domain>')
def browse_by_domain(domain):
    if not domain:
        abort(404)
    basequery = JobPost.query.filter_by(email_domain=domain)
    return index(basequery=basequery, domain=domain, title=domain, showall=True)

@app.route('/category/<name>', subdomain='<subdomain>')
@app.route('/category/<name>')
def browse_by_category(name):
    if name == 'all':
        return redirect(url_for('index'))
    ob = JobCategory.query.filter_by(name=name).first_or_404()
    basequery = JobPost.query.filter_by(category_id=ob.id)
    return index(basequery=basequery, category=ob, title=ob.title)

@app.route('/by/<md5sum>', subdomain='<subdomain>')
@app.route('/by/<md5sum>')
def browse_by_email(md5sum):
    if not md5sum:
        abort(404)
    basequery = JobPost.query.filter_by(md5sum=md5sum)
    return index(basequery=basequery, md5sum=md5sum, showall=True)

@app.route('/in/<location>', subdomain='<subdomain>')
@app.route('/in/<location>')
def browse_by_location(location):
    geodata = location_geodata(location)
    if not geodata:
        abort(404)
    basequery = JobPost.query.filter(db.and_(
        JobLocation.jobpost_id == JobPost.id, JobLocation.geonameid == geodata['geonameid']))
    return index(basequery=basequery, location=geodata, title=geodata['short_title'])

@app.route('/in/anywhere', subdomain='<subdomain>')
@app.route('/in/anywhere')
def browse_by_anywhere():
    basequery = JobPost.query.filter(JobPost.remote_location == True)
    return index(basequery=basequery)

@app.route('/feed', subdomain='<subdomain>')
@app.route('/feed')
def feed(basequery=None, type=None, category=None, md5sum=None, domain=None, location=None):
    title = "All jobs"
    if type:
        title = type.title
    elif category:
        title = category.title
    elif md5sum or domain:
        title = u"Jobs at a single employer"
    elif location:
        title = u"Jobs in {location}".format(location=location['short_title'])
    posts = list(getposts(basequery, showall=True))
    if posts:  # Can't do this unless posts is a list
        updated = posts[0].datetime.isoformat() + 'Z'
        if md5sum:
            title = posts[0].company_name
    else:
        updated = datetime.utcnow().isoformat() + 'Z'
    return Response(render_template('feed.xml', posts=posts, updated=updated, title=title),
        content_type='application/atom+xml; charset=utf-8')

@app.route('/type/<name>/feed', subdomain='<subdomain>')
@app.route('/type/<name>/feed')
def feed_by_type(name):
    if name == 'all':
        return redirect(url_for('feed'))
    ob = JobType.query.filter_by(name=name).first_or_404()
    basequery = JobPost.query.filter_by(type_id=ob.id)
    return feed(basequery=basequery, type=ob)

@app.route('/category/<name>/feed', subdomain='<subdomain>')
@app.route('/category/<name>/feed')
def feed_by_category(name):
    if name == 'all':
        return redirect(url_for('feed'))
    ob = JobCategory.query.filter_by(name=name).first_or_404()
    basequery = JobPost.query.filter_by(category_id=ob.id)
    return feed(basequery=basequery, category=ob)

@app.route('/by/<md5sum>/feed', subdomain='<subdomain>')
@app.route('/by/<md5sum>/feed')
def feed_by_email(md5sum):
    if not md5sum:
        abort(404)
    basequery = JobPost.query.filter_by(md5sum=md5sum)
    return feed(basequery=basequery, md5sum=md5sum)

@app.route('/at/<domain>/feed', subdomain='<subdomain>')
@app.route('/at/<domain>/feed')
def feed_by_domain(domain):
    if not domain:
        abort(404)
    basequery = JobPost.query.filter_by(email_domain=domain)
    return feed(basequery=basequery, domain=domain)

@app.route('/in/<location>/feed', subdomain='<subdomain>')
@app.route('/in/<location>/feed')
def feed_by_location(location):
    geodata = location_geodata(location)
    if not geodata:
        abort(404)
    basequery = JobPost.query.filter(db.and_(
        JobLocation.jobpost_id == JobPost.id, JobLocation.geonameid == geodata['geonameid']))
    return feed(basequery=basequery, location=geodata)

@app.route('/archive')
def archive():
    def sortarchive(order_by):
        current_order_by = request.args.get('order_by')
        reverse = request.args.get('reverse')
        if order_by == current_order_by:
            if reverse is None:
                reverse = False
            try:
                reverse = bool(int(reverse))
            except ValueError:
                reverse = False
            reverse = int(not reverse)
        return url_for('archive', order_by=order_by,
            reverse=reverse,
            start=request.args.get('start'),
            limit=request.args.get('limit'))

    order_by = {
        'date': JobPost.datetime,
        'headline': JobPost.headline,
        'company': JobPost.company_name,
        'location': JobPost.location,
        }.get(request.args.get('order_by'))
    reverse = request.args.get('reverse')
    start = request.args.get('start', 0)
    limit = request.args.get('limit', 100)
    if order_by is None and reverse is None:
        order_by = JobPost.datetime
        reverse = True
    try:
        if reverse is not None:
            reverse = bool(int(reverse))
    except ValueError:
        reverse = None
    try:
        if start is not None:
            start = int(start)
    except ValueError:
        start = 0
    try:
        if limit is not None:
            limit = int(limit)
    except ValueError:
        limit = 100
    count, posts = getallposts(order_by=order_by, desc=reverse, start=start, limit=limit)

    if request.is_xhr:
        tmpl = 'archive_inner.html'
    else:
        tmpl = 'archive.html'
    return render_template(tmpl, order_by=request.args.get('order_by'),
        posts=posts, start=start, limit=limit, count=count,
        # Pass some functions
        min=min, max=max, sortarchive=sortarchive)

@app.route('/sitemap.xml', subdomain='<subdomain>')
@app.route('/sitemap.xml')
def sitemap():
    sitemapxml = '<?xml version="1.0" encoding="UTF-8"?>\n'\
                 '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n'
    # Add job type pages to sitemap
    for item in JobType.query.filter_by(public=True).all():
        sitemapxml += '  <url>\n'\
                      '    <loc>%s</loc>\n' % url_for('browse_by_type', name=item.name, _external=True) + \
                      '  </url>\n'
    # Add job category pages to sitemap
    for item in JobCategory.query.filter_by(public=True).all():
        sitemapxml += '  <url>\n'\
                      '    <loc>%s</loc>\n' % url_for('browse_by_category', name=item.name, _external=True) + \
                      '  </url>\n'
    # Add live posts to sitemap
    for post in getposts(showall=True):
        sitemapxml += '  <url>\n'\
                      '    <loc>%s</loc>\n' % url_for('jobdetail', hashid=post.hashid, _external=True) + \
                      '    <lastmod>%s</lastmod>\n' % (post.datetime.isoformat() + 'Z') + \
                      '    <changefreq>monthly</changefreq>\n'\
                      '  </url>\n'
    sitemapxml += '</urlset>'
    return Response(sitemapxml, content_type='text/xml; charset=utf-8')

@app.route('/logo/<hashid>', subdomain='<subdomain>')
@app.route('/logo/<hashid>')
def logoimage(hashid):
    post = JobPost.query.filter_by(hashid=hashid).first_or_404()
    if not post.company_logo:
        # If there's no logo (perhaps it was deleted), don't try to show one
        abort(404)
    if post.status in [POSTSTATUS.REJECTED, POSTSTATUS.SPAM]:
        # Don't show logo if post has been rejected. Could be spam
        abort(410)
    return redirect(uploaded_logos.url(post.company_logo))

@app.route('/search', subdomain='<subdomain>')
@app.route('/search')
def search():
    now = datetime.utcnow()
    results = sorted(do_search(request.args.get('q', u''), expand=True),
        key=lambda r: getattr(r, 'datetime', now))
    results.reverse()
    return render_template('search.html', results=results, now=now, newlimit=newlimit)


#!/usr/bin/python3
import pymongo

my_client = pymongo.MongoClient("mongodb://localhost:27017/")
my_db = my_client["runoobdb"]
my_cal = my_db["sites"]

x = my_cal.find_one()
print(x)
print("--------------------------------------------")

# 查找所有
my_cal = my_db["site2"]
lists = my_cal.find()
for list in lists:
    print(list)
print("--------------------------------------------")

# 筛选字段
lists = my_cal.find({}, { "_id": 0, "name": 1, "address": 1})
for list in lists:
    print(list)
print("--------------------------------------------")

# 条件查找
condition = {"name" : "Google"}
lists = my_cal.find(condition)
for list in lists:
    print(list)
print("--------------------------------------------")


# -*- encoding: utf-8 -*-

"""
@File    : CardPole.py
@Time    : 2020/8/26 下午7:02
@Author  : dididididi
@Email   :
@Software: PyCharm
"""

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam
from collections import deque
import numpy as np
import random
import gym

episode = 5000

class DQNAgent:
    def __init__(self, state_size, action_size):
        self.state_size = state_size
        self.action_size = action_size
        self.memory = deque(maxlen=2000)
        self.gamma = 0.95
        self.epsilon_max = 1.
        self.epsilon_min = 0.01
        self.epsilon_decay = 0.995
        self.learning_rate = 0.001
        self.model = self.build_model()

    def build_model(self):
        # 创建一个构造器
        model = Sequential()
        model.add(Dense(24, input_dim=self.state_size, activation='relu'))
        model.add(Dense(24, activation='relu'))
        model.add(Dense(self.action_size, activation='linear'))
        model.compile(loss='mse', optimizer=Adam(lr=self.learning_rate), metrics=['accuracy'])

        return model

    def remember(self, state, action, reward, next_state, done):
        self.memory.append((state, action, reward, next_state, done))

    def act(self, state):
        if np.random.rand() <= self.epsilon_max:
            return random.randrange(self.action_size)
        else:
            q_values = self.model.predict(state)[0]
            return np.argmax(q_values)

    def replay(self, batch_size):
        mini_batch = random.sample(self.memory, batch_size)
        for step, action, reward, next_state, done in mini_batch:
            target = reward
            if not done:
                target = (reward + self.gamma * np.amax(self.model.predict(next_state)[0]))
            target_f = self.model.predict(state)
            target_f[0][action] = target
            self.model.fit(state, target_f, epochs=1, verbose=0)
        if self.epsilon_max > self.epsilon_min:
            self.epsilon_max = self.epsilon_max * self.epsilon_decay

if __name__ == '__main__':
    env = gym.make('CartPole-v1')
    state_size = env.observation_space.shape[0]
    action_size = env.action_space.n
    agent = DQNAgent(state_size, action_size)

    done = False
    batch_size = 8

    for e in range(episode):
        state = env.reset()
        state = np.reshape(state, [1, state_size])
        for time in range(500):
            env.render()
            # 选择动作
            action = agent.act(state)
            # 得到状态和回报
            next_state, reward, done, _ = env.step(action)
            reward = reward if not done else -10
            next_state = np.reshape(next_state, [1, state_size])

            # 记录信息到Q表
            agent.remember(state, action, reward, next_state, done)

            # 更新状态
            state = next_state

            if done:
                print("episode: {}/{}, score: {}, e: {:.2}"
                      .format(e, episode, time, agent.epsilon_max))
                break

            if len(agent.memory) > batch_size:
                agent.replay(batch_size)


from urllib import urlencode

from django.core import urlresolvers
from django.contrib import messages
from django.http import HttpResponseForbidden, HttpResponseRedirect
from django.shortcuts import render_to_response, redirect, get_list_or_404
from django.template import RequestContext
from django.utils.translation import ugettext_lazy as _

from appointmentbook import checks, forms, models, session, settings

MESSAGE_DICT = {
    'add_succeed': _(u'The appointment have been added successfully.'),
    'enable_cookies': _(u'Please enable cookies and try again.'),
    'operation_cancelled': _(u'The operation have been cancelled.'),
    'resource_exhausted': _(u"There are no more appointments availables for "
                             "the given date and subject."),
}

def index(request, add_view='appointmentbook:add',
                   remove_view='appointmentbook:remove',
                   add_form_class=forms.AddAppointmentStep0Form,
                   remove_form_class=forms.LoginForm,
                   check_credentials=checks.check_credentials,
                   check_new_appointment=checks.check_new_appointment,
                   template='index.html'):
    add_form = None
    remove_form = None
    if request.method == 'POST':
        cookie_test = request.session.test_cookie_worked()
        if add_form_class.submit in request.POST:
            add_form = add_form_class(request.POST)
            if add_form.is_valid() and cookie_test:
                try:
                    check_new_appointment(add_form.cleaned_data)
                except checks.UserAlreadyHaveAnAppointment as e:
                    add_form.append_non_field_error(e)
                else:
                    request.session.delete_test_cookie()
                    request.session.set_expiry(settings.SESSION_COOKIE_AGE)
                    session.save_form_data(request, add_form)
                    return redirect(add_view)

            elif not cookie_test:
                add_form.append_non_field_error(MESSAGE_DICT['enable_cookies'])

        elif remove_form_class.submit in request.POST:
            remove_form = remove_form_class(request.POST)
            if remove_form.is_valid() and cookie_test:
                person_id = remove_form.cleaned_data['person_id']
                key = remove_form.cleaned_data['key']
                try:
                    check_credentials(person_id, key)
                except checks.AppointmentDoesNotExist as e:
                    remove_form.append_non_field_error(e)
                else:
                    request.session.delete_test_cookie()
                    request.session.set_expiry(settings.SESSION_COOKIE_AGE)
                    session.save_form_data(request, remove_form)
                    return redirect(remove_view, key)

            elif not cookie_test:
                remove_form.append_non_field_error(
                    MESSAGE_DICT['enable_cookies'])

    if add_form is None:
        add_form = add_form_class()
    if remove_form is None:
        remove_form = remove_form_class()

    request.session.set_test_cookie()
    return render_to_response(template, {
        'add_form': add_form,
        'remove_form': remove_form,
        }, context_instance=RequestContext(request))

def add(request, success_view='appointmentbook:show',
                 form_class=forms.AddAppointmentForm,
                 checks_new_appointment=checks.checks_new_appointment,
                 template='add.html'):
    if request.method == 'POST':
        cookie_test = request.session.test_cookie_worked()
        form = form_class(request.POST)
        if form.is_valid() and cookie_test:
            form_data = form.cleaned_data
            try:
                checks_new_appointment(form_data)
            except checks.UserAlreadyHaveAnAppointment as e:
                form.append_non_field_error(e)
            else:
                if not 'person_id' in form_data:
                    if session.has_person_id(request):
                        form_data['person_id'] = session.get_person_id(request)
                elif not session.has_person_id(request):
                    session.set_person_id(request, form_data['person_id'])

                appointment = models.Appointment(form_data)
                try:
                    appointment.save(request=request)
                except models.ResourceExhaustedError:
                    form.append_non_field_error(
                        MESSAGE_DICT['resource_exhausted'])
                else:
                    request.session.delete_test_cookie()
                    messages.success(request, MESSAGE_DICT['add_succeed'])
                    return redirect(success_view, appointment.key)

        elif not cookie_test:
            form.append_non_field_error(MESSAGE_DICT['enable_cookies'])
    elif session.has_person_id(request):
        form = form_class(request.session)
    else:
        form = form_class()

    request.session.set_test_cookie()
    return render_to_response(template, {
        'form': form
        }, context_instance=RequestContext(request))

def remove(request, key, login_view='appointmentbook:login',
                         success_view='appointmentbook:logout',
                         cancel_view='appointmentbook:logout',
                         template='remove.html'):
    if session.has_person_id(request):
        appointment = get_list_or_404(models.Appointment, key=key)
        if not appointment.person_id == session.get_person_id(request):
            return HttpResponseForbidden()
        if request.method == 'POST':
            if 'ok' in request.POST:
                appointment.delete()
                messages.success(request, MESSAGE_DICT['remove_succeed'])
                return redirect(success_view)
            else:
                messages.success(request, MESSAGE_DICT['operation_cancelled'])
                return redirect(cancel_view)
        else:
            return render_to_response(template, {
                'appointment': appointment
                }, context_instance=RequestContext(request))
    else:
        query = {'continue': request.get_full_path()}
        return HttpResponseRedirect(urlresolvers.reverse(login_view) +
                                    urlencode(query))

def show(request, key, login_view='appointmentbook:login',
                       template='show.html',
                       logout_after=False):
    if session.has_person_id(request):
        appointment = get_list_or_404(models.Appointment, key=key)
        if not appointment.person_id == session.get_person_id(request):
            return HttpResponseForbidden()
        if logout_after:
            request.session.flush()
        return render_to_response(template, {
            'appointment': appointment
            }, context_instance=RequestContext(request))
    else:
        query = {'continue': request.get_full_path()}
        return HttpResponseRedirect(urlresolvers.reverse(login_view) +
                                    urlencode(query))

def login(request, login_view='appointmentbook:login',
                   show_view='appointmentbook:show',
                   form_class=forms.LoginForm,
                   checks_credentials=checks.checks_credentials,
                   template='login.html'):
    form = None
    if request.method == 'POST':
        cookie_test = request.session.test_cookie_worked()
        request_post = request.POST
        if 'continue' in request.GET:
            view, args, kwargs = urlresolvers.resolve(request.GET['continue'])
            if 'key' in kwargs:
                request_post['key'] = kwargs['key']
        form = form_class(request_post)
        if form.is_valid() and cookie_test:
            person_id = form.cleaned_data['person_id']
            key = form.cleaned_data['key']
            try:
                checks_credentials(person_id, key)
            except checks.AppointmentDoesNotExist as e:
                form.append_non_field_error(e)
            else:
                request.session.delete_test_cookie()
                request.session.set_expiry(settings.SESSION_COOKIE_AGE)
                session.save_form_data(request, form)
                if 'continue' in request.GET:
                    return redirect(request.GET['continue'])
                else:
                    return redirect(show_view, key)

        elif not cookie_test:
            form.append_non_field_error(MESSAGE_DICT['enable_cookies'])

    elif 'continue' in request.GET:
        view, args, kwargs = urlresolvers.resolve(request.GET['continue'])
        if 'key' in kwargs:
            form = form_class(hiden_fields=['key'])

    if form is None:
        form = form_class()

    request.session.set_test_cookie()
    return render_to_response(template, {
        'form': form
        }, context_instance=RequestContext(request))

def logout(request, top_view='appointmentbook:index'):
    request.session.flush()
    return redirect(top_view)