#
# Training program designed by H. Hamano version 2
# Mar 2 2018
#

import tensorflow as tf
import numpy as np
import pandas as pd
import cv2
import os
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec

from skimage.io import imread

import seaborn as sns

from sklearn.model_selection import StratifiedShuffleSplit, train_test_split
from sklearn.utils import shuffle

import keras
from keras.models import Sequential, Model
from keras.layers import Flatten, Dense, Lambda, Convolution2D, Cropping2D, Conv2D
from keras.layers.pooling import MaxPooling2D
from keras.optimizers import SGD, Adam, Nadam
from keras import backend as K

from utils import displayImage
from nVidiaModel import nVidiaModelClass
from Parameter import ParametersClass
from DataObject import DataObjectClass
from ImageObject import ImageDataObjectClass

def generator(X, y, baseDir, batch_size=32, straight_angle=None):

    num_samples = X.shape[0]
    cwd = os.path.join(os.getcwd(),baseDir)
    cwd = os.path.join(cwd,"IMG")

    while 1: # Loop forever so the generator never terminates

        #
        #    shuffle(samples)
        #
        X, y = shuffle(X, y)
        ridx = np.random.permutation(num_samples)

        imageDataObject = ImageDataObjectClass(X,y,cwd, straight_angle)

        for offset in range(0, num_samples, batch_size):

            offset_ridx = ridx[offset:offset+batch_size]
            #print("ridx",offset_ridx)
            X_train, y_train = imageDataObject.batch_next_random(offset_ridx)

            yield X_train, y_train

def generator2(X, y, baseDir, batch_size=32, straight_angle=None):

    num_samples = X.shape[0]
    cwd = os.path.join(os.getcwd(),baseDir)
    cwd = os.path.join(cwd,"IMG")

    while 1: # Loop forever so the generator never terminates

        #
        #    shuffle(samples)
        #
        X, y = shuffle(X, y)
        lengthOfData = X.shape[0]
        ridx = np.random.permutation(lengthOfData)

        imageDataObject = ImageDataObjectClass(X,y,cwd, straight_angle)

        for offset in range(0, num_samples, batch_size):

            X_train, y_train = imageDataObject.batch_next(offset,batch_size)

            yield X_train, y_train

def root_mean_squared_error(y_true, y_pred):
        return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1))

def main():

    BATCH_SIZE = 64
    #(after you are done with the model)
    #K.clear_session()
    paramCls = ParametersClass()
    params = paramCls.initialize()
    paramCls.checkParams()

    filename = "driving_log.csv"
    baseDir = params.directory

    print("-"*30)
    print("* EPOCHS *",params.epochs)
    print("* Learning RATE *",params.learning_rate)
    print("* load sample data", params.header)
    print("-"*30)

    myData = DataObjectClass()
    myData.loadCSVFullData(baseDir, filename, sample = params.header, straight_angle=params.straight_angle)
    X_train, X_test, y_train, y_test = myData.shuffleSplit(test_size=0.15)

    print(" Train / Test splitted size --> ", X_train.shape, X_test.shape, y_train.shape, y_test.shape)

    nVidia = nVidiaModelClass()
    model = nVidia.buildModel_Normal()
    model.summary()

    train_generator = generator(X_train, y_train, baseDir, batch_size=BATCH_SIZE)    #, straight_angle=params.straight_angle)
    validation_generator = generator(X_test, y_test, baseDir, batch_size=BATCH_SIZE) #, straight_angle=params.straight_angle)

    #optimizer = Nadam(lr=learning_rate)
    optimizer = Adam(lr=params.learning_rate)
    #model.compile(optimizer=optimizer, loss='mse')
    model.compile(optimizer = "rmsprop", loss = root_mean_squared_error,
              metrics =["accuracy"])

    if keras.__version__ == "1.2.1":
        history_object = model.fit_generator(train_generator, samples_per_epoch=X_train.shape[0], \
                validation_data=validation_generator,  \
                nb_val_samples=X_test.shape[0], nb_epoch=params.epochs, verbose=1)
    else:
        steps_per_epch = X_train.shape[0] // BATCH_SIZE
        valid_steps = X_test.shape[0] // BATCH_SIZE
        history_object = model.fit_generator(train_generator, steps_per_epoch=steps_per_epch , \
            validation_data=validation_generator,  \
            validation_steps= valid_steps  , epochs=params.epochs, verbose=2)

    print(history_object.history.keys())

    ### plot the training and validation loss for each epoch
    plt.plot(history_object.history['loss'])
    plt.plot(history_object.history['val_loss'])
    plt.title('model mean squared error loss')
    plt.ylabel('mean squared error loss')
    plt.xlabel('epoch')
    plt.legend(['training set', 'validation set'], loc='upper right')
    plt.show()

    model_file = os.path.join("convmodel","model.h5")
    model.save(model_file)

if __name__ == "__main__":
    main()


import os
import time
import pandas as pd
from io import StringIO
import werkzeug
werkzeug.cached_property = werkzeug.utils.cached_property
from flask import Flask, request, send_from_directory, make_response, Blueprint, url_for
from flask_restplus import Resource, Api, reqparse, fields
from flask_cors import CORS
from process import cleaning, fuzzymatch, src_dict, df_process
from werkzeug.datastructures import FileStorage
from utils import timeit, logger

app = Flask(__name__)
app.config.SWAGGER_UI_DOC_EXPANSION = 'list'
app.config.SWAGGER_UI_OPERATION_ID = True
app.config.SWAGGER_UI_REQUEST_DURATION = True
CORS(app)

if os.path.exists('./config.ini'):
    import configparser
    config = configparser.ConfigParser()
    config.read('./config.ini')
    PORT = config.get('config','APP_PORT')
elif os.getenv('APP_PORT'):
    PORT = int(str(os.getenv('APP_PORT')))
else:
    PORT=5000

##########################
#  Documentation Sphinx  #
##########################

blueprint_doc = Blueprint('documentation', __name__,
                          static_folder='../docs/build/html/_static',
                          url_prefix='/sivnorm/docs')

@blueprint_doc.route('/', defaults={'filename': 'index.html'})
@blueprint_doc.route('/<path:filename>')
def show_pages(filename):
    return send_from_directory('../docs/build/html', filename)

app.register_blueprint(blueprint_doc)

#################
#  API SWAGGER  #
#################

class Custom_API(Api):
    @property
    def specs_url(self):
        '''
        The Swagger specifications absolute url (ie. `swagger.json`)

        :rtype: str
        '''
        return url_for(self.endpoint('specs'), _external=False)

blueprint = Blueprint('api', __name__, url_prefix='/sivnorm')
api = Custom_API(
        blueprint, doc='/swagger', version='1.0', title='SivNorm',
        description="Match flou de la marque et modèle des véhicules dans la \
                carte grise à partir d'un référentiel")
app.register_blueprint(blueprint)

@timeit
def process_row(table_ref_name, marque, modele):
    if marque and modele:
        row = dict(modele=modele, marque=marque)
        for column in ['marque', 'modele']:
            row = fuzzymatch(cleaning(row, column), column, table_ref_name)
        row['score'] = (row['score_marque'] + row['score_modele']) / 200
        return row

@timeit
def process_csv(table_ref_name, input_file):

    num_workers = 16
    filename = "%s.csv" % ('output file')

    output_file = StringIO()

    df = pd.read_csv(input_file, names=['marque', 'modele'])

    df = df.fillna("")

    t1 = time.time()
    df = df_process(df, table_ref_name, num_workers)

    sec_wl = (num_workers*(time.time() - t1))/(df.shape[0])
    logger.debug(f'{sec_wl:.2f} seconds per worker per line')

    df.sort_index().to_csv(output_file, encoding='utf-8', index=False, header=False)
    output_csv = output_file.getvalue()
    output_file.close()

    resp = make_response(output_csv)
    resp.headers["Content-Disposition"] = ("attachment; filename=%s" % filename)
    resp.headers["Content-Type"] = "text/csv"

    return resp

parser = reqparse.RequestParser()
parser.add_argument('marque', type=str, location='args', help='Vehicle brand (eg: Renault)')
parser.add_argument('modele', type=str, location='args', help='Vehicle model (eg: Clio)')

parser_table = reqparse.RequestParser()
parser_table.add_argument('file', type=FileStorage, location='files', help='CSV file with multiple brand model lines with no header. Like: \n\n renault,clio \n renault,clio2 \n audi,TTs \n renault,renault clio7 \n renault,clio7 RT/RN')

NormOutput = api.model('NormalizedOutput', {
    'modele': fields.String(description='Matched model', example='Clio'),
    'marque': fields.String(description='Matched brand', example='Renault'),
    'score_marque': fields.Integer(description='Matching score for brand', min=0, max=100, example=100),
    'score_modele': fields.Integer(description='Matching score for model', min=0, max=100, example=100),
    'score': fields.Float(description='Global matching score. Combination of score_marque and score_modele', min=0, max=1, example=1),
})

CleanOutput = api.model('CleanOutput', {
    'modele': fields.String(description='Cleaned model', example='CLIO'),
    'marque': fields.String(description='Cleaned brand', example='RENAULT'),
})

@api.route('/norm/<string:table_ref_name>')
@api.doc(params={'table_ref_name': 'Reference table [siv,cardisiac,siv_caradisiac]'})
class Normalization(Resource):
    """Docstring for MyClass. """

    @api.expect(parser)
    @api.marshal_with(NormOutput, mask=None)
    def get(self, table_ref_name):
        """Normalize a single brand and model using a defined referential table"""
        marque = request.args.get('marque', '')
        modele = request.args.get('modele', '')

        logger.debug(f'MARQUE: {marque}')
        logger.debug(f'MODELE: {modele}')
        return process_row(table_ref_name, marque, modele)

    @api.expect(parser_table)
    @api.response(200, description='CSV file containing matching brand model and score like: \n\n RENAULT,CLIO,1.0\n RENAULT,CLIO,0.945 \n AUDI, TTS,1.0 \n RENAULT,CLIO,0.945 \n RENAULT,CLIO,0.95 \n RENAULT,CLIO,1.0')
    @api.produces(['text/csv'])
    def post(self, table_ref_name):
        """Normalize a table of brand and model using a defined referential table"""
        input_file = request.files['file']
        return process_csv(table_ref_name, input_file)

@api.route('/info/<string:table_ref_name>', doc=False)
@api.doc(params={'table_ref_name': 'Reference table [siv,cardisiac,siv_caradisiac]'})
class Information(Resource):
    """Docstring for MyClass. """

    @api.expect(parser)
    def get(self, table_ref_name):
        """Get brand and model image"""
        marque = request.args.get('marque', '')
        modele = request.args.get('modele', '')

        res_def = {'src': None, 'href': None}
        res = src_dict.get(table_ref_name, {}).get((marque, modele), res_def)
        return res

@api.route('/clean')
class Clean(Resource):
    """Docstring for MyClass. """

    @api.expect(parser)
    @api.marshal_with(CleanOutput, mask=None)
    def get(self):
        """Clean brand and model field"""
        marque = request.args.get('marque', '')
        modele = request.args.get('modele', '')
        row = dict(marque=marque, modele=modele)
        for column in ['marque', 'modele']:
            res = cleaning(row, column)
        return res

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0', port=PORT)


import numpy as np
from utils import labels_to_col_nr, facing_straight, unroll, normalize_by_eyes

def get_labelled_landmarks(data_row, col_nr, labels, face_straight=True, limit_deg=1,
                           face_id_clean=False):
    if face_straight and not facing_straight(
            [data_row[col_nr['headpose.yaw_angle']],
             data_row[col_nr['headpose.pitch_angle']]
             ],
            limit_deg):
        return None, None, False
    face_id = data_row[col_nr['face_id']]
    if not face_id_clean:
        face_id = face_id[11:-14]
    coordinates = [int(o) for o in
                   data_row[col_nr['contour_chin.y']:col_nr['right_eye_pupil.x']+1]
                   ]
    points = np.asarray([(p[1], -p[0]) for p in zip(coordinates[::2],
                                                    coordinates[1::2])])
    points_labels = [
        l[:-2] for l in labels[col_nr['contour_chin.y']:col_nr['right_eye_pupil.x']+1:2]]
    labelled_points = list(zip(points, points_labels))
    return face_id, labelled_points, True

def get_features(file, feature_func, labels_file, gender_file=None, face_id_clean=False, limit_deg=1):
    features = {}
    col_nr, labels = labels_to_col_nr(labels_file)
    if gender_file:
        gender = get_gender(gender_file)
    with open(file, 'r') as data_file:
        for row in data_file:
            data = row.split(',')
            face_id, labelled_data, ok = get_labelled_landmarks(
                data, col_nr, labels, face_id_clean=face_id_clean, limit_deg=limit_deg)
            if not ok or (gender_file and face_id not in gender):
                continue
            labelled_data = unroll(
                labelled_data, np.float(data[col_nr['headpose.roll_angle']]))
            labelled_data = normalize_by_eyes(labelled_data)
            features[face_id] = feature_func(labelled_data)
            if gender_file:
                features[face_id].append(gender[face_id])
            else:
                features[face_id].append(int(data[col_nr['gender']]))
    return features

def get_gender(file):
    genders = {}
    with open(file, 'r') as f:
        for row in f:
            data = row.split(',')
            face_id = data[0][1:-1]  # 1:-1 = remove ""
            gender = int(data[1])
            genders[face_id] = gender
    return genders


import scrapy
from datetime import date

class DafitiSpider(scrapy.Spider):

    name = 'dafiti'

    start_urls = ['https://www.dafiti.com.br/calcados/adidas/',
                  'https://www.dafiti.com.br/calcados/asics/',
                  'https://www.dafiti.com.br/calcados/mizuno/',
                  'https://www.dafiti.com.br/calcados/new-balance/',
                  'https://www.dafiti.com.br/calcados/nike/']

    download_delay = 2.0

    def parse(self, response):

        diaAtual = date.today()

        dafiti_xpath = '//div[contains(@class, "product-box-detail")]'

        for itens in response.xpath(dafiti_xpath):

            Loja = 'Dafiti'

            Nome = itens.xpath('.//p/text()').extract_first()

            Preco = itens.xpath('.//div[3]/span/text()').extract_first()

            Data = diaAtual

            Marca = itens.xpath('.//div/text()').extract_first()

            if Marca == 'adidas':
                Marca = 'Adidas'

            if (Marca == None) or (Nome == None):
                Marca = None
                Nome  = None
                Preco = None
                Loja  = None
                Data  = None

            yield dict(Nome = Nome , Preco = Preco , Data = Data, Loja = Loja, Marca = Marca)

            link_next = response.xpath('//a[contains(., "next")]/@href').extract_first()
            if link_next:
                yield scrapy.Request(response.urljoin(link_next))


#!/usr/bin/python

import pandas as pd
import math
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from matplotlib.backends.backend_pdf import PdfPages
from matplotlib.patches import Rectangle
from matplotlib.ticker import FormatStrFormatter
import argparse
from ast import literal_eval # to convert strings to lists when reading in csv

from get_scores_df import dump_scores

BAPS = ['Enobakhare..I wax it  James Broughton','Crazy on Ayew  Richard Mudd','Heaton Mess  Timothy Williams','Giroud - Sandstorm  Andrew Foster','Großly Underrated  Alasdair Winter']
KEY_PLAYERS = ['Owe me one Kanoti  Tamuka Kanoti','Lingardium Leviosa  Katy Williams','Don’tCallMeSchürrle  Abby O\'Rourke','Enobakhare..I wax it  James Broughton','Crazy on Ayew  Richard Mudd','Heaton Mess  Timothy Williams','Giroud - Sandstorm  Andrew Foster','Großly Underrated  Alasdair Winter']

def plot_on_page(pdf,series, x_label, y_label, players_to_show=None, log_y = False):
   fig, ax = plt.subplots(figsize=(11.69,8.27))
   ax.set_prop_cycle(color=['lightcoral','orangered','gold','darkolivegreen','limegreen','darkturquoise','midnightblue'])
   if players_to_show == None:
      al_var = 0.5
   else:
      al_var = 0.5
   # print(series)
   for i, ind in enumerate(series):
      # y_values = literal_eval(ind)
      if i < 7:
         if players_to_show is None or series.index[i] in players_to_show:
            plt.plot(ind,label=series.index[i],alpha=al_var)
         else:
            plt.plot(ind,label=series.index[i],linestyle='None',alpha=al_var)
      else:
         if players_to_show is None or series.index[i] in players_to_show:
            plt.plot(ind,label=series.index[i],linestyle='--',alpha=al_var)
         else:
            plt.plot(ind,label=series.index[i],linestyle='None',alpha=al_var)

   n_weeks = len(series[0])
   ax.set_ylabel(y_label,fontsize=10)
   ax.set_xlabel(x_label,fontsize=10)
   ax.set_xticks(range(0,n_weeks,2))
   if log_y:
      ax.set_yscale('log')
   plt.legend(fontsize=8,loc='best',handlelength=4)
   plt.grid(which='major', axis='both', alpha=0.2, linewidth = 1)
   pdf.savefig()

def calculate_goal_diff(in_row):
   goal_diff = []
   for i in range(len(in_row['points_for'])):
      goal_diff.append(in_row['points_for'][i] - in_row['points_against'][i])
   return goal_diff

def add_goal_diff(df):
   df['goal_diff'] = df.apply(lambda row: calculate_goal_diff(row), axis=1)

def calculate_form(in_list):
   result_list = in_list.copy()
   for i in range(len(result_list)):
      lower_lim = max(0,i-5)
      result_list[i] = in_list[i] - in_list[lower_lim]
   return result_list

def add_form(df, form_of_what, weeks_to_consider = 5):
   form_name = form_of_what + '_form'
   df[form_name] = df.apply(lambda row: calculate_form(row[form_of_what]),axis=1)

def calculate_cumulative(in_list):
   result_list = in_list.copy()
   for i in range(len(result_list)):
      result_list[i] = result_list[max(i-1,0)] + in_list[i]
   return result_list

def add_cumulative(df, cum_of_what):
   cum_name = cum_of_what + '_cum'
   df[cum_name] = df.apply(lambda row: calculate_cumulative(row[cum_of_what]),axis=1)

def calculate_league_points(in_goal_diff):
   results = []
   for diff in in_goal_diff:
      if diff >0: # win
         results.append(3)
      elif diff < 0: # loss
         results.append(0)
      else: # draw
         results.append(1)
   return results

def add_league_points(df):
   df['league_points'] = df.apply(lambda row: calculate_league_points(row['goal_diff']), axis=1)

def turn_strings_to_lists(df): # pandas reads the csv in as strings rather than lists
   cols = df.columns
   for col in cols:
      for i in range(len(df[col])):
         df[col][i] = literal_eval(df[col][i])

def main(args):
   print("In the main")
   if args.refresh_data == True:
      dump_scores()
      print("Score dumping finished")

   df = pd.read_csv('scores_dump.csv', index_col=0)
   turn_strings_to_lists(df)

   # Make the pdf output
   with PdfPages('Output.pdf') as pdf:
      firstPage = plt.figure(figsize=(11.69,8.27))
      firstPage.clf()
      img=mpimg.imread('alpine.jpg')
      plt.axis('off')
      imgplot = plt.imshow(img)
      txt = "Andy Foster's FPL insights, silently endorsed by Xheredan Shaqiri"
      firstPage.text(0.5,0.05,txt, transform=firstPage.transFigure, size=15, ha="center")
      pdf.savefig()
      plt.close()

      add_goal_diff(df)
      add_league_points(df)

      for col in df.columns:
         add_cumulative(df,col)

      add_form(df,'league_points_cum')

      print(df.columns)

      plot_on_page(pdf,df['points_for_cum'],'Gameweek','Total Points')
      plot_on_page(pdf,df['points_against_cum'],'Gameweek','Total Points Conceded')
      plot_on_page(pdf,df['goal_diff_cum'],'Gameweek','Goal Difference')
      plot_on_page(pdf,df['league_points_cum'],'Gameweek','Real Points')
      plot_on_page(pdf=pdf,series=df['league_points_cum_form'],x_label='Gameweek',y_label='5-Week Form', players_to_show=BAPS)
      plot_on_page(pdf=pdf,series=df['league_points_cum_form'],x_label='Gameweek',y_label='5-Week Form', players_to_show=KEY_PLAYERS)
# CAN I HAVE all players being in the legend still, but then only displaying the key players

      # plot_on_page(pdf,df['total_points_for'],'Gameweek','Total Points')
      # plot_on_page(pdf,df['total_points_against'],'Gameweek','Total Points Conceeded')
      # plot_on_page(pdf,df['total_points_diff'],'Gameweek','Goal Difference')

      # add_form(df,'cum_real_points')
      # plot_on_page(pdf,df['cum_real_points_form'],'Gameweek','Form')

def parse_args():
   description = \
   '''
   Makes some sweet stats about my FPL league
   '''
   parser = argparse.ArgumentParser(description=description)
   parser.add_argument('-r', '--refresh_data', default=False, action='store_true', help='Pass to refresh the data from the FPL site. Else we just use the previously saved csv as this is lots faster.', required=False)
   args = parser.parse_args()
   return args

if __name__ == '__main__':
   args = parse_args()
   main(args)


# Copyright The PyTorch Lightning team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os

import torch

import pytorch_lightning as pl
from pytorch_lightning.accelerators.accelerator import Accelerator
from pytorch_lightning.utilities.exceptions import MisconfigurationException

_log = logging.getLogger(__name__)

class GPUAccelerator(Accelerator):
    """Accelerator for GPU devices."""

    def setup_environment(self) -> None:
        super().setup_environment()
        if "cuda" not in str(self.root_device):
            raise MisconfigurationException(f"Device should be GPU, got {self.root_device} instead")
        torch.cuda.set_device(self.root_device)

    def setup(self, trainer: "pl.Trainer") -> None:
        """
        Raises:
            MisconfigurationException:
                If the selected device is not GPU.
        """
        self.set_nvidia_flags(trainer.local_rank)
        return super().setup(trainer)

    def on_train_start(self) -> None:
        # clear cache before training
        torch.cuda.empty_cache()

    @staticmethod
    def set_nvidia_flags(local_rank: int) -> None:
        # set the correct cuda visible devices (using pci order)
        os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
        all_gpu_ids = ",".join(str(x) for x in range(torch.cuda.device_count()))
        devices = os.getenv("CUDA_VISIBLE_DEVICES", all_gpu_ids)
        _log.info(f"LOCAL_RANK: {local_rank} - CUDA_VISIBLE_DEVICES: [{devices}]")

    def teardown(self) -> None:
        super().teardown()
        self._move_optimizer_state(torch.device("cpu"))


import xarray as xar
import os
import matplotlib.pyplot as plt
import numpy as np
import pdb
import time

def open_experiment(exp_folder_name, start_file, end_file, file_name):

    base_dir = os.environ['GFDL_DATA']

    folder_list = ['run%04d' % m for m in range(start_file, end_file+1)]
    files = [base_dir + '/' + exp_folder_name+ '/' + folder_list[i] + '/' + file_name for i in range(len(folder_list))]

    files_exist=[os.path.isfile(s) for s in files]

    if not(all(files_exist)):
        raise EOFError('EXITING BECAUSE OF MISSING FILES', [files[elem] for elem in range(len(files_exist)) if not files_exist[elem]])

    ds = xar.open_mfdataset(files, decode_times=False)

    return ds

def global_average_lat_lon(ds_in, var_name):

    try:
        ds_in['area_array']
    except KeyError:
        cell_area(ds_in)

    weighted_data = ds_in[var_name]*ds_in['area_array']

    area_average = weighted_data.mean(('lat', 'lon')) / ds_in['area_array'].mean(('lat','lon'))

    var_in_dims = ds_in[var_name].dims

    var_out_dims = tuple(x for x in var_in_dims if x!='lat' and x!='lon')

    ds_in[var_name+'_area_av'] = (var_out_dims, area_average)

def cell_area(dataset_in, radius = 6371.e3):

    lonb = dataset_in['lonb']
    latb = dataset_in['latb']

    lonb_1 = lonb[1::].values
    lonb_2 = lonb[0:-1].values

    delta_lon = lonb_1 - lonb_2

    latb_1 = latb[1::].values
    latb_2 = latb[0:-1].values

    delta_lat = latb_1 - latb_2

    dataset_in['delta_lon'] = (('lon'), delta_lon)
    dataset_in['delta_lat'] = (('lat'), delta_lat)

    xsize = radius*np.absolute(np.deg2rad(dataset_in['delta_lon']))*np.cos(np.deg2rad(dataset_in['lat']))
    ysize = radius*np.absolute(np.deg2rad(dataset_in['delta_lat']))

    area_array = xsize*ysize

    dataset_in['area_array'] = (('lat','lon'), area_array.transpose('lat','lon'))


from joblib import Parallel, delayed
import queue
import os

N_GPU = 1

q = queue.Queue(maxsize=N_GPU)
for i in range(N_GPU):
    q.put(i)

def runner(x):
    gpu = q.get()
    cmd = "python3 train.py --config_file ./confs/tuh1_512_wgan_5_otur_base.yml {} --cuda_device {}".format(x, gpu)
    os.system(cmd)
    q.put(gpu)

Parallel(n_jobs=N_GPU, backend="threading")(delayed(runner)(line.strip()) for line in open('runner.cfg'))


import sys
import csv
from PyQt5 import QtWidgets
from ui_proj import Ui_MainWindow
from ui_aide import Ui_HelpDialog
import rc_proj

class MaFenetre(QtWidgets.QMainWindow, Ui_MainWindow):

    def __init__(self):
        QtWidgets.QMainWindow.__init__(self)
        self.setupUi(self)
        self.datafile = "exhibits.csv"
        self.data = None
        self.connecteSignaux()
        self.achercher = ""
        self.montreData()
        return

    def montreAide(self, fichier):
        """
        crée une fenêtre de dialogue, avec le contenu du fichier affiché
        :param: fichier un fichier au format HTML
        """
        class monDialogue(QtWidgets.QDialog, Ui_HelpDialog):
            def __init__(self, fichier):
                QtWidgets.QDialog.__init__(self)
                self.setupUi(self)
                with open(fichier) as htmldata:
                    texte = htmldata.read()
                    self.textEdit.setText(texte)
                return
        dialogue = monDialogue(fichier)
        dialogue.exec_()

    def connecteSignaux(self):
        self.actionQuitter.triggered.connect(self.close)
        self.actionAide_F1.triggered.connect(self.montreAideF1)
        self.action_propos.triggered.connect(self.montreAidePropos)
        self.selectionEdit.editingFinished.connect(self.montreData)
        self.selectButton.clicked.connect(self.montreData)
        self.pushButton.clicked.connect(self.montrePushButton)
        self.pushButton_2.clicked.connect(self.montrePushButton_2)
        self.EraseButton.clicked.connect(self.effaceSelection)
        return

    def montreAideF1(self):
        self.montreAide("aide.html")
        return

    def montreAidePropos(self):
        self.montreAide("apropos.html")
        return

    def montrePushButton(self):
        self.montreAide("aide.html")
        return

    def montrePushButton_2(self):
        self.montreAide("apropos.html")
        return

    def appliqueSelection(self):
            self.achercher=self.selectionEdit.text()
            print("le texte était", self.achercher)
            self.montreData()
            return

    def effaceSelection(self):
            self.selectionEdit.setText("")
            self.achercher = ""
            self.montreData()
            print("la, il faudrait ajt du texte")
            return

    def montreData(self):
            self.achercher = self.selectionEdit.text()
            if not self.data:
                   self.data = []
                   with open(self.datafile) as csvfile:
                            lecteur = csv.reader(csvfile)
                            for ligne in lecteur:
                                   self.data.append(ligne)
            html = "
"
            html += "
" + "	".join(self.data[0])+"
"
            ok = False
            for ligne in self.data[1:]:
                   ok = self.achercher in "".join(ligne)
                   if ok:
                                 html += "
" + "	".join(ligne)+"
"
            html += "
"
            self.textEdit.setText(html)
            return

if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    f = MaFenetre()
    f.show()
    sys.exit( app.exec_() )


import argparse
import copy
from multiprocessing import Process
import os
from string import Template
import sys

parent_dir = os.path.abspath(".")
sys.path.append(parent_dir)

import numpy as np

from models import models
from lib import data_utils
from lib import utils

DATA_CATEGORIES = ["train", "val", "test"]

def load_data(data_directory, verbose=0):
    data = {}
    if verbose:
        print("Loading data from directory {}".format(data_directory))

    for category in DATA_CATEGORIES:
        path = "{}/{}.npz".format(data_directory, category)
        if verbose > 1:
            print("Loading datafile at path {}".format(path))

        with open(path, "rb") as f:
            npz = np.load(f)
            data["{}_{}".format(category, "x")] = npz["x"]
            data["{}_{}".format(category, "y")] = npz["y"]

            if verbose > 2:
                print("{}_{} shape: {}".format(category, "x", npz["x"].shape))
                print("{}_{} shape: {}".format(category, "y", npz["y"].shape))

    return data

def get_augment_function(data_augment_dict):
    functions = []
    if "zero" in data_augment_dict:
        zero = data_augment_dict["zero"]

        if "detectors" in zero:
            from scripts.generate_training_data import get_sensors_list
            detectors = zero["detectors"]
            detector_list = get_sensors_list(detectors["detector_list"])
            zero_detector_function = data_utils.zero_out_detectors(detectors["detectors"], detector_list)
            functions.append(zero_detector_function)
        if "days" in zero:
            proportion = zero["days"]["proportion"]
            seed = zero["days"].get("seed", 1)
            zero_days_function = data_utils.zero_out_days(proportion, seed)
            functions.append(zero_days_function)

    augment_function = utils.concatenate_functions(functions)

    return augment_function

def load_models(model_names):
    return [getattr(models, model_name) for model_name in model_names]

def run_models(data, model_configs, model_order=None, overwrite=False, debug=False, verbose=0):
    train_x = data["train_x"]
    train_y = data["train_y"]
    val_x = data["val_x"]
    val_y = data["val_y"]
    test_x = data["test_x"]
    test_y = data["test_y"]

    model_names = list(model_configs.keys()) if model_order is None else model_order
    models = load_models(model_names)

    if verbose:
        print("Starting to run models...")

    if verbose > 2:
        print("Max train_y: {}".format(np.max(train_y)))
        print("Min train_y: {}".format(np.min(train_y)))
        print("Max val_y: {}".format(np.max(val_y)))
        print("Min val_y: {}".format(np.min(val_y)))
        print("Max test_y: {}".format(np.max(test_y)))
        print("Min test_y: {}".format(np.min(test_y)))

    for i in range(len(models)):
        model_name = model_names[i]
        model_class = models[i]

        if isinstance(model_configs[model_name], list):
            model_kwargs = model_configs[model_name]
        elif isinstance(model_configs[model_name], dict) and model_configs[model_name].get("__named_models__", False):
            model_kwargs = [v for k, v in model_configs[model_name].items() if k != "__named_models__"]
        else:
            model_kwargs = [model_configs[model_name]]

        for kwargs in model_kwargs:
            if not overwrite and os.path.exists(kwargs["base_dir"])\
                    and "predictions.npz" in os.listdir(kwargs["base_dir"]):
                if verbose > 1:
                    print("Predictions exist in {}; skipping this model to not overwrite".format(kwargs["base_dir"]))

                continue
            elif debug:
                print("Debug: running model at base_dir {}".format(kwargs["base_dir"]))
                continue

            model = model_class(train_x, train_y, val_x, val_y, test_x, test_y,
                                verbose=verbose, **kwargs)
            if verbose:
                print("Created {} model".format(model_name))

            try:
                model.train()
            except Exception as e:
                model.close()
                print(e)
                continue

            if verbose:
                print("Trained {} model".format(model_name))

            errors = model.get_errors()

            if verbose > 1:
                for category in DATA_CATEGORIES:
                    if category in errors:
                        for key, value in errors[category].items():
                            print("{} {}: {}".format(category, key, value))

            if "base_dir" in kwargs:
                base_dir = kwargs["base_dir"]
                utils.verify_or_create_path(base_dir)
                path = os.path.join(base_dir, "predictions.npz")
                model.save_predictions(path)

            model.close()

def run_config(config, debug=False, verbose=0):
    data_directory = config["data_directory"]
    data = load_data(data_directory, verbose=verbose)
    model_configs = config["models"]
    model_order = config.get("model_order")
    data_augment = config.get("data_augment", {})
    overwrite = config.get("overwrite", False)

    if model_configs:
        if data_augment:
            augment_function = get_augment_function(data_augment)
            data_augmenter = utils.DataAugmenter(data_directory, augment_function)
            data_augmenter.copy()

            try:
                run_models(data, model_configs, model_order=model_order, overwrite=overwrite, debug=debug, verbose=verbose)
            finally:
                data_augmenter.restore()
        else:
            run_models(data, model_configs, model_order=model_order, overwrite=overwrite, debug=debug, verbose=verbose)

def run_configs(configs, debug=False, verbose=0):
    for config in configs:
        run_config(config, debug=debug, verbose=verbose)

def loop_config(config, debug=False, verbose=0):
    loop = config["loop"]
    values = loop["values"]
    keys = loop["keys"]
    is_parallel = "parallel" in loop
    exclude = loop.get("exclude", [])

    loop_mappings = utils.dictionary_product(values)

    if "substitute" in loop:
        for k, v in loop["substitute"].items():
            for mapping in loop_mappings:
                mapping[k] = v["map"][mapping[v["key"]]]

    if is_parallel:
        parallel = loop["parallel"]
        parallel_values = {value: values[value] for value in parallel}
        parallel_keys = utils.ordered_tuple_product(parallel_values, parallel)
        configs = {key: [] for key in parallel_keys}

    for mapping in loop_mappings:
        itr_config = copy.deepcopy(config)
        for loop_key in keys:
            if isinstance(loop_key, str):
                key = loop_key
                type_cast = str
            else:
                key = loop_key[0]
                type_cast = utils.get_type_by_name(loop_key[1])

            c = itr_config
            config_path = key.split("/")
            for k in config_path[:-1]:
                c = c[k]

            template = Template(c[config_path[-1]])
            c[config_path[-1]] = type_cast(template.substitute(mapping))

        for exclusions in exclude:
            exclude_values = exclusions["values"]
            if all([exclude_values[k] == mapping[k] for k in exclude_values]):
                c = itr_config
                key = exclusions["key"]
                config_path = key.split("/")
                for k in config_path[:-1]:
                    c = c[k]

                del c[config_path[-1]]

        if is_parallel:
            mapping_key = tuple(mapping[value] for value in parallel_values)
            configs[mapping_key].append(itr_config)
        else:
            run_config(itr_config, verbose=verbose)

    if is_parallel:
        processes = []
        for itr_configs in configs.values():
            p = Process(target=run_configs, args=(itr_configs,), kwargs={"debug": debug, "verbose": verbose})
            processes.append(p)

        utils.run_process_list_parallel(processes)

def main(args):
    verbose = args.verbose
    config = utils.load_yaml(args.config)
    debug = args.debug

    if "loop" in config.keys():
        loop_config(config, debug=debug, verbose=verbose)
    else:
        run_config(config, debug=debug, verbose=verbose)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("config", help="config file to specify data and model")
    parser.add_argument("-d", "--debug", action="store_true", help="Execute model runner without running models")
    parser.add_argument("-v", "--verbose", action="count", default=0, help="verbosity")
    args = parser.parse_args()

    main(args)


#!/usr/bin/python
# -------------------------------------------------------------------------------
# Name:        UnDirectedGraph
# Purpose:     Implement undirected graph in Python
#              Used to test the timings of
# Author:      Di Zhuang
# Created:     08/24/2015
# Version:     1.0
# Copyright:   (c) Di Zhuang 2015
# -------------------------------------------------------------------------------

from collections import deque
import numpy as np
from scipy.constants import golden
from time import time

def count(fn):
    def decorator(*args, **kwargs):
        decorator.called += 1
        start = time()
        result = fn(*args, **kwargs)
        t = time() - start
        decorator.time += t
        return result
    decorator.__name__ = fn.__name__
    decorator.called = 0
    decorator.time = 0
    return decorator

class Element(object):
    """
    Element is the internal data structure that holds the priority and object.

    self.obj: any object (e.g., a vertex or an edge for a graph algorithm)
    self.priority: priority of this node
    self.left, self.right: pointers to the siblings of this node
    self.parent: the parent of this node
    self.child: one of the children of this node
    self.degree: the degree of this node
    self.mark: whether this node has lost one of its children
    """

    def __init__(self, obj, priority):
        self.obj, self.priority = obj, priority
        self.left = self.right = self
        self.child = None
        self.parent = None
        self.degree = 0
        self.mark = False

    def get_value(self):
        """
        :return: object cached in this node
        """
        return self.obj

    def get_priority(self):
        """
        :return: priority of this node
        """
        return self.priority

    def children(self):
        """
        :return: all children of this node as a list
        """
        children = []
        x = self.child
        if x:
            while True:
                children.append(x)
                x = x.right
                if x == self.child:
                    break

        return children

    def siblings(self):
        """
        :return: all siblings of this node as a list
        """
        siblings = []
        x = self
        while True:
            siblings.append(x)
            x = x.right
            if x == self:
                break

        return siblings

    def __str__(self):
        # return "Element(obj={}, priority={})".format(self.obj, self.priority)  #  Fancier format
        return "({}, {})".format(self.obj, self.priority)

class FibonacciHeap(object):
    """
    Implements Fibonacci Heap
    """

    def __init__(self):
        """
        Creates an empty heap
        """
        self._size = 0
        self._min = None
        self._dict = {}

    def __getitem__(self, item):
        try:
            return self._dict[item]
        except KeyError:
            raise KeyError("Object %s no longer in heap!" % item)

    @count
    def insert(self, x, priority):
        """
        Insert a (x, priority) pair into the heap.
        Specifically, add the new node into the root list. If the priority
        of this node is the highest (lowest numerical value), update self._min_elem
        to reflect that.

        O(1) operation.

        :param x: obj associated with priority
        :param priority: priority of the object
        :return: an reference to the inserted node
        """

        elem = Element(x, priority)
        self._dict[x] = elem

        self._insert_to_root_list(elem)

        if self._min is None:  # if heap is empty
            self._min = elem
        elif self._min.priority > elem.priority:
            self._min = elem

        self._size += 1

        return elem

    def min(self):
        """
        :return: a reference to the minimal element in heap H
        """
        return self._min

    @count
    def extract_min(self):
        """
        Returns an reference to the minimal element in heap H and removes it from heap
        :return: an reference to element with the minimal priority value
        """
        z = self._min

        if z is not None:
            x = z.child

            if x is not None:
                # promote all children of the min to the root level
                while True:
                    next_child = x.right
                    x.parent = None
                    self._insert_to_root_list(x)
                    x = next_child
                    if x == z.child:
                        z.child = None
                        break

            if z == z.right:
                # if this is the only element in the heap, set self._min to None
                self._min = None
                self._remove_from_root_list(z)
            else:
                self._min = z.right
                self._remove_from_root_list(z)
                self._consolidate()

            self._size -= 1

        del self._dict[z.obj]

        return z

    @count
    def decrease_key(self, x, new_priority):
        """
        Assigns to element x within heap H the new key value k,
        which is assumed to be less than or equal to its current key value

        :param x: an reference to the node in the heap
        :param new_priority: new priority of x
        :return:
        :raise: ValueError if the new priority is not strictly less than the old priority
        """
        if new_priority >= x.priority:
            raise ValueError("Decrease key: new priority value (%d) must "
                             "be less than old priority (%d)!"
                             % (new_priority, x.priority))

        x.priority = new_priority
        y = x.parent

        # if x is not at the root level, then cut x from the tree
        # and hang it in the root list
        if y is not None and x.priority < y.priority:
            self._cut(x, y)
            self._cascading_cut(y)

        # if x's new priority is less than the current minimal priority, change
        # the minimal of heap to x
        if x.priority < self._min.priority:
            self._min = x

    @count
    def delete(self, x):
        """
        Delete element x from heap. To maintain heap invariant, this operation
        is just 1 decrease key and 1 extract min operation.  Therefore,
        this takes O(log(n)) time.

        :param x: element to be deleted
        :return:
        """
        self.decrease_key(x, -np.inf)
        self.extract_min()

    @count
    def size(self):
        """
        :return: number of nodes in the heap
        """
        return self._size

    @count
    def merge(self, heap):
        """
        Merge this Fibonacci heap with another one.
        The time is O(1) since it just splices two circular doubly linked lists together.
        :param heap: another Fibonacci heap
        :return: None
        """
        assert isinstance(heap, FibonacciHeap), \
            "Invalid heap!"

        min_one = self._min
        min_two = heap.min()

        if min_one:
            min_one_right = self._min.right

            if min_two != min_two.right:  # if more than 1 root node in heap
                min_one.right, min_two.right.left, min_two.right, min_one_right.left = \
                    min_two.right, min_one, min_one_right, min_two
            else:
                self._insert_to_root_list(min_two)
        else:
            self._min = min_two

        self._size += heap.size()
        heap._clear()

    @count
    def __len__(self):
        """
        :return: number of nodes in the heap
        """
        return self.size()

    @count
    def _insert_to_root_list(self, elem):
        """
        Insert element into the root level of the Fibonacci heap

        :param elem: a reference to the elem to be inserted
        :return:
        """
        if elem.parent is not None:
            elem.parent = None

        if self._min:
            self._doublylinkedlist_insert(self._min, elem)
        else:
            self._min = elem

    @staticmethod
    def _doublylinkedlist_insert(head, y):
        """
        Given a node x in a circular doubly linked list, add y to the list.
        :param head: a node in a circular doubly linked list
        :param y: node to be inserted into the list
        :return:
        """
        if head is not None:
            tail = head.right
            tail.left, head.right, y.left, y.right = y, y, head, tail

    def _remove_from_root_list(self, z):
        """
        Remove a node from the root list in the Fibonacci heap.
        Note, z's children are untouched.

        :param z: a root level node
        :return: None
        """
        assert z.parent is None, \
            "Node should be at root level!"
        if z.right == z:
            self._min = None
        else:
            if z == self._min:
                # if the removed node is the minimal node, designate an arbitrary one to the be new min
                self._min = z.right
            z.left.right = z.right
            z.right.left = z.left
            z.left = z.right = z

    @count
    def _consolidate(self):
        """
        Perform an operation to enforce the forest rule, which means at the end of
        the consolidation, no two root trees can have the same degree.
        To accomplish this, this operation merges two trees of the same degree (d)
        into 1 tree with degree (d+1) similar to addition in binary.
        :return:
        """
        array = [None] * int(np.ceil(np.log(self._size)/np.log(golden)))

        # enqueue all the root nodes into a queue since their pointers will be overwritten
        # root list will be empty at the end of the loop
        q = deque()
        while self._min:
            q.append(self._min)
            self._remove_from_root_list(self._min)

        while len(q):
            x = q.popleft()
            d = x.degree

            while array[d] is not None:
                # print d
                y = array[d]

                if x.priority > y.priority:
                    # this ensures that the node with the minimal priority will be the parent
                    x, y = y, x
                self._heap_link(y, x)
                array[d] = None
                d += 1

            array[d] = x

        # add all the new trees from the list back into the root list
        for elem in array:
            if elem is not None:
                if self._min is None:
                    self._insert_to_root_list(elem)
                else:
                    self._insert_to_root_list(elem)
                    if elem.priority < self._min.priority:
                        self._min = elem

    def _heap_link(self, y, x):
        """
        Link a node y to x, such that y become x's children
        :param y: y is to become the children of x
        :param x: x is to become the parent of y
        :return:
        """
        assert not y.priority < x.priority, \
            "heap link: priority invariant is violated!"
        if x.child:
            self._doublylinkedlist_insert(x.child, y)
        else:
            x.child = y
        y.parent = x
        x.degree += 1
        y.mark = False

    @count
    def _cut(self, x, y):
        """
        Cut x from the list of children in y and hang it in the root list
        :param x: a child of y
        :param y: the parent of x
        :return:
        """
        # Note that y.child may not be x if y has more than 1 children
        assert x.parent == y, "Cut: nodes are not parent-child!"

        if x.right == x:  # x is the only child of y
            y.child = None
        else:  # remove x from the list of children of y
            y.child = x.right
            x.right.parent = y
            x.right.left = x.left
            x.left.right = x.right

        self._insert_to_root_list(x)
        y.degree -= 1  # decrement the degree of y since it has just lost a child

        # x.parent = None
        x.mark = False

    @count
    def _cascading_cut(self, y):
        """
        Cascading cut operation in Fibonacci Heap. If y is not at
        the root level or if y's parent has already lost one of its children (a.k.a, marked),
        recursively cut y's parent.
        :param y: a node in the heap
        :return:
        """
        z = y.parent
        if z is not None:
            if not y.mark:
                y.mark = True
            else:
                self._cut(y, z)
                self._cascading_cut(z)

    def _clear(self):
        """
        Empty the heap. Note, the nodes are NOT actually deleted from memory.
        As such, this can lead to memory leaks.  However, this should be called
        by the merge operation.
        :return:
        """
        self._min = None
        self._size = 0


import ddt

from openshiftstoragelibs.baseclass import BaseClass
from openshiftstoragelibs import heketi_ops
from openshiftstoragelibs import heketi_version
from openshiftstoragelibs import openshift_ops

@ddt.ddt
class TestHeketiServerStateExamineGluster(BaseClass):

    def setUp(self):
        super(TestHeketiServerStateExamineGluster, self).setUp()
        self.node = self.ocp_master_node[0]
        version = heketi_version.get_heketi_version(self.heketi_client_node)
        if version < '8.0.0-7':
            self.skipTest("heketi-client package %s does not support server "
                          "state examine gluster" % version.v_str)

    def test_volume_inconsistencies(self):
        # Examine Gluster cluster and Heketi that there is no inconsistencies
        out = heketi_ops.heketi_examine_gluster(
            self.heketi_client_node, self.heketi_server_url)
        if ("heketi volume list matches with volume list of all nodes"
                not in out['report']):
            self.skipTest(
                "heketi and Gluster are inconsistent to each other")

        # create volume
        vol = heketi_ops.heketi_volume_create(
            self.heketi_client_node, self.heketi_server_url, 1, json=True)
        self.addCleanup(
            heketi_ops.heketi_volume_delete, self.heketi_client_node,
            self.heketi_server_url, vol['id'])

        # delete volume from gluster cluster directly
        openshift_ops.cmd_run_on_gluster_pod_or_node(
            self.node,
            "gluster vol stop %s force --mode=script" % vol['name'])
        openshift_ops.cmd_run_on_gluster_pod_or_node(
            self.node,
            "gluster vol delete %s --mode=script" % vol['name'])

        # verify that heketi is reporting inconsistencies
        out = heketi_ops.heketi_examine_gluster(
            self.heketi_client_node, self.heketi_server_url)
        self.assertNotIn(
            "heketi volume list matches with volume list of all nodes",
            out['report'])

    @ddt.data('', 'block')
    def test_compare_real_vol_count_with_db_check_info(self, vol_type):
        """Validate file/block volumes using heketi db check."""

        # Create File/Block volume
        block_vol = getattr(heketi_ops, 'heketi_%svolume_create' % vol_type)(
            self.heketi_client_node, self.heketi_server_url, 1, json=True)
        self.addCleanup(
            getattr(heketi_ops, 'heketi_%svolume_delete' % vol_type),
            self.heketi_client_node, self.heketi_server_url, block_vol["id"])

        # Check Heketi DB using Heketi CLI
        db_result = heketi_ops.heketi_db_check(
            self.heketi_client_node, self.heketi_server_url)
        vol_count = db_result["%svolumes" % vol_type]["total"]
        vol_list = getattr(heketi_ops, 'heketi_%svolume_list' % vol_type)(
            self.heketi_client_node, self.heketi_server_url, json=True)
        count = len(vol_list["%svolumes" % vol_type])
        self.assertEqual(
            count, vol_count,
            "%svolume count doesn't match expected "
            "result %s, actual result is %s" % (vol_type, count, vol_count))

    def test_compare_node_count_with_db_check_info(self):
        """Validate nodes count using heketi db check"""

        # Check heketi db
        db_result = heketi_ops.heketi_db_check(
            self.heketi_client_node, self.heketi_server_url)
        db_nodes_count = db_result["nodes"]["total"]
        nodes_list = heketi_ops.heketi_node_list(
            self.heketi_client_node, self.heketi_server_url, json=True)
        calculated_nodes_count = len(nodes_list)
        self.assertEqual(
            calculated_nodes_count, db_nodes_count,
            "Nodes count from 'DB check' (%s) doesn't match calculated nodes "
            "count (%s)." % (db_nodes_count, calculated_nodes_count))


# %load q02_best_k_features/build.py
# Default imports
import numpy as np
import pandas as pd

data = pd.read_csv('data/house_prices_multivariate.csv')

from sklearn.feature_selection import SelectPercentile
from sklearn.feature_selection import f_regression

# Write your solution here:
def percentile_k_features(data,k=20):
    X=data.iloc[:,:-1]
    y=data.iloc[:,-1]
    f_test, _ = f_regression(X, y)
    fs=SelectPercentile(percentile =k)
    X_fs=fs.fit_transform(X,y)
    col=np.asarray(X.columns.values)
    support=np.asarray(fs.get_support())
    col_with_support=col[support]
    fs_score = list(zip(features,scores))
    df = pd.DataFrame(fs_score,columns=['Name','Score'])
    return df.sort_values(['Score','Name'],ascending = [False,True])['Name'].tolist()

    return col_with_support

    sp = SelectPercentile(f_regression,percentile=k)
    sp.fit_transform(X,y)
    features = X.columns.values[sp.get_support()]
    scores = sp.scores_[sp.get_support()]
    fs_score = list(zip(features,scores))
    df = pd.DataFrame(fs_score,columns=['Name','Score'])
    return df.sort_values(['Score','Name'],ascending = [False,True])['Name'].tolist()

print(percentile_k_features(data))


import os
import glob
import SimpleITK as sitk
import numpy as np
import cv2

DATA_PATH = '/home/baiyu/Downloads/promise12'
IMAGE_PATH = '/home/baiyu/Downloads/promise12/images'

def normalize(img):
    img = img.astype(float)
    img = np.maximum(img,0) / img.max() * 255.0
    img = np.uint8(img)

    return img

for path in glob.iglob(os.path.join(DATA_PATH, '**', '*.mhd'), recursive=True):

    itkimage = sitk.ReadImage(path)
    mr_scan = sitk.GetArrayFromImage(itkimage)

    base_name = os.path.basename(path)

    for slc_idx, slc in enumerate(mr_scan):

        image_name = base_name.split('.')[0]

        if 'TrainingData_Part' in path:
            if 'segmentation' in path:
                image_name = "{}_slice{}_{}.png".format(image_name, slc_idx, 'train')
                slc *= 255
                slc = np.uint8(slc)
            else:
                image_name = "{}_slice{}_{}.jpg".format(image_name, slc_idx, 'train')
                slc = normalize(slc)

        else:
            image_name = "{}_slice{}_{}.jpg".format(image_name, slc_idx, 'test')
            slc = normalize(slc)

        save_path = os.path.join(IMAGE_PATH, image_name)

        cv2.imwrite(save_path, slc)


import requests
from bs4 import BeautifulSoup as bs
from urllib.parse import urljoin

def soup_check():
    url = 'http://toanchuccaothu.com/'
    r = requests.get(url)
    html_contents = r.text
    html_soup = bs(html_contents, 'html.parser')
    print(html_soup.find('h1'))

    for found in html_soup.find_all(['h1','h2']):
        print(found)

base_web_crawler_url ='http://webscrapingfordatascience.com/crawler/'
another_web_crawler_url ='https://vikings.gg/news'
links_seen = set()

def visit(url, links_seen):
    html = requests.get(url).text
    html_soup_crawl = bs(html,'html.parser')
    links_seen.add(url)
    for link in html_soup_crawl.find_all('a'):
        link_url = link.get('href')
        if link_url is None:
            continue
        full_url = urljoin(url, link_url)
        if full_url in links_seen:
            continue
        print('Found a new page:',full_url)
        #we store the results too
        visit(full_url,links_seen)

visit(another_web_crawler_url,links_seen)


"""
Responsible for grabbing images from four cameras.

Note
----
This will hopefully be extended to work for different camera configurations, but for now focuses
on the configuration we need for successful implementation of the project. It is not to be used
directly by the user and is instead meant to contain functions that are called programmatically.

Also, it is important to take into account that the order cameras are plugged into the device
matters a great deal with the current setup. You must plug them in clock-wise or counter-clockwise.
Direction should not matter, but consistency of direction does.

Another thing to mention, this is greatly simplified from previous work in order to make performance
as good as possible.

Examples
--------
This represents the part of the pipeline that interacts with the cameras. This module is meant to
be leveraged to grab frames for sending to other parts of the application.

"""

from __future__ import absolute_import, division, print_function

import os
import errno

import cv2 as opencv

# Make sure that range and xrange are used appropriately based on
# the current Python version.

try:
    xrange
except NameError:
    xrange = range # pylint: disable=W0622

def save_frames_from_frames_list(frames_list, out_path="out"):
    """
    Save frames in frames_list to output directory.

    Objective is to test in development whether or not real frames
    are being grabbed from the feeds. This is to distinguish between
    technically valid frames that may or not be the image coming from the
    cameras and those that definitely are.

    Args:
        frames_list: list of frames from camera feeds.

    Returns:
        Saves frames to output directory.
    """

    ensure_available_output_path(out_path)

    counter = 0
    for frame in frames_list:
        counter += 1
        opencv.imwrite("%s/%s.jpg" % (out_path, counter), frame)

def ensure_available_output_path(path):
    """
    Ensure that provided output path exists in current directory.
    """

    try:
        os.makedirs(path)
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise

def grab_frames_from_feeds(feed_list):
    """
    Grab frame from each of available four CameraFeeds.

    Does not verify if feeds in list are open or valid.
    Should be paired with create_camera_feeds() to
    ensure that CameraFeeds are verified.

    Args:
        feed_list: list of camera feeds

    Returns:
        frames_list: list of frames from camera feeds
    """
    frames_list = []

    for feed in feed_list:
        if feed is not None:
            frames_list.append(feed.get_next())

    return frames_list

def grab_frame_from_feed(feed):
    """
    Grab frame from individual feed.

    Does not verify if feed is open or valid.
    Should be paired with create_camera_feeds() to
    ensure that CameraFeeds are verified.

    Args:
        feed: a single camera feed

    Returns:
        frame: a single frame from the provided camera feed
    """
    frame = feed.get_next()

    return frame

def create_camera_feeds():
    """
    Create four CameraFeeds and return in list if all are open and valid.

    Implicitly verifies that returned list of feeds contains open
    and valid feeds.

    For linux, first four feeds are used. For OS X, because webcam
    index switches from 0 to 2 when four cameras are plugged in (for
    no apparent reason), indices 0, 1, 3, and 4 are used.

    Returns:
        feed_list: list of four open and valid camera feeds
    """

    feed_list = []

    # Add unvalidated camera feeds to feed list.
    for feed_index in xrange(0, 5):
        feed = CameraFeed(feed_index)
        if feed.camera_feed.isOpened():
            feed_list.append(feed)
            print("Index %s is valid!" % feed.feed_index)

    list_length = len(feed_list)
    if list_length > 1:
        print("There are %s feeds available." % list_length)
    else:
        print("There is %s feed available." % list_length)
    return feed_list

class CameraFeed(object):
    """
    Define behavior of CameraFeed.
    """

    def __init__(self, feed_index):
        self.feed_index = feed_index
        self.check_index()
        self.camera_feed = opencv.VideoCapture(feed_index)

    def get_next(self):
        """
        Get the next frame from the CameraFeed. If frame does not exist, throw error.
        """

        is_valid, frame = self.camera_feed.read()

        if is_valid:
            return frame
        else:
            raise IOError('Could not grab a valid frame.')

    def check_index(self):
        """
        Check to make sure index is greater than -1 and less than 5.
        """

        if not(self.feed_index > -1 and self.feed_index < 5):
            raise ValueError('Camera index is not valid.')

    def ramp(self, num_frames=30):
        """
        Ramp feed to prepare for capture and data relay.
        """

        for _ in xrange(num_frames):
            self.get_next()

    def show(self):
        """
        Show CameraFeed in desktop gui window.

        Repeatedly get frame from CameraFeed, resize frame,
        and display frame in window.
        """

        while self.camera_feed.isOpened():
            frame = self.get_next()
            frame = resize_frame(frame)
            title = "Camera Feed %s" % self.feed_index
            opencv.imshow(title, frame)
            key = opencv.waitKey(1) & 0xFF
            if key == ord("q"):
                break
        opencv.destroyAllWindows()

    def close(self):
        """
        Close the CameraFeed.
        """

        self.camera_feed.release()

def resize_frame(frame):
    """
    Resize frame to have a width of 400 pixels.

    Note:
        Our specific camera setup outputs frames via usb that are 2448 rows tall
        and 3264 columns wide. We resize in a manner that keeps the ratio consistent,
        setting number of rows to 300 and number of columns to 400.

    Args:
        frame: height of 2448 pixels and width of 3264 pixels.

    Returns:
        frame: height of 300 pixels and width of 400 pixels.
    """

    return opencv.resize(frame, (400, 300))


import numpy as np
from matplotlib import pyplot as plt

from pyschieber.card import from_string_to_card
from pyschieber.deck import Deck
from pyschieber.trumpf import Trumpf

'''
|  36 hand cards  |   table cards (36 cards per player)   |   history (36 cards per stich per player)    | trumpf |
|-----------------|---------------------------------------|----------------------------------------------|--------|

|          36 table cards per each player           |
|---------------------------------------------------|
|| player 1  |  player 2  |  player 3  |  player 4 ||
|------------|------------|------------|------------|
-->> This is the current stich, because that are the most important cards for the current desicion
-->> That is the reason too, why we make the history just to 8 stichs

|          36 cards per stich per player (history of the stichs from the current game without the last stich)         |
|---------------------------------------------------------------------------------------------------------------------|
|           stich 1         ||           stich 2         ||           stich 3         |...|           stich 8         |
|---------------------------||---------------------------||---------------------------|...|---------------------------|
|   p1 |   p2 |   p3 |   p4 ||   p1 |   p2 |   p3 |   p4 ||   p1 |   p2 |   p3 |   p4 |...|   p1 |   p2 |   p3 |   p4 |

36 + 4 * 36 + 8 * 4 * 36 + 6 = 1'338
'''

class InputHandler:
    deck = Deck()
    trumpfs = [trumpf.name for trumpf in list(Trumpf)]

    nr_cards = 36
    nr_trumpf_modes = 6
    amount_players = 4
    amount_card_sets = 37

    pos_hand_cards = 0
    pos_player_on_table = [1 * nr_cards, 2 * nr_cards, 3 * nr_cards, 4 * nr_cards]
    # pos_players_per_stich is the history
    pos_trumpf = amount_card_sets * nr_cards

    input_size = amount_card_sets * nr_cards + nr_trumpf_modes  # 1338 input neurons
    output_size = nr_cards

    def __init__(self):
        self.state = None
        self.reset()
        self.pos_players_per_stich = self.generate_player_per_stich(self.amount_players + 1, self.amount_card_sets)

    def reset(self):
        self.state = np.zeros(self.input_size, dtype='float32')

    def update_state_stich(self, game_state, cards, player_id):
        table_list = analyse_table(game_state['table'])
        self.set_history(table_list, cards)

    def update_state_choose_card(self, game_state, cards, player_id):
        table_list = analyse_table(game_state['table'])
        self.set_hand(cards)
        self.set_table(table_list)
        self.set_history(table_list, cards)
        self.set_trumpf(trumpf=game_state['trumpf'])

    def set_history(self, table, cards):
        max_history_stich = 8
        amount_hand_cards = len(cards)
        stich_number = max_history_stich - amount_hand_cards
        if stich_number < max_history_stich:
            for player_id, card in table:
                self.state[self.pos_players_per_stich[stich_number][player_id] + card_to_index(card)] = 1.

    def set_table(self, table):
        self.state[self.pos_player_on_table[0]:self.pos_player_on_table[3] + self.nr_cards] = 0
        for player_position, card in table:
            self.state[self.pos_player_on_table[player_position] + card_to_index(card)] = 1.

    def set_hand(self, cards):
        self.state[self.pos_hand_cards:self.pos_hand_cards + self.nr_cards] = 0.
        for card in cards:
            self.state[self.pos_hand_cards + card_to_index(card)] = 1.

    def set_trumpf(self, trumpf):
        self.state[self.pos_trumpf + self.trumpfs.index(trumpf)] = 1.

    def generate_player_per_stich(self, start_range, end_range):
        player_collector = []
        pos_players_in_stich = []

        for player in range(start_range, end_range):
            player_collector.append(player * self.nr_cards)
            if len(player_collector) == 4 or player == end_range:
                pos_players_in_stich.append(player_collector)
                player_collector = []

        return pos_players_in_stich

def card_to_index(card):
    return InputHandler.deck.cards.index(card)

def index_to_card(index):
    return InputHandler.deck.cards[index]

def analyse_table(table):
    table_list = []
    for played_card in table:
        player_id = played_card['player_id']
        card = from_string_to_card(played_card['card'])
        table_list.append((player_id, card))
    return table_list

def print_state(input_state, player_id):
    y = np.reshape(input_state, (1, InputHandler.input_size))
    plt.imshow(y, cmap='gray')
    title_obj = plt.title('Player: {}'.format(player_id))
    plt.setp(title_obj, color='black')
    plt.show()


import cv2
import numpy as np
from sklearn import mixture
import sys
import os
from matplotlib import pyplot as plt
from numba import jit

'''
Note:
1. Using 5 cluster GMM to improve pre-maxflow segmentation accuracy
2. Using F-F with the help from numba to speed up repetitive calculation
'''

####################
# Helper functions #
####################

@jit
def DS(img,i): # Downsampling

    img_small = cv2.resize(img,None,fx=1/i, fy=1/i, interpolation = cv2.INTER_CUBIC)
    dimen_1, dimen_2, _ = img_small.shape
    img_flat = np.concatenate((img_small[:,:,0].flatten().reshape(-1,1),
                               img_small[:,:,1].flatten().reshape(-1,1),
                               img_small[:,:,2].flatten().reshape(-1,1)),axis=1)

    return img_flat, dimen_1, dimen_2

@jit
def GMM(img_flat, dimen_1, dimen_2): # GMM clustering, 5 clusters in total

    gmm_tmp = mixture.GaussianMixture(n_components=5, covariance_type='full', max_iter=500, n_init=5)
    gmm_tmp.fit(img_flat)
    tmp = np.argmax(gmm_tmp.predict_proba(img_flat),axis=1).reshape(-1,1)
    tmp0 = tmp.reshape(dimen_1, dimen_2).astype('float16')

    return tmp0

@jit
def BFS(graph,e): # BFS is written in function becuase it is recursivly called
    q = [0]
    pred = [0] * len(graph) # prevent a vertice been visited twice
    pred[0]=1
    while len(q)>0:
        cur = q.pop(0)
        for index, item in enumerate(graph[cur]):
            if pred[index]==0 and index!=0 and item>0:
                pred[index]=1
                q.append(index)
                e[index]=cur # record path to sink

    if pred[len(graph)-1]==0:
        return False, pred, e
    else:
        return True, pred, e

@jit
def F_F(graph): # Maxflow

    max_flow = 0 # initialize flow
    e = [-1]*len(graph) # record all paths

    # Find a sortest path in residual graph
    path_exist, _, _ = BFS(graph,e)
    while path_exist:
        df = float('inf')

        # Find the minimum flow on the path in the residual graph
        end = num_of_vertices-1
        while end!=0:
            df = min(df, graph[e[end]][end])
            end = e[end]

        # update residual graph in the opposite direction of flow graph
        end = num_of_vertices-1
        while end!=0:
            start = e[end]
            graph[end][start] += df
            graph[start][end] -= df
            end = start

        max_flow += df
        path_exist, _, _ = BFS(graph,e)

    return graph

@jit
def pplot1(i,j,tmp0): # Merge two clusters

    tmp1=np.zeros((dimen_1, dimen_2))
    tmp1[tmp0==i]=1; tmp1[tmp0==j]=1

    return tmp1.astype('int16')

@jit
def pplot2(i,tmp0): # Merge three clusters

    tmp1=np.zeros((dimen_1, dimen_2))
    tmp1[tmp0==i]=1

    return tmp1.astype('int16')

########
# Main #
########

# Read image
img = cv2.imread(sys.argv[1])

global height, width
dimen_limit = 300

# current_path = os.getcwd()
# image_path = current_path.replace('Python','Images')
# img = cv2.imread(image_path + '\\' + 'FB' + '\\'+ '4' + '.jpg') # 4,6,9

# Find downsampling dimension for GMM
height, width, _ = img.shape
large = max(height,width)
for i in range(1,100):
    if large/i < dimen_limit:
        break

# Downsampling for GMM
img_flat, dimen_1, dimen_2 = DS(img,i)
# Five cluster GMM
tmp0 = GMM(img_flat, dimen_1, dimen_2)

# Book keeping all 15 GMM result so user can chose one of them to perform maxflow
a1 =pplot1(0,1,tmp0); a2 =pplot1(0,2,tmp0);a3 =pplot1(0,3,tmp0);a4 =pplot1(0,4,tmp0);a5 =pplot1(1,2,tmp0);a6 =pplot1(1,3,tmp0)
a7 =pplot1(1,4,tmp0);a8 =pplot1(2,3,tmp0);a9 =pplot1(2,4,tmp0);a10 =pplot1(3,4,tmp0);a11 =pplot2(0,tmp0);a12 =pplot2(1,tmp0);
a13 =pplot2(2,tmp0);a14 =pplot2(3,tmp0);a15 =pplot2(4,tmp0)
img_choice = {'1':a1,'2':a2,'3':a3,'4':a4,'5':a5,'6':a6,'7':a7,'8':a8,'9':a9,
              '10':a10,'11':a11,'12':a12,'13':a13,'14':a14,'15':a15}

# Plot all 15 GMM images for user to select the most correct one
fig,axe=plt.subplots(4,5,figsize=(20,12))
axe[0,1].imshow(img);axe[0,1].set_title('Original Image');axe[0,1].set_xticklabels([]);axe[0,1].set_yticklabels([])
axe[0,3].imshow(np.ceil(tmp0*255).astype('int32'), cmap="hot");axe[0,3].set_title('Clustered Image');
axe[0,3].set_xticklabels([]);axe[0,3].set_yticklabels([])
axe[1,0].imshow(cv2.resize(pplot1(0,1,tmp0), (width,height)), cmap="hot");axe[1,0].set_title('1');axe[1,0].set_xticklabels([])
axe[1,0].set_yticklabels([])
axe[1,1].imshow(cv2.resize(pplot1(0,2,tmp0), (width,height)), cmap="hot");axe[1,1].set_title('2');axe[1,1].set_xticklabels([])
axe[1,1].set_yticklabels([])
axe[1,2].imshow(cv2.resize(pplot1(0,3,tmp0), (width,height)), cmap="hot");axe[1,2].set_title('3');axe[1,2].set_xticklabels([])
axe[1,2].set_yticklabels([])
axe[1,3].imshow(cv2.resize(pplot1(0,4,tmp0), (width,height)), cmap="hot");axe[1,3].set_title('4');axe[1,3].set_xticklabels([])
axe[1,3].set_yticklabels([])
axe[1,4].imshow(cv2.resize(pplot1(1,2,tmp0), (width,height)), cmap="hot");axe[1,4].set_title('5');axe[1,4].set_xticklabels([])
axe[1,4].set_yticklabels([])
axe[2,0].imshow(cv2.resize(pplot1(1,3,tmp0), (width,height)), cmap="hot");axe[2,0].set_title('6');axe[2,0].set_xticklabels([])
axe[2,0].set_yticklabels([])
axe[2,1].imshow(cv2.resize(pplot1(1,4,tmp0), (width,height)), cmap="hot");axe[2,1].set_title('7');axe[2,1].set_xticklabels([])
axe[2,1].set_yticklabels([])
axe[2,2].imshow(cv2.resize(pplot1(2,3,tmp0), (width,height)), cmap="hot");axe[2,2].set_title('8');axe[2,2].set_xticklabels([])
axe[2,2].set_yticklabels([])
axe[2,3].imshow(cv2.resize(pplot1(2,4,tmp0), (width,height)), cmap="hot");axe[2,3].set_title('9');axe[2,3].set_xticklabels([])
axe[2,3].set_yticklabels([])
axe[2,4].imshow(cv2.resize(pplot1(3,4,tmp0), (width,height)), cmap="hot");axe[2,4].set_title('10');axe[2,4].set_xticklabels([])
axe[2,4].set_yticklabels([])
axe[3,0].imshow(cv2.resize(pplot2(0,tmp0), (width,height)), cmap="hot");axe[3,0].set_title('11');axe[3,0].set_xticklabels([])
axe[3,0].set_yticklabels([])
axe[3,1].imshow(cv2.resize(pplot2(1,tmp0), (width,height)), cmap="hot");axe[3,1].set_title('12');axe[3,1].set_xticklabels([])
axe[3,1].set_yticklabels([])
axe[3,2].imshow(cv2.resize(pplot2(2,tmp0), (width,height)), cmap="hot");axe[3,2].set_title('13');axe[3,2].set_xticklabels([])
axe[3,2].set_yticklabels([])
axe[3,3].imshow(cv2.resize(pplot2(3,tmp0), (width,height)), cmap="hot");axe[3,3].set_title('14');axe[3,3].set_xticklabels([]);
axe[3,3].set_yticklabels([])
axe[3,4].imshow(cv2.resize(pplot2(4,tmp0), (width,height)), cmap="hot");axe[3,4].set_title('15');axe[3,4].set_xticklabels([])
axe[3,4].set_yticklabels([])
fig.delaxes(axe[0,0]);fig.delaxes(axe[0,2]);fig.delaxes(axe[0,4])
plt.show()

#######
# F-F #
#######

# Choose the image by user
which_img = input('Which image is the most similar to the correct answer? ')
ff_input = img_choice[which_img]
downsampleto = 150
img_gmm = cv2.resize(ff_input,(downsampleto,downsampleto), interpolation = cv2.INTER_CUBIC)

# Set up flow capacity
background_cap = img_gmm * 10
background_cap_flat = background_cap.astype('int32').flatten()

# Convert graph into adjacency matrix
num_of_vertices = 2 + downsampleto**2
graph_capacity = np.zeros((num_of_vertices,num_of_vertices))
graph_capacity[0,1:downsampleto**2+1] = background_cap_flat
graph_capacity[1:downsampleto**2+1,downsampleto**2+1] = (10 - background_cap_flat)

for i in range(downsampleto):
    for j in range(downsampleto):
        tmp_img = np.zeros((downsampleto,downsampleto))
        if i-1>=0:
            tmp_img[i-1][j] = 10
        if j-1>=0:
            tmp_img[i][j-1] = 10
        if j+1<downsampleto:
            tmp_img[i][j+1] = 10
        if i+1<downsampleto:
            tmp_img[i+1][j] = 10
        graph_capacity[i*downsampleto+j+1,1:downsampleto**2+1] = tmp_img.flatten()

# residual graph started with max capasity
graph = graph_capacity.copy()

graph = F_F(graph)

# Run BFS to find the segmentations
_, pred, _ = BFS(graph, [-1]*(num_of_vertices-1))
tmp = np.asarray(pred)[1:num_of_vertices-1].reshape(downsampleto,downsampleto).astype('int16')

# Form mask to segment image
shrink_ratio = 3
img_third = cv2.resize(img, (round(width/shrink_ratio), round(height/shrink_ratio)), interpolation=cv2.INTER_CUBIC)
resized_image = cv2.resize(tmp.astype('float32'),
                           (round(width/shrink_ratio), round(height/shrink_ratio)),
                           interpolation=cv2.INTER_CUBIC).round()
mask1 = np.zeros((round(height/shrink_ratio), round(width/shrink_ratio),3))
mask2 = np.zeros((round(height/shrink_ratio), round(width/shrink_ratio),3))

mask1[:,:,0] = mask1[:,:,2] = mask1[:,:,1] = 1-resized_image
mask2[:,:,0] = mask2[:,:,2] = mask2[:,:,1] = resized_image

seg_img1 = cv2.resize(np.multiply(mask1.astype('bool'),img_third),(round(width), round(height)), interpolation=cv2.INTER_CUBIC)
seg_img2 = cv2.resize(np.multiply(mask2.astype('bool'),img_third),(round(width), round(height)), interpolation=cv2.INTER_CUBIC)

fig,axe = plt.subplots(1,2,figsize=(9,4))
axe[0].imshow(seg_img1)
axe[1].imshow(seg_img2)
plt.show()
fig.savefig('EXT2.jpg')


#   Copyright 2014-2015 PUNCH Cyber Analytics Group
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""
Overview
========

Monitor a directory for files extracted by Suricata

"""

import os

from time import sleep
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler

from stoq.plugins import StoqSourcePlugin

class SuricataDirmonSource(StoqSourcePlugin, FileSystemEventHandler):

    def __init__(self):
        super().__init__()

    def activate(self, stoq):
        self.stoq = stoq
        super().activate()

    def on_created(self, event):

        # Skip this file if it is not the meta file created by Suricata
        if not event.src_path.endswith(".meta") and not event.src_path.endswith(".meta.tmp"):
            return

        # A bit of a race condition here. Suricata writes a tmp file, until the stream is complete.
        # At that point, suricata moves file to a filename minus the .tmp extension which is not
        # detected by the filesystem as an on_created() event, thus we will never see it. Let's
        # assume that if the file name ends with a ".tmp" extension, by the time we get to loading
        # the file itself, suricata has finished writing to disk. Ugly? yes.
        if event.src_path.endswith(".tmp"):
            meta_filename = os.path.splitext(event.src_path)[0]
        else:
            meta_filename = event.src_path

        # We should be left with only the file that was carved, time to
        # open the meta file, parse it, and make sure we pass the content
        # as metadata back into stoQ.
        meta = {}

        # Make sure we define the filename correctly
        path = os.path.splitext(meta_filename)[0]

        max_attempts = 3
        attempts = 0
        # More trying to fix a race condition when files are written to disk. In some fringe cases,
        # the file may not exist yet. Let's retry loading the file if it is None.
        while attempts < max_attempts:
            content = self.stoq.get_file(meta_filename)
            if content:
                break
            attempts +=1
            sleep(.3)

        if not content:
            self.log.warning(
                "Attempted to open {}, but no content was found, skipping...".format(
                    meta_filename))
            return

        # Normalize the metadata content so we can store is in a dict
        for line in content.splitlines():
            meta_line = line.split(b':', 1)

            # Normalize the key
            meta_key = meta_line[0].decode()
            meta_key = meta_key.lower()
            meta_key = meta_key.replace(" ", "_")

            # Attempt to decode the value of the key, if we can't due to the
            # encoding, we will leave it as bytes().
            try:
                meta_value = meta_line[1].strip().decode()
            except UnicodeDecodeError:
                meta_value = meta_line[1].strip()

            meta[meta_key] = meta_value

        meta['src'] = 'suricata'

        self.stoq.worker.multiprocess_put(path=path, archive='file', **meta)

    def ingest(self):
        """
        Monitor a directory for files extracted by Suricata

        """

        observer = Observer()
        observer.schedule(self, self.stoq.worker.path, recursive=False)
        observer.start()
        self.log.info("Monitoring {} for extracted files...".format(self.stoq.worker.path))

        try:
            while True:
                sleep(2)
        except KeyboardInterrupt:
            observer.stop()
        observer.join()


#!/usr/bin/env python
#coding:utf-8
import argparse,os,collections,re
parser = argparse.ArgumentParser(description="This Script is used to get the gene expression metrix or transcripts expression metrix by the gene abundance txt file or transcripts.txt file ")
parser.add_argument("-i","--input",type=str,help="the input gene_abundance.txt files generated by 'stringTie -e' or input transcripts.gtf file generated by 'stringTie -e'",required = True,nargs='*')
parser.add_argument("-o","--outfile",type=str,help="The output file",required = True)
parser.add_argument("-t","--type",type=str,help="The expression quality value type, default: 'TPM'",default = "TPM",choices = ["TPM","FPKM"])
parser.add_argument("-v","--version",action="version",version='%(prog)s 1.0')
args = parser.parse_args()

def get_gene_expression_dict(type,abundance_file_list):
    d = collections.OrderedDict()
    samples = [os.path.basename(os.path.dirname(os.path.abspath(i))) for i in abundance_file_list]
    for i in abundance_file_list:
        sn  = samples[abundance_file_list.index(i)]
        with open(i) as fi:
            header = fi.next()
            index = header.strip("\n").split("\t").index(type)
            for line in fi:
                line = line.strip("\n").split("\t")
                d.setdefault(line[0],{})[sn] = line[index]
    for k in d:
        for sn in samples:
            d[k].setdefault(sn,str(float(0)))
    return d

def get_trans_expression_dict(type,abundance_file_list):
    d = collections.OrderedDict()
    samples = [os.path.basename(os.path.dirname(os.path.abspath(i))) for i in abundance_file_list]
    for i in abundance_file_list:
        sn = samples[abundance_file_list.index(i)]
        with open(i) as fi:
            for line in fi:
                if re.search('transcript_id "(.+?)";.+?%s "(.+?)";'%type,line):
                    transcript_id,value = re.search('transcript_id "(.+?)";.+?%s "(.+?)";'%type,line).group(1,2)
                    d.setdefault(transcript_id,{})[sn] = value
    for k in d:
        for sn in samples:
            d[k].setdefault(sn,str(float(0)))
    return d

def main():
    samples = [os.path.basename(os.path.dirname(os.path.abspath(i))) for i in args.input]
    if args.input[0].endswith(".gtf"):
        expression = get_trans_expression_dict(args.type,args.input)
        with open(args.outfile,"w") as fo:
            fo.write("Transcript id\t" + "\t".join(samples) + "\n")
            for trans in expression:
                #if any([float(expression[trans][sn]) for sn in samples]):  ## 去除样本中表达值全为0的转录本
                    fo.write(trans + "\t" + "\t".join([expression[trans][sn] for sn in samples]) + "\n")
    else:
        expression = get_gene_expression_dict(args.type,args.input)
        with open(args.outfile,"w") as fo:
            fo.write("Gene id\t" + "\t".join(samples) + "\n")
            for gene in expression:
                #if any([float(expression[gene][sn]) for sn in sampes]):  ## 去除样本中表达值全为0的基因
                    fo.write(gene + "\t" + "\t".join([expression[gene][sn] for sn in samples]) + "\n")

if __name__ == "__main__":
    main()


#!/usr/bin/env python
import rospy
import time
import yaml
from pathlib import Path

from os import listdir
from os.path import isfile, join, isdir

from geometry_msgs.msg import Point

class LocationsParser:
    def __init__(self, file_path = "building101/firstFloor_locations.yaml", ):
        self.relative_path = file_path
        self.my_path = Path(__file__).resolve()  # resolve to get rid of any symlinks
        self.load_dict()

    def load_dict(self):
        self.locations_path = self.my_path.parent / self.relative_path
        self.locationsDictionary = self.__loadDictionaryFromYaml()

    def find_database_directories(self):
        onlydirs = [directory for directory in listdir(self.my_path.parent) if isdir(join(self.my_path.parent,directory))
                    and 'cache' not in directory ]
        return onlydirs

    def get_fileNames(self, database_directory="sim"):
        directories = self.find_database_directories()
        assert database_directory in directories , print("Directory %s not found" %database_directory)

        files_path = self.my_path.parent/database_directory
        try:
             listdir(files_path)
             onlyfiles = [file for file in listdir(files_path) if isfile(join(files_path, file))]
             return onlyfiles
        except :
            print("No such path!")
            return Exception("Path does not exist")

    def get_relativePath(self):
        return self.relative_path

    def set_relativePath(self, relative_path):
        try:
            self.relative_path = relative_path
            self.load_dict()
            return True
        except: return False

    def find_Location(self, desiredLocation_name):
        assert desiredLocation_name.strip() != "" , "Location name is empty!"
        possibleLocationNames = set()
        for name in self.locationsDictionary['positions']:
            if name == desiredLocation_name.strip(): return name
            elif name in desiredLocation_name.strip() or desiredLocation_name.strip() in name:
                possibleLocationNames.add(name)
        return possibleLocationNames

    def insert_Location(self,location_name, coordinates_point ):
        assert  location_name.strip() != "" , "Location name is empty!"
        # assert  X, Y
        coordinatesDict = {'x':coordinates_point.x,'y':coordinates_point.y, 'z':coordinates_point.z}
        self.locationsDictionary['positions'].update({location_name : coordinatesDict })
        with open( self.locations_path  , 'w') as outfile:
          print('writing to file')
          yaml.dump(self.locationsDictionary, outfile, default_flow_style=False)
          outfile.close()

    def delete_Location(self, location_name):
        assert location_name.strip() != "" , "Location name is empty!"
        del self.locationsDictionary['positions'][location_name]
        with open( self.locations_path  , 'w') as outfile:
         print ('writing to file')
         yaml.dump(self.locationsDictionary, outfile, default_flow_style=False)
         outfile.close()

    def get_XY_Coordinates(self, location_name ):
        assert location_name.strip() != "" , "Location name is empty!"
        return self.__getCoordinates(self.locationsDictionary['positions'],location=location_name )

    def get_positionNames(self):
        positionNames = set()
        for positionName in self.locationsDictionary['positions']:
            positionNames.add(positionName)
        return positionNames

    def dump_positions(self):
        for positionName in self.locationsDictionary['positions']:
            # print(positionName)
            coordinatesList = self.locationsDictionary['positions'][positionName]
            print(positionName, 'X : ', coordinatesList['x'], '   Y : ', coordinatesList['y'])

    def __loadDictionaryFromYaml(self):
        with self.locations_path.open() as locations_file:
            locationsDictionary = yaml.safe_load(locations_file)
        return locationsDictionary

    def __getCoordinates(self, locations_dictionary, location ):
        locationDict = locations_dictionary[location]
        return [locationDict['x'] , locationDict['y']]

if __name__ == "__main__":

    """

    These are tests for the location parser.

    -> These commands are not to be executed while operating the robot. Instead this only serves as a place to
        write unit tests for certain functions

    """

    sectionSplitter = '----------' # UI Element [IGNORE]

    file_path = "sim/house_locations.yaml"

    parser  = LocationsParser(file_path=file_path)

    print("\n\nRunning parser test... If you're not expecting to see this please run a node for example [ goals_publisher | user_interface ] \n\n")

    print(parser.get_relativePath())

    #TESTING BASE FUNCTIONALITY [Retreiving Information]
    print('The following are all positions for the given path:', file_path)
    print(sectionSplitter)
    parser.dump_positions()
    print(sectionSplitter)

    #TESTING ADDING NEW ITEM
    appendingLocationName = 'fakeRoomz'
    print('\n\nNext test is to add this location to the DB:',appendingLocationName)
    print( sectionSplitter )
    try:
        parser.insert_Location(location_name=appendingLocationName,coordinates_point= Point(x=1, y=1, z=0))
        print("Inserted item into database with no issues faced...")
        parser.insert_Location(location_name=appendingLocationName,coordinates_point= Point(x=23, y=43, z=0))
        print("Overwritten from X = 4 , Y = 4        to      X = 23  ,  Y =  43")
        print(sectionSplitter)
        print(sectionSplitter)
    except:
        print("Something went  wrong!")
    parser.dump_positions()
    print(sectionSplitter)

    #TESTING DELETING THAT ADDED ITEM
    print('\n\nNext test is to delete room from the DB:',appendingLocationName)
    print( sectionSplitter )
    try:parser.delete_Location(appendingLocationName)
    except:print('Something went wrong!')
    parser.dump_positions()
    print(sectionSplitter)

#   TESTING DELETING OF A NON-EXISTING ITEM
    nanItem = "HelloWorldLocation"
    print('\n\nNext test is to delete a non existing room from the DB:',nanItem)
    print( sectionSplitter )
    try:parser.delete_Location(nanItem)
    except:print('Something went wrong!')
    parser.dump_positions()
    print(sectionSplitter)

    #TESTING QUERY SUGGESTIONS
    testLocationName = 'room'
    print('\n\nNext test is to find all similar locations for givin desired location:',testLocationName)
    suggestedLocationNames = parser.find_Location(testLocationName)
    if type(suggestedLocationNames) == set :
     print('Listed below are the possible location names for "%s"' %testLocationName)
     print('\n%s' %sectionSplitter )
     for name in suggestedLocationNames:
        print(name)
     print(sectionSplitter,'\n')

    elif type(suggestedLocationNames) == str:
        print("Unique item found!", testLocationName )

    #TESTING QUERY FOR EXISTING ITEM
    testLocationName = 'study_room'
    print('\n\nNext test is to find all similar locations for givin desired location:',testLocationName)
    suggestedLocationNames = parser.find_Location(testLocationName)
    if type(suggestedLocationNames) == set() :
     print('Listed below are the possible location names for "%s"' %testLocationName)
     print( sectionSplitter )
     for name in suggestedLocationNames:
        print(name)
        print(sectionSplitter)
    elif type(suggestedLocationNames) == str:
        print("Unique item found!", testLocationName )

    print("\n\n\nNext test is to find all database directories in locations directory")

    directories = parser.find_database_directories()
    print(directories)
    dirSelection=""
    while True:
        dirSelection = input('\n\nPlease select a database:')
        print(directories)
        if dirSelection in directories:break
        else: print("Please ensure that you inputted the right directory name!")

    files = parser.get_fileNames(database_directory=dirSelection)
    print(files)
    fileSelection = ""
    while True:
        fileSelection = input('\n\nPlease select a file:')
        print(files)
        if fileSelection in files:break
        else: print("Please ensure that you inputted the right directory name!")

    new_path=dirSelection+'/'+fileSelection
    print("Switching DB to", new_path)
    parser.set_relativePath(new_path)

    time.sleep(2)
    parser.dump_positions()

    print('\nTests complete...\n')


import numpy as np
from libs.file import netcdf as nc
from libs.geometry import project as p
from datetime import datetime
from libs.console import *
import os
localpath = os.path.dirname(__file__)
gdal_supported = False
try:
	from osgeo import gdal
	gdal_supported = True
except Exception:
	pass

source = "http://limie.com.ar/gersolar_collab/linketurbidity.nc"
filename = source.split("/")[-1]
destiny = localpath + "/" + filename

def initial_configuration():
	from libs.file.toolbox import download, unzip
	say("Downloading "+filename+" package... ")
	download(source,destiny)

if not os.path.exists(destiny):
	initial_configuration()

def get_month(month):
	if gdal_supported:
		ds = gdal.Open(localpath + "/tifs/"+str(month).zfill(2)+"_longlat_wgs84.tif")
		linke = ds.ReadAsArray()
	else:
		root, n = nc.open(destiny)
		data = nc.getvar(root, "linketurbidity")
		linke = data[month -1]
	# The linke turbidity is obtained when the image pixel value is divied by 20.
	return linke/20.

def project_coordinates(lat, lon):
	shape = get_month(1).shape
	return p.pixels_from_coordinates(lat, lon, shape[0], shape[1])

def cut_month(x, y, month):
	linke = get_month(month)
	result = p.transform_data(linke,x,y)
	return np.float32(result)

def cut_projected(root):
	lat = nc.getvar(root, 'lat')
	lon = nc.getvar(root, 'lon')
	time = nc.getvar(root, 'data_time')
	months = list(set([ (datetime.fromtimestamp(int(t))).month for t in time ]))
	months_d = nc.getdim(root, 'monthing')
	months_cut = nc.getvar(root, 'months', 'i2', ('monthing',))
	dims = list(lat.dimensions)
	dims.insert(0, 'monthing')
	linke = nc.getvar(root, 'linketurbidity', 'f4', tuple(dims),4)
	linke_x, linke_y = project_coordinates(lat[:], lon[:])
	months_cut[:] = np.array(list(months))
	for i in range(len(months)):
		linke[i] = cut_month(linke_x, linke_y, months[i])
	return linke[:], linke_x, linke_y