# -*- coding: utf-8 -*-
"""
Created on Mon Feb 26 17:15:36 2018

@author: Raj
"""

import pycroscopy as px
from ffta.utils import hdf_utils
import numpy as np

from ffta import pixel

import warnings

def FFT_testfilter(hdf_file, parameters={}, DC=True, linenum = 0, show_plots = True,
               narrowband = False, noise_tolerance = 5e-7, bandwidth=-1, check_filter=True):
    """
    Applies FFT Filter to the file at a specific line and displays the result

    Usage:
    >> h5_ll = hdf_utils.get_line(h5_file, linenum, avg=True)
    >> filt_sig, freq_filts, _,_ = filtering.FFT_testfilter(h5_avg,
                                                            parameters,
                                                            narrowband=True,
                                                            noise_tolerance=1e-5,
                                                            show_plots=True)

    Some important notes:
        1) This works on any line. However, you should have the filter work on a single signal
        2) Meaning,

    hdf_file : h5Py file or Nx1 NumPy array (preferred is NumPy array)
        hdf_file to work on, e.g. hdf.file['/FF-raw'] if that's a Dataset
        if ndarray, uses passed or default parameters
        Use ndarray.flatten() to ensure correct dimensions

    DC : bool, optional
        Determines whether to remove mean (DC-offset)

    parameters : dict, optional
        Contains parameters in FF-raw file for constructing filters. Automatic if a Dataset/File
        Must contain num_pts and samp_rate to be functional

    linenum : int, optional
        For extracting a specific line to do FFT Filtering on

    show_plots : bool, optional
        Turns on FFT plots from Pycroscopy

    narrowband : bool, optional
        Sets noiseband filter to be a narrow pass filter centered on the drive_frequency
        Sometimes fails for very large datasets for unknown reasons.

    noise_tolerance : float 0 to 1
        Amount of noise below which signal is set to 0

    bandwidth : int, optional
        Total bandwidth (each side is half-bandwidth) is capped at 2500 Hz for computational reasons
        This overrides the parameters value

    Returns
    -------
    filt_line : numpy.ndarray
        Filtered signal of hdf_file

    freq_filts : list
        The filter parameters to be passed to SignalFilter

    fig_filt, axes_filt: matplotlib controls
        Only functional if show_plots is on
    """

    reshape = False
    ftype = str(type(hdf_file))
    if ('h5py' in ftype) or ('Dataset' in ftype):   #hdf file

        parameters = hdf_utils.get_params(hdf_file)
        hdf_file = hdf_utils.get_line(hdf_file, linenum, array_form=True, transpose=False)
        hdf_file = hdf_file.flatten()

    if len(hdf_file.shape) == 2:

        sh = hdf_file.shape
        reshape = True
        hdf_file = hdf_file.flatten()

    num_pts = hdf_file.shape[0]
    drive = parameters['drive_freq']
    lpf_cutoff = np.round(drive/1e5, decimals=0)*2*1e5     # 2times the drive frequency, round up
    samp_rate = parameters['sampling_rate']

    #default filtering, note the bandwidths --> DC filtering and certain noise peaks
    lpf = px.processing.fft.LowPassFilter(num_pts, samp_rate, lpf_cutoff)
    nbf = px.processing.fft.NoiseBandFilter(num_pts, samp_rate,
                                            [10E3, 50E3, 100E3, 150E3, 200E3],
                                            [20E3, 1E3, 1E3, 1E3, 1E3])

    freq_filts = [lpf, nbf]

    # Remove DC Offset
    if DC == True:

        hdf_file -= hdf_file.mean(axis=0)

    # Generate narrowband signal
    if narrowband == True:

        if bandwidth == -1: #unspecified
            try:
                bandwidth = parameters['filter_bandwidth']
                if bandwidth > 2500:
                    warnings.warn('Bandwidth of that level might cause errors')
                    bandwidth = 2500
            except:
                print ('No bandwidth parameters')
                bandwidth = 2500

        nbf = px.processing.fft.HarmonicPassFilter(num_pts, samp_rate, drive, bandwidth, 7)
        freq_filts = [nbf]

    composite_filter = px.fft.build_composite_freq_filter(freq_filts)
    print('Composite filter of len:', len(composite_filter))

    # Test filter on a single line:
    filt_line, fig_filt, axes_filt = px.processing.gmode_utils.test_filter(hdf_file,
                                                                           frequency_filters=freq_filts,
                                                                           noise_threshold=noise_tolerance,
                                                                           show_plots=show_plots)

    # If need to reshape
    if reshape == True:
        filt_line = np.reshape(filt_line, sh)

    # Test filter out in Pixel
    if check_filter:
        h5_px_filt = pixel.Pixel(filt_line, parameters)
        h5_px_filt.clear_filter_flags()
        h5_px_filt.analyze()
        h5_px_filt.plot(newplot=True)

        h5_px_filt = pixel.Pixel(filt_line, parameters)

        h5_px_filt.analyze()
        h5_px_filt.plot(newplot=True)

        h5_px_raw = pixel.Pixel(hdf_file, parameters)
        h5_px_raw.analyze()
        h5_px_raw.plot(newplot=True,c1='b', c2='k')

        h5_px_raw = pixel.Pixel(hdf_file, parameters)
        h5_px_raw.clear_filter_flags()
        h5_px_raw.analyze()
        h5_px_raw.plot(newplot=True,c1='r', c2='y')

#    h5_px_raw_unfilt = pixel.Pixel(hdf_file, parameters)
#    h5_px_raw_unfilt.clear_filter_flags()
#    h5_px_raw_unfilt.analyze()
#    h5_px_raw_unfilt.plot(newplot=False,c1='y', c2='c')

    return filt_line, freq_filts, fig_filt, axes_filt

def FFT_filter(h5_main, freq_filts, noise_tolerance=5e-7, make_new=False):
    """
    Stub for applying filter above to the entire FF image set

    h5_main : h5py.Dataset object
        Dataset to work on, e.g. h5_main = px.hdf_utils.getDataSet(hdf.file, 'FF_raw')[0]

    freq_filts : list
        List of frequency filters usually generated in test_line above

    noise_tolerance : float, optional
        Level below which data are set to 0. Higher values = more noise (more tolerant)

    make_new : bool, optional
        Allows for re-filtering the data by creating a new folder

    Returns
    -------

    h5_filt : Dataset
        Filtered dataset within latest -FFT_Filtering Group

    """

    h5_filt_grp = px.hdf_utils.check_for_old(h5_main, 'FFT_Filtering')

    if h5_filt_grp == None or make_new == True:

        sig_filt = px.processing.SignalFilter(h5_main, frequency_filters=freq_filts,
                                              noise_threshold=noise_tolerance,
                                              write_filtered=True, write_condensed=False,
                                              num_pix=1,verbose=True, cores=2, max_mem_mb=512)

        h5_filt_grp = sig_filt.compute()

    else:
        print('Taking previously computed results')

    h5_filt = h5_filt_grp['Filtered_Data']
    px.hdf_utils.copyAttributes(h5_main.parent, h5_filt)
    px.hdf_utils.copyAttributes(h5_main.parent, h5_filt.parent)

    return h5_filt

Old G Lab Code and my simulations/ffta/utils/filtering.py


#!/usr/bin/env python

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from math import ceil

import tensorflow as tf
import numpy as np

# from tensorflow.python.ops import math_ops
# from tensorflow.python.ops.metrics_impl import _remove_squeezable_dimensions
# from tensorflow.python.framework import dtypes

# from tensorflow.contrib.estimator import stop_if_no_decrease_hook

RANDOM_SEED = 42

tf.set_random_seed(RANDOM_SEED)
np.random.seed(RANDOM_SEED)

tf.logging.set_verbosity(tf.logging.INFO)

CSV_COLUMNS = ' ,id,age,concentration,hearing_impairments,musical_expertise,nationality,artistic,fault,imagination,lazy,nervous,outgoing,reserved,stress,thorough,trusting,activity,engagement,familiarity,like_dislike,positivity,tension,sex,hallelujah_reaction,location,language,music_pref_none,music_pref_hiphop,music_pref_dance,music_pref_world,music_pref_rock,music_pref_pop,music_pref_classical,music_pref_jazz,music_pref_folk,music_pref_traditional_irish'.split(',')
LABEL_COLUMN = 'hallelujah_reaction'
KEY_FEATURE_COLUMN = None
DEFAULTS = [[], [''], [], [], [], [], [''], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [''], [0], [''], [''], [], [], [], [], [], [], [], [], [], []]

# These are the raw input columns, and will be provided for prediction also
INPUT_COLUMNS = [
    # Categorical features
    tf.feature_column.indicator_column(tf.feature_column.categorical_column_with_vocabulary_list('language', ['N/A', 'en', 'zh_TW'])),
    tf.feature_column.indicator_column(tf.feature_column.categorical_column_with_vocabulary_list('location', ['dublin', 'taipei_city', 'taichung_city'])),
    tf.feature_column.indicator_column(tf.feature_column.categorical_column_with_vocabulary_list('nationality', ['other', 'irish', 'british', 'taiwanese', 'indonesian', 'japanese', 'chinese', 'singaporean', 'american', 'algerian', 'thai'])),
    tf.feature_column.indicator_column(tf.feature_column.categorical_column_with_vocabulary_list('sex', ['female', 'male'])),

    # Numeric features
    tf.feature_column.numeric_column('activity'),
    tf.feature_column.numeric_column('artistic'),
    tf.feature_column.numeric_column('age'),
    tf.feature_column.numeric_column('concentration'),
    tf.feature_column.numeric_column('engagement'),
    tf.feature_column.numeric_column('familiarity'),
    tf.feature_column.numeric_column('fault'),
    tf.feature_column.numeric_column('hearing_impairments'),
    tf.feature_column.numeric_column('imagination'),
    tf.feature_column.numeric_column('lazy'),
    tf.feature_column.numeric_column('like_dislike'),
    tf.feature_column.numeric_column('musical_expertise'),
    tf.feature_column.numeric_column('music_pref_classical'),
    tf.feature_column.numeric_column('music_pref_dance'),
    tf.feature_column.numeric_column('music_pref_folk'),
    tf.feature_column.numeric_column('music_pref_hiphop'),
    tf.feature_column.numeric_column('music_pref_jazz'),
    tf.feature_column.numeric_column('music_pref_none'),
    tf.feature_column.numeric_column('music_pref_pop'),
    tf.feature_column.numeric_column('music_pref_rock'),
    tf.feature_column.numeric_column('music_pref_traditional_irish'),
    tf.feature_column.numeric_column('music_pref_world'),
    tf.feature_column.numeric_column('nervous'),
    tf.feature_column.numeric_column('outgoing'),
    tf.feature_column.numeric_column('positivity'),
    tf.feature_column.numeric_column('reserved'),
    tf.feature_column.numeric_column('stress'),
    tf.feature_column.numeric_column('tension'),
    tf.feature_column.numeric_column('thorough'),
    tf.feature_column.numeric_column('trusting')

    # Engineered features that are created in the input_fn
]

# Build the estimator
def build_estimator(model_dir, nbuckets, hidden_units, learning_rate=0.001, beta1=0.9, beta2=0.999, dropout=None, activation_function='relu', checkpoint_secs=90):
    """
    Build an estimator starting from INPUT COLUMNS.
    These include feature transformations and synthetic features.
    The model is a wide-and-deep model.
    """

    # Input columns
    (language, location, nationality, sex, activity, artistic, age, concentration, engagement, familiarity, fault, hearing_impairments, imagination, lazy, like_dislike, musical_expertise, music_pref_classical, music_pref_dance, music_pref_folk, music_pref_hiphop, music_pref_jazz, music_pref_none, music_pref_pop, music_pref_rock, music_pref_traditional_irish, music_pref_world, nervous, outgoing, positivity, reserved, stress, tension, thorough, trusting) = INPUT_COLUMNS

    # Wide columns and deep columns
    wide_columns = [
        # Feature crosses

        # Sparse columns

        # Anything with a linear relationship
    ]

    deep_columns = [
        # Embedding columns

        # Numeric columns
        language, location, nationality, sex, activity, artistic, age, concentration, engagement, familiarity, fault, hearing_impairments, imagination, lazy, like_dislike, musical_expertise, music_pref_classical, music_pref_dance, music_pref_folk, music_pref_hiphop, music_pref_jazz, music_pref_none, music_pref_pop, music_pref_rock, music_pref_traditional_irish, music_pref_world, nervous, outgoing, positivity, reserved, stress, tension, thorough, trusting
    ]

    session_config = tf.ConfigProto(
        allow_soft_placement=True
    )

    run_config = tf.estimator.RunConfig(
        tf_random_seed=RANDOM_SEED,
        save_checkpoints_secs=checkpoint_secs,  # Save checkpoints every 90 seconds
        keep_checkpoint_max=100,       # Retain the 10 most recent checkpoints.
        session_config=session_config
    )

    activation_functions = {
        'elu': tf.nn.elu,
        'relu': tf.nn.relu,
        'leaky_relu': tf.nn.leaky_relu
    }

    optimizer = tf.train.AdamOptimizer(learning_rate, beta1, beta2)

    estimator = tf.estimator.DNNClassifier(
        model_dir=model_dir,
        feature_columns=deep_columns,
        hidden_units=hidden_units,
        config=run_config,
        optimizer=optimizer,
        dropout=dropout,
        activation_fn=activation_functions[activation_function]
    )

    estimator = tf.contrib.estimator.add_metrics(estimator, additional_metrics)
    return estimator

# Create input function to load data into datasets
def read_dataset(args, mode):
    batch_size = args['train_batch_size']

    if mode == tf.estimator.ModeKeys.TRAIN:
        input_paths = args['train_data_paths']
    else:
        input_paths = args['eval_data_paths']

    def _input_fn():
        def decode_csv(value_column):
            columns = tf.decode_csv(value_column, record_defaults=DEFAULTS)
            features = dict(zip(CSV_COLUMNS, columns))
            label = features.pop(LABEL_COLUMN)
            return features, label

        # Create list of files that match pattern
        file_list = tf.gfile.Glob(input_paths)

        # Create dataset from file list
        dataset = tf.data.TextLineDataset(file_list).map(decode_csv)

        if mode == tf.estimator.ModeKeys.TRAIN:
            num_epochs = None  # indefinitely
            dataset = dataset.shuffle(buffer_size=10 * batch_size, reshuffle_each_iteration=True)
            dataset = dataset.repeat(num_epochs).batch(batch_size)
        else:
            num_epochs = 1  # end-of-input after this
            # This batching is super brittle
            dataset = dataset.repeat(num_epochs).batch(args['num_train_examples'])

        return dataset.make_one_shot_iterator().get_next()

    return _input_fn

# Create estimator train and evaluate function
def train_and_evaluate(args):
    # Calculate number of training steps required
    # Here, we treat train_steps as epochs
    total_training_steps = ceil(((args['num_train_examples'] * 1.0) / args['train_batch_size']) * args['train_steps'])

    hidden_units = [str(args['num_nodes']) for _ in range(args['num_layers'])]

    estimator = build_estimator(args['output_dir'], args['nbuckets'], hidden_units,
                                args['learning_rate'], args['beta1'], args['beta2'], args['dropout'],
                                args['activation_function'])
    train_spec = tf.estimator.TrainSpec(
        input_fn=read_dataset(args, tf.estimator.ModeKeys.TRAIN),
        max_steps=total_training_steps)
    eval_spec = tf.estimator.EvalSpec(
        input_fn=read_dataset(args, tf.estimator.ModeKeys.EVAL),
        steps=args['eval_steps'],
        start_delay_secs=5,
        throttle_secs=5)
    tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)

    if args['optimize']:
        return estimator.evaluate(read_dataset(args, tf.estimator.ModeKeys.EVAL), steps=1)
    else:
        return

def additional_metrics(labels, predictions):
    precision, precision_op = tf.metrics.precision(labels, predictions['class_ids'])
    recall, recall_op = tf.metrics.recall(labels, predictions['class_ids'])

    true_positives, true_positives_op = tf.metrics.true_positives(labels, predictions['class_ids'])
    true_negatives, true_negatives_op = tf.metrics.true_negatives(labels, predictions['class_ids'])
    false_positives, false_positives_op = tf.metrics.false_positives(labels, predictions['class_ids'])
    false_negatives, false_negatives_op = tf.metrics.false_negatives(labels, predictions['class_ids'])

    f1 = 2. / ((1. / precision) + (1. / recall))

    # positive_support = math_ops.equal(labels, True)
    # negative_support = math_ops.equal(labels, False)
    #
    positive_precision, positive_precision_op = tf.metrics.precision(labels, predictions['class_ids'], labels)
    positive_recall, positive_recall_op = tf.metrics.recall(labels, predictions['class_ids'], labels)
    #
    # negative_precision, negative_precision_op = tf.metrics.precision(labels, predictions['class_ids'],
    #                                                                  math_ops.cast(
    #                                                                      math_ops.logical_not(
    #                                                                          math_ops.cast(labels, dtypes.bool)
    #                                                                      ),
    #                                                                      dtypes.int32)
    #                                                                  )
    # negative_recall, negative_recall_op = tf.metrics.recall(labels, predictions['class_ids'],
    #                                                         math_ops.cast(
    #                                                             math_ops.logical_not(
    #                                                                 math_ops.cast(labels, dtypes.bool)
    #                                                             ),
    #                                                             dtypes.int32)
    #                                                         )

    # positive_precision = (1. * true_positives) / (true_positives + false_positives)
    # positive_recall = (1. * true_positives) / (true_positives + false_negatives)
    # positive_f1 = 2. / ((1. / positive_precision) + (1. / positive_recall))
    #
    # negative_precision = (1. * true_negatives) / (true_negatives + false_negatives)
    # negative_recall = (1. * true_negatives) / (true_negatives + false_positives)
    # negative_f1 = 2. / ((1. / negative_precision) + (1. / negative_recall))

    # weighted_f1 = (positive_f1 * ((1. * positive_support) / (positive_support + negative_support))) + (
    #             negative_f1 * ((1. * negative_support) / (positive_support + negative_support)))

    # tf.logging.info('***** predictions["class_ids"]:')
    # tf.logging.info(predictions['class_ids'])

    return {
        'f1_score': (f1, tf.group(precision_op, recall_op)),
        'true_positives': (true_positives, true_positives_op),
        'true_negatives': (true_negatives, true_negatives_op),
        'false_positives': (false_positives, false_positives_op),
        'false_negatives': (false_negatives, false_negatives_op),
        # 'positive_precision': (positive_precision, positive_precision_op),
        # 'positive_recall': (positive_recall, positive_recall_op),
        # 'f1_score_positive': positive_f1,
        # 'negative_precision': (negative_precision, negative_precision_op),
        # 'negative_recall': (negative_recall, negative_recall_op)
        # 'f1_score_negative': negative_f1,
        # 'f1_score_weighted': weighted_f1
    }


from setuptools import setup
import os

required_packages = [
    'Django>=1.7',
    'djangorestframework>=3',
    'django-datetime-widget',
    'django-crispy-forms',
    'pandas',
    'matplotlib',
]

setup(name='',
      version='1.0',
      description='',
      author='',
      author_email='',
      url='http://www.python.org/sigs/distutils-sig/',
      install_requires=required_packages,
)


def login_disponivel(string, lista):
    k = False
    j = 0
    for i in range(len(lista)):
        if string == lista[i]:
            j = int(j) + 1
            k = True
    if k:
        string = string + str(j)
        return string
    else:
        return string


import sys
import multiprocessing as mp

print("Python version %s " % str(sys.version))
print("Cores          %s " % str(mp.cpu_count()))

if True:
  print("tady")
else:
  print("tady nikdy")
  a = 0
  b = 1
  c = b / a


import carregabaseApache as cbe
import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import roc_curve,auc, accuracy_score
from imblearn.over_sampling import SMOTE
from imblearn.over_sampling import RandomOverSampler
from imblearn.under_sampling import RandomUnderSampler

X, y, data = cbe.carrega_base("mimiciii_apache_total.csv")
#1 / (1 + exp(- (-4.4360 + 0.04726*(APSIII) )))
X_train = np.array(X)
X_apa = 1 / (1 + np.exp(- (-4.4360 + 0.04726*(X_train) )))
y_train = np.array(y)

#sm=SMOTE(random_state=42)
sm=RandomUnderSampler(random_state=42)
X_res, y_res = sm.fit_sample(X_apa, y_train)

mean_fpr = np.linspace(0, 1, 100)
i = 0
H= 0
# Compute ROC curve and area the curve
fpr, tpr, thresholds = roc_curve(y_res, X_res)
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, lw=1, alpha=0.8, color='r',
            label='AUC = %0.4f S= %0.4f E=%0.4f'% (roc_auc, np.mean(tpr), np.mean(1-fpr)))

plt.plot([0, 1], [0, 1], lw=2, color='b',
         label='Randomico', alpha=.2)

plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('Taxa de falsos positivos')
plt.ylabel('Taxa de verdadeiros positivos')
plt.title('Curvas')
plt.legend(loc="lower right")

plt.show()


import pandas as pd

from bokeh.plotting import ColumnDataSource, figure
from bokeh.io import output_file, show
from bokeh.models import CategoricalColorMapper, HoverTool
from bokeh.layouts import column, row

output_file('pop-life.html')

file = 'country-pops.csv'

countries = pd.read_csv(file)

country_data = ColumnDataSource(countries)

color_mapper = CategoricalColorMapper(factors=['Asia', 'Africa', 'Antarctica', 'Australia', 'Central America','Europe', 'North America', 'Oceania', 'South America'],
                                    palette=['#00FF00', '#FFD343', 'darkgray', 'brown', 'cyan', 'crimson', 'red',   '#0000FF', 'purple'])

TOOLTIPS = 'pan, wheel_zoom,box_zoom, reset, hover,save'

plot_life_expect = figure(x_axis_label='Population', y_axis_label='Life Expectancy', tools= TOOLTIPS, title= 'Population vs. Life Expectancy')

plot_birth_rate = figure(x_axis_label='Population', y_axis_label='Birth Rate', title='Population vs. Birth Rate',  tools=TOOLTIPS)

plot_death_rate = figure(x_axis_label='Population', y_axis_label='Death Rate', title='Population vs. Death Rate',  tools=TOOLTIPS)

plot_birth_rate.circle(x='Population', y='Birthrate', source=country_data, size=10, color=dict(field='Continent',     transform=color_mapper), legend='Continent')

plot_death_rate.triangle(x='Population', y='Deathrate', source=country_data, size=10, color=dict(field='Continent',     transform=color_mapper), legend='Continent')

plot_life_expect.diamond(x='Population', y='Life_expectancy', source=country_data, size=10, color=dict(field='Continent',     transform=color_mapper), legend='Continent')

hover = plot_life_expect.select_one(HoverTool)
hover_birth = plot_birth_rate.select_one(HoverTool)
hover_death = plot_death_rate.select_one(HoverTool)

hover.tooltips = [('Country Name English', '@Country_English'), ('Population', '@Population'), ('Life Expectancy (years)', '@Life_expectancy')]

hover_birth.tooltips = [('Country Name English', '@Country_English'), ('Population', '@Population'), ('Birth Rate', '@Birthrate')]

hover_death.tooltips = [('Country Name English', '@Country_English'), ('Population', '@Population'), ('Death Rate', '@Deathrate')]

plot_life_expect.legend.location = 'bottom_right'
plot_life_expect.legend.background_fill_color = 'lightgrey'

plot_birth_rate.legend.location = 'bottom_right'
plot_birth_rate.legend.background_fill_color = 'lightgrey'

plot_death_rate.legend.location = 'bottom_right'
plot_death_rate.legend.background_fill_color = 'lightgrey'

plot_birth_rate.x_range = plot_life_expect.x_range
plot_death_rate.x_range = plot_life_expect.x_range

show(row(column(plot_life_expect, plot_birth_rate), column(plot_death_rate)))


import os
import numpy as np
import pandas as pd
import joblib
import yaml
import json

params_path = "params.yaml"
schema_path = os.path.join("prediction_service","schema_in.json")

class NotInRange(Exception):
    def __init__(self,message = "values entered are not in range"):
        self.message = message
        super().__init__(self.message)

class NotInCols(Exception):
    def __init__(self,message = "entered feature is in the feature list"):
        self.message = message
        super().__init__(self.message)

def read_params(config_path=params_path):
    with open(config_path) as f:
        config = yaml.safe_load(f)
    return config

def predict(data):
    config = read_params(params_path);
    model_dir_path = config["webapp_model_dir"]
    model = joblib.load(model_dir_path)
    prediction = model.predict(data).tolist()[0]
    #return prediction
    try:
        if 3 <= prediction <= 8:
            return prediction
        else:
            raise NotInRange
    except NotInRange:
        return "Unexpected result"

def get_schema(schema_path=schema_path):
    with open(schema_path) as f:
        schema = json.load(f)
    return schema

def validate_input(dict_req):
    def _validate_cols(col):
        schema = get_schema()
        if col not in schema.keys():
            raise NotInCols

    def _validate_values(col,val):
        schema = get_schema()
        if not(schema[col]["min"] <= float(dict_req[col]) <= schema[col]["max"]):
            raise NotInRange

    for col,val in dict_req.items():
        _validate_cols(col)
        _validate_values(col,val)

    return True

def form_response(dict_req):
    if validate_input(dict_req):
        data = dict_req.values()
        data = [list(map(float,data))]
        responsne = predict(data)
        return responsne

def api_response(dict_req):
    try:
        if validate_input(dict_req):
            data = np.array([list(dict_req.values())])
            response = predict(data)
            response = {"response":response}
            return response

    except NotInRange as e:
        response = {"the_expected_range":get_schema(),"response": str(e)}
        return response

    except NotInCols as e:
        response = {"the_expected_cols":get_schema().keys(),"response" : str(e)}
        return response

    except Exception as e:
        response = {"response": str(e)}
        return response


from __future__ import absolute_import
from __future__ import print_function
from __future__ import division

import os
import os.path as osp
import torch
import torch.nn as nn
from data import init_dataset
from data.sampler import IdentitySampler, RandomIdentitySampler
from data import ImageDataset, TestDataset, RandomErasing
from torch.utils.data import DataLoader
from model import SDAN
from model.netutils import WarmupMultiStepLR
from loss import SmoothCrossEntropyLoss
from loss import BatchHardTripletLoss, CenterLoss,TripletLoss_WRT, BiBatchHardTripletLoss

from argparse import ArgumentParser
from datetime import datetime
import time
import torch.backends.cudnn as cudnn
import numpy as np
import shutil

from metrics import compute_distance_matrix, evaluate

import torchvision.transforms as transforms
from tensorboardX import SummaryWriter
from tools import Logger, str_boolean, AverageMeter
import pdb

def main(args):
    log_path = osp.join(args.log_path, args.dataset, 'log.txt')
    logger = Logger(logger_path=log_path)
    logger("{} Begin training...".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
    logger(args)

    writer = SummaryWriter(args.vis_log_dir)

    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)
    # Load data, produce DataLoader
    logger("{} Loading data...".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
    train_transformer = transforms.Compose([
        transforms.Resize((args.img_height, args.img_width), interpolation=3),
        transforms.Pad(10),
        transforms.RandomCrop((args.img_height, args.img_width)),
        transforms.RandomHorizontalFlip(),
        # transforms.ColorJitter(brightness=0.2, contrast=0.15, saturation=0, hue=0),
        # transforms.RandomGrayscale(p=0.5),
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
        RandomErasing(probability=args.erasing_p, mean=[0.485, 0.456, 0.406])
    ])

    test_transformer = transforms.Compose([
        transforms.Resize((args.img_height, args.img_width), interpolation=3),
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
    ])

    data_src = init_dataset(name=args.dataset, mode=args.eval_mode, shot=args.num_shot)
    # select sampler method, identity_sampler selects batch_size person, each person with 1 images/modality, random
    # sampler select batch_size person and each person with batch_instance images/modality.
    if args.sampler == 'identity':
        # sampler = IdentitySampler(data_src.train_rgb_imgs, data_src.train_ir_imgs, batch_size=args.train_batch)
        sampler = IdentitySampler(data_src.train_rgb_imgs, data_src.train_ir_imgs, batch_size=args.train_batch, batch_instance=args.num_instance)
    elif args.sampler == 'random':
        sampler = RandomIdentitySampler(data_src.train_rgb_imgs, data_src.train_ir_imgs, batch_pid=args.train_batch, batch_instance=args.num_instance)
    else:
        raise ValueError("Sampler {} is not support. Please select from 'identity' and '' random...".format(args.sampler))

    # data loader
    train_loader = DataLoader(ImageDataset(data_src.train_rgb_imgs, data_src.train_ir_imgs, width=args.img_width,
                                           height=args.img_height, transform=train_transformer),
                              batch_size=args.train_batch, sampler=sampler,
                              num_workers=args.num_workers, drop_last=True)
    test_rgb_loader = DataLoader(TestDataset(data_src.gallery_imgs, width=args.img_width, heigh=args.img_height,
                                            transform=test_transformer), batch_size=args.test_batch, shuffle=False,
                                 num_workers=args.num_workers)  # rgb
    test_ir_loader = DataLoader(TestDataset(data_src.query_imgs, width=args.img_width, heigh=args.img_height,
                                          transform=test_transformer), batch_size=args.test_batch, shuffle=False,
                                num_workers=args.num_workers)  # ir

    cudnn.benchmark = True
    devices = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    model = SDAN(num_strips=args.num_strips, num_features=args.num_feat, num_classes=args.num_classes, drop=args.dropout)
    model = nn.DataParallel(model).to(devices)
    if args.mode =='eval':
        logger('{} Evaluating...'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
        checkpoint = torch.load(osp.join(args.save_path, 'best_model.pth.tar'))
        model.load_state_dict(checkpoint['state_dict'])
        rank1 = eval(model, test_rgb_loader, test_ir_loader, logger, args.max_epoch, args, ranks=[1, 5, 10, 20])
        return rank1

    elif args.mode == 'resume':
        logger('{} Resume training...'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
        checkpoint = torch.load(osp.join(args.save_path, args.ckpt_name))
        model.load_state_dict(checkpoint['state_dict'])
        start_epoch = int(args.ckpt_name.split('_')[1].split('-')[1])

    elif args.mode == 'train':
        start_epoch = 0

    else:
        raise ValueError("{} The mode '{}' does not support, please select from train, resume and eval...".format(
            datetime.now().strftime('%Y-%m-%d %H:%M:%S'), args.mode))

# Define losses
    criterion = {}
    sxent = SmoothCrossEntropyLoss(num_classes=args.num_classes)
    if args.triplet == 'hard':
        tripletloss = BatchHardTripletLoss(margin=args.margin)
        bitripletloss = BiBatchHardTripletLoss(margin=args.margin)
    elif args.triplet == 'soft':
        tripletloss = TripletLoss_WRT()
    else:
        raise ValueError("{} is not support".format(args.triplet))
    xent = nn.CrossEntropyLoss()

    criterion['sxent'] = sxent.cuda()
    criterion['triplet'] = tripletloss.cuda()
    criterion['xent'] = xent.cuda()
    criterion['bitriplet'] = bitripletloss.cuda()
    if str_boolean(args.center):
        logger("{}: Center loss is adopted...".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
        centerloss = CenterLoss(num_classes=args.num_classes, feat_dim=args.num_feat * (args.num_strips + 1))
        criterion['center'] = centerloss.cuda()

# Define optimizer
    base_param_ids = list(map(id, model.module.base_rgb.parameters())) + list(map(id, model.module.base_ir.parameters()))
    base_params = list(model.module.base_rgb.parameters()) + list(model.module.base_ir.parameters())
    new_params = [p for p in model.parameters() if id(p) not in base_param_ids]

    param_groups = [{'params': base_params, 'lr_multi': 0.1},
                    {'params': new_params, 'lr_multi': 1.0}]

    optimizers = {}
    if args.optim == 'sgd':
        optimizer = torch.optim.SGD(param_groups, lr=args.lr, weight_decay=args.weight_decay,
                                    momentum=0.9, nesterov=True)
    elif args.optim == 'adam':
        optimizer = torch.optim.Adam(param_groups, lr=args.lr, weight_decay=args.weight_decay)
    else:
        raise ValueError('{} Optimizer {} does not support...'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                                                                      args.optim))
    optimizers['optim'] = optimizer
    if str_boolean(args.center):
        center_optimizer = torch.optim.SGD(centerloss.parameters(), lr=0.5)
        optimizers['c_optim'] = center_optimizer

    best_accu = 0
    for epoch in range(start_epoch, args.max_epoch):
        train(model, train_loader, optimizers, criterion, logger, writer, epoch=epoch, args=args)
        if (epoch+1) % args.eval_step == 0 or epoch + 1 == args.max_epoch:
            logger('{}: Performing evaluation...'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
            accu = eval(model, gallery_loader=test_rgb_loader, query_loader=test_ir_loader, logger=logger, epoch=epoch,
                        args=args, ranks=[1, 5, 10, 20])
            is_best = accu > best_accu
            if is_best: best_accu = accu
            if not os.path.exists(args.save_path):
                os.makedirs(args.save_path)
            ckpt_name = os.path.join(args.save_path, 'ckpt_epoch-{}_sampler-{}.pth.tar'.format(epoch + 1, args.sampler))
            state_dict = model.state_dict()
            torch.save({'state_dict':state_dict, 'epoch': epoch + 1}, ckpt_name)
            if is_best:
                shutil.copyfile(ckpt_name, os.path.join(args.save_path, 'best_model.pth.tar'))
            logger('{}: Best Accu: {}'.format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'), best_accu))
    writer.close()

# def adjust_lr(optimizer, epoch):
#     if epoch <= 30:
#         lr = args.lr
#     elif 30 < epoch <= 50:
#         lr = args.lr * 0.1
#     else:
#         lr = args.lr * 0.01
#     for g in optimizer.param_groups:
#         g['lr'] = lr * g.get('lr_multi', 1.0)

def adjust_lr(optimizer, epoch):
    if epoch < 10:
        lr = args.lr * (epoch + 1) / 10
    elif 10 <= epoch < 20:
        lr = args.lr
    elif 20 <= epoch < 50:
        lr = args.lr * 0.1
    else:
        lr = args.lr * 0.01
    for g in optimizer.param_groups:
        g['lr'] = lr * g.get('lr_multi', 1.0)

def adjust_lr_adam(optimizer, epoch):
    if epoch <= 10:
        lr = args.lr * (epoch + 1) / 10
    elif 10 < epoch <= 40:
        lr = args.lr
    elif 40 < epoch <= 70:
        lr = args.lr * 0.1
    else:
        lr = args.lr * 0.01
    for g in optimizer.param_groups:
        g['lr'] = lr * g.get('lr_multi', 1.0)

# def warmup_lr_scheduler(optimizer, milestones, gamma=0.1, warmup_factor=1.0/3, warmup_iters=500, warmup_method='linear', last_epoch=-1):
#     scheduler = WarmupMultiStepLR(optimizer, milestones, gamma=gamma, warmup_factor=warmup_factor, warmup_iters=warmup_iters,
#                                   warmup_method=warmup_method, last_epoch=last_epoch)
#     return scheduler

def train(model, data_loader, optimizers, criterions, logger, writer, epoch, args):
    adjust_lr(optimizers['optim'], epoch)
    model.train()
    logger("{} Epoch {}/{}".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), epoch + 1, args.max_epoch))
    TlossMeter = AverageMeter()
    IDlossMeter = AverageMeter()
    DAlossMeter =AverageMeter()
    AGlossMeter = AverageMeter()
    TPlossMeter = AverageMeter()
    i = 0

    for idx, (rgb_img, rgb_labels, rgb_camid, ir_img, ir_labels, ir_camid) in enumerate(data_loader):
        p = float(i + epoch * len(data_loader)) / args.max_epoch / len(data_loader)
        alpha = 2. / (1. + np.exp(-10 * p)) - 1
        rgb_img = rgb_img.float().cuda()
        ir_img = ir_img.float().cuda()
        rgb_labels = rgb_labels.long().cuda()
        ir_labels = ir_labels.long().cuda()
        labels = torch.cat((rgb_labels, ir_labels))
        domain_label = torch.cat((torch.ones_like(rgb_labels), torch.zeros_like(ir_labels))).long().cuda()
        domain_logit, feat, aligned_feat, feat_list, logit_list = model(rgb_img, ir_img, alpha, out_feature=False, norm=str_boolean(args.norm))
        if str_boolean(args.label_smooth):
            id_criterion = criterions['sxent']
        else:
            id_criterion = criterions['xent']

        part_id_loss = sum([id_criterion(logit_list[idx], labels) for idx in range(args.num_strips)]) / args.num_strips
        # part_id_loss = [id_criterion(logit_list[idx], labels) for idx in range(args.num_strips)]

        glob_id_loss = id_criterion(logit_list[args.num_strips], labels)

        id_loss = 0.5 * part_id_loss + 0.5 * glob_id_loss

        if args.align_norm == 'fro':
            aligned_loss = torch.sum(torch.norm(aligned_feat, p='fro', dim=1, keepdim=False))/args.train_batch
        elif args.align_norm == 'abs':
            aligned_loss = torch.sum(torch.norm(aligned_feat, p=1, dim=1, keepdim=False))/args.train_batch
        else:
            raise ValueError("Wrong input for the normalization, Pls select from 'fro' and 'abs'...")

        domain_loss = criterions['xent'](domain_logit, domain_label)
        # triplet_loss = criterions['triplet'](feat, labels)
        rgb_tripletloss = criterions['triplet'](feat[:args.train_batch, :], rgb_labels)
        ir_tripletloss = criterions['triplet'](feat[args.train_batch:, :], ir_labels)
        bitriplet_loss = criterions['bitriplet'](feat, labels)
        # if str_boolean(args.center):
        #     center_loss = criterions['center'](feat, labels)
        #     embed_loss = triplet_loss + 0.0005 * center_loss
        # else:
        #     embed_loss = triplet_loss

        loss = id_loss + domain_loss + bitriplet_loss + 0.0001 * aligned_loss

        optimizers['optim'].zero_grad()
        loss.backward()
        optimizers['optim'].step()

        if str_boolean(args.center):
            optimizers['c_optim'].zero_grad()
            for param in criterions['center'].parameters():
                param.grad.data *= (1. / 0.0005)
            optimizers['c_optim'].step()

        # update loss
        TlossMeter.update(loss, args.train_batch)
        DAlossMeter.update(domain_loss, args.train_batch)
        TPlossMeter.update(bitriplet_loss, args.train_batch)
        AGlossMeter.update(aligned_loss, args.train_batch)
        IDlossMeter.update(id_loss, args.train_batch)

        if idx % 50 == 0:
            logger("{}: Epoch: [{}/{}], Iter: [{}/{}] ID_loss: {:.5f}, Tri_loss: {:.5f}, DA_loss: {:.5f}, "
                   "Align_loss: {:.5f}, Tot_loss: {:.5f}".format(datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
                                                                 epoch + 1, args.max_epoch, idx, len(data_loader),
                                                                 id_loss, bitriplet_loss, domain_loss, aligned_loss, loss))

    writer.add_scalar('total_loss', TlossMeter.avg, epoch + 1)
    writer.add_scalar('identity_loss', IDlossMeter.avg, epoch + 1)
    writer.add_scalar('triplet_loss', TPlossMeter.avg, epoch + 1)
    writer.add_scalar('align_loss', AGlossMeter.avg, epoch + 1)
    writer.add_scalar('domain_loss', DAlossMeter.avg, epoch + 1)

def eval(model, gallery_loader, query_loader, logger, epoch, args, ranks=[1, 5, 10, 20]):
    model.eval()
    logger("{}: Extracting gallery feature...".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
    gf, gf_glob, g_pids, g_camids = [], [], [], []
    start_time = time.time()
    with torch.no_grad():
        for idx, (img, labels, camid) in enumerate(gallery_loader):
            # pdb.set_trace()
            img = img.float().cuda()
            feat_list = model(img, img, alpha=None, out_feature=True, norm=str_boolean(args.norm))
            feature = torch.cat(tuple(feat_list.values()), dim=1)[0]
            # pdb.set_trace()
            gf_glob.append(feat_list[args.num_strips][0].detach().cpu())
            gf.append(feature.detach().cpu())
            g_pids.extend(labels)
            g_camids.extend(camid.numpy())

        gf = torch.stack(gf)
        gf_glob = torch.stack(gf_glob)
        g_pids = np.asarray(g_pids)
        g_camids = np.asarray(g_camids)
    logger("Extracted features for gallery set, obtained {}-by-{} matrix".format(gf.size(0), gf.size(1)))
    logger('Extracting Time:\t {:.3f}'.format(time.time() - start_time))

    logger("{}: Extracting probe feature...".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
    qf, qf_glob, q_pids, q_camids = [], [], [], []
    start_time = time.time()
    with torch.no_grad():
        for idx, (img, labels, camid) in enumerate(query_loader):
            img = img.float().cuda()
            feat_list = model(img, img, alpha=None, out_feature=True, norm=str_boolean(args.norm))
            feature = torch.cat(tuple(feat_list.values()), dim=1)[1]
            qf_glob.append(feat_list[args.num_strips][1].detach().cpu())
            qf.append(feature.detach().cpu())
            q_pids.extend(labels)
            q_camids.extend(camid.numpy())
        qf = torch.stack(qf)
        qf_glob = torch.stack(qf_glob)
        q_pids = np.asarray(q_pids)
        q_camids = np.asarray(q_camids)
    logger("Extracted features for query set, obtained {}-by-{} matrix".format(qf.size(0), qf.size(1)))
    logger('Extracting Time:\t {:.3f}'.format(time.time() - start_time))

    logger("Computing distance matrix")
    if args.metric == 'euclidean':
         distmat = compute_distance_matrix(qf, gf, metric='euclidean')
         glob_distmat = compute_distance_matrix(qf_glob, gf_glob, metric='euclidean')
    elif args.metric == 'cosine':
        distmat = compute_distance_matrix(qf, gf, metric='cosine')
        glob_distmat = compute_distance_matrix(qf_glob, gf_glob, metric='cosine')
    else:
        raise ValueError("Type of distance {} is not supported. Pls select from euclidean and cosine...".format(args.metric))

    cmc, mAP, mINP = evaluate(distmat, q_pids, g_pids, q_camids, g_camids, max_rank=20, dataset=args.dataset)
    cmc_glob, mAP_glob, mINP_glob = evaluate(glob_distmat, q_pids, g_pids, q_camids, g_camids, max_rank=20, dataset=args.dataset)

    logger("Epoch: [{}] Results ----------".format(epoch + 1))
    logger("mAP: {:.1%}, GmAP: {:.1%}".format(mAP, mAP_glob))
    logger("mINP: {:.1%}, GmINP: {:.1%}".format(mINP, mINP_glob))
    logger("CMC curve")
    for r in ranks:
        logger("Rank-{:<3}: {:.1%}, {:.1%}".format(r, cmc[r - 1], cmc_glob[r -1]))
    logger("------------------")

    return cmc[0]

if __name__ == "__main__":
    parser = ArgumentParser(description="Parameters for SDAN")
    parser.add_argument('--mode', type=str, default='train', help='executive mode, train, resume or eval')
    parser.add_argument('--dataset', type=str, default='sysu', help='the used dataset')
    parser.add_argument('--seed', type=int, default=0, help='random seed')

    # image related parameters
    parser.add_argument('--img_height', type=int, default=288, help='image height')
    parser.add_argument('--img_width', type=int, default=144, help='image width')
    parser.add_argument('--num_strips', type=int, default=6, help='number of horizonal body strips')
    parser.add_argument('--num_feat', type=int, default=256, help='number of features')
    parser.add_argument('--num_classes', type=int, default=395, help='number of training classes')
    parser.add_argument('--erasing_p', type=float, default=0.5, help='possibility of random erasing')
    parser.add_argument('--trail', type=int, default=1, help='number of trails for RegDB cross-validation, 1,...,10')

    # training parameters
    parser.add_argument('--sampler', type=str, default='identity', help='sampler is used, identity or random')
    parser.add_argument('--train_batch', type=int, default=64, help='number of images in a batch')
    parser.add_argument('--num_instance', type=int, default=4, help='number of images per person in a batch')
    parser.add_argument('--optim', type=str, default='sgd', help='optimizer is used, sgd or adam')
    parser.add_argument('--weight_decay', type=float, default=5e-4, help='weight decay for optimizer')
    parser.add_argument('--lr', type=float, default=0.1, help='learning rate， 0。00035 for Adam, 0.1 for sgd')
    parser.add_argument('--num_workers', type=int, default=8, help='number of workers to load data')
    parser.add_argument('--max_epoch', type=int, default=100, help='maximum training epochs')

    # parameters for loss
    parser.add_argument('--margin', type=float, default=0.5, help='margin of triplet loss')
    parser.add_argument('--dropout', type=float, default=0, help='dropout possibility')
    parser.add_argument('--triplet', type=str, default='hard', help='type of triplet loss')
    parser.add_argument('--label_smooth', type=str, default='true', help='whether label smooth is used')
    parser.add_argument('--align_norm', type=str, default='fro', help='normalization manner, select from fro and abs')
    parser.add_argument('--center', type=str, default='false', help='whether center loss is used')
    parser.add_argument('--norm', type=str, default='false', help='whether feature is l2 normalized')

    # eval parameters
    parser.add_argument('--test_batch', type=int, default=1, help='batch size for testing, default is 1')
    parser.add_argument('--eval_step', type=int, default=10, help='number of epoch to perform evaluation')
    parser.add_argument('--num_shot', type=str, default='single', help='number of shots for evaluation, single or all')
    parser.add_argument('--eval_mode', type=str, default='all', help='evaluation mode: all and indoor search')

    parser.add_argument('--metric', type=str, default='cosine', help='evaluation distance, euclidean or cosine')
    # parameters for model saving
    parser.add_argument('--save_path', type=str, default='./logs/0512/', help='the path to save model')
    parser.add_argument('--ckpt_name', type=str, default='ckpt_epoch-100_sampler-identity.pth.tar')
    parser.add_argument('--log_path', type=str, default='./logs/0512/', help='the path to store logs')
    parser.add_argument('--vis_log_dir', type=str, default='./logs/0512/', help='the path to store visualization log')
    args = parser.parse_args()
    main(args)


#!/usr/bin/env python

import binascii
import hashlib
import logging.config
import math
import os
import struct
from argparse import ArgumentParser
from datetime import datetime

# static variables
COMMAND_DELAY = .008    # seconds
TIMESTAMP_LEN = 14      # len(now)
NODE_AMT_LEN = 4
NODE_ID_LEN = 10
SEED_LEN = 10
assert NODE_ID_LEN == SEED_LEN
PS_DATA_LEN = 50

log_parser = ArgumentParser()
log_parser.add_argument('--logfile', dest='log_file', help='log to filename', default='ps_multi_nodes.log')
args, unknown = log_parser.parse_known_args()
logging.config.fileConfig('logging.ini', defaults={'log_file': args.log_file})
logger = logging.getLogger()

class PerfectScheme:
    alloc_frame = []
    seed = None
    nodes_expect_time = []

    def __init__(self, broadcast_id, send_pkt_id, node_slot_time):
        self.ps_broadcast_id = broadcast_id
        self.ps_send_pkt_id = send_pkt_id
        self.node_slot_time = node_slot_time

    def compute_alloc_index(self, node_amount, node_id, salt):
        """
        Math hashing: h(Node ID, seed)

        Returns: alloc index

        """
        # Python official: hashlib.<crpyto_hash> (openssl_<crypto_hash) is 3x faster than pbkdf2_hmac
        hashed = hashlib.sha256(node_id + salt).hexdigest()
        # derived_key = hashlib.pbkdf2_hmac('sha256', node_id, seed, 10000)
        # hashed = binascii.hexlify(derived_key)
        hashed_bin = int(hashed, 16)
        alloc_index = hashed_bin % node_amount
        return alloc_index

    def get_random_seed(self):
        """
        Random seeds at size of NODE_ID_LEN. Prefix with leading zeros if chosen number has less length.
        Note: Seed length must be same as Node ID length.

        Returns: seed number
        """
        # seed = '{0:8}'.format(randint(1, 99999999))
        # # replace prefix spaces with 0, if any
        # return seed.replace(' ', '0')
        # return os.urandom(NODE_ID_LEN)

        # Hex seed will double its size, so half the size first
        return binascii.b2a_hex(os.urandom(NODE_ID_LEN/2))

    def generate_perfect_seed(self, node_id_list):          # BS only
        assert all(len(node_id) == NODE_ID_LEN for node_id in node_id_list),\
            "All Node ID length must be {}".format(NODE_ID_LEN)

        salt = None
        has_collision = True
        calc_count = 0
        while has_collision:
            calc_count += 1
            self.alloc_frame = ['0'] * len(node_id_list)
            salt = self.get_random_seed()
            for node_id in node_id_list:
                a_index = self.compute_alloc_index(len(node_id_list), node_id, salt)
                self.alloc_frame[a_index] = node_id

            # Check if any calculated node index is collided
            has_collision = any(n == '0' for n in self.alloc_frame)
            # logger.debug("salt {}, alloc {}".format(salt, self.alloc_frame))

        logger.info("Calculated perfect seed: {}, calculation count: {}, \nAlloc Frame: {}".format(
                    salt, calc_count, self.alloc_frame))
        self.seed = salt

    def broadcast_ps_pkt(self, my_tb, pkt_size, node_amount, pktno=1):     # BS only
        # payload = prefix + now + ps_broadcast + node_amount + seed + node_begin_time + dummy

        assert self.seed is not None, "Seed is None!"

        payload_prefix = struct.pack('!H', pktno & 0xffff)
        broadcast = struct.pack('!H', self.ps_broadcast_id & 0xffff)
        node_amount_str = str(node_amount).zfill(NODE_AMT_LEN)
        assert len(node_amount_str) <= NODE_AMT_LEN, "Too much nodes to handle: {}".format(node_amount)
        data_size = len(payload_prefix) + TIMESTAMP_LEN + len(broadcast) + len(node_amount_str) + len(self.seed)\
            + TIMESTAMP_LEN
        dummy = (pkt_size - data_size) * chr(pktno & 0xff)
        now_timestamp = my_tb.sink.get_time_now().get_real_secs()
        now_timestamp_str = '{:.3f}'.format(now_timestamp)
        begin_timestamp = math.ceil(now_timestamp + 1)      # begin time buffer 1 second, let all nodes to be ready
        begin_timestamp_str = '{:.3f}'.format(begin_timestamp)
        # For BS recv PS_PKT session timer use
        self.nodes_expect_time = []
        for i in range(node_amount):
            # Each node time slot: 0.5 seconds
            begin_at = begin_timestamp + (self.node_slot_time * i)
            end_at = begin_at + 0.5 - 0.0001
            self.nodes_expect_time.append((self.alloc_frame[i], begin_at, end_at))
        payload = payload_prefix + now_timestamp_str + broadcast + node_amount_str + self.seed + begin_timestamp_str\
            + dummy
        my_tb.txpath.send_pkt(payload)
        logger.info("{} send PS_BROADCAST {}, Total nodes: {}, Seed: {}, Node begin: {}, \nSession: {}".format(
                    str(datetime.fromtimestamp(now_timestamp)), pktno, node_amount, self.seed,
                    str(datetime.fromtimestamp(begin_timestamp)), self.nodes_expect_time))

    def send_ps_pkt(self, node_id, my_tb, pkt_size, ps_data, pktno=1):               # Node only
        # payload = prefix + now + ps_pkt + data + dummy

        assert len(ps_data) <= PS_DATA_LEN, "wrong ps_data len {}".format(len(ps_data))
        ps_data = ps_data.rjust(PS_DATA_LEN)    # padding with spaces

        payload_prefix = struct.pack('!H', pktno & 0xffff)
        ps_pkt = struct.pack('!H', self.ps_send_pkt_id & 0xffff)
        data_size = len(payload_prefix) + TIMESTAMP_LEN + len(ps_pkt) + PS_DATA_LEN
        dummy = (pkt_size - data_size) * chr(pktno & 0xff)
        now_timestamp = my_tb.sink.get_time_now().get_real_secs()
        now_timestamp_str = '{:.3f}'.format(now_timestamp)
        payload = payload_prefix + now_timestamp_str + ps_pkt + ps_data + dummy
        my_tb.txpath.send_pkt(payload)
        logger.info("{} PS: Node {} send data {}".format(str(datetime.fromtimestamp(now_timestamp)), node_id, pktno))

    def get_node_amount(self, payload):
        node_amount_str = payload[2+TIMESTAMP_LEN+2:2+TIMESTAMP_LEN+2+NODE_AMT_LEN]
        node_amount = int(node_amount_str)
        return node_amount

    def get_seed(self, payload):
        seed = payload[2+TIMESTAMP_LEN+2+NODE_AMT_LEN:2+TIMESTAMP_LEN+2+NODE_AMT_LEN+SEED_LEN]
        return seed

    def get_begin_time_str(self, payload):
        begin_timestamp_str = payload[2+TIMESTAMP_LEN+2+NODE_AMT_LEN+SEED_LEN:
                                      2+TIMESTAMP_LEN+2+NODE_AMT_LEN+SEED_LEN+TIMESTAMP_LEN]
        return begin_timestamp_str

    def get_node_data(self, payload):
        ps_data = payload[2+TIMESTAMP_LEN+2:2+TIMESTAMP_LEN+2+PS_DATA_LEN]
        return ps_data.lstrip()


#!/usr/bin/env python3
from functools import reduce

#248345-746315

start=[2,4,8,3,4,5]
end  =746315

count=0

def onlyRising(list):
    for i in range(1,len(list)):
        if list[i-1] > list[i]:
            list[i] = list[i-1]

    return list

def hasDouble(list):
    i=0
    while i < len(list)-1:
        lookahead = 1
        matched=0
        while True:
            if i+lookahead > len(list)-1:
                break
            if list[i] == list[i+lookahead]:
                matched+=1
                lookahead+=1
            else:
                break
        if matched == 1:
            return True
        i+=lookahead
    return False

def reachedLimit(current, limit):
    reduced = reduce((lambda x, y: int(str(x) + str(y))), current)
    return reduced>=limit

def increment(list):
    index=len(list)-1

    list[index] = (list[index]+1)%10
    wrapped = list[index] == 0
    while wrapped:
        index-=1
        list[index] = (list[index]+1)%10
        wrapped = list[index] == 0
    wasWrapped = index < len(list)-1
    return list, wasWrapped

current = onlyRising(start)

while True:
    if hasDouble(current):
        count+=1

    current, wrapped = increment(current)
    if wrapped:
        current = onlyRising(current)

    if reachedLimit(current,end):
        break

print(current)
print(count)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
from compilation import junit_generation
from compilation import files

def main():
    test_suites = files.parse_jsonl(files.read_file('test_summary.jsonl'))
    xml = junit_generation.generate_junit_xml(test_suites)
    files.create_text_file('log.xml', xml)

if __name__ == "__main__":
    main()


from sys import stdin ,stdout
n=int(stdin.readline())
rows,cols=(n,n)
a=[[0 for i in range(cols)] for j in range(rows)]
for i in range(n):
    b=stdin.readline()
    for j in range(n):
        a[i][j]=int(b.split()[j])
for i in range(n):
    stdout.write(str(i+1))
    for j in range(n):
        if(a[i][j]==1):
            stdout.write(" ")
            stdout.write(str(j+1))
    stdout.write('\n')


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

pd.set_option('display.max_columns', None)
pd.set_option('display.width', 150)

sns.set()
ROOT = '/work/workspaces/pycharm/training_data_science/data/'
df = pd.read_csv(ROOT + 'police.csv')
months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']

df.rename(
    columns={'driver_age': 'age', 'driver_age_raw': 'year_of_birth', 'driver_race': 'race', 'driver_gender': 'gender'},
    inplace=True)

print(df.info())

df.drop('county_name', axis=1, inplace=True)

df.groupby('gender')['gender'].count()
df['gender'].fillna(method='pad', inplace=True)

# search type is related to
df[df['search_type'].notna() & df['search_conducted']].count()
df.loc[df['search_type'].isna(), ['search_type']] = 'NO SEARCH'

# drop NaNs because there is same number in almost all columns
df.dropna(inplace=True)
df['age'] = df['age'].astype(np.int)
df['year_of_birth'] = df['year_of_birth'].astype(np.int)
df['is_arrested'] = df['is_arrested'].astype(np.bool)

df.stop_duration.replace(['1','2'],'30+ Min', inplace=True )

mapping = {'0-15 Min':8, '16-30 Min':23, '30+ Min':45}
df['stop_minutes'] = df.stop_duration.map(mapping)

# add multiindex from dates
# mi = pd.MultiIndex.from_frame(df[['stop_date','stop_time']])
df.index = pd.to_datetime(df['stop_date'] + ' ' + df['stop_time'])
df.index.name='stop_date_time'
df.drop(['stop_date', 'stop_time'], axis=1, inplace=True)

#group by month and gender
month_gen = df.groupby([df.index.month,'gender'])['gender'].count().unstack()
month_gen.index = months
month_gen.plot()
plt.show()

#get hourly stop by gender
hour_gen = df.groupby([df.index.hour,'gender'])['gender'].count().unstack()
hour_gen.plot()
plt.show()

#bar plot by violation and age
age_periods= pd.cut(df['age'],5, precision=1)
violation_pt = df.pivot_table(df[['stop_outcome']],index='violation',columns=age_periods, aggfunc='count')
violation_pt.plot(kind='bar')
plt.show()

#return violation kind
df.groupby(['violation', 'stop_outcome'])['violation'].count().unstack(level=1).plot(kind='bar')

#group violations by age group
us = df.groupby([age_periods,'violation'])['violation'].count().unstack(level=0)
per_age_proc = us.transform((lambda x: (x/x.sum()) * 100), axis=0).round(1)
per_age_proc.T.plot.bar()
plt.show()

#per violation
per_viol = us.transform((lambda x: (x/x.sum()) * 100), axis=1).round(1)
per_viol.T.plot.bar()
plt.show()

#relationship between violation and stop outcome
viol_outcome_mins = df.pivot_table(values='stop_minutes', columns='stop_outcome', index='violation').round(1)
pd.concat([viol_outcome_mins.idxmin(), viol_outcome_mins.min()], axis=1,keys=['violation', 'time'])

####currently not used cleaning functions

# fix driver raw age, replace all Nan, missing and not relevant with mean date
# df['year_of_birth'].fillna(0, inplace=True)
# df['year_of_birth'] = df['year_of_birth'].astype(np.int)
# df.loc[df['year_of_birth'] > 2020, 'year_of_birth'] = 0
# df.loc[df['year_of_birth'] < 1920, 'year_of_birth'] = 0
# print(df.loc[df['year_of_birth'] > 0, 'year_of_birth'].describe())
#
# # set missing year of birth to mean
# df.loc[df['year_of_birth'] == 0, 'year_of_birth'] = round(df.loc[df['year_of_birth'] > 0, 'year_of_birth'].mean())

# fixing age
# df['age'].fillna(0, inplace=True)
# df['age'] = df['age'].astype(np.int)
# df.loc[df['age'] > 100, 'age'] = 0
# df.loc[df['age'] < 0, 'age'] = 0
# df.loc[df['age'] == 0, 'age'] = round(df.loc[df['age'] > 0, 'age'].mean())
#
# # process race
# df.groupby('race')['race'].count()
# df['race'].describe()
# df['race'].fillna(method='pad', inplace=True)
#
# # because volation an violation raw have equal nr of null check for intersections
# df.loc[(df['violation'].notna() & df['violation_raw'].notna()), ['violation_raw', 'violation']].count()
#
# # violation raw
# df.groupby('violation_raw')['violation_raw'].count()
# df['violation_raw'].describe()
# df['violation_raw'].fillna(method='pad', inplace=True)
#
# # violation
# df.groupby('violation')['violation'].count()
# df['violation'].describe()
# df['violation'].fillna(method='pad', inplace=True)


import  pandas

upload = pandas.read_csv("linked.csv", encoding='utf-8')
def printja(data):
    l = []
    if data == "None" or data == None:
        l.append("None")
    else:
        for i in data:
            l.append(upload['url'][i])

    return l

#a = printja({0,1,2})
#print(a)


#!/usr/bin/python
# Solved by Bogdan Trif @   Completed on Wed, 21 Sep 2016, 22:21
#The  Euler Project  https://projecteuler.net
'''
Ordered fractions       -       Problem 71
Consider the fraction, n/d, where n and d are positive integers. If n<d and HCF(n,d)=1, it is called a reduced proper fraction.
If we list the set of reduced proper fractions for d ≤ 8 in ascending order of size, we get:
1/8, 1/7, 1/6, 1/5, 1/4, 2/7, 1/3, 3/8, 2/5, 3/7, 1/2, 4/7, 3/5, 5/8, 2/3, 5/7, 3/4, 4/5, 5/6, 6/7, 7/8
It can be seen that 2/5 is the fraction immediately to the left of 3/7.
By listing the set of reduced proper fractions for d ≤ 1,000,000 in ascending order of size,
find the numerator of the fraction immediately to the left of 3/7.
'''
import time
print('------------------   MY SOLUTION, Corrected in 2016-09-24, 16:00 ---------------------------')
t1  = time.time()

#   using  DYNAMIC RANGING :

max_nr = 0.4
iter=1
for i in range (999900, 1000000):
    for j in range(int(i*3/7), int(i*3/7)+1) :
        iter+=1
        if ( max_nr < j/i < 3/7  ):         # ATTENTION !!!  What an ERROR LOGIC CAUGHT ME HERE
            max_nr, a , b = j/i , j, i                                          # I put max_nr < 3/7 instead of j/i < 3/7
            #print(str(iter)+'.   j , i = ',j ,', ' , i  , '  ;   ',max_nr,'  ;  ', j / i,'  ; 3/7 = ',3/7)            # Correct : 428570   /  999997

print('\nAnd the ANSWER is :     ',a, b, max_nr)
#time.sleep(1)
t2  = time.time()
print('\nCompleted in :', (t2-t1)*1000, 'ms')                       #Completed in : 3.00 ms

print('-------'*20)

print('\n===============  SOLUTIONS FROM THE EULER FORUM ===============')
print('\n--------------SOLUTION 1 ----------Heureka, from Russia------------')
# Incredible solution
# https://en.wikipedia.org/wiki/Stern%E2%80%93Brocot_tree
t1  = time.time()

M = 1000000

n = 2
d = 5

while d + 7 <= M:
    n = n + 3
    d = d + 7

print(n, d)

t2  = time.time()
print('\nCompleted in :', (t2-t1)*1000, 'ms')

print('\n--------------------SOLUTION 2 ---------------------')

t1  = time.time()

from fractions import Fraction

d = 1000000
lower = int ((3/7 * d) - 50) #as close as possible to 3/7
set1 = set()

while d > 0:
    for i in range (lower,d):
        if i/d < 3/7:
            set1.add(Fraction(i,d))
        else: break
    d -= 1

listAns = sorted(list(set1))
print(listAns[-1].numerator)

t2  = time.time()
print('\nCompleted in :', (t2-t1)*1000, 'ms')

print('\n--------------------SOLUTION 3 ---------------------')
t1  = time.time()

from fractions import Fraction

a=max([Fraction(int((3000000-i)/7),1000000)for i in range(1,100)])
print (max([Fraction(x,y) for x in range(a.numerator-3,a.numerator+3) for y in range (a.denominator-3,a.denominator+1) if float(x)/y<float(3)/7]))

t2  = time.time()
print('\nCompleted in :', (t2-t1)*1000, 'ms')

print('\n--------------------SOLUTION 4 - Brute Force, by davidmcdavid from England ---------------------')
t1  = time.time()

#Brute force with floats in Python3:
target = 3/7
nearest = float('inf')
for d in range(1, 1000000+1):
    n = d*3//7
    distance = target-n/d
    if nearest > distance and distance != 0:
        nearest = distance
        answer = n
print(answer)
assert answer==428570

t2  = time.time()
print('\nCompleted in :', (t2-t1)*1000, 'ms')

print('\n--------------------SOLUTION 5 - Brute Force 2---------------------')
t1  = time.time()

#Brute force in Python 3.
# For i in range(1,000,000), calculate the closest number to 3/7 without going over and then compare it to a running total of the closest.

upper = [3, 7]
lower = [1,3]

for i in range(2,10**6):
    num = (upper[0] * i - 1) // upper[1]
    potential = [num, i]
    if potential[0] * lower[1] > potential[1] * lower[0]:
        lower = potential
print(lower)

t2  = time.time()
print('\nCompleted in :', (t2-t1)*1000, 'ms')

Project EULER/pb071 Ordered fractions.py


# -*- coding: utf-8 -*-
# Author: sunmengxin
# time: 10/18/18
# file: 单对单最短路径-dijkstra算法.py
# description: 一步一步更新经多经过当前点的最短路径

import math
import copy as cp

if __name__ == '__main__':

    n, m = input().split()
    n, m = int(n), int(m)

    vis = [0 for _ in range(n)]
    map = [[math.inf for _ in range(n)] for _ in range(n)]
    for i in range(n):
        for j in range(n):
            if i == j:
                map[i][j] = 0

    for _ in range(m):
        x = input()
        a, b, w = x.split(' ')
        map[int(a)][int(b)] = int(w)
        map[int(b)][int(a)] = int(w)

    dis = cp.deepcopy(map[0])
    vis[0] = 1 # v0 是起点,算作是已经被访问过了
    print(0,end = '\t')
    for i in range(1,n):
        # 找到一个ie最近的顶点
        min_j = -1
        min_dis = math.inf
        for k in range(1,n):
            if vis[k] == 0 and dis[k] < min_dis:
                min_j = k
                min_dis = dis[k]
        if min_j == -1:
            break
        else:
            vis[min_j] = 1
            print(min_j,end = '\t')
            for k in range(n):
                if vis[k] == 0 and dis[k] > min_dis + map[min_j][k]:
                    dis[k] = min_dis + map[min_j][k]
'''
9 16
0 1 1
0 2 5
1 2 3
1 3 7
1 4 5
2 4 1
2 5 7
3 4 2
3 6 3
4 5 3
4 6 6
4 7 9
5 7 5
6 7 2
6 8 7
7 8 4
'''


import random

import numpy as np

from rl.action import Action
from rl.agent.agent import Agent
from rl.domain import Domain
from rl.state import State
from rl.task import Task
from rl.valuefunction import FeatureExtractor
from rl.valuefunction.linear import LinearVFA

class SarsaVFA(Agent):
    def __init__(self, domain: Domain, task: Task, feature_extractor: FeatureExtractor, epsilon=0.1, alpha=0.6,
                 gamma=1.0, name="Sarsa"):
        """
        :param domain: The world the agent is placed in.
        :param task: The task in the world, which defines the reward function.
        """
        super().__init__(domain, task, name)
        self.world = domain
        self.task = task
        self.epsilon = epsilon
        self.alpha = alpha
        self.gamma = gamma
        self.previousaction = None
        self.previousstate = None

        actions = domain.get_actions()
        self.feature_extractor = feature_extractor
        self.value_function = LinearVFA(self.feature_extractor.num_features(), actions)

        self.current_cumulative_reward = 0.0

    def act(self):
        """Execute one action on the world, possibly terminating the episode.

        """
        state = self.domain.get_current_state()
        action = self.choose_action(state)

        self.world.apply_action(action)

        # For the first time step, we won't have received a reward yet.
        # We're just notifying the learner of our starting state and action.
        if self.previousstate is None and self.previousaction is None:
            ()
        else:
            self.update(self.previousstate, self.previousaction, state, action)

        self.previousaction = action
        self.previousstate = state

        state_prime = self.world.get_current_state()
        if self.task.stateisfinal(state_prime):
            self.update(state, action, state_prime, None, terminal=True)

    def update(self, state: State, action: Action, state_prime: State, action_prime: Action,
               terminal=False):
        reward = self.task.reward(state, action, state_prime)

        phi = self.feature_extractor.extract(state, action)

        value = self.value_function.value(phi)

        # Terminal states are defined to have value 0
        if terminal:
            phi_prime = np.zeros(len(phi))
        else:
            phi_prime = self.feature_extractor.extract(state_prime, action_prime)
        value_prime = self.value_function.value(phi_prime)

        self.value_function.weights += self.alpha * (reward + self.gamma * value_prime - value) * phi
        self.current_cumulative_reward += reward

    def choose_action(self, state) -> Action:
        """Given a state, pick an action according to an epsilon-greedy policy.

        :param state: The state from which to act.
        :return:
        """
        if random.random() < self.epsilon:
            actions = self.domain.get_actions()
            return random.sample(actions, 1)[0]
        else:
            best_actions = self.value_function.bestactions(state, self.feature_extractor)
            return random.sample(best_actions, 1)[0]

    def get_cumulative_reward(self):
        return self.current_cumulative_reward

    def episode_ended(self):
        # Observe final transition if needed
        self.current_cumulative_reward = 0.0
        self.previousaction = None
        self.previousstate = None
        self.epsilon *= 0.99999

    def logepisode(self):
        print("Episode reward: " + str(self.current_cumulative_reward))


import argparse
import torch
import torch.optim as optim
import os
import pandas as pd
import torch.nn.functional as F
import numpy as np
import matplotlib.pyplot as plt
from dataset_binary import AgricultureDataset
from dice_loss import BinaryDice
from unet import Unet_ResNet34 as Unet
from PIL import Image
import time

def save_checkpoint(state, path):
    torch.save(state, path)

def train(model, device, train_loader, optimizer, epoch, History: dict):
    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        # print(data.shape)

        '''
         shape of test-data:torch.Size([1, 3, 512, 512]),test-target:torch.Size([1, 512, 512])
        shape:torch.Size([16, 4, 512, 512])  16是batch_size,4是三分类+一个背景
        '''
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)

        '''
        weight表示4种分类各自在loss中占有的权重
        NLLLoss的结果就是把上面的输出output与Label对应的那个值(索引)拿出来，再去掉负号，再求均值。
        log_softmax+NLLLoss的操作可由CrossEntropyLoss直接合并
        '''
        output = F.sigmoid(output)
        loss = BinaryDice(output, target)
        loss.backward()
        optimizer.step()

        if batch_idx % 20 == 0:  # 每50个batchs打印一下训练状态
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                       100. * batch_idx / len(train_loader), loss.item()))

    if epoch % 1 == 0:
        img_origin = (data.detach()[0, :, :, :].cpu().numpy() * 255).astype(np.uint8)
        img_origin = np.transpose(img_origin, (1, 2, 0))
        img_origin = Image.fromarray(img_origin)
        img_pre = output.detach()[0, :, :, :].cpu()
        img_pre = np.around(img_pre.byte().numpy()) * 220
        img_pre = Image.fromarray(img_pre)
        img_real = target.detach()[0, :, :, :].cpu()
        img_real = np.around(img_real.byte().numpy()) * 220
        img_real = Image.fromarray(img_real)
        img_origin.save('./results/tmp/img{}.png'.format(epoch))
        img_pre.save('./results/tmp/predict{}.png'.format(epoch))
        img_real.save('./results/tmp/real{}.png'.format(epoch))

def evaluate(model, device, data_loader, data_catag, History: dict, class_num=4):
    model.eval()
    eval_loss = 0
    max_batch = len(data_loader)
    with torch.no_grad():
        for batch_idx, (data, target) in enumerate(data_loader):
            data, target = data.to(device), target.to(device)

            output = model(data)
            output = F.sigmoid(output)
            loss = BinaryDice(output, target).to('cuda')
            eval_loss += loss

    eval_loss = eval_loss / max_batch
    eval_acc = 1 - eval_loss

    History[data_catag + "_loss"].append(eval_loss)
    History[data_catag + "_acc"].append(eval_acc)
    print("{} Loss is {:.6f}, mean precision is: {:.4f}".format(data_catag, eval_loss, eval_acc))

def main():
    # Training settings
    parser = argparse.ArgumentParser(description='Scratch segmentation Example')
    parser.add_argument('--class_num', type=int, default=4, metavar='N',
                        help='input class number of label including background (default: 4)')
    parser.add_argument('--class_name', type=1, default=1, metavar='N',
                        help='input the class name which you want to seg')
    parser.add_argument('--train_batch_size', type=int, default=8, metavar='N',
                        help='input batch size for training (default: 64)')
    parser.add_argument('--test_batch_size', type=int, default=8, metavar='N',
                        help='input batch size for testing (default: 1000)')
    parser.add_argument('--epochs', type=int, default=30, metavar='N',
                        help='number of epochs to train (default: 10)')
    parser.add_argument('--lr', type=float, default=0.001, metavar='LR',
                        help='learning rate (default: 0.01)')
    parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
                        help='SGD momentum (default: 0.5)')
    parser.add_argument('--results_path', type=str,
                        help='input your path to save the results')
    parser.add_argument('--image_path', type=str,
                        help='input your path to load the image')
    parser.add_argument('--label_path', type=str,
                        help='input your path to load the label')
    parser.add_argument('--data_csv_path', type=str,
                        help='input your path to load the date_cvs file')
    parser.add_argument('--no_cuda', action='store_true', default=False,
                        help='disables CUDA training')
    parser.add_argument('--seed', type=int, default=1, metavar='S',
                        help='random seed (default: 1)')

    args = parser.parse_args()

    if not os.path.exists(args.results_path + "_" + args.class_num):
        os.mkdir(args.results_path + "_" + args.class_name)
    if not os.path.exists(args.results_path + os.sep + "checkpoint_path"):
        os.mkdir(args.results_path + os.sep + "checkpoint_path")
    if not os.path.exists(args.results_path + os.sep + "model"):
        os.mkdir(args.results_path + os.sep + "model")
    if not os.path.exists(args.results_path + os.sep + 'loss_curve'):
        os.mkdir(args.results_path + os.sep + 'loss_curve')
    if not os.path.exists(args.results_path + os.sep + 'tmp'):
        os.mkdir(args.results_path + os.sep + 'tmp')

    use_cuda = not args.no_cuda and torch.cuda.is_available()
    torch.manual_seed(args.seed)
    device = torch.device("cuda" if use_cuda else "cpu")
    print('my device is :', device)

    start_epoch = 0
    best_precision = 0

    History = {"train_loss": [],
               "train_acc": [],
               "val_loss": [],
               "val_acc": []
               }

    model = Unet(args.class_num).to(device)
    optimizer = optim.Adam(model.parameters(), lr=args.lr)

    resume = False
    # 4.4 restart the training process
    if resume:
        checkpoint = torch.load(args.results_path + os.sep + "checkpoint_path" + os.sep + "_checkpoint.pth.tar")
        start_epoch = checkpoint["epoch"]
        best_precision = checkpoint["best_precision"]
        model.load_state_dict(checkpoint["state_dict"])
        History = checkpoint["History"]
        print("---------------loading checkpoint----------------")
        optimizer.load_state_dict(checkpoint["optimizer"])

    data_list = pd.read_csv(args.data_csv_path)
    data_list = data_list.sample(frac=1)

    kwargs = {'num_workers': 0, 'pin_memory': True} if use_cuda else {}
    train_loader = torch.utils.data.DataLoader(AgricultureDataset(image_path=args.image_path,
                                                                  label_path=args.label_path,
                                                                  datalist=data_list,
                                                                  class_name=args.class_name,
                                                                  mode="train",
                                                                  train_ratio=0.8),
                                               batch_size=args.train_batch_size,
                                               shuffle=True,
                                               drop_last=True, **kwargs)
    valid_loader = torch.utils.data.DataLoader(AgricultureDataset(image_path=args.image_path,
                                                                  label_path=args.label_path,
                                                                  datalist=data_list,
                                                                  class_name=args.class_name,
                                                                  mode="valid",
                                                                  is_aug=False),
                                               batch_size=args.test_batch_size,
                                               shuffle=True,
                                               drop_last=True, **kwargs)

    for epoch in range(start_epoch, args.epochs + 1):

        train(model, device, train_loader, optimizer, epoch, History)

        save_checkpoint({
            "epoch": epoch + 1,
            "state_dict": model.state_dict(),
            "best_precision": best_precision,
            "optimizer": optimizer.state_dict(),
            "History": History
        }, args.results_path + os.sep + "checkpoint_path" + os.sep + "/_checkpoint.pth.tar")

        if (epoch + 0) % 1 == 0:
            print("epoch:{}--------------------\n".format(epoch))

            evaluate(model, device, train_loader, "train", History, args.class_num)
            evaluate(model, device, valid_loader, "val", History, args.class_num)

            torch.save(model,
                       args.results_path + os.sep + "model" + os.sep + "model_epoch{}.pth".format(epoch))
            if History["val_loss"][-1] < best_precision:
                best_precision = History["val_loss"][-1]
                torch.save(model, args.results_path + os.sep + "model" + os.sep + "model_best.pth")
            plt.style.use("ggplot")
            plt.figure()
            N = epoch + 1
            plt.plot(np.arange(0, N), History["train_loss"], label="train_loss")
            plt.plot(np.arange(0, N), History["val_loss"], label="val_loss")
            plt.plot(np.arange(0, N), History["train_acc"], label="train_acc")
            plt.plot(np.arange(0, N), History["val_acc"], label="val_acc")
            plt.title("Training Loss and Accuracy on Unet Satellite Seg")
            plt.xlabel("Epoch #")
            plt.ylabel("Loss/Accuracy")
            plt.legend(loc="lower left")
            plt.savefig(args.results_path + os.sep + 'loss_curve' + os.sep + 'curve_{}.jpg'.format(epoch))
            plt.close()

if __name__ == '__main__':
    main()


from __future__ import unicode_literals
import frappe
from frappe import _

def get_fiscal_year(fiscal_year):
    return frappe.db.sql(""" select start_date, end_date from `tabFiscal Year` where name=%s """, fiscal_year, as_dict=1)

def get_columns(fiscal_year):
    columns = [
    {
        "fieldname": "account",
        "label": _("Accounts"),
        "fieldtype": "Link",
        "options": "Accounts",
        "width": 300
    },
    {
        "fieldname": "account_balance",
        "label" : fiscal_year,
        "fieldtype": "Currency",
        "width": 200
    }]
    return columns

def get_data(account_type, fiscal_year):
    accounts = get_accounts(account_type)
    if not accounts:
        return None
    accounts, accounts_by_name, parent_children_map = filter_accounts(accounts)
    gl_entries_by_account = {}
    for root in frappe.db.sql("""select lft, rgt from tabAccounts
            where account_type=%s and ifnull(parent_accounts, '') = ''""", account_type, as_dict=1):

        set_gl_entries_by_account(fiscal_year[0].start_date, fiscal_year[0].end_date, root.lft, root.rgt, gl_entries_by_account, ignore_closing_entries=False)

    accumulate_values_into_parents(accounts, accounts_by_name)
    out = prepare_data(accounts, fiscal_year)

    return out

def get_accounts(account_type):
    return frappe.db.sql("""
        select *
        from `tabAccounts`
        where account_type=%s order by lft""", (account_type), as_dict=1)


import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tellmewhen.settings")

from apscheduler.schedulers.blocking import BlockingScheduler
from watcher.models import Watcher
import redis
from rq import Worker, Queue, Connection

sched = BlockingScheduler()
listen = ['high', 'default', 'low']

redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379')

conn = redis.from_url(redis_url)

with Connection(conn):
    worker = Worker(map(Queue, listen))
    worker.work()

@sched.scheduled_job('interval', minutes=1)
def queue_object_check():
    Queue(connection=conn).enqueue(Watcher.objects.check_for_new_objects)

sched.start()

listen = ['high', 'default', 'low']

redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379')

conn = redis.from_url(redis_url)

with Connection(conn):
    worker = Worker(map(Queue, listen))
    worker.work()


# Горохова Елена КНИТ 16-А
# Сформировать функцию подсчета количества разных представлений заданного натурального числа в виде
# суммы не менее двух попарно различных натуральных слагаемых
import numpy as np
def rec_decompose(c):
    '''
    Функция подсчета количества различных представлений заданого числа. Рекурсивно беез повторений
    :param c: число
    :return: количество различных представлений числа
    '''
    def func(c, id):
        if c == 0 and id == 0:
            return 1
        if c <= 0 or id <= 0:
            return 0
        return func(c - id, id) + func(c - 1, id - 1)
    stack = []
    if c == 0 or c == 1:
        return 1
    for i in range(1, c):
        stack.append(func(c, i))
    return max(stack)
def number_decompose_iter(n):
    """ Функция подсчета количества различных представлений заданого числа.
     Итерационно без повторений

    :param n: число
    :return: количество различных представлений числа
    """
    a = 0
    n1 = n
    c = 0  # количество представлений числа n
    while n1 >= 0:
        a += 1
        n1 -= a
    mas = np.zeros(a - 1, dtype=int)
    for i in range(0, a - 1):
        mas[i] = i + 1
    if n1 != 0:
        mas[a - 2] = mas[a - 3] + a + 1 + n1
    a -= 1
    for i in range(0, a):
        pass
    c += 1
    while a > 1:
        for j in range(a - 2, -1, -1):
            temp1 = mas[j]
            temp2 = mas[a - 1]
            while mas[j] + 1 < mas[a - 1] - 1:
                mas[j] += 1
                mas[a - 1] -= 1
                n1 = 0
                for i in range(0, a - 1):
                    for i1 in range(i + 1, a):
                        if mas[i] == mas[i1]:
                            n1 = 1
                if n1 == 0:
                    for i in range(0, a):
                        pass
                    c += 1
            mas[j] = temp1
            mas[a - 1] = temp2
        mas[a - 2] += mas[a - 1]
        a -= 1
    if n > 5:
        return c + 1
    else:
        return c

def rec(n, m):
    '''
    Функция рекурсивно считает количество разных представлений числа
    :param n: число
    :param m: число (копия числа n)
    :return: количество различных представлений числа
    '''
    if m == 0:
        if n == 0:
            return 1
        return 0
    if m <= n:
        return rec(n, m-1) + rec(n - m, m)
    return rec(n,n)
def iter(n, m):
    '''
    Функция рекурсивно считает количество разных представлений числа
    :param n: число
    :param m: число (копия числа n)
    :return: количество различных представлений числа
    '''
    s = 0
    a = []
    a.append((n, m))
    while a:
        r = a.pop()
        b = r[0]
        c = r[1]
        if c == 0:
            if b == 0:
                s += 1
        elif c > b:
            a.append((b, b))
        elif c <= b:
            a.append((b, c - 1))
            a.append((b - c, c))
    return s
while True:
    try:
        n = m = int(input('N = '))
        if n <= 0 or m <= 0:
            print('Не правильное число! введите число больше 0')
        else:
            kol = rec(n,m)
            kol1 = iter(n, m)
            print('Количество представлений заданного числа: ',"\nрекурсивно - ",kol, "\nитерационно - ", kol1)
            print("без 1+1+1... рекурсивно = ", rec_decompose(n) )
            print("без 1+1+1... итерационно = ", number_decompose_iter(n))
        w = input('\nХотите начать работу с программой заново? [1 - да]: ')
        if w == '1':
            print()
            continue
        else:
            print("пока!")
            break
    except (ValueError, RecursionError, MemoryError):
        print("Введите коректные данные!")


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Fri Oct  6 13:32:41 2017

@author: xuduo
"""

import numpy as np
import matplotlib.pyplot as plt
from scipy import optimize
import scipy
from astropy.io import fits
import glob, os
import time

def readfits(file_name):
    rot_median = fits.open(file_name+'.fits')
    rot_median=rot_median[0].data
    return rot_median

#parameter_all=np.loadtxt('fitting_gaussian.txt')
#PA_all=np.loadtxt('PA.txt')
parameter_all=np.loadtxt('fitting_gaussian_2.txt')
PA_all=np.loadtxt('PA_2.txt')
x_cen_all=parameter_all[:,2]
y_cen_all=parameter_all[:,1]

#save_path='../newfits/ROXs42B/'
save_path='../newfits/ROXs12/'

image_all=np.zeros([len(y_cen_all),1024,1024])

for ctt in range(0,len(y_cen_all)):
    hdulist = fits.open(save_path+str(ctt)+'_shift.fits')
#    hdulist = fits.open(save_path+str(ctt)+'_rot.fits')
    image=hdulist[0].data

    image_all[ctt,:,:]=image

plt.figure(1)
plt.clf()
plt.imshow(np.sum(image_all,axis=0),origin='lower')
#plt.xlim([590,630])
#plt.ylim([451,491])
plt.show()
#    plt.savefig('../image/2/'+str(ctt)+'_shift.png')

plt.figure(2)
plt.clf()
plt.imshow(np.median(image_all,axis=0),origin='lower')
#plt.xlim([590,630])
#plt.ylim([451,491])
plt.show()

#fits.writeto('shift_sum.fits',np.sum(image_all,axis=0),overwrite=True)
#fits.writeto('shift_median.fits',np.median(image_all,axis=0),overwrite=True)
#fits.writeto('rot_sum.fits',np.sum(image_all,axis=0),overwrite=True)
#fits.writeto('rot_median.fits',np.median(image_all,axis=0),overwrite=True)

fits.writeto('shift_sum_12.fits',np.sum(image_all,axis=0),overwrite=True)
fits.writeto('shift_median_12.fits',np.median(image_all,axis=0),overwrite=True)
#fits.writeto('rot_sum_12.fits',np.sum(image_all,axis=0),overwrite=True)
#fits.writeto('rot_median_12.fits',np.median(image_all,axis=0),overwrite=True)

#"""
image_size=150
#file_all=['rot_sum','rot_median','shift_median','shift_sum']
file_all=['rot_sum_12','rot_median_12','shift_median_12','shift_sum_12']
for ctt in range(4):
    image=readfits(file_all[ctt])
    plt.figure(1)
    plt.clf()
    plt.imshow(image,origin='lower')
    #plt.xlim([590,630])
    #plt.ylim([451,491])
    plt.show()
#    plt.savefig(file_all[ctt]+'_42B.png')
    plt.savefig(file_all[ctt]+'_12.png')

    plt.figure(2)
    plt.clf()
    plt.imshow(image,origin='lower')
    plt.xlim([611-image_size,611+image_size])
    plt.ylim([470-image_size,470+image_size])
    plt.show()
#    plt.savefig(file_all[ctt]+'_42B_crop.png')
    plt.savefig(file_all[ctt]+'_12_crop.png')
#"""


import serpy

import falcon
import json

class DictField(serpy.Field):
    def to_value(self,value):
        try:
            return json.loads(value)
        except TypeError:
            return json.dumps(value)

class BaseSerializer(serpy.DictSerializer):

    id = serpy.StrField( required = False)
    created_at = serpy.StrField( required = False)
    updated_at = serpy.StrField( required = False)

    class Meta:
        read_protected_fields = ()
        write_protected_fields = ('id','created_at','updated_at')

    def __is_valid(self,internal = False):
        try:
            self.data #this raises exception if data required is not given
            return True
        except KeyError as ke:
            if internal:
                raise KeyError
            raise falcon.HTTPBadRequest(title = "Missing Request Parameters", description = "{field} is a required field".format( field = ke) )

    def __remove_from_data(self,fields, data):
        for k in list(data.keys()):
            if k in fields:
                del data[k]

        return data

    def protect_data(self, protected_fields, data):
        try:
            #clean remove read protextect fields
            self.__remove_from_data( fields = protected_fields,data = data)

        except AttributeError:
            #probably data is list of dicts
            for d in data:
                self.__remove_from_data( fields = protected_fields ,data = d)

        return data

    @property
    def valid_read_data(self):
        if self.__is_valid():
            valid_data =  self.protect_data( protected_fields = self.Meta.read_protected_fields, data = self.data )
            return valid_data

    @property
    def valid_write_data(self):
       if self.__is_valid():
            valid_data =  self.protect_data( protected_fields = self.Meta.write_protected_fields, data = self.data )
            return valid_data


import datetime
import requests
import whois
import argparse
from urllib.parse import urlparse

def load_urls4check(filepath):
    with open(filepath, 'r') as file:
        urls_lst = file.read().split('\n')
    return urls_lst

def is_server_respond_ok(url):
    try:
        return requests.get(url).ok
    except requests.exceptions.RequestException:
        return False

def get_domain_name(url):
    parsed_url = urlparse(url)
    domain = parsed_url.netloc
    return domain

def get_domain_expiration_date(domain_name):
    domain_info = whois.whois(domain_name)
    if type(domain_info.expiration_date) is list:
        return domain_info.expiration_date[0]
    else:
        return domain_info.expiration_date

def get_days_before_expiration(exp_date):
    if exp_date:
        return (exp_date - datetime.datetime.today()).days
    else:
        return 0

def is_expiration_date_ok(days_before_expiration, days_limit=30):
    return days_before_expiration > days_limit

def get_parsed_argument():
    parser = argparse.ArgumentParser()
    parser.add_argument('filepath', type=str, help='path to text-file')
    return parser.parse_args()

def print_check_message(
        is_server_respond,
        is_exp_date_correct,
        domain, days_before_exp
):
    if is_server_respond and is_exp_date_correct:
        general_check_status = 'SUCCESS!'
    else:
        general_check_status = 'FAILED!'
    server_response_status = 'OK' if is_server_respond else 'NO RESPONSE'
    print('for domain "{}"  :  check status: {}'.format(
        domain,
        general_check_status
    ))
    print('server response : {}'.format(server_response_status))
    print('days before expiration : {}'.format(days_before_exp))

if __name__ == '__main__':
    separator = '*' * 30
    arg = get_parsed_argument()
    filepath = arg.filepath
    try:
        urls_for_check_list = load_urls4check(filepath)
    except(FileNotFoundError, UnicodeDecodeError):
        exit('incorrect path to file / not a text file')
    for url in urls_for_check_list:
        is_server_respond = is_server_respond_ok(url)
        domain = get_domain_name(url)
        exp_date = get_domain_expiration_date(domain)
        days_before_exp = get_days_before_expiration(exp_date)
        is_exp_date_correct = is_expiration_date_ok(days_before_exp)
        print_check_message(
            is_server_respond,
            is_exp_date_correct,
            domain,
            days_before_exp
        )
        print(separator)


#!/usr/bin/env python3

import sys
import os
import os.path as P
import subprocess
import shutil
from subprocess import PIPE

crates = {
    "ndarray": ("docs", ""),
    "ndarray-rand": ("", "ndarray-rand"),
    #"ndarray-rblas": ("", "ndarray-rblas"),
}

def crate_name(s):
    """Return crate name (with underscores) """
    return s.replace("-", "_")

def manifest_path(crate):
    home = P.join(P.dirname(sys.argv[0]), "..")
    return P.normpath(P.join(home, crates[crate][1], "Cargo.toml"))

def run_get(cmd):
    print(cmd)
    return subprocess.Popen(cmd, stdout=PIPE).communicate()[0]

def run(cmd):
    print(cmd)
    subprocess.check_call(cmd)

def run_shell(cmd_string):
    print(cmd_string)
    subprocess.getoutput(cmd_string)

def version(crate):
    manifest = manifest_path(crate)
    pkgid = run_get(["cargo", "pkgid", "--manifest-path", manifest])
    pkgid = pkgid.decode("utf-8").strip()
    last = pkgid.rsplit("#")[-1]
    return last.rsplit(":")[-1]

def target_dir():
    home = P.join(P.dirname(sys.argv[0]), "..")
    return P.join(home, "target")

def dest_dir():
    home = P.join(P.dirname(sys.argv[0]), "..")
    return P.join(home, "master")

def doc_home():
    return P.dirname(sys.argv[0])

def image_dir():
    return P.join(doc_home(), "images")

def mkdocs():
    for crate in crates:
        run(["cargo", "doc", "-v", "--no-deps",
            "--manifest-path", manifest_path(crate),
            "--features", crates[crate][0]])
        docdir = P.join(target_dir(), "doc", crate_name(crate))
        run_shell(r'find %s -name "*.html" -exec sed -i -e "s/<title>\(.*\) - Rust/<title>%s - \1 - Rust/g" {} \;'
                  % (docdir, version(crate)))
    dest = dest_dir()
    target_doc = P.join(target_dir(), "doc")
    run_shell("rm -rvf ./%s" % dest)
    run_shell("cp -r %s %s" % (target_doc, dest))
    run_shell("cat %s/custom.css >> %s/main.css" % (doc_home(), dest))
    # remove empty files
    run_shell("find %s -size 0 -delete" % (dest, ))

def main():
    path = sys.argv[0]
    os.putenv("CARGO_TARGET_DIR", target_dir())
    mkdocs()

if __name__ == "__main__":
    main()


import json
import os
import requests

CACHE_FNAME = "flickr_cache.json"

try:
    cache_file = open(CACHE_FNAME, 'r')
    cache_contents = cache_file.read()
    CACHE_DICTION = json.loads(cache_contents)
    cache_file.close()

except:
    CACHE_DICTION = {}

def fetch_from_cache(identifier):
    # find thing, return if there,
    # return None if not there
    pass

def add_to_cache(identifier):
    # Codes here are just to get you started. Please review
    # This function here needs to add api data to cache.
    CACHE_DICTION[unique_ident] = json.loads(resp.text)
    dumped_json_cache = json.dumps(CACHE_DICTION, indent=2)
    fw = open(CACHE_FNAME,"w")
    fw.write(dumped_json_cache)
    fw.close()
    pprint(CACHE_DICTION[unique_ident])
    return CACHE_DICTION[unique_ident]


#!/usr/bin/python3

# Register  |       Descriptoin              |  Resoulution
# ----------|--------------------------------|----------------------
# 0x0000    |   High voltage alarm threshold |  1LSB correspond
#           |   (5-350V), default is 300V    |  to 0.01V
#-----------|--------------------------------|----------------------
# 0x0001    |   Low voltage alarm threshold  |  1LSB correspond
#           |   (1-350V) , defalut is 7V     |  to 0.01V
#-----------|--------------------------------|----------------------
# 0x0002    |   ModBus-RTU Address           |  Range is
#           |                                |  0x0001 - 0x00F7
#-----------|--------------------------------|----------------------
# 0x0003    |   The current range            |  0x0001  100A
#           |   (only PZEM-017)              |  0x0002   50A
#           |                                |  0x0003  200A
#           |                                |  0x0004  300A
#           |                                |

import minimalmodbus
from time import sleep

pz                  =   minimalmodbus.Instrument('/dev/ttyUSB0', 248, debug=0)
pz.serial.baudrate  =   9600                                 #    ^--- Modbus RTU address 1 - 247, if only one device on the bus, also works 248.
pz.serial.timeout   =   1
pz.serial.bytesize  =   8
pz.serial.parity    =   minimalmodbus.serial.PARITY_NONE
pz.serial.stopbits  =   2

x                   =   0

while 1:
    try:
        Shunts      =   ["100 A"," 50 A","200 A","300 A"]
        data        =   pz.read_registers(0,4,3)

        print("\n High Voltage alarm  : ","{:8.2f}".format(float((data[0] * 0.01))), "V\n",
              "Low Voltage alarm   : ",   "{:8.2f}".format(float((data[1] * 0.01))), "V\n",
              "Modbus-RTU Address  : ",   "{:8.0f}".format(data[2]),                  "\n",
              "Current Range (017) :      ",                        Shunts[data[3]],  "\n")

        print('Erfog.')
        break
    except:
        x = x + 1
        sleep(1)
        if x >= 5:
           print('Scotty, we have a problem ... :( ')
           break