#### Project 2 ---- Predictive Analysis
#### Analytics Students
#### Hongyang Zheng, Heng Zhou, Zhengqian Xu, Youyou Xie

# Import libraries
###############################################################################
import sys
import pandas as pd
import matplotlib.pyplot as plt
from pprint import pprint
from sklearn import preprocessing

from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.naive_bayes import GaussianNB
from sklearn import svm
from sklearn.metrics import roc_curve, auc

from scipy import stats
import statsmodels.api as sm
from statsmodels.formula.api import ols
###############################################################################

#### Main function
def main(itself):

    # load dataset3
    df3=pd.read_csv("dataset3.csv")

    # Call function for Parametric Statistical Tests
    testing(df3)

    # We use desicion tree and random forest to do the hypothesis1
    hypothesis1(df3)

    # Classification method - knn,bayes and svm
    classification(df3)

###############################################################################
#             Hypothesis Testing -- Parametric Statistical Tests              #
###############################################################################
# Function for Parametric Statistical Tests
def testing(df3):

    # Hypothesis 1
    # Null hypothesis: the mean of beta for Oil & Gas - Integrated is same as the mean of beta for Oil & Gas - Midstream
    # Alternative hypothesis: the mean of beta for Oil & Gas - Integrated is not same as the mean of beta for Oil & Gas - Midstream
    # Conduct t-test for Beta between Oil&Gas-Integrated and Oil&Gas-Midstream
    OilGas_integrated = df3[df3['sub_industry'] == ' Integrated']['Beta']
    OilGas_midstream = df3[df3['sub_industry'] == ' Midstream']['Beta']
    ttest1=str(stats.ttest_ind(OilGas_integrated, OilGas_midstream))
    pprint(ttest1)

    # Hypothesis 2
    # Null hypothesis: the mean of ytdChange for 11 sectors are same
    # Alternative hypothesis: the mean of ytdChange for at least one sector of 11 sectors are not same as others
    # Conduct ANOVA test for average total volume between 11 sectors
    mod=ols("ytdChange ~ sector",data=df3).fit()
    table = str(sm.stats.anova_lm(mod, typ=2))
    pprint(table)

    # Hypothesis 3
    # Null hypothesis: latestPrice and avgTotalVolume are independent
    # Alternative hypothesis: latestPrice and avgTotalVolume are not independent
    # Conduct Linear Regression for latestPrice and avgTotalVolume
    # Linear Regression
    X = df3['latestPrice']
    y = df3['avgTotalVolume']
    model = sm.OLS(y, X).fit()
    pprint(model.summary())

    # Hypothesis 4
    # Null hypothesis: avgTotalVolume, latestPrice, ytdChange and Beta are independent
    # Alternative hypothesis: at least one variable of latestPrice, ytdChange, Beta is not independent from avgTotalVolume
    # Conduct Linear Regression for latestPrice, ytdChange and avgTotalVolume
    X = df3[['latestPrice','ytdChange','Beta']]
    y = df3['avgTotalVolume']
    model = sm.OLS(y, X).fit()
    pprint(model.summary())

###############################################################################
#               Hypothesis Testing -- Predictive Model Methods                #
###############################################################################
#### The fisrt hypothesis is when classifying, the result of decision tree is the same as the result of random forest.
#### We use desicion tree and random forest to do the hypothesis
def hypothesis1(df_new):
    # Use 'risk' as classification
    # Select features for analysis
    dfdt= pd.concat([df_new['sector'],df_new['avgTotalVolume'],df_new['latestPrice'], df_new['industry'],df_new['MktCap'],df_new['Risk']],axis=1)

    #### Convert categorical variable'industry' into numerical variable
    # List categories of Sector
    cate2=list(dfdt['industry'].unique())
    n2=len(cate2)

    # Assign different number to each category
    m=0
    while m < n2:
        for i in range(len(dfdt.index)):
                if (dfdt.loc[i,"industry"])==cate2[m] :
                    dfdt.loc[i,"industry"]=(m+1)
        m=m+1

    # Convert class into two category for later ROC curve
    # dfdt.loc[dfdt['Risk'] == 'small', 'Risk'] = 'positive'
    # dfdt.loc[dfdt['Risk'] == 'big', 'Risk'] = 'positive'

    # Returns a numpy array
    x = dfdt.values

    # Choose first 5 columns as attributes and the last column as classification
    X = x[:,0:5]
    Y = x[:,5]

    # Choose test size
    test_size = 0.20
    # Set seed
    seed = 7
    X_train, X_validate, Y_train, Y_validate = train_test_split(X, Y, test_size=test_size, random_state=seed)

    # Setup 10-fold cross validation to estimate the accuracy of different models
    # Split data into 10 parts
    # Test options and evaluation metric
    num_folds = 10
    scoring = 'accuracy'

    ######################################################
    # Use different algorithms to build models
    ######################################################

    # Add each algorithm and its name to the model array
    models = []
    models.append(('CART', DecisionTreeClassifier()))
    models.append(('RandomForest', RandomForestClassifier()))

    # Evaluate each model, add results to a results array,
    # Print the accuracy results
    results = []
    names = []
    for name, model in models:
        # k fold
        kfold = KFold(n_splits=num_folds, random_state=seed, shuffle=False)

        # Use cross-validation for traning data
        cv_results = cross_val_score(model, X_train, Y_train, cv=kfold, scoring=scoring)
        results.append(cv_results)
        names.append(name)
        print("\nThe cross_val_score is:\n")
        # Print the result of cross validation
        msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
        print(msg)

    #### Here is Validation test
    #### Decision tree test validation
    print('\n\n********For decision tree, we use test data to validate it :**********')
    cart = DecisionTreeClassifier()
    predictions1 = cart.fit(X_train, Y_train).predict(X_validate)

    # Print the accuracy of cart
    print("\nthe accuracy for cart:",accuracy_score(Y_validate, predictions1))
    print("\nthe confusion matrix for cart is:\n",confusion_matrix(Y_validate, predictions1))
    print("\nclassification report for cart:\n",classification_report(Y_validate, predictions1))

    #### Random forest test validation
    print('\n\n********For random forest, we use test data to validate it :**********')
    rf = RandomForestClassifier()
    rf.fit(X_train, Y_train)
    predictions2 = rf.predict(X_validate)

    # Print the accuracy of cart
    print("\nthe accuracy for random forest:",accuracy_score(Y_validate, predictions2))
    print("\nthe confusion matrix for random forest is:\n",confusion_matrix(Y_validate, predictions2))
    print("\nclassification report for random forest:\n",classification_report(Y_validate, predictions2))

    #### Plot ROC Curve
    #### For decision tree
    # Calculate y_score
    y_score1 = cart.fit(X_train, Y_train).predict_proba(X_validate)

    #### Compute ROC curve and ROC area for each class
    # For the first label-big
    fpr11,tpr11,_ = roc_curve(Y_validate, y_score1[:,0],pos_label='big')
    roc_auc11 = auc(fpr11,tpr11)

    plt.figure()
    plt.plot(fpr11, tpr11,color='orange',lw=2, label='ROC curve in desicon tree(area = %0.2f)' % roc_auc11)
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC for decision tree(big label)')
    plt.legend(loc="lower right")
    plt.show()

    # For the first label-negative
    fpr12,tpr12,_ = roc_curve(Y_validate, y_score1[:,1],pos_label='negative')
    roc_auc12 = auc(fpr12,tpr12)

    plt.figure()
    plt.plot(fpr12, tpr12,color='orange',lw=2, label='ROC curve in desicon tree(area = %0.2f)' % roc_auc12)
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC for decision tree(negative label)')
    plt.legend(loc="lower right")
    plt.show()

    # For the first label-small
    fpr13,tpr13,_ = roc_curve(Y_validate, y_score1[:,2],pos_label='small')
    roc_auc13 = auc(fpr13,tpr13)

    plt.figure()
    plt.plot(fpr13, tpr13,color='orange',lw=2, label='ROC curve in desicon tree(area = %0.2f)' % roc_auc13)
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC for decision tree(small label)')
    plt.legend(loc="lower right")
    plt.show()

    #### For random forest
    y_score2 = rf.fit(X_train, Y_train).predict_proba(X_validate)

    #### Compute ROC curve and ROC area for each class
    # For the first label-big
    fpr21,tpr21,_ = roc_curve(Y_validate, y_score2[:,0],pos_label='big')
    roc_auc21 = auc(fpr21,tpr21)

    plt.figure()
    plt.plot(fpr21, tpr21,color='orange',lw=2, label='ROC curve in random forest(area = %0.2f)' % roc_auc21)
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC for random forest(big label)')
    plt.legend(loc="lower right")
    plt.show()

    # For the first label-negative
    fpr22,tpr22,_ = roc_curve(Y_validate, y_score2[:,1],pos_label='negative')
    roc_auc22 = auc(fpr22,tpr22)

    plt.figure()
    plt.plot(fpr21, tpr21,color='orange',lw=2, label='ROC curve in random forest(area = %0.2f)' % roc_auc22)
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC for random forest(negative label)')
    plt.legend(loc="lower right")
    plt.show()

    # For the first label-small
    fpr23,tpr23,_ = roc_curve(Y_validate, y_score2[:,2],pos_label='small')
    roc_auc23 = auc(fpr23,tpr23)

    plt.figure()
    plt.plot(fpr23, tpr23,color='orange',lw=2, label='ROC curve in random forest(area = %0.2f)' % roc_auc23)
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC for random forest(small label)')
    plt.legend(loc="lower right")
    plt.show()

#### Classification method - knn,bayes and svm
def classification(df_new):

    # Convert categorical variable into numerical variable
    Industry1 = list(df_new['industry'].unique())
    n2 = len(Industry1)
    # Assign different number to each category
    m = 0
    while m < n2:
        for i in range(len(df_new.index)):
            if (df_new.loc[i,"industry"])==Industry1[m] :
                df_new.loc[i,"industry"]=(m+1)
        m=m+1
    # Add the new column
    df_new['Volumes'] = None

    # Add the values for the new column
    df_new.loc[df_new['Volume_level'] == 'A','Volumes'] = 'High'
    df_new.loc[df_new['Volume_level'] == 'B','Volumes'] = 'High'
    df_new.loc[df_new['Volume_level'] == 'C','Volumes'] = 'High'
    df_new.loc[df_new['Volume_level'] == 'D','Volumes'] = 'Low'
    df_new.loc[df_new['Volume_level'] == 'F','Volumes'] = 'Low'
    # Local the numeric values
    x = df_new.values
    X = x[:, [2,4,9,13,14,15]]
    # Local the Volumn_level
    Y = x[:,18]
    # Then make the validation set 20% of the entire
    test_size = 0.20
    # Set seed
    seed = 123
    X_train, X_validate, Y_train, Y_validate = train_test_split(X, Y, test_size=test_size, random_state=seed)
    # Transform df into Min-Max Normalization for later calculating
    min_max_scaler = preprocessing.MinMaxScaler()
    x_scaled = min_max_scaler.fit_transform(X)
    normalizedDataFrame = pd.DataFrame(x_scaled)
    '''
    #normalize the data
    Normalize = preprocessing.scale(X)
    #read the normalized data to the dataframe
    normalizedDataFrame = pd.DataFrame(Normalize)
    '''
    pprint(normalizedDataFrame[:10])

    # Select 10 folds
    num_folds = 10
    seed = 123
    scoring = 'accuracy'
    # Add each algorithm and its name to the model array
    models = []
    models.append(('KNN', KNeighborsClassifier()))
    models.append(('Naive Bayes', GaussianNB()))
    models.append(('svm',svm.SVC()))

    # Evaluate each model, add results to a results array,
    # Print the accuracy results (remember these are averages and std
    results = []
    names = []
    for name, model in models:
        kfold = KFold(n_splits=num_folds, random_state=seed, shuffle=False)
        # Do cross validation
        cv_results = cross_val_score(model, X_train, Y_train, cv=kfold, scoring=scoring)
        results.append(cv_results)
        names.append(name)
        # Print the result of these methods
        msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
        print(msg)

    # Validation test
    ######################################################
    # Make predictions on validation dataset in knn
    knn = KNeighborsClassifier()
    knn.fit(X_train, Y_train)
    predictions3 = knn.predict(X_validate)
    # Print the accuracy for knn
    print('\n\n********For knn, we use test data to validate it :**********')
    print('The accurary_score of knn is:\n',accuracy_score(Y_validate, predictions3))
    print('The confusion_matrix of knn is:\n',confusion_matrix(Y_validate, predictions3))
    print('The classification of knn is:\n', classification_report(Y_validate, predictions3))

    # Make predictions on validation dataset in bayes
    bayes = GaussianNB()
    bayes.fit(X_train, Y_train)
    predictions4 = bayes.predict(X_validate)
    # Print the accuracy for bayes
    print('\n\n********For bayes, we use test data to validate it :**********')
    print('The accurary_score of bayes is:\n',accuracy_score(Y_validate, predictions4))
    print('The confusion_matrix of bayes is:\n',confusion_matrix(Y_validate, predictions4))
    print('The classification of bayes is:\n',classification_report(Y_validate, predictions4))

    # Make predictions on validation dataset in SVM
    s = svm.SVC()
    s.fit(X_train, Y_train)
    predictions5 = s.predict(X_validate)
    # Print the accuracy for svm
    print('\n\n********For svm, we use test data to validate it :**********')
    print("\nthe accuracy for svm:",accuracy_score(Y_validate, predictions5))
    print("\nthe confusion matrix for svm is:\n",confusion_matrix(Y_validate, predictions5))
    print("\nclassification report for svm:\n",classification_report(Y_validate, predictions5))

    #### 3 learn to predict each class against each other compute the Y_score
    # knn method
    Y_score3 = knn.fit(X_train,Y_train).predict_proba(X_validate)
    # Compute ROC curve and ROC area for each class
    fpr3,tpr3,threshold3 = roc_curve(Y_validate,Y_score3[:,1],pos_label='Low')
    roc_auc3 = auc(fpr3,tpr3)
    # Plot the figure about the ROC in knn method
    plt.figure()
    plt.plot(fpr3, tpr3, color='orange',lw=2, label='ROC curve in knn(area = %0.2f)' % roc_auc3)
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC for knn')
    plt.legend(loc="lower right")
    plt.show()

    # Bayes method
    Y_score4 = bayes.fit(X_train,Y_train).predict_proba(X_validate)
    # Compute ROC curve and ROC area for each class
    fpr4,tpr4,threshold4 = roc_curve(Y_validate,Y_score4[:,1],pos_label='Low')
    roc_auc4 = auc(fpr4,tpr4)
    # Plot the figure about the ROC in bayes method
    plt.figure()
    plt.plot(fpr4, tpr4, color='orange',lw=2, label='ROC curve in Bayes(area = %0.2f)' % roc_auc4)
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC for bayes')
    plt.legend(loc="lower right")
    plt.show()

    # svm method
    Y_score5 = s.fit(X=X_train,y=Y_train).decision_function(X_validate)
    # Compute ROC curve and ROC area for each class
    fpr5,tpr5,threshold5 = roc_curve(Y_validate,Y_score5,pos_label='Low')
    roc_auc5 = auc(fpr5,tpr5)
    # Plot the figure about the ROC in svm method
    plt.figure()
    plt.plot(fpr5, tpr5, color='orange',lw=2, label='ROC curve in svm(area = %0.2f)' % roc_auc5)
    plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC for bayes')
    plt.legend(loc="lower right")
    plt.show()

# Call main function
if __name__ == "__main__":
    main(sys.argv)


""" This module contains general helper functions. """
import os
from typing import Union, Optional, Tuple, Iterator
import uuid
import datetime
import re
import numpy as np
import numpy.ma as ma
from scipy import stats, ndimage
from scipy.interpolate import RectBivariateSpline, griddata
import pytz
import requests

SECONDS_PER_MINUTE = 60
SECONDS_PER_HOUR = 3600
SECONDS_PER_DAY = 86400

def seconds2hours(time_in_seconds: np.ndarray) -> np.ndarray:
    """Converts seconds since some epoch to fraction hour.

    Args:
        time_in_seconds: 1-D array of seconds since some epoch that starts on midnight.

    Returns:
        Time as fraction hour.

    Notes:
        Excludes leap seconds.

    """
    seconds_since_midnight = np.mod(time_in_seconds, SECONDS_PER_DAY)
    fraction_hour = seconds_since_midnight/SECONDS_PER_HOUR
    if fraction_hour[-1] == 0:
        fraction_hour[-1] = 24
    return fraction_hour

def seconds2time(time_in_seconds: float) -> list:
    """Converts seconds since some epoch to time of day.

    Args:
        time_in_seconds: seconds since some epoch.

    Returns:
        list: [hours, minutes, seconds] formatted as '05' etc.

    """
    seconds_since_midnight = np.mod(time_in_seconds, SECONDS_PER_DAY)
    hours = seconds_since_midnight // SECONDS_PER_HOUR
    minutes = seconds_since_midnight % SECONDS_PER_HOUR // SECONDS_PER_MINUTE
    seconds = seconds_since_midnight % SECONDS_PER_MINUTE
    time = [hours, minutes, seconds]
    return [str(t).zfill(2) for t in time]

def seconds2date(time_in_seconds: float, epoch: Optional[tuple] = (2001, 1, 1)) -> list:
    """Converts seconds since some epoch to datetime (UTC).

    Args:
        time_in_seconds: Seconds since some epoch.
        epoch: Epoch, default is (2001, 1, 1) (UTC).

    Returns:
        [year, month, day, hours, minutes, seconds] formatted as '05' etc (UTC).

    """
    epoch_in_seconds = datetime.datetime.timestamp(datetime.datetime(*epoch, tzinfo=pytz.utc))
    timestamp = time_in_seconds + epoch_in_seconds
    return datetime.datetime.utcfromtimestamp(timestamp).strftime('%Y %m %d %H %M %S').split()

def time_grid(time_step: Optional[int] = 30) -> np.ndarray:
    """Returns decimal hour array between 0 and 24.

    Computes fraction hour time vector 0-24 with user-given
    resolution (in seconds).

    Args:
        time_step: Time resolution in seconds, greater than 1. Default is 30.

    Returns:
        Time vector between 0 and 24.

    Raises:
        ValueError: Bad resolution as input.

    """
    if time_step < 1:
        raise ValueError('Time resolution should be >= 1 seconds')
    half_step = time_step/SECONDS_PER_HOUR/2
    return np.arange(half_step, 24+half_step, half_step*2)

def binvec(x: Union[np.ndarray, list]) -> np.ndarray:
    """Converts 1-D center points to bins with even spacing.

    Args:
        x: 1-D array of N real values.

    Returns:
        ndarray: N + 1 edge values.

    Examples:
        >>> binvec([1, 2, 3])
            [0.5, 1.5, 2.5, 3.5]

    """
    edge1 = x[0] - (x[1]-x[0])/2
    edge2 = x[-1] + (x[-1]-x[-2])/2
    return np.linspace(edge1, edge2, len(x)+1)

def rebin_2d(x_in: np.ndarray,
             array: ma.MaskedArray,
             x_new: np.ndarray,
             statistic: Optional[str] = 'mean',
             n_min: Optional[int] = 1) -> ma.MaskedArray:
    """Rebins 2-D data in one dimension.

    Args:
        x_in: 1-D array with shape (n,).
        array: 2-D input data with shape (n, m).
        x_new: 1-D target vector (center points) with shape (N,).
        statistic: Statistic to be calculated. Possible statistics are 'mean', 'std'.
            Default is 'mean'.
        n_min: Minimum number of points to have good statistics in a bin. Default is 1.

    Returns:
        Rebinned data with shape (N, m).

    Notes:
        0-values are masked in the returned array.

    """
    edges = binvec(x_new)
    result = np.zeros((len(x_new), array.shape[1]))
    array_screened = ma.masked_invalid(array, copy=True)  # data may contain nan-values
    for ind, values in enumerate(array_screened.T):
        mask = ~values.mask
        if ma.any(values[mask]):
            result[:, ind], _, bin_no = stats.binned_statistic(x_in[mask],
                                                               values[mask],
                                                               statistic=statistic,
                                                               bins=edges)
            if n_min > 1:
                unique, counts = np.unique(bin_no, return_counts=True)
                result[unique[counts < n_min]-1, ind] = 0

    result[~np.isfinite(result)] = 0
    return ma.masked_equal(result, 0)

def rebin_1d(x_in: np.ndarray,
             array: ma.MaskedArray,
             x_new: np.ndarray,
             statistic: Optional[str] = 'mean') -> ma.MaskedArray:
    """Rebins 1D array.

    Args:
        x_in: 1-D array with shape (n,).
        array: 1-D input data with shape (m,).
        x_new: 1-D target vector (center points) with shape (N,).
        statistic: Statistic to be calculated. Possible statistics are 'mean', 'std'.
            Default is 'mean'.

    Returns:
        Rebinned data with shape (N,).

    """
    edges = binvec(x_new)
    result = np.zeros(len(x_new))
    array_screened = ma.masked_invalid(array, copy=True)  # data may contain nan-values
    mask = ~array_screened.mask  # pylint: disable=E1101
    if ma.any(array_screened[mask]):
        result, _, _ = stats.binned_statistic(x_in[mask],
                                              array_screened[mask],
                                              statistic=statistic,
                                              bins=edges)
    result[~np.isfinite(result)] = 0
    return ma.masked_equal(result, 0)

def filter_isolated_pixels(array: np.ndarray) -> np.ndarray:
    """From a 2D boolean array, remove completely isolated single cells.

    Args:
        array: 2-D boolean array containing isolated values.

    Returns:
        Cleaned array.

    Examples:
        >>> filter_isolated_pixels([[0, 0, 0], [0, 1, 0], [0, 0, 0]])
            array([[0, 0, 0],
                   [0, 0, 0],
                   [0, 0, 0]])

    """
    structure = np.ones((3, 3))
    return _filter(array, structure)

def filter_x_pixels(array: np.ndarray) -> np.ndarray:
    """From a 2D boolean array, remove cells isolated in x-direction.

    Args:
        array: 2-D boolean array containing isolated pixels in x-direction.

    Returns:
        Cleaned array.

    Notes:
        Stronger cleaning than `filter_isolated_pixels()`

    Examples:
        >>> filter_x_pixels([[1, 0, 0], [0, 1, 0], [0, 1, 1]])
            array([[0, 0, 0],
                   [0, 1, 0],
                   [0, 1, 0]])

    """
    structure = np.array([[0, 1, 0], [0, 1, 0], [0, 1, 0]])
    return _filter(array, structure)

def _filter(array: np.ndarray, structure: np.ndarray) -> np.ndarray:
    filtered_array = np.copy(array)
    id_regions, num_ids = ndimage.label(filtered_array, structure=structure)
    id_sizes = np.array(ndimage.sum(array, id_regions, range(num_ids + 1))).astype(int)
    area_mask = id_sizes == 1
    filtered_array[area_mask[id_regions]] = 0
    return filtered_array

def isbit(array: np.ndarray, nth_bit: int) -> np.ndarray:
    """Tests if nth bit (0,1,2..) is set.

    Args:
        array: Integer array.
        nth_bit: Investigated bit.

    Returns:
        Boolean array denoting values where nth_bit is set.

    Raises:
        ValueError: negative bit as input.

    Examples:
        >>> isbit(4, 1)
            False
        >>> isbit(4, 2)
            True

    See also:
        utils.setbit()

    """
    if nth_bit < 0:
        raise ValueError('Negative bit number')
    mask = 1 << nth_bit
    return array & mask > 0

def setbit(array: np.ndarray, nth_bit: int) -> int:
    """Sets nth bit (0, 1, 2..) on number.

    Args:
        array: Integer array.
        nth_bit: Bit to be set.

    Returns:
        Integer where nth bit is set.

    Raises:
        ValueError: negative bit as input.

    Examples:
        >>> setbit(1, 1)
            3
        >>> setbit(0, 2)
            4

    See also:
        utils.isbit()

    """
    if nth_bit < 0:
        raise ValueError('Negative bit number')
    mask = 1 << nth_bit
    array |= mask
    return array

def interpolate_2d(x: np.ndarray,
                   y: np.ndarray,
                   z: np.ndarray,
                   x_new: np.ndarray,
                   y_new: np.ndarray) -> np.ndarray:
    """Linear interpolation of gridded 2d data.

    Args:
        x: 1-D array.
        y: 1-D array.
        z: 2-D array at points (x, y).
        x_new: 1-D array.
        y_new: 1-D array.

    Returns:
        Interpolated data.

    Notes:
        Does not work with nans. Ignores mask of masked data. Does not extrapolate.

    """
    fun = RectBivariateSpline(x, y, z, kx=1, ky=1)
    return fun(x_new, y_new)

def interpolate_2d_mask(x: np.ndarray,
                        y: np.ndarray,
                        z: ma.MaskedArray,
                        x_new: np.ndarray,
                        y_new: np.ndarray) -> ma.MaskedArray:
    """Interpolates 2D masked array preserving the mask.

    Args:
        x: 1D array, x-coordinates.
        y: 1D array, y-coordinates.
        z: 2D masked array, data values.
        x_new: 1D array, new x-coordinates.
        y_new: 1D array, new y-coordinates.

    Returns:
        Interpolated 2D masked array.

    Notes:
        Points outside the original range will be nans (and masked). Uses linear interpolation.
        Input data may contain nan-values.

    """
    z = ma.masked_invalid(z, copy=True)
    # Interpolate ignoring masked values:
    valid_points = ~z.mask
    xx, yy = np.meshgrid(y, x)
    x_valid = xx[valid_points]
    y_valid = yy[valid_points]
    z_valid = z[valid_points]
    xx_new, yy_new = np.meshgrid(y_new, x_new)
    data = griddata((x_valid, y_valid), z_valid.ravel(), (xx_new, yy_new), method='linear')
    # Preserve mask:
    mask_fun = RectBivariateSpline(x, y, z.mask[:], kx=1, ky=1)
    mask = mask_fun(x_new, y_new)
    mask[mask < 0.5] = 0
    masked_array = ma.array(data, mask=mask.astype(bool))
    masked_array = ma.masked_invalid(masked_array)
    return masked_array

def interpolate_2d_masked(array: ma.MaskedArray,
                          ax_values: Tuple[np.ndarray, np.ndarray],
                          ax_values_new: Tuple[np.ndarray, np.ndarray]) -> ma.MaskedArray:
    """Interpolates 2D array preserving the mask.

    Notes:
        Deprecated. Use interpolate_2d_mask instead which is more accurate.

    """
    def _mask_invalid_values(data_in):
        data_range = (np.min(array), np.max(array))
        return ma.masked_outside(data_in, *data_range)

    data_interp = interpolate_2d(*ax_values, array, *ax_values_new)
    return _mask_invalid_values(data_interp)

def calc_relative_error(reference: np.ndarray, array: np.ndarray) -> np.ndarray:
    """Calculates relative error (%)."""
    return ((array - reference) / reference) * 100

def db2lin(array: np.ndarray, scale: Optional[int] = 10) -> np.ndarray:
    """dB to linear conversion."""
    if ma.isMaskedArray(array):
        return ma.power(10, (array/scale))
    return np.power(10, (array/scale))

def lin2db(array: np.ndarray, scale: Optional[int] = 10) -> np.ndarray:
    """Linear to dB conversion."""
    if ma.isMaskedArray(array):
        return scale*ma.log10(array)
    return scale*np.log10(array)

def mdiff(array: np.ndarray) -> float:
    """Returns median difference of 1-D array."""
    return ma.median(ma.diff(array))

def l2norm(*args: any) -> ma.MaskedArray:
    """Returns l2 norm.

    Args:
       *args: Variable number of data (*array_like*) with the same shape.

    Returns:
        The l2 norm.

    """
    ss = 0
    for arg in args:
        if isinstance(arg, ma.MaskedArray):
            arg[~arg.mask] = arg[~arg.mask] ** 2
        else:
            arg = arg ** 2
        ss = ss + arg
    return ma.sqrt(ss)

def l2norm_weighted(values: tuple, overall_scale: float, term_weights: tuple) -> float:
    """Calculates scaled and weighted Euclidean distance.

    Calculated distance is of form: scale * sqrt((a1*a)**2 + (b1*b)**2 + ...)
    where a, b, ... are terms to be summed and a1, a2, ... are optional weights
    for the terms.

    Args:
        values: Tuple containing the values.
        overall_scale: Scale factor for the calculated Euclidean distance.
        term_weights: Weights for the terms. Must be single float or a list of numbers
            (one per term).

    Returns:
        Scaled and weighted Euclidean distance.

    TODO: Use masked arrays instead of tuples.

    """
    generic_values = ma.array(values, dtype=object)
    weighted_values = ma.multiply(generic_values, term_weights)
    return overall_scale * l2norm(*weighted_values)

def cumsumr(array: np.ndarray, axis: Optional[int] = 0) -> np.ndarray:
    """Finds cumulative sum that resets on 0.

    Args:
        array: Input array.
        axis: Axis where the sum is calculated. Default is 0.

    Returns:
        Cumulative sum, restarted at 0.

    Examples:
        >>> x = np.array([0, 0, 1, 1, 0, 0, 0, 1, 1, 1])
        >>> cumsumr(x)
            [0, 0, 1, 2, 0, 0, 0, 1, 2, 3]

    """
    cums = array.cumsum(axis=axis)
    return cums - np.maximum.accumulate(cums * (array == 0), axis=axis)  # pylint: disable=E1101

def ffill(array: np.ndarray, value: Optional[int] = 0) -> np.ndarray:
    """Forward fills an array.

    Args:
        array: 1-D or 2-D array.
        value: Value to be filled. Default is 0.

    Returns:
        ndarray: Forward-filled array.

    Examples:
        >>> x = np.array([0, 5, 0, 0, 2, 0])
        >>> ffill(x)
            [0, 5, 5, 5, 2, 2]

    Notes:
        Works only in axis=1 direction.

    """
    ndims = len(array.shape)
    ran = np.arange(array.shape[ndims - 1])
    idx = np.where((array != value), ran, 0)
    idx = np.maximum.accumulate(idx, axis=ndims-1)  # pylint: disable=E1101
    if ndims == 2:
        return array[np.arange(idx.shape[0])[:, None], idx]
    return array[idx]

def init(n_vars: int,
         shape: tuple,
         dtype: Optional[type] = float,
         masked: Optional[bool] = True) -> Iterator[Union[np.ndarray, ma.MaskedArray]]:
    """Initializes several numpy arrays.

    Args:
        n_vars: Number of arrays to be generated.
        shape: Shape of the arrays, e.g. (2, 3).
        dtype: The desired data-type for the arrays, e.g., int. Default is float.
        masked: If True, generated arrays are masked arrays, else ordinary numpy arrays.
            Default is True.

    Yields:
        Iterator containing the empty arrays.

    Examples:
        >>> a, b = init(2, (2, 3))
        >>> a
            masked_array(
              data=[[0., 0., 0.],
                    [0., 0., 0.]],
              mask=False,
              fill_value=1e+20)

    """
    fun = ma.zeros if masked else np.zeros
    for _ in range(n_vars):
        yield fun(shape, dtype=dtype)

def n_elements(array: np.ndarray, dist: float, var: Optional[str] = None) -> int:
    """Returns the number of elements that cover certain distance.

    Args:
        array: Input array with arbitrary units or time in fraction hour. *x* should be evenly
            spaced or at least close to.
        dist: Distance to be covered. If x is fraction time, *dist* is in minutes. Otherwise *x*
            and *dist* should have the same units.
        var: If 'time', input is fraction hour and distance in minutes, else inputs have the same
            units. Default is None (same units).

    Returns:
        Number of elements in the input array that cover *dist*.

    Examples:
        >>> x = np.array([2, 4, 6, 8, 10])
        >>> n_elements(x, 6)
            3

        The result is rounded to the closest integer, so:

        >>> n_elements(x, 6.9)
            3
        >>> n_elements(x, 7)
            4

        With fraction hour time vector:

        >>> x = np.linspace(0, 1, 61)
        >>> n_elements(x, 10, 'time')
            10

    """
    n = dist/mdiff(array)
    if var == 'time':
        n = n/60
    return int(np.round(n))

def isscalar(array: any) -> bool:
    """Tests if input is scalar.

    By "scalar" we mean that array has a single value.

    Examples:
        >>> isscalar(1)
            True
        >>> isscalar([1])
            True
        >>> isscalar(np.array(1))
            True
        >>> isscalar(np.array([1]))
            True

    """
    arr = ma.array(array)
    if not hasattr(arr, '__len__') or arr.shape == () or len(arr) == 1:
        return True
    return False

def get_time() -> str:
    """Returns current UTC-time."""
    return datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')

def date_range(start_date: datetime.date,
               end_date: datetime.date) -> Iterator[datetime.date]:
    """Returns range between two dates (datetimes)."""
    for n in range(int((end_date - start_date).days)):
        yield start_date + datetime.timedelta(n)

def get_uuid() -> str:
    """Returns unique identifier."""
    return uuid.uuid4().hex

def get_wl_band(radar_frequency: Union[float, int]) -> int:
    """Returns integer corresponding to radar frequency.

    Args:
        radar_frequency: Radar frequency (GHz).

    Returns:
        0 = 35GHz radar, 1 = 94Ghz radar.

    """
    return 0 if (30 < radar_frequency < 40) else 1

def transpose(data: np.ndarray) -> np.ndarray:
    """Transposes numpy array of (n, ) to (n, 1)."""
    if data.ndim != 1 or len(data) <= 1:
        raise ValueError('Invalid input array shape')
    return data[:, np.newaxis]

def del_dict_keys(data: dict, keys: Union[tuple, list]) -> dict:
    """Deletes multiple keys from dictionary.

    Args:
        data: A dictionary.
        keys: Keys to be deleted.

    Returns:
        Dictionary without the deleted keys.

    """
    temp_dict = data.copy()
    for key in keys:
        if key in data:
            del temp_dict[key]
    return temp_dict

def array_to_probability(array: ma.MaskedArray,
                         loc: float,
                         scale: float,
                         invert: Optional[bool] = False) -> np.ndarray:
    """Converts continuous variable into 0-1 probability.

    Args:
        array: Masked numpy array.
        loc: Center of the distribution. Values smaller than this will have small probability.
            Values greater than this will have large probability.
        scale: Width of the distribution, i.e., how fast the probability drops or increases from
            the peak.
        invert: If True, large values have small probability and vice versa. Default is False.

    Returns:
        Probability with the same shape as the input data.

    """
    arr = ma.copy(array)
    prob = np.zeros(arr.shape)
    ind = ~arr.mask
    if invert:
        arr *= -1
        loc *= -1
    prob[ind] = stats.norm.cdf(arr[ind], loc=loc, scale=scale)
    return prob

def range_to_height(range_los: np.ndarray, tilt_angle: float) -> np.ndarray:
    """Converts distances from a tilted instrument to height above the ground.

    Args:
        range_los: Distances towards the line of sign from the instrument.
        tilt_angle: Angle in degrees from the zenith (0 = zenith).

    Returns:
        Altitudes of the LOS points.

    Notes:
        Uses plane parallel Earth approximation.

    """
    return range_los * np.cos(np.deg2rad(tilt_angle))

def find_first_empty_line(file_name: str) -> int:
    """Finds first text file line that is empty."""
    line_number = 1
    with open(file_name) as file:
        for line in file:
            if is_empty_line(line):
                break
            line_number += 1
    return line_number

def is_empty_line(line: str) -> bool:
    """Tests if a line (of a text file) is empty."""
    if line in ('\n', '\r\n'):
        return True
    return False

def is_timestamp(timestamp: str) -> bool:
    """Tests if the input string is formatted as -yyyy-mm-dd hh:mm:ss"""
    reg_exp = re.compile(r'-\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}')
    if reg_exp.match(timestamp) is not None:
        return True
    return False

def get_sorted_filenames(file_path: str, extension: str) -> list:
    """Returns full paths of files with some extension, sorted by filename."""
    extension = extension.lower()
    all_files = os.listdir(file_path)
    files = ['/'.join((file_path, file)) for file in all_files if file.lower().endswith(extension)]
    files.sort()
    return files

def str_to_numeric(value: str) -> Union[int, float]:
    """Converts string to number (int or float)."""
    try:
        return int(value)
    except ValueError:
        return float(value)

def fetch_cloudnet_model_types() -> list:
    """Finds different model types."""
    url = f"https://cloudnet.fmi.fi/api/models"
    data = requests.get(url=url).json()
    models = [model['id'] for model in data]
    model_types = [model.split('-')[0] for model in models]
    return list(set(model_types))

def get_epoch(units: str) -> tuple:
    """Finds epoch from units string."""
    fallback = (2001, 1, 1)
    try:
        date = units.split()[2]
    except IndexError:
        return fallback
    date = date.replace(',', '')
    try:
        date_components = [int(x) for x in date.split('-')]
    except ValueError:
        try:
            date_components = [int(x) for x in date.split('.')]
        except ValueError:
            return fallback
    year, month, day = date_components
    current_year = datetime.datetime.today().year
    if (1900 < year <= current_year) and (0 < month < 13) and (0 < day < 32):
        return tuple(date_components)
    return fallback


#coding=utf-8
from xml.etree import ElementTree
from win32com.client import Dispatch
import win32com.client
import sys,os

def ChangeReturnKeyInString(str):
    print("str:[",str,"]")
    retStr = str.strip().replace('<p>','')
    retStr = retStr.replace('</p>','\n')
    return retStr

class easy_excel:
    def __init__(self,filename=None):
        self.xlApp=win32com.client.Dispatch('Excel.Application')
        if filename:
            self.filename=filename
            self.xlBook=self.xlApp.Workbooks.Open(self.filename)
        else:
            self.xlBook=self.xlApp.Workbooks.Add()
            self.filename=''
    def save(self,newfilename=None):
        if newfilename:
            self.filename=newfilename
            self.xlBook.SaveAs(newfilename)
        else:
            self.xlBook.Save()

    def close(self):
        self.xlBook.Close(SaveChanges=0)
    def getCell(self,sheet,row,col):
        sht=self.xlBook.Worksheets(sheet)
        return sht.Cell(row,col).Value
    def setCell(self,sheet,row,col,value):
        sht=self.xlBook.Worksheets(sheet)
        sht.Cells(row,col).Value=value
        sht.Cells(row,col).HorizontalAlignment=3
        sht.Rows(row).WrapText=True
    def mergeCells(self,sheet,row1,col1,row2,col2):
        sht=self.xlBook.Worksheets(sheet)
        sht.Range(sht.Cells(row1,col1),sht.Cells(row2,col2)).Merge()
    def setBorder(self,sheet,row,col):
        sht=self.xlBook.Worksheets(sheet)
        sht.Cells(row,col).Borders.LineStyle=1
    def set_col_width(self,sheet):
        sht=self.xlBook.Worksheets(sheet)
        sht.Columns("E:H").ColumnWidth=30

if __name__ =='__main__':
    if (len(sys.argv) == 1):
        print("Please specified a xml file")
        os.system("pause")
        sys.exit(0)
    else:
        tmpInFile = os.getcwd()+"\\"+sys.argv[1]
        inFile = tmpInFile
        if not tmpInFile.endswith(".xml"):
            outFile = tmpInFile + "-tcexp.xlsx"
            inFile  = tmpInFile + ".xml"
        else:
            outFile =  tmpInFile[:-4] +"-tcexp.xlsx"

    xls=easy_excel()
    xls.setCell('Sheet1',1,1,u"一级目录")
    xls.setCell('Sheet1',1,2,u"二级目录")
    xls.setCell('Sheet1',1,3,u"用例名称")
    xls.setCell('Sheet1',1,4,u"用例编号")
    xls.setCell('Sheet1',1,5,u"用例概要说明")
    xls.setCell('Sheet1',1,6,u"预置条件")
    xls.setCell('Sheet1',1,7,u"操作步骤")
    xls.setCell('Sheet1',1,8,u"预期结果")
    xls.set_col_width('Sheet1')

    tree=ElementTree.parse(inFile)
    root = tree.getroot()

    row_flag=1
    #一级目录
    for sub1Testsuite in root.findall("testsuite"):
        sub1SuiteId = sub1Testsuite.get("id")
        sub1SuiteName = sub1Testsuite.get("name")
        for sub1TestCase in sub1Testsuite.findall("testcase"):
            row_flag = row_flag + 1
            sub2SuiteName = ""
            title = sub1TestCase.get("name")
            externalid = sub1TestCase.find("externalid").text
            summary = ChangeReturnKeyInString(sub1TestCase.find("summary").text)
            print("sub1TestCase:[",sub1TestCase,"]")
            preconditions = ChangeReturnKeyInString(sub1TestCase.find("preconditions").text)

            xls.setCell('Sheet1',row_flag,1,sub1SuiteName)
            xls.setCell('Sheet1',row_flag,2,sub2SuiteName)
            xls.setCell('Sheet1',row_flag,3,title)
            xls.setCell('Sheet1',row_flag,4,externalid)
            xls.setCell('Sheet1',row_flag,5,summary)
            xls.setCell('Sheet1',row_flag,6,preconditions)
            stepsNode=sub1TestCase.find("steps")
            actions = ""
            expectedresults = ""
            for stepNode in stepsNode.findall("step"):
                print("stepNode:[",stepNode,"]")
                step_number = stepNode.find('step_number').text
                actions = actions+stepNode.find('actions').text.strip()
                expectedresults = expectedresults + stepNode.find('expectedresults').text.strip()
            actions = ChangeReturnKeyInString(actions)
            expectedresults = ChangeReturnKeyInString(expectedresults)
            xls.setCell('Sheet1',row_flag,7,actions)
            xls.setCell('Sheet1',row_flag,8,expectedresults)

        for sub2Testsuite in sub1Testsuite.findall("testsuite"):
            sub2SuiteId = sub2Testsuite.get("id")
            sub2SuiteName = sub2Testsuite.get("name")
            for sub2TestCase in sub2Testsuite.findall("testcase"):
                row_flag = row_flag + 1
                title = sub2TestCase.get("name")
                externalid = sub2TestCase.find("externalid").text
                summary = ChangeReturnKeyInString(sub2TestCase.find("summary").text)
                preconditions = ChangeReturnKeyInString(sub2TestCase.find("preconditions").text)

                xls.setCell('Sheet1',row_flag,1,sub1SuiteName)
                xls.setCell('Sheet1',row_flag,2,sub2SuiteName)
                xls.setCell('Sheet1',row_flag,3,title)
                xls.setCell('Sheet1',row_flag,4,externalid)
                xls.setCell('Sheet1',row_flag,5,summary)
                xls.setCell('Sheet1',row_flag,6,preconditions)
                actions = ""
                expectedresults = ""
                stepsNode=sub2TestCase.find("steps")
                for stepNode in stepsNode.findall("step"):
                    step_number = stepNode.find('step_number').text
                    actions =actions + stepNode.find('actions').text
                    expectedresults = expectedresults + stepNode.find('expectedresults').text
                actions = ChangeReturnKeyInString(actions)
                expectedresults = ChangeReturnKeyInString(expectedresults)
                xls.setCell('Sheet1',row_flag,7,actions)
                xls.setCell('Sheet1',row_flag,8,expectedresults)

    for row in range(2,row_flag):
        for col in range(1,9):
            xls.setBorder('Sheet1',row,col)
    xls.save(outFile)
    xls.close()

    print("finished.")
    sys.exit(0)


import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE','moviesproject1.settings')
import django
django.setup()

from testapp.models import Jobs
from random import *
from faker import Faker
fake=Faker()
def phonenumbergen():
    d1=randint(6,9)
    num=str(d1)
    for i in range(9):
        num=num+str(randint(0,9))
    return int(num)
def populate(n):
    for i in range(n):
        fdate=fake.date()
        fcompany=fake.company()
        ftitle=fake.random_element(elements=('project Manager','TL','clark','se','jr.asst'))
        feligibility=fake.random_element(elements=('MCA','BTECT','MSC','SSC'))
        faddress=fake.address()
        femail=fake.email()
        fphoneno=phonenumbergen()
        jobs_record=Jobs.objects.get_or_create(date=fdate,companyname=fcompany,title=ftitle,eligibility=feligibility,
        address=faddress,email=femail,phoneno=fphoneno)
populate(30)


import numpy as np
import pygame
import sys
from queue import PriorityQueue

# colors
colors = {
          'white': [255, 255, 255],
          'black': [0, 0, 0],
          'red'  : [255, 0, 0],
          'green': [0, 255, 0],
          'blue' : [0, 0, 255],
         }

# no. of dimensions
nD = 2

directions = {
              'horiz': np.array([1, 0]),
              'vert' : np.array([0, 1]),
             }

def normalize(v):
    norm = np.linalg.norm(v)
    if norm == 0:
        return v
    else:
        return v / norm

def distance(v1, v2):
    return(np.linalg.norm(v2-v1))

def calc_normal(v):
    return normalize(np.array([-v[1], v[0]]))

# Check if a, b, c are counter-clockwise
def ccw(a, b, c):
    return (c[1] - a[1]) * (b[0] - a[0]) > (b[1] - a[1]) * (c[0] - a[0])

# Check if two line segments l1 and l2 intersect
def intersect(l1_start, l1_end, l2_start, l2_end):
    c1 = ccw(l1_start, l2_start, l2_end) != ccw(l1_end, l2_start, l2_end)
    c2 = ccw(l1_start, l1_end, l2_start) != ccw(l1_start, l1_end, l2_end)
    return c1 and c2

# Get intersection point of line segments l1 and l2
def intersection_point(l1_start, l1_end, l2_start, l2_end):
    x1, x2 = l1_start[0], l1_end[0]
    y1, y2 = l1_start[1], l1_end[1]
    x3, x4 = l2_start[0], l2_end[0]
    y3, y4 = l2_start[1], l2_end[1]
    t = ((y3-y4) * (x1-x3) + (x4-x3) * (y1-y3)) / ((x4-x3) * (y1-y2) - (x1-x2) * (y4-y3))
    return l1_start + t * (l1_end - l1_start)

class ball:
    def __init__(self,
                 pos   = np.zeros(nD),
                 vel   = np.zeros(nD),
                 rad   = 1.0,
                 mass  = 1.0,
                 color = colors['black'],
                 ID    = -1):

        self.pos    = pos.flatten()
        self.vel    = vel.flatten()
        self.rad    = rad
        self.mass   = mass
        self.color  = color
        self.events = []
        self.ID     = ID

    def draw(self, surface):
        pygame.draw.circle(surface, self.color, self.pos.astype(int), self.rad)

    def move(self, dt):
        self.pos = self.pos + self.vel * dt

    def bounce(self, wall):
        self.vel = self.vel - 2 * np.dot(self.vel, wall.normal) * wall.normal

class wall:
    def __init__(self,
                 direction = directions['horiz'],
                 start     = np.zeros(nD),
                 length    = 1,
                 width     = 5,
                 color     = colors['black'],
                 ID        = -1):

        self.direction = direction
        self.normal    = calc_normal(direction)
        self.start     = start
        self.length    = length
        self.end       = start + length * direction
        self.width     = width
        self.color     = color
        self.ID        = ID

    def draw(self, surface):
        pygame.draw.line(surface, self.color, self.start, self.end, self.width)

def put_balls(balls, N, r, min_pos = 0, max_pos = 800, min_v = 0, max_v = 100, color = colors['black']):
    # Enforce no overlaps
    i = 0
    while i < N:
        c = np.random.uniform(min_pos, max_pos, size=(1,nD))
        overlap = False
        for b in balls:
            if distance(c, b.pos) < 2*r:
                overlap = True
                break
        if overlap is False:
            if color == 'random':
                color = np.random.randint(100, 255, size=(1,3)).flatten()
            balls.append(ball(pos = c,
                              vel = np.random.uniform(min_v, max_v, size=(1,nD)),
                              rad = r,
                              color = color,
                              ID    = i))
            i += 1

    # Enforce static center of mass for system
    v_com = np.zeros(nD)
    for b in balls:
        v_com += b.vel
    balls[0].vel -= v_com

def ball_collision(b1, b2):
    dr = b2.pos - b1.pos
    dv = b2.vel - b1.vel
    J_size = 2 * b1.mass * b2.mass * np.dot(dv, dr) / ((b1.rad + b2.rad) * (b1.mass + b2.mass))
    J = J_size * dr / (b1.rad + b2.rad)
    b1.vel = b1.vel + J/b1.mass
    b2.vel = b2.vel - J/b2.mass

def time_ball_collision(b1, b2, time):
    dr  = b2.pos - b1.pos
    dv  = b2.vel - b1.vel
    dr2 = np.dot(dr, dr)
    dv2 = np.dot(dv, dv)
    dvr = np.dot(dv, dr)
    sig = b1.rad + b2.rad
    d   = dvr**2 - dv2 * (dr2 - sig**2)
    if dvr >= 0 or d < 0:
        return float('inf')
    else:
        return -1 * (dvr + np.sqrt(d)) / dv2 + time

def time_wall_collision(b, w, time):
    l1_start = b.pos
    l1_end   = b.pos + 1E10 * normalize(b.vel)
    if intersect(l1_start, l1_end, w.start, w.end):
        p = intersection_point(l1_start, l1_end, w.start, w.end)
        d = distance(b.pos, p) - b.rad
        v = np.linalg.norm(b.vel)
        if v != 0:
            return d / v + time
        else:
            return float('inf')
    else:
        return float('inf')

class event:
    def __init__(self, ball1, ball2=None, wall=None, time=0):
        self.ball1  = ball1
        self.ball2  = ball2
        self.wall   = wall
        self.active = True
        if ball2 is None:
            if wall is None:
                print('Error: No second ball nor wall provided.')
                return None
            else:
                self.time = time_wall_collision(ball1, wall, time)
        else:
            if wall is not None:
                print('Error: Second ball and wall provided.')
                return None
            else:
                self.time = time_ball_collision(ball1, ball2, time)

    def __lt__(self, other):
        return self.time < other.time

def create_events(i, ball1, balls, walls, queue, current_time):
    for e in ball1.events:
        e.active = False
    ball1.events = []
    for j, ball2 in enumerate(balls):
        if i < j < len(balls):
            e = event(ball1, ball2, wall=None, time=current_time)
            queue.put(e)
            ball1.events.append(e)

    for wall in walls:
        e = event(ball1, None, wall, current_time)
        queue.put(e)
        ball1.events.append(e)

scr_size = 800
dt = 0.05
N_balls = 10
balls = []
put_balls(balls,
          N       = N_balls,
          r       = 10,
          min_pos = 150,
          max_pos = 600,
          min_v   = -40,
          max_v   = 40,
          color   = 'random')

wall0 = wall(start  = np.array([100, 100]),
             length = 600,
             width  = 5,
             color  = colors['blue'],
             ID     = 0)

wall1 = wall(start  = np.array([100, 100]),
             direction = directions['vert'],
             length = 600,
             width  = 5,
             color  = colors['blue'],
             ID     = 1)

wall2 = wall(start  = np.array([100, 700]),
             length = 600,
             width  = 5,
             color  = colors['blue'],
             ID     = 2)

wall3 = wall(start  = np.array([700, 100]),
             direction = directions['vert'],
             length = 600,
             width  = 5,
             color  = colors['blue'],
             ID     = 3)

walls = [wall0, wall1, wall2, wall3]

current_time = 0
delta_time = 0
event_queue = PriorityQueue()
for i, ball1 in enumerate(balls):
    create_events(i, ball1, balls, walls, event_queue, current_time)

pygame.init()
screen = pygame.display.set_mode ((scr_size, scr_size))
while True:
    for ev in pygame.event.get():
        if ev.type == pygame.QUIT:
            pygame.quit()
            sys.exit()
        if ev.type == pygame.KEYDOWN:
            if ev.key == pygame.K_q:
                pygame.quit()
                sys.exit()

    if not event_queue.empty():
        next_event = event_queue.get()
        if next_event.time > current_time and next_event.active:
            print(current_time, next_event.time)
            delta_time = next_event.time - current_time
            current_time = next_event.time
            ball1 = next_event.ball1
            ball2 = next_event.ball2
            wall  = next_event.wall

            #a, b = -1, -1
            #if ball2 is not None:
            #    a = ball2.ID
            #if wall is not None:
            #    b = wall.ID
            #print(ball1.ID, a, b, delta_time, current_time)

            if ball2 is not None:
                ball_collision(ball1, ball2)
                create_events(ball2.ID, ball2, balls, walls, event_queue, current_time)
            if wall is not None:
                ball1.bounce(wall)
            create_events(ball1.ID, ball1, balls, walls, event_queue, current_time)
        else:
            print('passed.')

    screen.fill(colors['white'])
    for ball in balls:
        ball.move(delta_time)
        ball.draw(screen)
    for wall in walls:
        wall.draw(screen)
    pygame.display.update()
    pygame.time.delay(500)


# -*- coding: utf8 -*-
import MapModel as Mm
from MapObjects.MovingObject import MovingObject
import Statistic

class Enemy(MovingObject):
    dx = [0, 1, 0, -1]
    dy = [1, 0, -1, 0]

    def __init__(self, health, width, height):
        super().__init__()
        self.gold = 2
        self.able_to_go = {Mm.Player, Mm.Ground, Mm.Arrow,
                           Mm.HeartStone, Mm.Trap, Mm.Wall, Mm.SpiralTower}
        self.unpretty = 10000
        self.damage = 1
        self.health = health
        self.field = [[-1 for _ in range(height)] for _ in range(width)]
        self.lazy_collision_init = self.collision_init

    def on_dead(self):
        Statistic.total_killed_enemies += 1
        Statistic.player_gold += self.gold
        return self.get_from_below()

    def get_info(self):
        info = "Панда\n"
        info += "Здоровье: {0}\n".format(self.health)
        info += "Урон: {0}\n".format(self.damage)
        return info

    def collision_init(self):
        # noinspection PyUnusedLocal
        @self.collide_registrar(Mm.Ground)
        def ground_collide(obj, ground):
            return None, obj

        @self.collide_registrar(Mm.HeartStone)
        def heartstone_collide(obj, heartstone):
            heartstone.attack(obj.damage * (obj.health / heartstone.defence))
            return None, heartstone

        @self.collide_registrar(Mm.Arrow)
        def arrow_collide(obj, arrow):
            obj.health -= arrow.damage
            if self.health > 0:
                return None, obj
            else:
                self.on_dead()
                return None, obj.get_from_below()

        @self.collide_registrar(Mm.Player)
        def player_collide(obj, player):
            player.health -= obj.damage * (obj.health / player.damage)
            obj.health -= player.damage * (obj.health / player.damage)
            if player.health > 0:
                obj.on_dead()
                return None, player
            else:
                return None, obj

        @self.collide_registrar(Mm.Trap)
        def trap_collide(obj, structure):
            structure.act_on_movable(obj)
            if obj.health > 0:
                obj.from_below = structure
                return None, obj
            else:
                obj.on_dead()
                return None, structure.check()

        @self.collide_registrar(Mm.SpiralTower)
        @self.collide_registrar(Mm.Wall)
        def wall_collide(obj, wall):
            damage = obj.damage * obj.health
            if damage > wall.health:
                obj.health -= wall.health / obj.damage
                return None, obj
            else:
                wall.health -= damage
                return None, wall.check()


import random

import numpy

from firm import Firm
#from firm_result import FirmResult
from firm_labormarket_result import FirmLaborMarketResult
from firm_goodmarket_result import FirmGoodMarketResult
from worker import Worker
from world import World

def invert(x):
    if x != 0:
        return 1 / x
    else:
        return 0

class BasicWorld(World):
    def manage_firm_actions(self, firm_actions):
        """
        Basic World! Contains basic algorithms for labor market and good market.
        :param firm_actions:
        :return:
        """
        salaries = numpy.array(list(firm_action.salary if firm_action.salary >= 0 else 0 for firm_action in self.firm_actions))

        self.fire()

        new_workers, quited = self.manage_job_offers()

        sales = self.manage_sales()
        # Aggregate firm results
        #  for firm in self.firms:
        #     self.firm_results[firm.id] = FirmResult(new_workers[firm.id], quited[firm.id], salaries[firm.id],
        #                                            sales[firm.id])

    def manage_job_offers(self):
        initial_salaries = numpy.array(list(firm_action.salary if firm_action.salary >= 0 and firm_action.offer_count > 0
                                            else 0 for firm_action in self.firm_labormarket_actions))
        salaries = numpy.array(list(firm_action.salary if firm_action.salary > 0 and firm_action.offer_count > 0 else 0
                                    for firm_action in self.firm_labormarket_actions))
        max_salary = max(salaries)

        quited = [[] for i in range(len(self.firms))]
        new_workers = [[] for i in range(len(self.firms))]

        potential_candidates = []
        for worker in self.workers:
            if worker.employer is None or worker.salary < max_salary:
                potential_candidates.append(worker)

        while len(potential_candidates) > 0 and sum(salaries) > 0:
            worker = random.choice(potential_candidates)
            employer = numpy.random.choice(self.firms, replace=False, p=salaries / sum(salaries))
            assert isinstance(employer, Firm)

            potential_candidates.remove(worker)
            if worker.salary < salaries[employer.id]:
                new_workers[employer.id].append(worker)
                if worker.employer is not None:
                    quited[worker.employer].append(worker)
            if self.firm_labormarket_actions[employer.id].offer_count == len(new_workers[employer.id]):
                salaries[employer.id] = 0

        for firm in self.firms:
            self.firm_labormarket_results[firm.id] = FirmLaborMarketResult(new_workers[firm.id], quited[firm.id], initial_salaries[firm.id])

    def manage_sales(self):
        # Basic selection algorithm for good market
        prices = numpy.array(list(firm_action.price if firm_action.price >= 0 and firm_action.production_count > 0
                                  else 0 for firm_action in self.firm_goodmarket_actions))
        inverted_prices = numpy.array(list(invert(x) for x in prices))
        sales = [0] * len(self.firms)
        money = self.money
        for firm in self.firms:
            if self.firm_goodmarket_actions[firm.id].production_count > self.firms[firm.id].stock:
                self.firm_goodmarket_actions[firm.id].production_count = self.firms[firm.id].stock
        production_counts = numpy.array(list(firm_action.production_count for firm_action in self.firm_goodmarket_actions))
        while sum(prices) > 0 and money > 0:
            seller = numpy.random.choice(self.firms, replace=False, p=inverted_prices / sum(inverted_prices))
            assert isinstance(seller, Firm)

            sales[seller.id] += 1
            production_counts[seller.id] -= 1
            money -= prices[seller.id]
            if production_counts[seller.id] <= 0:
                prices[seller.id] = 0
                inverted_prices[seller.id] = 0
        for firm in self.firms:
            self.firm_goodmarket_results[firm.id] = FirmGoodMarketResult(sales[firm.id])

    def fire(self):
        fired_workers = list(firm_action.fire_people for firm_action in self.firm_actions)

# for worker in fired_workers:
#            assert isinstance(worker, Worker)
#            worker.employer = None
#            worker.salary = 0


import os
import time
from datetime import datetime
import argparse
import torch.multiprocessing as mp
import torchvision
import torchvision.transforms as transforms
import torch
import torch.nn as nn
import torch.distributed as dist
import torch.nn.functional as F
import torchvision.models as models
import sys

parser = argparse.ArgumentParser()
parser.add_argument('--epochs',default=1, type=int)
parser.add_argument('--batch-size',default=100, type=int)
parser.add_argument('--size',default=224, type=int)
parser.add_argument('--optimizer', default="SGD")
parser.add_argument('--momentum', default=0.0, type=float)
parser.add_argument('--beta1', default=0.9, type=float)
parser.add_argument('--beta2', default=0.999, type=float)
parser.add_argument('--lr', default=0.001, type=float)
parser.add_argument('--weight-decay', default=0.0, type=float)
parser.add_argument('--arch',default="vgg11")
parser.add_argument('--pretrained', action='store_true')
parser.add_argument('--amp', action='store_true')
parser.add_argument('--summary', action='store_true')
parser.add_argument('--test', action='store_true')
parser.add_argument('--p-hori', default=0.0, type=float)
parser.add_argument('--rotation', default=0.0, type=float)
parser.add_argument('--shear', default=0.0, type=float)
parser.add_argument('--min-scale', default=1.0, type=float)
parser.add_argument('--max-scale', default=1.0, type=float)
parser.add_argument('--brightness', default=0.0, type=float)
parser.add_argument('--contrast', default=0.0, type=float)
parser.add_argument('--saturation', default=0.0, type=float)
parser.add_argument('--hue', default=0.0, type=float)
parser.add_argument('--dataset', default="CIFAR10", type=str)

def main():
    args = parser.parse_args()
    epochs=args.epochs
    batch_size = args.batch_size
    print(batch_size)
    rank = int(os.getenv('OMPI_COMM_WORLD_RANK', '0'))
    world_size = int(os.getenv("OMPI_COMM_WORLD_SIZE", '1'))
    gpu = rank % torch.cuda.device_count()

    master_ip = os.getenv('MASTER_ADDR', 'localhost')
    master_port = os.getenv('MASTER_PORT', '6000')
    init_method = 'tcp://' + master_ip + ':' + master_port
    train(epochs,batch_size,rank,world_size,gpu,master_ip,master_port,init_method,args)

#class LeNet(nn.Module):
class ConvNet(nn.Module):
  def __init__(self):
    super().__init__()
    self.conv1 = nn.Conv2d(3, 16, 3, 1, padding=1)
    self.conv2 = nn.Conv2d(16, 32, 3, 1, padding=1)
    self.conv3 = nn.Conv2d(32, 64, 3, 1, padding=1)
    self.fc1 = nn.Linear(4*4*64, 500)
    self.dropout1 = nn.Dropout(0.5)
    self.fc2 = nn.Linear(500, 10)

  def forward(self, x):
    x = F.relu(self.conv1(x))
    x = F.max_pool2d(x, 2, 2)
    x = F.relu(self.conv2(x))
    x = F.max_pool2d(x, 2, 2)
    x = F.relu(self.conv3(x))
    x = F.max_pool2d(x, 2, 2)
    x = x.view(-1, 4*4*64)
    x = F.relu(self.fc1(x))
    x = self.dropout1(x)
    x = self.fc2(x)
    return x

def train(epochs,batch_size,rank,world_size,gpu,master_ip,master_port,init_method,args):
    #rank = args.nr * args.gpus + gpu
    torch.cuda.set_device(gpu)
    dist.init_process_group(backend='nccl', init_method=init_method, world_size=world_size, rank=rank)
    torch.manual_seed(0)
    #model = ConvNet()
    if "vgg" in args.arch:
        if(args.arch=="vgg11"):
            model = models.vgg11(pretrained=args.pretrained)
        elif(args.arch=="vgg11_bn"):
            model = models.vgg11_bn(pretrained=args.pretrained)
        elif(args.arch=="vgg13"):
            model = models.vgg13(pretrained=args.pretrained)
        elif(args.arch=="vgg13_bn"):
            model = models.vgg13_bn(pretrained=args.pretrained)
        elif(args.arch=="vgg16"):
            model = models.vgg16(pretrained=args.pretrained)
        elif(args.arch=="vgg16_bn"):
            model = models.vgg16_bn(pretrained=args.pretrained)
        elif(args.arch=="vgg19"):
            model = models.vgg19(pretrained=args.pretrained)
        elif(args.arch=="vgg19_bn"):
            model = models.vgg19_bn(pretrained=args.pretrained)
        if(args.dataset=="CIFAR10"):
            model.avgpool=nn.Identity()
            model.classifier[0] = nn.Linear(512 * 1 * 1, 4096)  #(512 * 7 * 7, 4096)
            num_ftrs = model.classifier[6].in_features
            model.classifier[6] = torch.nn.Linear(num_ftrs, 10)
    elif "resnet" in args.arch and "resnext" not in args.arch and "wide" not in args.arch:
        if(args.arch=="resnet18"):
            model = models.resnet18(pretrained=args.pretrained)
        elif(args.arch=="resnet34"):
            model = models.resnet34(pretrained=args.pretrained)
        elif(args.arch=="resnet50"):
            model = models.resnet50(pretrained=args.pretrained)
        elif(args.arch=="resnet101"):
            model = models.resnet101(pretrained=args.pretrained)
        elif(args.arch=="resnet152"):
            model = models.resnet152(pretrained=args.pretrained)
        if(args.dataset=="CIFAR10"):
            num_ftrs = model.fc.in_features
            model.fc=torch.nn.Linear(num_ftrs, 10)
    elif "mobilenet_v2" in args.arch:
        model = models.mobilenet_v2(pretrained=args.pretrained)
        if(args.dataset=="CIFAR10"):
            num_ftrs = model.classifier[1].in_features
            model.classifier[1] = torch.nn.Linear(num_ftrs, 10)
    elif "mobilenet_v3" in args.arch:
        if(args.arch=="mobilenet_v3_large"):
            model = models.mobilenet_v3_large(pretrained=args.pretrained)
        elif(args.arch=="mobilenet_v3_small"):
            model = models.mobilenet_v3_small(pretrained=args.pretrained)
        if(args.dataset=="CIFAR10"):
            num_ftrs = model.classifier[3].in_features
            model.classifier[3] = torch.nn.Linear(num_ftrs, 10)
    elif "resnext" in args.arch:
        if(args.arch=="resnext50_32x4d"):
            model = models.resnext50_32x4d(pretrained=args.pretrained)
        elif(args.arch=="resnext101_32x8d"):
            model = models.resnext101_32x8d(pretrained=args.pretrained)
        if(args.dataset=="CIFAR10"):
            num_ftrs = model.fc.in_features
            model.fc=torch.nn.Linear(num_ftrs, 10)
    elif "wide" in args.arch:
        if(args.arch=="wide_resnet50_2"):
            model = models.wide_resnet50_2(pretrained=args.pretrained)
        elif(args.arch=="wide_resnet101_2"):
            model = models.wide_resnet101_2(pretrained=args.pretrained)
        if(args.dataset=="CIFAR10"):
            num_ftrs = model.fc.in_features
            model.fc=torch.nn.Linear(num_ftrs, 10)
    elif "densenet" in args.arch:
        if(args.arch=="densenet121"):
            model = models.densenet121(pretrained=args.pretrained)
        elif(args.arch=="densenet169"):
            model = models.densenet169(pretrained=args.pretrained)
        elif(args.arch=="densenet161"):
            model = models.densenet161(pretrained=args.pretrained)
        elif(args.arch=="densenet201"):
            model = models.densenet201(pretrained=args.pretrained)
        if(args.dataset=="CIFAR10"):
            num_ftrs = model.classifier.in_features
            model.classifier=torch.nn.Linear(num_ftrs, 10)
    model.to(gpu)

    if(args.summary==True):
        from torchsummary import summary
        if(args.dataset=="CIFAR10"):
            print(args.dataset)
            print(args.arch)
            summary(model,(3,32,32))
        else:
            print(args.dataset)
            print(args.arch)
            print(args.size)
            summary(model,(3,args.size,args.size)) # summary(model,(channels,H,W))
        exit()

    # define loss function (criterion) and optimizer
    criterion = nn.CrossEntropyLoss().cuda(gpu)
    criterion2 = nn.CrossEntropyLoss(reduction='sum').cuda(gpu)
    # Wrap the model
    # Data loading code
    # Initialization
    model = nn.parallel.DistributedDataParallel(model, device_ids=[gpu])
    #if rank==0:
    #    wandb.watch(model)
###########################################################################################################################
    if(args.dataset=="CIFAR10"):
        transform_train = transforms.Compose([transforms.RandomCrop(32, padding=4),
                                              transforms.RandomHorizontalFlip(p=args.p_hori),
                                              transforms.RandomAffine(args.rotation, shear=args.shear, scale=(args.min_scale,args.max_scale)),
                                              transforms.ColorJitter(brightness=args.brightness, contrast=args.contrast, saturation=args.saturation,hue=args.hue),
                                              transforms.ToTensor(),
                                              transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
                                             ])

        transform = transforms.Compose([transforms.Resize((32,32)),
                                        transforms.ToTensor(),
                                        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
                                       ])

        train_dataset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train)
        validation_dataset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform)

        #training_loader = torch.utils.data.DataLoader(training_dataset, batch_size=100, shuffle=True)
        #validation_loader = torch.utils.data.DataLoader(validation_dataset, batch_size = 80, shuffle=False)
    ###########################################################################################################################

        #train_dataset = torchvision.datasets.MNIST(root='./data',train=True,transform=transforms.ToTensor(),download=True)
        if(args.test==True):
            train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset,num_replicas=world_size,rank=rank,shuffle=False)
        else:
            train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset,num_replicas=world_size,rank=rank)
        train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
                                                   batch_size=batch_size,
                                                   shuffle=False,
                                                   num_workers=5,
                                                   pin_memory=True,
                                                   sampler=train_sampler)

        validation_sampler = torch.utils.data.distributed.DistributedSampler(validation_dataset,num_replicas=world_size,rank=rank,shuffle=False)
        validation_loader = torch.utils.data.DataLoader(dataset=validation_dataset,
                                                   batch_size=batch_size,
                                                   shuffle=False,
                                                   num_workers=5,
                                                   pin_memory=True,
                                                   sampler=validation_sampler)
    elif(args.dataset=="Imagenet"):
        traindir="/fs2/groups2/gac50562/image-net/train"
        valdir="/fs2/groups2/gac50562/image-net/val"

        transform_train = transforms.Compose([transforms.RandomResizedCrop(args.size),
                                              transforms.RandomHorizontalFlip(),
                                              transforms.ToTensor(),
                                              transforms.Normalize((0.485, 0.456, 0.406),(0.229, 0.224, 0.225)),
                                             ])

        transform = transforms.Compose([transforms.Resize(256),
                                        transforms.CenterCrop(args.size),
                                        transforms.ToTensor(),
                                        transforms.Normalize((0.485, 0.456, 0.406),(0.229, 0.224, 0.225)),
                                       ])

        train_dataset = torchvision.datasets.ImageFolder(traindir,transform_train)
        validation_dataset = torchvision.datasets.ImageFolder(valdir,transform)
        if(args.test==True):
            train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset,num_replicas=world_size,rank=rank,shuffle=False)
        else:
            train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset,num_replicas=world_size,rank=rank)
        train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
                                                   batch_size=batch_size,
                                                   shuffle=False,
                                                   num_workers=5,
                                                   pin_memory=True,
                                                   sampler=train_sampler)

        validation_sampler = torch.utils.data.distributed.DistributedSampler(validation_dataset,num_replicas=world_size,rank=rank,shuffle=False)
        validation_loader = torch.utils.data.DataLoader(dataset=validation_dataset,
                                                   batch_size=batch_size,
                                                   shuffle=False,
                                                   num_workers=5,
                                                   pin_memory=True,
                                                   sampler=validation_sampler)
################################################################################################################################
    if rank==0:
        import wandb
        run=wandb.init(project="my-project")
        wandb.config.world_size=world_size
        wandb.config.total_batch_size=args.batch_size*world_size
        wandb.config.update(args)

    if(args.test==True):
        test_i=0

    lr=args.lr
    start = datetime.now()
    for epoch in range(epochs):
        train_sampler.set_epoch(epoch)
        if(args.dataset=="CIFAR10"):
            if(epoch==149):
                lr=0.1*lr
            if(epoch==224):
                lr=0.1*lr
        else:
            if(epoch==29):
                lr=0.1*lr
            if(epoch==59):
                lr=0.1*lr
            if(epoch==89):
                lr=0.1*lr
        if(args.optimizer=="SGD"):
            optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=args.momentum, weight_decay=args.weight_decay)
        elif(args.optimizer=="Adam"):
            optimizer = torch.optim.Adam(model.parameters(), lr=lr,betas=(args.beta1,args.beta2), weight_decay=args.weight_decay)
        elif(args.optimizer=="AdamW"):
            optimizer = torch.optim.AdamW(model.parameters(), lr=lr,betas=(args.beta1,args.beta2), weight_decay=args.weight_decay)
        model.train()
        running_corrects = 0.0
        epoch_acc=0.0
        epoch_loss=0.0
        epoch_loss_validation=0.0
        epoch_acc_validation=0.0
        number_val_data=0
        if(args.amp==True):
            scaler = torch.cuda.amp.GradScaler()

        for i, (images, labels) in enumerate(train_loader):
            if(args.test==True):
                test_i+=1
                #start = datetime.now()
            total_step = len(train_loader)
            images = images.to(gpu,non_blocking=True)
            labels = labels.to(gpu,non_blocking=True)
            # Forward pass
            optimizer.zero_grad()
            if(args.amp==True):
                with torch.cuda.amp.autocast():
                    outputs = model(images)
                    loss = criterion(outputs, labels)
                scaler.scale(loss).backward()
                scaler.step(optimizer)
                scaler.update()
            else:
                outputs = model(images)
                loss = criterion(outputs, labels)
                # Backward and optimize
                loss.backward()
                optimizer.step()

            _, preds = torch.max(outputs, 1)
            running_corrects_t = torch.sum(preds == labels.data).float()/len(labels.data)
            #if (i + 1) % 10 == 0:
            print('RANK[{}],GPU[{}], Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}, Acc: {:.4f}'.format(rank,gpu,epoch + 1, epochs, i + 1, total_step,
                                                                         loss.item(),running_corrects_t.item()))

            dist.all_reduce(loss)
            epoch_loss+=loss.item()/world_size
            dist.all_reduce(running_corrects_t)
            epoch_acc+= running_corrects_t.item()/world_size
            if(args.test==True):
                if(test_i>99):
                    break
        if(args.test==True):
            if(test_i>99):
                break
        epoch_acc=epoch_acc/ total_step
        epoch_loss=epoch_loss/total_step

        model.eval()
        with torch.no_grad():
            for i, (images, labels) in enumerate(validation_loader):
                number_val_data+=len(labels.data)
                images = images.to(gpu,non_blocking=True)
                labels = labels.to(gpu,non_blocking=True)
                # Forward pass
                if(args.amp==True):
                    with torch.cuda.amp.autocast():
                        outputs = model(images)
                        loss = criterion2(outputs, labels)
                else:
                    outputs = model(images)
                    loss = criterion2(outputs, labels)
                _, preds = torch.max(outputs, 1)
                dist.all_reduce(loss)
                epoch_loss_validation+=loss.item()
                epoch_acc_validation_t=torch.sum(preds == labels.data).float()
                dist.all_reduce(epoch_acc_validation_t)
                epoch_acc_validation+= epoch_acc_validation_t.item()
                #print(type(epoch_acc_validation))
        epoch_acc_validation=epoch_acc_validation/(world_size*number_val_data)
        epoch_loss_validation=epoch_loss_validation/(world_size*number_val_data)

        if rank==0:
            wandb.log({'loss_train': epoch_loss,'accuracy_train': epoch_acc, 'loss_val': epoch_loss_validation,'accuracy_val': epoch_acc_validation})
            print('RANK[{}],GPU[{}], Epoch [{}/{}],  Loss: {:.4f}, Acc:{:.4f},  Loss_val: {:.4f}, Acc_val:{:.4f}'.format(rank,gpu, epoch + 1,  epochs, epoch_loss,epoch_acc, epoch_loss_validation,epoch_acc_validation))

    if rank==0:
        training_time=datetime.now() - start
        print("Training complete in: " + str(training_time))
        wandb.config.training_time=training_time
        run.finish()

if __name__ == '__main__':
    main()
    print("Finished")
    #sys.exit(0)


'''
These functions goes side by side to main app.py Python file
in the same folder.
'''
# ---------------------------------------------- #
# Project starts at: 2020 02 01                  #
# Lithuana, by Vytautas Bielinskas               #
# ---------------------------------------------- #

# Import modules and packages
import matplotlib.pyplot as plt
import numpy as np
import urllib.request as ur
#import cv2
import io
import requests
import json
import datetime, time
from PIL import Image

# System variables
BAD_CHARS = ['(', ')']

# download the image, convert it to a NumPy array, and then read
# it into OpenCV format
def url_to_image(url):

	print('| Given URL: {}'.format(url))

	resp = ur.urlopen(url)
	image = np.asarray(bytearray(resp.read()), dtype="uint8")
	image = cv2.imdecode(image, cv2.IMREAD_COLOR)

	#img_to_plot = .thumbnail((64, 64), Image.ANTIALIAS)
	#plt.imshow(image, interpolation='nearest', cmap='hot')
	#plt.show()

	# return the image
	return image

# Extracting data from image
# Read Image File
def extract_text(img, API_KEY):
	'''
	Args:
		-- image - a given image of floorplan.
	'''

	if type(img) != None:

		x = input('Debuging station')

		try:
			# Get Size of Image
			height, width, _ = img.shape

			# Cutting Image
			#roi = img[1: height-1, 1: width-1]
			roi = img

			# Ocr
			url_api = 'https://api.ocr.space/parse/image'
			_, compressedimage = cv2.imencode('.jpg', roi, [1, 90])
			file_bytes = io.BytesIO(compressedimage)
			time.sleep(3)
			result = requests.post(url_api,
			files = {'--.jpg': file_bytes},
					 data = {'apikey' : API_KEY})

			result = result.content.decode()
			result = json.loads(result)

			#print(result)

			try:
				text_detected = str(result.get('ParsedResults')[0].get('ParsedText')).split('\n')
				#print(text_detected)
				return text_detected
			except:
				print('Result = {}'.format(result))
				return None

			# Show the file
			#cv2.imshow('Img-Roi', roi)
			#cv2.waitKey(0)
			#cv2.destroyAllWindows()

			return result.get('ParsedResults')[0].get('ParsedText')

		except:
			x = input('Debuging station')
			return None

	else:
		print('No Image found.')
		return None

def has_numbers(line):
	'''
	Args:
		-- a given subset of whole string that possibly contains numbers
	'''
	return any(char.isdigit() for char in line)

def get_float_numbers_from_string(line):
	'''
	Args:
		-- a given subset of whole string that possibly contains numbers
	'''
	if has_numbers(line):

		for this_bad_char in BAD_CHARS:
			line = line.replace(this_bad_char, '')

		print('line = {}'.format(line.split()))

		a = []
		for word in line.split():

			try:
				a.append(float(word))
			except ValueError:
				pass

		return a

	else:
		print('--> No found numbers here. <--')
		return None

def text_parse(extracted_text):
	'''
	Args:
		-- extracted_text : a list of string that has been extracted.
	'''
	possible_area = None

	for this_segment in extracted_text:
		if len(this_segment) > 5:

			x = input('Debuging station')

			print('--> {}'.format(this_segment))

			if 'total'.upper() in this_segment.upper() and 'area'.upper() in this_segment.upper() or\
			   'internal'.upper() in this_segment.upper() and 'area'.upper() in this_segment.upper() or\
			   'approximate'.upper() in this_segment.upper() and 'area'.upper() in this_segment.upper():
				print('----> {}'.format(this_segment))

				possible_area = get_float_numbers_from_string(this_segment)

	if possible_area != None and type(possible_area) != None:
		try:
			print('Possible area = {}'.format(possible_area))
			if len(possible_area) > 0:
				return possible_area
			else:
				print('\n(!) Possible area is not available (!).\n')
				return [float(0)]
		except:
			print('Possible area is not available.')
			return [float(0)]

	else:
		return None

def save_data(data):
	'''
	Args:
		-- data : a given Pandas DataFrame with extracted values
	'''
	timeNow = str(datetime.datetime.now()).replace(":","-")[:-10]
	date = '2020'
	filename = "{}-AreaValues_Zoopla.csv".format(timeNow)

	data.to_csv(filename, encoding='utf-8')
	print('\nThe file is saved to external CSV file: {}.'.format(filename))

	return None


"""
  Capstone Project.  Code written by Joey Hatfield.
  Fall term, 2018-2019.
"""

import rosebotics_new as rb
import time
import tkinter
from tkinter import ttk
import mqtt_remote_method_calls as com

def main():
    #colorsensortest()
    #polygontest(5)
    #arm_and_claw_test()
    beacon_go_to_test()

def colorsensortest():
    """ Runs YOUR specific part of the project """
    colors = [rb.Color.RED.value, rb.Color.WHITE.value, rb.Color.BLUE.value]
    robot = rb.Snatch3rRobot()
    print(time.localtime())
    robot.color_sensor.wait_until_intensity_is_greater_than(20)
    print(time.localtime())
    robot.color_sensor.wait_until_intensity_is_less_than(80)
    print(time.localtime())
    robot.color_sensor.wait_until_color_is_one_of(colors)
    print(time.localtime())
    robot.color_sensor.wait_until_color_is(rb.Color.WHITE.value)
    print(time.localtime())
def polygontest(n):
    robot = rb.Snatch3rRobot()
    for k in range(n):
        robot.drive_system.go_straight_inches(10)
        time.sleep(.5)
        robot.drive_system.spin_in_place_degrees(180-((180*(n-2))//n))
        time.sleep(.5)
        print(180-((180*(n-2)//n)))
def arm_and_claw_test():
    robot = rb.Snatch3rRobot()
    robot.arm.raise_arm_and_close_claw()
    robot.arm.calibrate()
    robot.arm.move_arm_to_position(69000000)
def beacon_go_to_test():
    #robot = rb.Snatch3rRobot()
    root = tkinter.Tk()
    root.title('Beacon Control')
    frame = ttk.Frame(root, padding=10)
    frame.grid()
    button = ttk.Button(frame, text='Infrared Beacon')
    button.grid()
    button['command'] = lambda: print('going')#go_places(robot)
    #while True:
    #   print(robot.beacon_sensor.get_distance_to_beacon(),robot.beacon_sensor.get_heading_to_beacon(),robot.beacon_sensor.get_heading_and_distance_to_beacon())
    root.mainloop()
# def go_places(robot):
#     turn = robot.beacon_sensor.get_heading_to_beacon()
#     go = robot.beacon_sensor.get_distance_to_beacon()

main()


import pandas as pd
import sys
from musixmatch_api import *
from emotion_predictor import *
df = pd.read_csv("slim-2.csv")

terminado = False
while not terminado:
    print()
    print("SONGS & LYRICS")
    print("[1] Buscar")
    print("[2] Descubrir")
    print("[3] Salir del programa")
    menu_principal = int(input())
    if menu_principal == 1:
        print()
        print("[1] Buscar cancion")
        print("[2] Buscar recomendiación por género")
        submenu = int(input())
        if submenu == 1:
            print()
            print("Buscar canción por: ")
            print("[1] Título y Artista")
            print("[2] Letra")
            print("[3] (¡Especial!) Buscar sentimientos de la canción")
            submenu_1 = int(input())
            if submenu_1 == 1:
                print()
                titulo = input("Título de la canción: ")
                artista = input("Artista: ")
                music_matcher(
                    {
                        'by_lyrics': False,
                        'track': titulo,
                        'artist': artista
                    }
                )
            elif submenu_1 == 2:
                print()
                titulo = input("Letra de la canción: ")
                artista = input("Escribe el nombre del artista. Si no te sabes el nombre del artista, escribe None: ")
                if artista != "None":
                    music_matcher(
                        {
                            'by_lyrics': True,
                            "artist": artista,
                            'lyrics': titulo
                        }
                    )
                else:
                    music_matcher(
                        {
                            'by_lyrics': True,
                            "artist": None,
                            'lyrics': titulo
                        }
                    )
            elif submenu_1 == 3:
                print()
                letra = input("Ingrese la letra que desea encontrar el sentimiento: ")
                print()
                for emotion, percentage in predict_emotion(letra).items():
                    print(f'{emotion}: {percentage}')
            else:
                print()
                print("Oops, ocurrió un error. Intenta nuevamente.")
        elif submenu == 2:
            print()
            print('Lista de géneros recomendados:')
            print('- Pop')
            print('- K-Pop')
            print('- Rock')
            print('- Hard Rock')
            print('- Heavy Metal')
            print('- Rap')
            print('- Country')
            print('- Classical')
            print()
            genero = input("Escribe tu genero: ")
            recommendations_genre(
                {
                'top': '10',
                'genre': genero
                }
            )
        else:
            print()
            print("Oops, parece que ocurrió un error")
    elif menu_principal == 2:
        print()
        print("Descubrir Top Charts de:")
        print("[1] Canciones")
        print("[2] Artistas")
        submenu_2 = int(input())
        if submenu_2 in [1, 2]:
            x = input("Ingrese el nombre del país: ")
            print()
            print("[1] Top 100")
            print("[2] Top 25")
            print("[3] Top 10")
            top_charts = int(input())
            if top_charts == 1:
                top = "100"
            elif top_charts == 2:
                top = "25"
            elif top_charts == 3:
                top = "10"
            else:
                print()
                print("Oops, ocurrió un error. Intente nuevamente.")
            iso_3166 = df.drop("country-code", axis = 1)
            iso_3166.columns = ["Nombre", "Código"]
            iso_3166.set_index("Nombre", inplace=True)
            fila = iso_3166.loc[x]
            país = fila["Código"]
            if submenu_2 == 1:
                print()
                top_tracks(
                    {
                        'top': top,
                        'country': país,
                    }
                )
            elif submenu_2 == 2:
                print()
                top_artists(
                {
                    'top': top,
                    'country': país
                }
                )
            else:
                print()
                print("Oops, parece que ocurrió un error")
        else:
            print()
            print("Oops, parece que ocurrió un error")
    elif menu_principal == 3:
        print()
        print("Gracias por utilizar SONGS & LYRICS")
        sys.exit()
    else:
        print()
        print("Oops, parece que ocurrió un error")


#!/usr/bin/env python

# This script is used by the server to initiate a remote shell on the client.
# It should never be called directly.

import argparse
from contextlib import ExitStack
from daemon import DaemonContext
import sys

from penguindome import set_gpg
from penguindome.client import get_logger
from penguindome.shell import (
    InteractionBroker,
    PenguinDomeServerPeer,
)
from penguindome.shell.client import PexpectPeer

def background_code(args):
    log = get_logger('shell')
    pipe_id = args.pipe_id
    log.info('Initiating shell with pipe ID {}', pipe_id)
    try:
        set_gpg('client')
        with PenguinDomeServerPeer('client', pipe_id) as remote, \
                PexpectPeer('/bin/bash') as shell:
            broker = InteractionBroker(shell, remote)
            broker.interact()
    except:
        log.exception()
        sys.exit(1)

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--foreground', action='store_true')
    parser.add_argument('pipe_id', metavar='PIPE-ID')
    args = parser.parse_args()
    return args

def main():
    args = parse_args()
    if args.foreground:
        manager = ExitStack
    else:
        manager = DaemonContext
    with manager():
        background_code(args)

if __name__ == '__main__':
    main()