# Opus/UrbanSim urban simulation software.
# Copyright (C) 2010-2011 University of California, Berkeley, 2005-2009 University of Washington
# See opus_core/LICENSE
from numpy import arange, array, where, logical_and, concatenate
from opus_core.misc import get_distinct_names, unique
from opus_core.datasets.dataset import DatasetSubset
from opus_core.variables.attribute_type import AttributeType
from opus_core.logger import logger
from urbansim.models.household_transition_model import HouseholdTransitionModel
from copy import copy

class RegionalHouseholdTransitionModel(HouseholdTransitionModel):
    """Creates and removes households from household_set. It runs the urbansim HTM with control totals for each region."""

    model_name = "Regional Household Transition Model"

    def run(self, year, household_set, control_totals, characteristics, resources=None):
        self._do_initialize_for_run(household_set)
        control_totals.get_attribute("total_number_of_households") # to make sure they are loaded
        self.characteristics = characteristics
        self.all_categories = self.characteristics.get_attribute("characteristic")
        self.all_categories = array(map(lambda x: x.lower(), self.all_categories))
        self.scaled_characteristic_names = get_distinct_names(self.all_categories).tolist()
        self.marginal_characteristic_names = copy(control_totals.get_id_name())
        index_year = self.marginal_characteristic_names.index("year")
        self.marginal_characteristic_names.remove("year")
        self.marginal_characteristic_names.remove("large_area_id")
        large_area_ids = control_totals.get_attribute("large_area_id")
        households_large_area_ids = household_set.compute_variables("washtenaw.household.large_area_id")
        unique_large_areas = unique(large_area_ids)
        is_year = control_totals.get_attribute("year")==year
        all_households_index = arange(household_set.size())
        for area in unique_large_areas:
            idx = where(logical_and(is_year, large_area_ids == area))[0]
            self.control_totals_for_this_year = DatasetSubset(control_totals, idx)
            households_index = where(households_large_area_ids == area)[0]
            households_for_this_area = DatasetSubset(household_set, households_index)
            logger.log_status("HTM for area %s (currently %s households)" % (area, households_for_this_area.size()))
            last_remove_idx = self.remove_households.size
            last_new_hhs_idx = self.mapping_existing_hhs_to_new_hhs.size
            self._do_run_for_this_year(households_for_this_area)
            add_hhs_size = self.new_households[self.location_id_name].size-self.new_households["large_area_id"].size
            remove_hhs_size = self.remove_households.size-last_remove_idx
            logger.log_status("add %s, remove %s, total %s" % (add_hhs_size, remove_hhs_size,
                                                               households_for_this_area.size()+add_hhs_size-remove_hhs_size
                                                               ))
            self.new_households["large_area_id"] = concatenate((self.new_households["large_area_id"],
                                            array(add_hhs_size*[area], dtype="int32")))
            # transform indices of removing households into indices of the whole dataset
            self.remove_households[last_remove_idx:self.remove_households.size] = all_households_index[households_index[self.remove_households[last_remove_idx:self.remove_households.size]]]
            # do the same for households to be duplicated
            self.mapping_existing_hhs_to_new_hhs[last_new_hhs_idx:self.mapping_existing_hhs_to_new_hhs.size] = all_households_index[households_index[self.mapping_existing_hhs_to_new_hhs[last_new_hhs_idx:self.mapping_existing_hhs_to_new_hhs.size]]]

        self._update_household_set(household_set)
        idx_new_households = arange(household_set.size()-self.new_households["large_area_id"].size, household_set.size())
        households_large_area_ids = household_set.compute_variables("washtenaw.household.large_area_id")
        households_large_area_ids[idx_new_households] = self.new_households["large_area_id"]
        household_set.delete_one_attribute("large_area_id")
        household_set.add_attribute(households_large_area_ids, "large_area_id", metadata=AttributeType.PRIMARY)
        # return an index of new households
        return idx_new_households

    def _do_initialize_for_run(self, household_set):
        HouseholdTransitionModel._do_initialize_for_run(self, household_set)
        self.new_households["large_area_id"] = array([], dtype="int32")

from opus_core.tests import opus_unittest
from opus_core.resources import Resources
from opus_core.storage_factory import StorageFactory
from numpy import array, logical_and, int32, int8, zeros
from numpy import ma
from urbansim.datasets.household_dataset import HouseholdDataset
from urbansim.datasets.household_characteristic_dataset import HouseholdCharacteristicDataset
from urbansim.datasets.control_total_dataset import ControlTotalDataset
class Tests(opus_unittest.OpusTestCase):

    def setUp(self):
        #1) 3000 households in large_area=1 with age_of_head < 50, income < 40,000, persons < 3.
        #2) 3000 households in large_area=2 with age_of_head < 50, income < 40,000, persons < 3.
        #3) 1000 households in large_area=1 with age_of_head < 50, income < 40,000, persons >= 3.
        #4) 1000 households in large_area=2 with age_of_head < 50, income < 40,000, persons >= 3.
        #5) 1500 households in large_area=1 with age_of_head < 50, income >= 40,000, persons < 3.
        #6) 1500 households in large_area=2 with age_of_head < 50, income >= 40,000, persons < 3.
        #7) 2000 households in large_area=1 with age_of_head < 50, income >= 40,000, persons >= 3.
        #8) 2000 households in large_area=2 with age_of_head < 50, income >= 40,000, persons >= 3.
        #9) 1000 households in large_area=1 with age_of_head >= 50, income < 40,000, persons < 3.
        #10) 1000 households in large_area=2 with age_of_head >= 50, income < 40,000, persons < 3.
        #11) 2500 households in large_area=1 with age_of_head >= 50, income < 40,000, persons >= 3.
        #12) 2500 households in large_area=2 with age_of_head >= 50, income < 40,000, persons >= 3.
        #13) 1500 households in large_area=1 with age_of_head >= 50, income >= 40,000, persons < 3.
        #14) 1500 households in large_area=2 with age_of_head >= 50, income >= 40,000, persons < 3.
        #15) 4000 households in large_area=1 with age_of_head >= 50, income >= 40,000, persons >= 3.
        #16) 4000 households in large_area=2 with age_of_head >= 50, income >= 40,000, persons >= 3.

        self.households_data = {
            "household_id":arange(33000)+1,
            "grid_id":       array(6000*[1] + 2000*[2] + 3000*[3] + 4000*[4] + 2000*[5] + 5000*[6] +
                                3000*[10]+ 8000*[100], dtype=int32),
            "large_area_id": array(3000*[1] + 3000*[2]+ 1000*[1] + 1000*[2] + 1500*[1] + 1500*[2] +
                                   2000*[1] + 2000*[2] + 1000*[1] + 1000*[2] + 2500*[1] + + 2500*[2] +
                                1500*[1]+  1500*[2]+ 4000*[1] + 4000*[2], dtype=int32),
            "age_of_head": array(6000*[40] + 2000*[45] + 3000*[25] + 4000*[35] + 2000*[50] + 5000*[60] +
                                3000*[75]+ 8000*[65], dtype=int32),
            "income": array(6000*[35000] + 2000*[25000] + 3000*[40000] + 4000*[50000] + 2000*[20000] +
                                5000*[25000] + 3000*[45000]+ 8000*[55000], dtype=int32),
            "persons": array(6000*[2] + 2000*[3] + 3000*[1] + 4000*[6] + 2000*[1] + 5000*[4] +
                                3000*[1]+ 8000*[5], dtype=int8),
            }
        self.household_characteristics_for_ht_data = {
            "characteristic": array(2*['age_of_head'] + 2*['income'] + 2*['persons']),
            "min": array([0, 50, 0, 40000, 0, 3]),
            "max": array([49, 100, 39999, -1, 2, -1])
            }

    def test_same_distribution_after_household_addition(self):
        """Using the control_totals and no marginal characteristics,
        add households and ensure that the distribution within each group stays the same
        """

        annual_household_control_totals_data = {
            "year": array([2000, 2000]),
            "total_number_of_households": array([20000, 30000]),
            "large_area_id": array([1,2])
            }

        storage = StorageFactory().get_storage('dict_storage')

        storage.write_table(table_name = 'hh_set', table_data = self.households_data)
        hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')

        storage.write_table(table_name = 'hct_set', table_data = annual_household_control_totals_data)
        hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what="household")

        storage.write_table(table_name = 'hc_set', table_data = self.household_characteristics_for_ht_data)
        hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')

        model = RegionalHouseholdTransitionModel()
        model.run(year=2000, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)

        #check that there are 20000 (area 1) and 30000 (area 2) total households after running the model
        areas = hh_set.get_attribute("large_area_id")
        results = array([0,0])
        for iarea in [0,1]:
            results[iarea] = where(areas == [1,2][iarea])[0].size
        should_be = [20000, 30000]
        self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
                         True, "Error, should_be: %s, but result: %s" % (should_be, results))

        #check that the number of unplaced households is exactly the number of new households created
        results = where(hh_set.get_attribute("grid_id")<=0)[0].size
        should_be = [17000]
        self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
                         True, "Error, should_be: %s, but result: %s" % (should_be, results))

        #check that the distribution of households in each group and each area is the same as before running the model
        results = self.get_count_all_groups(hh_set)
        should_be = array([
                    # area 1
                     3000.0/16500.0*20000.0, 1000.0/16500.0*20000.0, 1500.0/16500.0*20000.0, 2000.0/16500.0*20000.0,
                     1000.0/16500.0*20000.0, 2500.0/16500.0*20000.0, 1500.0/16500.0*20000.0, 4000.0/16500.0*20000.0,
                     # area 2
                     3000.0/16500.0*30000.0, 1000.0/16500.0*30000.0, 1500.0/16500.0*30000.0, 2000.0/16500.0*30000.0,
                     1000.0/16500.0*30000.0, 2500.0/16500.0*30000.0, 1500.0/16500.0*30000.0, 4000.0/16500.0*30000.0])
        self.assertEqual(ma.allclose(results, should_be, rtol=0.1),
                         True, "Error, should_be: %s, but result: %s" % (should_be, results))
        # check the types of the attributes
        self.assertEqual(hh_set.get_attribute("age_of_head").dtype, int32,
                         "Error in data type of the new household set. Should be: int32, is: %s" % str(hh_set.get_attribute("age_of_head").dtype))
        self.assertEqual(hh_set.get_attribute("income").dtype, int32,
                         "Error in data type of the new household set. Should be: int32, is: %s" % str(hh_set.get_attribute("income").dtype))
        self.assertEqual(hh_set.get_attribute("persons").dtype, int8,
                         "Error in data type of the new household set. Should be: int8, is: %s" % str(hh_set.get_attribute("persons").dtype))

    def test_same_distribution_after_household_subtraction(self):
        """Using the control_totals and no marginal characteristics,
        subtract households and ensure that the distribution within each group stays the same
        """
        annual_household_control_totals_data = {
            "year": array([2000, 2000]),
            "total_number_of_households": array([8000, 12000]),
             "large_area_id": array([1,2])
            }

        storage = StorageFactory().get_storage('dict_storage')

        storage.write_table(table_name = 'hh_set', table_data = self.households_data)
        hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')

        storage.write_table(table_name = 'hct_set', table_data = annual_household_control_totals_data)
        hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what="household")

        storage.write_table(table_name = 'hc_set', table_data = self.household_characteristics_for_ht_data)
        hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')

        model = RegionalHouseholdTransitionModel()
        model.run(year=2000, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)

        #check that there are indeed 8000 (area 1) and 12000 (area 2) total households after running the model
        areas = hh_set.get_attribute("large_area_id")
        results = array([0,0])
        for iarea in [0,1]:
            results[iarea] = where(areas == [1,2][iarea])[0].size
        should_be = [8000, 12000]
        self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
                         True, "Error, should_be: %s, but result: %s" % (should_be, results))

        #check that the distribution of households in each group is the same as before running the model
        results = self.get_count_all_groups(hh_set)
        should_be = array([# area 1
                     3000.0/16500.0*8000.0, 1000.0/16500.0*8000.0, 1500.0/16500.0*8000.0, 2000.0/16500.0*8000.0,
                     1000.0/16500.0*8000.0, 2500.0/16500.0*8000.0, 1500.0/16500.0*8000.0, 4000.0/16500.0*8000.0,
                     # area 2
                     3000.0/16500.0*12000.0, 1000.0/16500.0*12000.0, 1500.0/16500.0*12000.0, 2000.0/16500.0*12000.0,
                     1000.0/16500.0*12000.0, 2500.0/16500.0*12000.0, 1500.0/16500.0*12000.0, 4000.0/16500.0*12000.0])
        self.assertEqual(ma.allclose(results, should_be, rtol=0.1),
                         True, "Error, should_be: %s,\n but result: %s" % (should_be, results))

    def test_controlling_with_one_marginal_characteristic(self):
        """Using the age_of_head as a marginal characteristic, which would partition the 8 groups into two larger groups
        (those with age_of_head < 40 and >= 40), ensure that the control totals are met and that the distribution within
        each large group is the same before and after running the model
        """

        #IMPORTANT: marginal characteristics grouping indices have to start at 0!
        #i.e. below, there is one marg. char. "age_of_head". here we indicate that the first "large group" (groups 1-4),
        #consisting of those groups with age_of_head < 40 should total 25000 households after running this model for one year,
        #and the second large group, those groups with age_of_head > 40, should total 15000 households
        annual_household_control_totals_data = {
            "year": array([2000, 2000, 2000, 2000]),
            "age_of_head": array([0, 1, 0, 1]),
            "total_number_of_households": array([20000, 10000, 5000, 5000]),
            "large_area_id": array([1, 1, 2, 2] )
            }

        storage = StorageFactory().get_storage('dict_storage')

        storage.write_table(table_name = 'hh_set', table_data = self.households_data)
        hh_set = HouseholdDataset(in_storage=storage, in_table_name='hh_set')

        storage.write_table(table_name = 'hct_set', table_data = annual_household_control_totals_data)
        hct_set = ControlTotalDataset(in_storage=storage, in_table_name='hct_set', what='household')

        storage.write_table(table_name = 'hc_set', table_data = self.household_characteristics_for_ht_data)
        hc_set = HouseholdCharacteristicDataset(in_storage=storage, in_table_name='hc_set')

        model = RegionalHouseholdTransitionModel()
        model.run(year=2000, household_set=hh_set, control_totals=hct_set, characteristics=hc_set)

        #check that there are indeed 40000 total households after running the model
        areas = hh_set.get_attribute("large_area_id")
        results = array([0,0])
        for iarea in [0,1]:
            results[iarea] = where(areas == [1,2][iarea])[0].size
        should_be = [30000, 10000]
        self.assertEqual(ma.allclose(should_be, results, rtol=1e-1),
                         True, "Error, should_be: %s, but result: %s" % (should_be, results))

        #check that the number of households within the groups correspond to the control totals
        results = self.get_count_all_groups(hh_set)
        should_be = [20000, 10000, 5000, 5000]
        idx1 = arange(0,4)
        idx2 = arange(4,8)
        idx3 = arange(8,12)
        idx4 = arange(12,16)
        self.assertEqual(ma.allclose([results[idx1].sum(), results[idx2].sum(), results[idx3].sum(), results[idx4].sum()], should_be, rtol=1e-1),
                         True, "Error, should_be: %s, but result: %s" % (should_be,
                                 array([results[idx1].sum(), results[idx2].sum(), results[idx3].sum(), results[idx4].sum()])))

        #check that the distribution of households within the groups are the same before and after
        #running the model, respectively

        should_be = [# area 1
                     3000.0/7500.0*20000.0, 1000.0/7500.0*20000.0, 1500.0/7500.0*20000.0, 2000.0/7500.0*20000.0,
                     1000.0/9000.0*10000.0, 2500.0/9000.0*10000.0, 1500.0/9000.0*10000.0, 4000.0/9000.0*10000.0,
                     # area 2
                     3000.0/7500.0*5000.0, 1000.0/7500.0*5000.0, 1500.0/7500.0*5000.0, 2000.0/7500.0*5000.0,
                     1000.0/9000.0*5000.0, 2500.0/9000.0*5000.0, 1500.0/9000.0*5000.0, 4000.0/9000.0*5000.0]
        self.assertEqual(ma.allclose(results, should_be, rtol=0.1),
                         True, "Error, should_be: %s, but result: %s" % (should_be, results))

    def get_count_all_groups(self, hh_set):
        res = zeros(16)
        i=0
        for area in [1,2]:
            is_area = hh_set.get_attribute('large_area_id') == area
            for age_conditional in ["<", ">="]:
                tmp1 = eval("hh_set.get_attribute('age_of_head') %s 50" % age_conditional)
                for income_conditional in ["<", ">="]:
                    tmp2 = logical_and(tmp1, eval("hh_set.get_attribute('income') %s 40000" % income_conditional))
                    for persons_conditional in ["<", ">="]:
                        tmp3 = logical_and(tmp2, eval("hh_set.get_attribute('persons') %s 3" % persons_conditional))
                        res[i] = (logical_and(tmp3, is_area)).sum()
                        i+=1
        return res

if __name__=='__main__':
    opus_unittest.main()


# -*- coding: utf-8 -*-
"""
Created on Wed Feb 14 12:55:25 2018

@author: Susan
"""

from pylab import *
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cbook as cbook
import time
from scipy.misc import imread
from scipy.misc import imresize
import matplotlib.image as mpimg
from scipy.ndimage import filters
import urllib
from numpy import random
import os
import pickle as pickle
from scipy.io import loadmat

#Load the data
def loadData():
    M = loadmat("mnist_all.mat")
    return M

#Helper function to load the training data from matrix
def loadTrain(M):
    #Initialize training data
    training = M["train0"]
    #Initialize labels
    labels = np.zeros((10,training.shape[0]))
    labels[0,:]=1
    #Get training data and labels
    i = 1
    while i<10:
        next = M["train"+str(i)]
        new_labels = np.zeros((10,next.shape[0]))
        new_labels[i,:]=1
        training = np.vstack((training,next))
        labels = np.hstack((labels,new_labels))
        i+=1
    training = np.transpose(training)
    training = training/255.0
    return training, labels

#Helper function to split data
def split(data,labels, percentage):
    combined = np.vstack((data,labels))
    combined = np.transpose(combined)
    np.random.seed(1)
    np.random.shuffle(combined)
    combined = np.transpose(combined)
    trainData, validData = combined[:-10,:int(percentage*combined.shape[1])-1], combined[:-10,int(percentage*combined.shape[1]):int(combined.shape[1])]
    trainLabels, validLabels = combined[784:,:int(percentage*combined.shape[1])-1], combined[784:,int(percentage*combined.shape[1]):int(combined.shape[1])]

    return trainData, validData, trainLabels, validLabels

def traintestsplit(data_,labels_,ratio):
    Full=np.vstack([data_,labels_])
    size_=int(data_.shape[1]*ratio)
    np.random.seed(1)
    np.random.shuffle(Full.T)
    Labels_=Full[784:,:]
    Data_=Full[0:784,:]
    train_x=Data_[:,0:size_]
    train_y=Labels_[:,0:size_]
    test_x=Data_[:,size_:]
    test_y=Labels_[:,size_:]
    return train_x,train_y,test_x,test_y

#Helper function to display an MNIST data point.
def display(x):
    imshow(x.reshape((28,28)),cmap = cm.gray)
    show()

#Provided Softmax function
def softmax(y):
    return exp(y)/tile(sum(exp(y),0), (len(y),1))

#Tanh function
def tanh(y,W,b):
    return np.tanh(np.dot(W.T,y)+b)

#Negative log-loss
def NLL(y,p):
    return -np.sum(y*np.log(softmax(p)))

def get_loss(w1,w2,x,y,w_train,b_train):
    #y=trainLabels
    #x=trainData
    #np.random.seed(1)
    #W = np.random.normal(0.0,0.2,(784,10))
    w_train_=w_train.copy()
    w_train_[490,3]=w1
    w_train_[491,3]=w2
    p=compute(x,w_train_,b_train)
    return -np.sum(y*np.log(softmax(p)))

#Forward propagation
def forward(x, W0, b0, W1, b1):
    L0 = tanh(x, W0, b0)
    L1 = dot(W1.T, L0) + b1
    output = softmax(L1)
    return L0, L1, output

#-------------------------------Test Code Implementation------------------------------------------------------#
def test():
    #Load sample weights for the multilayer neural network
    M = loadData()
    file = open(os.path.join(os.getcwd(),'snapshot50.pkl'),'rb')
    snapshot = pickle.load(file,encoding="latin1")
    W0 = snapshot["W0"]
    print("W0 Shape:")
    print(W0.shape)
    b0 = snapshot["b0"].reshape((300,1))
    print("b0 Shape:")
    print(b0.shape)
    W1 = snapshot["W1"]
    print("W1 Shape:")
    print(W1.shape)
    b1 = snapshot["b1"].reshape((10,1))
    print("b1 Shape:")
    print(b1.shape)

    #Load one example from the training set, and run it through the
    #neural network
    x = (M["train5"][148:200].T)/255.0
    print("x Shape:")
    print(x.shape)
    L0, L1, output = forward(x, W0, b0, W1, b1)
    #get the index at which the output is the largest
    print("Output shape:")
    print(output.shape)
    print("L0 shape:")
    print(L0.shape)
    print("L1 shape:")
    print(L1.shape)
    y = argmax(output,axis=0)
    print("Prediction:")
    print(y)

#-------------------------------Part 2 Function Implementation------------------------------------------------------#
def compute(X,W,b):
    hypothesis = np.matmul(W.T,X)
    hypothesis = hypothesis+b
    return hypothesis

#-------------------------------Part 2 Test------------------------------------------------------#

def testPart2():
    #Load Data
    M = loadData()
    train = loadTrain(M)
    #Testing computation function
    #Initialize weights and bias to random normal variables
    W = np.random.normal(0.0,0.1,(784,10))
    b = np.random.normal(0,0.1,(10,1))
    hypothesis = compute(train,W,b)
    print(softmax(hypothesis))

#-------------------------------Part 3 Function Implementation------------------------------------------------------#
#Computes the gradient of negative log-loss function for weights only
def grad_NLL_W(y,o,layer):

    p = softmax(o)
    grad = p - y
    grad = np.matmul(grad,np.transpose(layer))
    return np.transpose(grad)

#Computes the gradient of negative log-loss function for biases only
def grad_NLL_b(y,o):
    p = softmax(o)
    grad = p-y
    grad = np.sum(grad,axis=1,keepdims = True)
    return grad

#-------------------------------Part 3 Test------------------------------------------------------#

def testPart3():
    #Test gradient functionality
    y = np.zeros((10,1))
    y[1,:]=1

    #Create a test matrix
    M = loadData()
    test0 = ((M["train1"][130].T)/255.0).reshape((784,1))
    W = np.random.normal(0, 0.2, (784,10))
    b = np.random.normal(0,0.2, (10,1))
    #print(np.where(test0 != 0))

    #Create a finite difference
    h = 0.00001

    #Weight testing
    finite_W = np.zeros((784,10))
    finite_W[542,0]=h
    finite_d = (NLL(y,compute(test0,W+finite_W,b))-NLL(y,compute(test0,W,b)))/(h)
    print("Cost for row 542, column 0: " + str(finite_d))
    gradient = grad_NLL_W(y,compute(test0,W,b),test0)
    print("Gradient for row 542, column 0: " + str(gradient[542,0]))

    #Bias testing
    finite_b = np.zeros((10,1))
    finite_b[1,:] = h
    finite_d = (NLL(y,compute(test0,W,b+finite_b))-NLL(y,compute(test0,W,b)))/(h)
    print("Cost for second element in bias: " + str(finite_d))
    gradient = grad_NLL_b(y,compute(test0,W,b))
    print("Gradient matrix: " + str(gradient))

    #Reinitialize test variables for another test
    finite_W = np.zeros((784,10))
    finite_b = np.zeros((10,1))
    y = np.zeros((10,1))
    test1 = ((M["train9"][130].T)/255.0).reshape((784,1))
    y[9,:]=1

    #Weight testing
    finite_W[300,4] = h
    finite_d = (NLL(y,compute(test1,W+finite_W,b))-NLL(y,compute(test1,W,b)))/(h)
    print("Cost for row 300, column 4: " + str(finite_d))
    gradient = grad_NLL_W(y,compute(test1,W,b),test1)
    print("Gradient for row 300, column 4: " + str(gradient[300,4]))

    #Bias testing
    finite_b[4,:] = h
    finite_d = (NLL(y,compute(test1,W,b+finite_b))-NLL(y,compute(test1,W,b)))/(h)
    print("Cost for fifth element in bias: " + str(finite_d))
    gradient = grad_NLL_b(y,compute(test1,W,b))
    print("Gradient matrix: " + str(gradient))

#-------------------------------Part 4 Implementation------------------------------------------------------#
def grad_descent(NLL, grad_NLL_W, grad_NLL_b, x, y, init_w, init_b, alpha, iterations):
    iters = list()
    costy = list()

    EPS = 1e-5
    prev_w = init_w-10*EPS
    prev_b = init_b-10*EPS
    w = init_w.copy()
    b = init_b.copy()
    max_iter = iterations
    iter = 0
    while np.linalg.norm(w-prev_w) > EPS and iter < max_iter and np.linalg.norm(b-prev_b) > EPS:
        prev_w=w.copy()
        prev_b = b.copy()
        w-=alpha*(grad_NLL_W(y, compute(x,w,b),x))
        b-=alpha*(grad_NLL_b(y,compute(x,w,b)))
        if iter % 10 == 0:
            cost_i = NLL(y,compute(x,w,b))
            iters.append(iter)
            costy.append(cost_i)
            print("Iteration: " + str(iter))
            print("Cost: "+str(cost_i))
        iter += 1

    x_plot = iters
    y1_plot = costy
    plt.xlabel("Iterations")
    plt.ylabel("Log Loss")
    plt.title("Iterations vs. Log-loss")
    plt.plot(x_plot, y1_plot, 'r--')
    plt.show()

    return w, b

#-------------------------------Part 6 Implementation------------------------------------------------------#

def grad_descent_6(x, y, init_w, init_b, alpha, iterations): #,WW

    #gd_traj = [(init_w1, init_w2), (step1_w1, step1_w2), ...]

    EPS = 1e-5
    prev_w = init_w-10*EPS
    prev_b = init_b-10*EPS
    w = init_w.copy()
    b = init_b.copy()

    w1=init_w[490,3].copy() #these are the only weights to be changed
    w2=init_w[491,3].copy()

    w1+=0.15
    w2-=0.15

    Weights = list()
    Weights.append((w1, w2))
    print (type(Weights))

    max_iter = iterations
    iter = 0
    #while np.linalg.norm(w-prev_w) > EPS and iter < max_iter and np.linalg.norm(b-prev_b) > EPS:
    while iter < max_iter: #and np.linalg.norm(b-prev_b) > EPS:
        prev_w=w.copy()
        prev_b = b.copy()
        w-=alpha*(grad_NLL_W(y, compute(x,w,b),x))
        #b-=alpha*(grad_NLL_b(y,compute(x,w,b)))

        #Update Weights
        w1=w[490,3].copy()
        w2=w[491,3].copy()
        Weights.append((w1, w2))

        #Reset - Keeping all other Weights constant
        w=prev_w

        w[490,3]=w1
        w[491,3]=w2

        if iter % 100 == 0:
            cost_i = NLL(y,compute(x,w,b))
            print (type(Weights))

            print("Iteration: " + str(iter))
            print("Cost: "+str(cost_i))
        iter += 1

    return Weights

def grad_descent_6b(x, y, init_w, init_b, alpha, iterations): #,WW

    #gd_traj = [(init_w1, init_w2), (step1_w1, step1_w2), ...]

    v = np.zeros((784,10))
    momentum_rate = 0.5
    v[:,:]=0.000001 #Initialize

    EPS = 1e-5
    prev_w = init_w-10*EPS
    prev_b = init_b-10*EPS
    w = init_w.copy()
    b = init_b.copy()

    w1=init_w[490,3].copy() #these are the only weights to be changed
    w2=init_w[491,3].copy()

    w1+=0.15
    w2-=0.15

    Weights_m = list()
    Weights_m.append((w1, w2))
    print (type(Weights_m))

    max_iter = iterations
    iter = 0
    #while np.linalg.norm(w-prev_w) > EPS and iter < max_iter and np.linalg.norm(b-prev_b) > EPS:
    while iter < max_iter: #and np.linalg.norm(b-prev_b) > EPS:
        prev_w=w.copy()
        prev_b = b.copy()
        grad_desc=alpha*(grad_NLL_W(y, compute(x,w,b),x))
        mv = np.multiply(momentum_rate,v)
        v = np.add(mv, grad_desc)
        w-= v
        #b-=alpha*(grad_NLL_b(y,compute(x,w,b)))

        #Update Weights
        w1=w[490,3].copy()
        w2=w[491,3].copy()
        Weights_m.append((w1, w2))

        #Reset - Keeping all other Weights constant
        w=prev_w

        w[490,3]=w1
        w[491,3]=w2

        if iter % 100 == 0:
            cost_i = NLL(y,compute(x,w,b))
            print (type(Weights_m))

            print("Iteration: " + str(iter))
            print("Cost: "+str(cost_i))
        iter += 1

    return Weights_m

"""Produce a contour plot of the cost function, when the weights w1 and w2 are allowed to vary around the values
that you obtained in Part 5.
Plot the contour of the cost function. The cost function will be a function of the two weights.
The two weights w1 and w2 should vary around the value obtained in part 5. Label your axes"""

def Weights_6():

    w1_=w_train[490,3].copy()
    w2_=w_train[491,3].copy()

    Weights_a = list()

    w1_-=0.1
    w2_-=0.1

    Weights_a.append((w1_, w2_))

    i=0
    for i in range (0,20):
       w1_+=0.01
       w2_+=0.01
       Weights_a.append((w1_, w2_))
       i+=1

    return Weights_a

#Inital Weights

#-------------------------------Part 4 Test------------------------------------------------------#

def testPart4():
    #Load data
    M = loadData()
    #Initialize training data
    data, labels = loadTrain(M)
    trainData, validData, trainLabels, validLabels = split(data,labels, 0.2)
    #trainData,trainLabels,validData,validLabels=traintestsplit(data,labels, 0.7)

    x=trainData.copy()
    y=trainLabels.copy()

    print(trainData.shape)
    print(trainLabels.shape)
    print(validData.shape)
    print(validLabels.shape)

    np.random.seed(1)
    W = np.random.normal(0.0,0.2,(784,10))
    b = np.random.normal(0, 0.2,(10,1))

    w_train, b_train = grad_descent_w(NLL, grad_NLL_W, grad_NLL_b, trainData, trainLabels, W, b, 0.00001, 500)

    trainPred = compute(trainData, w_train,b_train)
    trainPred = np.argmax(trainPred,axis = 0)
    trainLabels = np.argmax(trainLabels,axis = 0)
    print("Training accuracy: " + str(np.sum(np.equal(trainPred,trainLabels))/(float(trainData.shape[1]))))

    validPred = compute(validData,w_train,b_train)
    validPred = np.argmax(validPred,axis = 0)
    validLabels = np.argmax(validLabels,axis = 0)

    print("Validation accuracy: " + str(np.sum(np.equal(validPred,validLabels))/(float(validData.shape[1]))))

    print (W.shape)
    print (W)

    return x,y,w_train,b_train

#-------------------------------Part 5 Implementation------------------------------------------------------#
def grad_descent_w(NLL, grad_NLL_W, grad_NLL_b, x, y, init_w, init_b, alpha, iterations):
    iters = list()
    costy = list()

    EPS = 1e-5
    prev_w = init_w-10*EPS
    prev_b = init_b-10*EPS
    w = init_w.copy()
    b = init_b.copy()
    max_iter = iterations
    iter = 0

    v = np.zeros((784,10))
    v[:,:]=0.000001 #Initialize
    momentum_rate = 0.5

    while np.linalg.norm(w-prev_w) > EPS and iter < max_iter and np.linalg.norm(b-prev_b) > EPS:
        prev_w=w.copy()
        prev_b = b.copy()
        grad_desc=alpha*(grad_NLL_W(y, compute(x,w,b),x))
        mv = np.multiply(momentum_rate,v)
        v = np.add(mv, grad_desc)
        w-= v
        b-=alpha*(grad_NLL_b(y,compute(x,w,b)))
        if iter % 100 == 0:
            print("Iteration: " + str(iter))
            print("Cost: "+str(NLL(y,compute(x,w,b))))
            cost_i = NLL(y,compute(x,w,b))
            iters.append(iter)
            costy.append(cost_i)
        iter += 1

    x_plot = iters
    y1_plot = costy
    plt.xlabel("Iterations")
    plt.ylabel("Log Loss")
    plt.title("Iterations vs. Log-loss")
    plt.plot(x_plot, y1_plot, 'r--')
    plt.show()
    return w, b

def Plot_Contour(Weights,Weights_m,x,y,w_train,b_train):
    gd_traj = Weights
    mo_traj = Weights_m
    w1s = np.arange(-0.65, -0.1, 0.05)
    w2s = np.arange(-0.65, -0.1, 0.05)
    w1z, w2z = np.meshgrid(w1s, w2s)
    C = np.zeros([w1s.size, w2s.size])
    for i, w1 in enumerate(w1s):
        for j, w2 in enumerate(w2s):
            C[j,i] = get_loss(w1, w2,x,y,w_train,b_train)
    CS = plt.contour(w1z, w2z, C, camp=cm.coolwarm)
    plt.plot([a for a, b in gd_traj], [b for a,b in gd_traj], 'ro-', label="No Momentum")
    plt.plot([a for a, b in mo_traj], [b for a,b in mo_traj], 'bo-', label="Momentum")
    plt.legend(loc='lower right')
    plt.title('Contour plot')

def testPart6():
    x,y,w_train,b_train=testPart4()
    Weights=grad_descent_6(x,y,w_train,b_train,0.017,20)
    Weights_m=grad_descent_6b(x,y,w_train,b_train,0.017,20)
    Plot_Contour(Weights,Weights_m,x,y,w_train,b_train)

#Main Function

def main():
    np.random.seed(1)
    #testPart2()
    #testPart3()
    #test()
    #testPart4()
    testPart6()

if __name__ == "__main__":
    main()


from simple_net import SimpleNet
from torch.utils.data import DataLoader
import torch.nn.functional as F
import torch
from torchvision import datasets
from torchvision.transforms import ToTensor
from mnist_dataset import MNIST

def test():
    BATCH_SIZE = 100

    working_set = MNIST(is_train=False)

    dataloader = DataLoader(working_set, batch_size=BATCH_SIZE, shuffle=True)
    model = SimpleNet()
    model.load_state_dict(torch.load("models/lin.h5"))
    model.eval()
    correct = 0
    total = 0
    with torch.no_grad():
        for data, y_true in dataloader:
            data = data.reshape(data.shape[0], 1, data.shape[1],data.shape[2])
            y_pred = model(data)
            pred = torch.argmax(y_pred, dim=1, keepdim=True)
            correct += pred.eq(y_true.data.view_as(pred)).sum()
            total += 1

    print(f"{correct} correct among {len(working_set)}")

test()


import numpy as np
import scipy.io.wavfile as w
import matplotlib.pyplot as plt

fs,data = w.read('tuning_fork.wav')
k = len(data)
data = np.asarray(data)
n = np.random.uniform(-20000,20000,(k,2))
d_n = data+n
r = np.zeros((k,2),dtype=float)
for h in range(2):
	for i in range(k):
		s=0
		for j in range(k):
			if(i+j < k):
				s = s+d_n[i+j,h]
		r[i,h] = s/k

plt.plot(r)
plt.show()


import os
import sys
from selenium import webdriver
from selenium.webdriver.remote.remote_connection import RemoteConnection
from selenium.webdriver.common.keys import Keys
from AmazonSesSample import EmailService
from os.path import join, dirname
from dotenv import load_dotenv
from bs4 import BeautifulSoup
import time
import unittest
import random
import requests
import base64
import json
import atexit
import shutil

dotenv_path = join(dirname(__file__), '.env')
load_dotenv(dotenv_path)
USERNAME = os.environ.get("USERNAME")
API_KEYS = os.environ.get("API_KEYS")
KOBITON_SERVER_URL = "https://" + USERNAME + ":" + API_KEYS + "@api.kobiton.com/wd/hub"
session_timeout = 40

###############FetchFavoriteDevices##############################
def FetchFavoriteDevices():
    api_key = (USERNAME + ':' + API_KEYS)
    auth = base64.b64encode(api_key.encode())
    headers = {
    'Authorization': 'Basic ' + auth.decode(),
    'Accept': 'application/json'
    }
    response = requests.get('https://api.kobiton.com/v1/devices', params={
    }, headers = headers)
    data = response.json()
    return data

def SetUpKobiton(self,file_name):
    # Delete file in the report folder
    target_dir = 'report'
    shutil.rmtree(target_dir)
    os.mkdir(target_dir)
    FavoriteDevices = FetchFavoriteDevices()
    for FavoriteDevice in FavoriteDevices['favoriteDevices']:
        if FavoriteDevice['isBooked'] == False and FavoriteDevice['isOnline'] == True:
            browserName = FavoriteDevice['installedBrowsers'][0]['name']
            deviceName = FavoriteDevice['deviceName']
            platformName = FavoriteDevice['platformName']
            platformVersion = FavoriteDevice['platformVersion']
            print(deviceName)
            break
        else:
            continue
    try:
        browserName
    except:
        print('Device is no available')

    desired_caps = {
        # The generated session will be visible to you only.
        'sessionName':        file_name,
        'sessionDescription': '',
        'deviceOrientation':  'portrait',
        'captureScreenshots': True,
        'browserName':        browserName,
        'deviceGroup':        'KOBITON',
        # For deviceName, platformVersion Kobiton supports wildcard
        # character *, with 3 formats: *text, text* and *text*
        # If there is no *, Kobiton will match the exact text provided
        # 'groupId':            944,
        'deviceName':         deviceName,
        'platformName':       platformName,
        'platformVersion':    platformVersion
    }
    self.driver = webdriver.Remote(KOBITON_SERVER_URL,desired_caps)
    self.driver.implicitly_wait(session_timeout)
    print(file_name)
    kobitonSessionId = self.driver.desired_capabilities.get('kobitonSessionId')
    print("https://portal.kobiton.com/sessions/%s" % (kobitonSessionId))

session_list = []

# ###############Finish Test on Kobiton#################################
def QuitKobiton(self):
    self.driver.quit()
    kobitonSessionId = self.driver.desired_capabilities.get('kobitonSessionId')
    session_list.append(kobitonSessionId)

    # After finising program and generating Xml file as results, fetchTestResult() going to run.
    atexit.register(fetchTestResult, session_list)

###########################################################################
###############Get Test Result and Send mail###############################
###########################################################################

path = os.getcwd()
files = os.listdir(path)

# fond file witch match 'test_'
test_files = [file for file in files if 'test' in file]
test_files.sort()
files_count = len(test_files)
count = 0
# make two-dimensional list
result_lists = [[] for i in range(files_count)]

email_text = "

File name	Status	Device Name	Session URL	result
"
def fetchTestResult(kobitonSessionIds):
    session_count = len(kobitonSessionIds)
    for kobitonSessionId in kobitonSessionIds:
        print("https://portal.kobiton.com/sessions/%s" % (kobitonSessionId))
        global count
        global email_text
        global files

        api_key = (USERNAME+ ':' + API_KEYS)
        auth = base64.b64encode(api_key.encode())
        headers = {
                'Authorization': 'Basic ' + auth.decode(),
                'Accept': 'application/json'
                }
        response = requests.get('https://api.kobiton.com/v1/sessions/' + str(kobitonSessionId), params={
        }, headers = headers)
        data = response.json()

        xml_files = os.listdir(path + "/report")
        if session_count == 1:
            result_lists[count].append(data['executionData']['desired']['sessionName'])
        else:
            result_lists[count].append(test_files[count])
        result_lists[count].append(data['state'])
        result_lists[count].append(data['executionData']['desired']['deviceName'])
        result_lists[count].append("https://portal.kobiton.com/sessions/"+ str(kobitonSessionId))

        for xml_file in xml_files:
            if  test_files[count][0:-3] in xml_file:
                with open("report/" + xml_file) as f:
                    soup = BeautifulSoup(f, 'xml')
                for e in soup.find_all('error'):
                    result_lists[count].append(e)
        count += 1
        print(count)
        print(session_count)
        if count == session_count:
            if session_count == 1:
                if len(result_lists[0]) == 4:
                    email_text += "
" + result_lists[0][0] + "	"
                    email_text += "" + result_lists[0][1] + "	"
                    email_text += "" + result_lists[0][2] + "	"
                    email_text += "" + result_lists[0][3] + "	"
                    email_text += "Passed
"
                else:
                    email_text += "
" + result_lists[0][0] + "	"
                    email_text += "" + result_lists[0][1] + "	"
                    email_text += "" + result_lists[0][2] + "	"
                    email_text += "" + result_lists[0][3] + "	"
                    email_text += "Failed
"
                    email_text += "
Error Description
"
                    email_text += "
" + str(result_lists[0][4]) + "
"
                email_text += "
"
                EmailService().send_result_mail(email_text,kobitonSessionId)

            for result_list in result_lists:
                print(result_list[0])
                if len(result_list) == 4:
                    email_text += "
" + result_list[0] + "	"
                    email_text += "" + result_list[1] + "	"
                    email_text += "" + result_list[2] + "	"
                    email_text += "" + result_list[3] + "	"
                    email_text += "Passed
"
                else:
                    email_text += "
" + result_list[0] + "	"
                    email_text += "" + result_list[1] + "	"
                    email_text += "" + result_list[2] + "	"
                    email_text += "" + result_list[3] + "	"
                    email_text += "Failed
"
                    email_text += "
Error Description
"
                    email_text += "
" + str(result_list[4]) + "
"
            email_text += "
"
            EmailService().send_result_mail(email_text,kobitonSessionId)
        else:
            print("Test is still in progress...")


from django.forms import ModelForm
from django import forms
from django.utils.translation import ugettext_lazy as _

from .models import Signature

import uuid

class SignatureForm(ModelForm):

    class Meta:
        model = Signature
        fields = ['first_name', 'last_name', 'phone', 'email', 'subscribed_to_mailinglist']
        widgets = {
            'first_name': forms.TextInput(attrs={'placeholder': _('First name *'), 'class': 'form-control eaFormTextField eaFullWidthContent'}),
            'last_name': forms.TextInput(attrs={'placeholder': _('Last name *'), 'class': 'form-control eaFormTextField eaFullWidthContent'}),
            'phone': forms.TextInput(attrs={'placeholder': _('Phone number'), 'class': 'form-control eaFormTextField eaFullWidthContent'}),
            'email': forms.EmailInput(attrs={'placeholder': _('Email address *'), 'class': 'form-control eaFormTextField eaFullWidthContent'})
        }

        labels = { f : '' for f in  ['first_name', 'last_name', 'phone', 'email'] }

    def __init__(self, petition=None, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.instance.petition = petition
        if not petition.has_newsletter:
            del self.fields['subscribed_to_mailinglist']
        else:
            self.fields['subscribed_to_mailinglist'].label = self.instance.petition.newsletter_text

    def save(self, commit=True):
        object = super().save(commit=False)
        hashstring = str(uuid.uuid4())
        object.confirmation_hash = hashstring
        object.confirmed = False
        if commit:
            object.save()
        return object


# Create your views here.

from django.shortcuts import render
from projects.models import Project
from projects.forms import InputForm

def project_index(request):
    projects = Project.objects.all()
    context = {
        'projects': projects}
    return render(request, 'project_index.html', context)

# Create your views here.
def home_view(request):
    context ={}
    context['form']= InputForm()
    return render(request, "home.html", context)

def corona_view(request):
    eyesymptom="null"
    context = {}
    metrics={}
    if request.method == "POST":
        # Get the posted form
        form = InputForm(request.POST)
        if form.is_valid():
            eyesymptom = form.cleaned_data.get('eye_symptom')
            chestsymptom=form.cleaned_data.get('chest_symptom')
            soarsymptom = form.cleaned_data.get('soar_symptom')
            runnynosesymptom = form.cleaned_data.get('runnynose_symptom')
            weaknesssymptom=form.cleaned_data.get('weakness_symptom')
            achessymptom = form.cleaned_data.get('aches_symptom')
            headachesymptom = form.cleaned_data.get('headache_symptom')
            coughsymptom = form.cleaned_data.get('cough_symptom')
            breathingsymptom=form.cleaned_data.get('breathing_symptom')
            sleepsymptom=form.cleaned_data.get('sleep_symptom')
            meetppl=form.cleaned_data.get('meetppl')
            coronatest=form.cleaned_data.get('corona_test')
            corona14=form.cleaned_data.get('corona14')
            disease=form.cleaned_data.get('disease')
            gone_out=form.cleaned_data.get('gone_out')
            gone_out1 = form.cleaned_data.get('gone_out1')
            gone_out2 = form.cleaned_data.get('gone_out2')
            spent_time=form.cleaned_data.get('spent_time')
            more_meet=form.cleaned_data.get('more_meet')
            crowd = form.cleaned_data.get('crowd')
            transport=form.cleaned_data.get('transport')
            home=form.cleaned_data.get('home')
            any=form.cleaned_data.get('any')
            prec=form.cleaned_data.get('precautions')
            times=form.cleaned_data.get('times')
            gen=form.cleaned_data.get('gender')
            smoke=form.cleaned_data.get('smoke')
            alc=form.cleaned_data.get('alc')
            handwash=form.cleaned_data.get('handwash')
            handwash2 = form.cleaned_data.get('handwash2')
            age=form.cleaned_data.get('age')
            context = {'eyesymptom': eyesymptom,'chestsymptom':chestsymptom,'soarsymptom':soarsymptom,'runnynosesymptom':runnynosesymptom,'weaknesssymptom':weaknesssymptom,'achessymptom':achessymptom,'headachesymptom':headachesymptom
                       ,'coughsymptom':coughsymptom,'breathingsymptom':breathingsymptom,'sleepsymptom':sleepsymptom
                       ,'meetppl':meetppl,'coronatest':coronatest,'corona14':corona14,'disease':disease
                       ,'gone_out':gone_out,'gone_out1':gone_out1,'gone_out2':gone_out2,'spent_time':spent_time,'more_meet':more_meet
                       ,'crowd':crowd,'transport':transport,'home':home,'any':any,'prec':prec,'times':times,'gen':gen,'smoke':smoke
                       ,'alc':alc,'handwash':handwash,'handwash2':handwash2,'age':age}

            print("context['eyesymptom']", context['eyesymptom'])
            metrics=covid_prediction(context)

            context['eyesymptom']="No" if context['eyesymptom'] == 0 else "Yes"
            #print(context['eyesymptom'])
            context['chestsymptom'] = "No" if context['chestsymptom'] == 0 else "Yes"
            context['soarsymptom'] = "No" if context['soarsymptom'] == 0 else "Yes"
            context['runnynosesymptom'] = "No" if context['runnynosesymptom'] == 0 else "Yes"
            context['weaknesssymptom'] = "No" if context['weaknesssymptom'] == 0 else "Yes"
            context['achessymptom'] = "No" if context['achessymptom'] == 0 else "Yes"
            context['headachesymptom'] = "No" if context['headachesymptom'] == 0 else "Yes"
            context['coughsymptom'] = "No" if context['coughsymptom'] == 0 else "Yes"
            context['breathingsymptom'] = "No" if context['breathingsymptom'] == 0 else "Yes"
            context['sleepsymptom'] = "No" if context['sleepsymptom'] == 0 else "Yes"
            context['meetppl'] = "No" if context['meetppl'] == 0 else "Yes"
            context['coronatest'] = "No" if context['coronatest'] == 0 else "Yes"
            context['corona14'] = "No" if context['corona14'] == 0 else "Yes"
            if context['disease'] ==2:
                context['disease'] = 'High BP-2'
            elif context['disease'] == 3:
                context['disease'] = 'Asthama-3'
            else:
                context['disease'] = "None"
            context['gone_out'] = "No" if context['gone_out'] == 0 else "Yes"
            context['gone_out1'] = "No" if context['gone_out1'] == 0 else "Yes"
            context['gone_out2'] = "No" if context['gone_out2'] == 0 else "Yes"
            context['spent_time'] = "No" if context['spent_time'] == 0 else "Yes"
            context['more_meet'] = "No" if context['more_meet'] == 0 else "Yes"
            context['crowd'] = "No" if context['crowd'] == 0 else "Yes"
            context['transport'] = "No" if context['transport'] == 0 else "Yes"
            context['home'] = "No" if context['home'] == 0 else "Yes"
            context['any'] = "No" if context['any'] == 0 else "Yes"
            if context['prec'] == 2:
                context['prec'] = 'Sometimes'
            elif context['prec'] == 3:
                context['prec'] = 'Often'
            elif context['prec'] == 4:
                context['prec'] = 'Always'
            else:
                context['prec'] = "Rarely"

            if context['times'] == 0:
                context['times'] = 'None of the time'
            elif context['times'] == 3:
                context['times'] = 'Most of the time'
            elif context['times'] == 4:
                context['times'] = 'All of the time'
            else:
                context['times'] = "Some of the time"
            context['gen'] = "Female" if context['any'] == 0 else "Male"
            context['smoke'] = "No" if context['smoke'] == 0 else "Yes"
            context['alc'] = "No" if context['alc'] == 0 else "Yes"
            context['handwash'] = "No"  if context['handwash'] == 0 else "Yes"
            context['handwash2'] = "No"  if context['handwash2'] == 0 else "Yes"
            if context['age'] == 0:
                context['age'] = '18-25'
            elif context['age'] == 1:
                context['age'] = '26-32'
            elif context['age'] == 2:
                context['age'] = '33-45'
            else:
                context['age'] = ">45"
            metrics['testSetValues1'] = "Potential No" if metrics['testSetValues1'] == 0 else "Potential Yes"
            metrics['testSetValues2'] = "Potential No" if metrics['testSetValues2'] == 0 else "Potential Yes"
            metrics['testSetValues3'] = "Potential No" if metrics['testSetValues3'] == 0 else "Potential Yes"
            metrics['testSetValues4'] = "Potential No" if metrics['testSetValues4'] == 0 else "Potential Yes"

            context.update(metrics)
            print(metrics)
            print(context)
            print("context['eyesymptom']",context['eyesymptom'])
        return render(request, "poutput.html", context)
    else:
        context['form'] = InputForm()
    return render(request, "home.html",context)

def covid_prediction(pcontext):
    import numpy as np
    import pandas as pd
    import statsmodels.api as sm
    import matplotlib.pyplot as plt
    #%matplotlibinline
    import time
    from sklearn.model_selection import train_test_split
    from sklearn.naive_bayes import GaussianNB, BernoulliNB, MultinomialNB
    from sklearn.metrics import confusion_matrix
    from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score, fbeta_score
    from sklearn.tree import DecisionTreeClassifier
    from sklearn.ensemble import RandomForestClassifier
    from sklearn.linear_model import LogisticRegression
    from random import randint

    metrics = {}
    link='https://raw.githubusercontent.com/PraseedaSaripalle/Symptoms_Cleaned_Dataset_/main/DATASET_Symptoms_CovidPred.csv'
    #link2='https://www.kaggle.com/jayasreethyadi/covid19-symptoms'
    #df=pd.read_csv("/datasets/DATASET_Symptoms_CovidPred.csv")
    df=pd.read_csv(link)
    #df=pd.read_csv(link2)
    # Split dataset in training and test datasets. in this X_train, X_test,are testing related feature set and label set. y_train, y_test are testing feature set and label set
    size = df.shape[0]
    testsize = 0.2
    print("test size of records", testsize)
    X_train, X_test, y_train, y_test = train_test_split(
        df.drop(['If you have been tested, did this test find that you had coronavirus (COVID-19)?'], axis=1),
        df['If you have been tested, did this test find that you had coronavirus (COVID-19)?'], test_size=testsize)
    gnb = GaussianNB()
    # Train classifier
    gnb.fit(X_train, y_train)
    # predict using X_Test

    y_pred = gnb.predict(X_test)
    print(y_pred)
    ypredlenth = len(y_pred)
    print("Last person contacted covid or not", y_pred[ypredlenth - 1])
    print("Last but 1 person contacted covid or not", y_pred[ypredlenth - 2])
    print("Last but 2 person contacted covid or not", y_pred[ypredlenth - 3])
    # to know the lenght of predicted
    print(len(y_pred))
    accuracyScore = round(accuracy_score(y_test, y_pred), 2)
    confusion_matrix(y_test, y_pred)
    precisionScore = round(precision_score(y_test, y_pred), 2)
    recallScore = round(recall_score(y_test, y_pred), 2)
    f1Score = round(f1_score(y_test, y_pred), 2)
    fbetaScore = round(fbeta_score(y_test, y_pred, beta=0.5, average='micro'), 2)

    print("precisionScore", precisionScore)
    print("recallScore", recallScore)
    print("f1Score", f1Score)
    print("accuracyScore", accuracyScore)
    print("fbetaScore", fbetaScore)

    dtc = DecisionTreeClassifier(max_depth=None, random_state=None)
    dtc.fit(X_train, y_train)
    # predict using X_Test
    y_pred = dtc.predict(X_test)
    print(y_pred)
    ypredlenth = len(y_pred)
    # print("Last Project Success or Fail", y_pred[ypredlenth-1])
    # print("Last but 1 Project Success or Fail", y_pred[ypredlenth - 2])
    # print("Last but 2 Project Success or Fail", y_pred[ypredlenth - 3])
    # to know the lenght of predicted
    # print(len(y_pred))

    # now find out precision_score, recall_score, f1_score
    # to build confusion matrix actual vs predicted
    daccuracyScore = round(accuracy_score(y_test, y_pred), 2)
    confusion_matrix(y_test, y_pred)
    dprecisionScore = round(precision_score(y_test, y_pred), 2)
    drecallScore = round(recall_score(y_test, y_pred), 2)
    df1Score = round(f1_score(y_test, y_pred), 2)
    dfbetaScore = round(fbeta_score(y_test, y_pred, beta=0.5), 2)

    print(daccuracyScore, dprecisionScore, drecallScore, df1Score, dfbetaScore)

    rfc = RandomForestClassifier(max_depth=None, random_state=None)
    rfc.fit(X_train, y_train)
    # predict using X_Test
    y_pred = rfc.predict(X_test)
    print(y_pred)
    ypredlenth = len(y_pred)
    # print("Last Project Success or Fail", y_pred[ypredlenth-1])
    # print("Last but 1 Project Success or Fail", y_pred[ypredlenth - 2])
    # print("Last but 2 Project Success or Fail", y_pred[ypredlenth - 3])
    # to know the lenght of predicted
    # print(len(y_pred))
    # brining back to normal shape
    # now find out precision_score, recall_score, f1_score
    # to build confusion matrix actual vs predicted
    raccuracyScore = round(accuracy_score(y_test, y_pred), 2)
    confusion_matrix(y_test, y_pred)
    rprecisionScore = round(precision_score(y_test, y_pred), 2)
    rrecallScore = round(recall_score(y_test, y_pred), 2)
    rf1Score = round(f1_score(y_test, y_pred), 2)
    rfbetaScore = round(fbeta_score(y_test, y_pred, beta=0.5, average='micro'), 2)

    print(raccuracyScore, rprecisionScore, rrecallScore, rf1Score, rfbetaScore)

    lrs = LogisticRegression()
    lrs.fit(X_train, y_train)
    # predict using X_Test
    y_pred = lrs.predict(X_test)
    print(y_pred)
    ypredlenth = len(y_pred)
    # print("Last Project Success or Fail", y_pred[ypredlenth-1])
    # print("Last but 1 Project Success or Fail", y_pred[ypredlenth - 2])
    # print("Last but 2 Project Success or Fail", y_pred[ypredlenth - 3])
    # to know the lenght of predicted
    # print(len(y_pred))
    # now find out precision_score, recall_score, f1_score
    # to build confusion matrix actual vs predicted
    laccuracyScore = round(accuracy_score(y_test, y_pred), 2)
    confusion_matrix(y_test, y_pred)
    lprecisionScore = round(precision_score(y_test, y_pred), 2)
    lrecallScore = round(recall_score(y_test, y_pred), 2)
    lf1Score = round(f1_score(y_test, y_pred), 2)
    lfbetaScore = round(fbeta_score(y_test, y_pred, beta=0.5, average='micro'), 2)
    print(laccuracyScore, lprecisionScore, lrecallScore, lf1Score, lfbetaScore)

    pcontext['eyesymptom']=0 if pcontext['eyesymptom']=="No" else 1
    print(pcontext['eyesymptom'])
    pcontext['chestsymptom'] = 0 if pcontext['chestsymptom'] == "No" else 1
    pcontext['soarsymptom'] = 0 if pcontext['soarsymptom'] == "No" else 1
    pcontext['runnynosesymptom'] = 0 if pcontext['runnynosesymptom'] == "No" else 1
    pcontext['weaknesssymptom'] = 0 if pcontext['weaknesssymptom'] == "No" else 1
    pcontext['achessymptom'] = 0 if pcontext['achessymptom'] == "No" else 1
    pcontext['headachesymptom'] = 0 if pcontext['headachesymptom'] == "No" else 1
    pcontext['coughsymptom'] = 0 if pcontext['coughsymptom'] == "No" else 1
    pcontext['breathingsymptom'] = 0 if pcontext['breathingsymptom'] == "No" else 1
    pcontext['sleepsymptom'] = 0 if pcontext['sleepsymptom'] == "No" else 1
    pcontext['meetppl'] = 0 if pcontext['meetppl'] == "No" else 1
    pcontext['coronatest'] = 0 if pcontext['coronatest'] == "No" else 1
    pcontext['corona14'] = 0 if pcontext['corona14'] == "No" else 1
    if pcontext['disease']=='High BP-2':
        pcontext['disease']=2
    elif pcontext['disease']=='Asthama-3':
        pcontext['disease']=3
    else:
        pcontext['disease']=1
    pcontext['gone_out'] = 0 if pcontext['gone_out'] == "No" else 1
    pcontext['gone_out1'] = 0 if pcontext['gone_out1'] == "No" else 1
    pcontext['gone_out2'] = 0 if pcontext['gone_out2'] == "No" else 1
    pcontext['spent_time'] = 0 if pcontext['spent_time'] == "No" else 1
    pcontext['more_meet'] = 0 if pcontext['more_meet'] == "No" else 1
    pcontext['crowd'] = 0 if pcontext['crowd'] == "No" else 1
    pcontext['transport'] = 0 if pcontext['transport'] == "No" else 1
    pcontext['home'] = 0 if pcontext['home'] == "No" else 1
    pcontext['any'] = 0 if pcontext['any'] == "No" else 1
    if pcontext['prec']=='Sometimes':
        pcontext['prec']=2
    elif pcontext['prec']=='Often':
        pcontext['prec']=3
    elif pcontext['prec']=='Always':
        pcontext['prec']=4
    else:
        pcontext['prec']=0

    if pcontext['times'] == 'None of the time':
        pcontext['times'] = 0
    elif pcontext['times'] == 'Most of the time':
        pcontext['times'] = 3
    elif pcontext['times'] == 'All of the time':
        pcontext['times'] = 4
    else:
        pcontext['times'] = 2
    pcontext['gen'] = 0 if pcontext['any'] == "Female" else 1
    pcontext['smoke'] = 0 if pcontext['smoke'] == "No" else 1
    pcontext['alc'] = 0 if pcontext['alc'] == "No" else 1
    pcontext['handwash'] = 0 if pcontext['handwash'] == "No" else 1
    pcontext['handwash2'] = 0 if pcontext['handwash2'] == "No" else 1
    if pcontext['age']=='18-25':
        pcontext['age']=0
    elif pcontext['age']=='26-32':
        pcontext['age']=1
    elif pcontext['age']=='33-45':
        pcontext['age']=2
    else:
        pcontext['age']=3

    # trying to predict new projects sucess using this algorithm
    testSet = [[[pcontext['eyesymptom'],pcontext['chestsymptom'],pcontext['soarsymptom'],
                 pcontext['runnynosesymptom'],pcontext['weaknesssymptom'],pcontext['achessymptom'],
                 pcontext['headachesymptom'],pcontext['coughsymptom'],pcontext['breathingsymptom'],
                 pcontext['sleepsymptom'],pcontext['meetppl'],pcontext['coronatest'],
                 pcontext['corona14'],pcontext['disease'],pcontext['gone_out'],
                 pcontext['gone_out1'],pcontext['gone_out2'],pcontext['spent_time'],
                 pcontext['more_meet'],pcontext['crowd'],pcontext['transport'],
                 pcontext['home'],pcontext['any'],pcontext['prec'],pcontext['times'],pcontext['gen'],
                 pcontext['smoke'],pcontext['alc'],pcontext['handwash'],pcontext['handwash2'],pcontext['age']
               ]]]
    # print(testSet[0][0][0])
    print(testSet)
    predTest = []
    predTest = testSet
    testSetValues = []
    print("project scores are ")
    newy_pred1 = lrs.predict(testSet[0])
    testSetValues.append(newy_pred1[0])
    print("lrs newy_pred", newy_pred1)
    newy_pred1 = rfc.predict(testSet[0])
    testSetValues.append(newy_pred1[0])
    print("rfc newy_pred", newy_pred1)
    newy_pred1 = gnb.predict(testSet[0])
    testSetValues.append(newy_pred1[0])
    print("gnb newy_pred", newy_pred1)
    newy_pred1 = dtc.predict(testSet[0])
    testSetValues.append(newy_pred1[0])
    print("dtc newy_pred", newy_pred1)

    metrics =  {'precisionScore': precisionScore, 'recallScore': recallScore, 'f1Score': f1Score,
                   'accuracyScore': accuracyScore, 'fbetaScore': fbetaScore, 'daccuracyScore': daccuracyScore,
                   'dprecisionScore': dprecisionScore, 'drecallScore': drecallScore, 'df1Score': df1Score,
                   'dfbetaScore': dfbetaScore, 'raccuracyScore': raccuracyScore, 'rprecisionScore': rprecisionScore,
                   'rrecallScore': rrecallScore, 'rf1Score': rf1Score, 'rfbetaScore': rfbetaScore,
                   'laccuracyScore': laccuracyScore, 'lprecisionScore': lprecisionScore, 'lrecallScore': lrecallScore,
                   'lf1Score': lf1Score, 'lfbetaScore': lfbetaScore, 'predTest': predTest, 'testSetValues1': testSetValues[0],
                    'testSetValues2': testSetValues[1],'testSetValues3': testSetValues[2],'testSetValues4': testSetValues[3]
                   }
    return metrics


#!/usr/bin/python

import logging
import sys
import os.path
import json
import re
import PIL
from PIL import Image
from pprint import pprint

#KMS4498,http://cspic.smk.dk/globus/40412247/img0083.jpg

logging.basicConfig(filename='logs/thw2.log',level=logging.DEBUG)

f = open( sys.argv[1], 'rU' )
lines=f.readlines()

for line in lines:
  lineSplit=line.split(',')
  idTag=lineSplit[0]
  outname = "newtry/" + idTag + ".txt"
  testoutname = "/home/thw/git/colorsearch/colsearch/newcolfiles/trash/" + idTag + ".*"

  if os.path.exists(testoutname):
    logging.debug("Continue: " + line)
    continue

  url=lineSplit[1]
  filePost=url.split("/")
  tmpfileLine='/'.join(filePost[3:])
  fileLine = '/mnt/cifs/Globus/' + tmpfileLine.rstrip()

  try:
    logging.debug("OKopen: " + fileLine)
    image = Image.open(fileLine)
  except:
    logging.debug("Err: " + fileLine)
    continue

  w,h=image.size
  colors=image.getcolors(w*h)

  if len(colors) > 256:
    logging.info("C: " + line)
    hist={}
    testdcit = {"hello": "world"}

    for i,c in enumerate(colors):
        hex='%02x%02x%02x' % (c[1][0],c[1][1],c[1][2])
        hist[hex]=c[0]

    #pprint(hist)
    try:
      with open(outname, 'w') as outfile:
        logging.debug("OK: " + outname)
        json.dump(hist, outfile, sort_keys=True, indent=4, separators=(',', ': '))
    except:
      logging.debug("Err: " + outname)
      continue

  else:
    logging.info("B: " + line)
    continue


#!/usr/bin/python3

from jsonsocket import JsonServer
from jsonsocket import JsonClient
from socket import gethostname
from sys import argv
from time import sleep
import resource
import random
from datetime import datetime

def log(file, s):
    file.write('{}: {}\n'.format(datetime.now().__str__(), s))

port = int(argv[2])
RAM = int(argv[3]) * 1024 * 1024
file = open('/root/runtime/t11_r.log', 'w')
resource.setrlimit(resource.RLIMIT_AS, (RAM, RAM))
try:
    server = JsonServer(gethostname(), port, 1)
    log(file, 'Waiting for input...')
    data = server.serve()
    server.close()
    log(file, 'Received input, closing server')
    sleep_time = int(argv[4])
    if sleep_time != 0:
        log(file, 'Sleeping for {} sec'.format(sleep_time))
        sleep(sleep_time)
        log(file, 'Finish sleeping')
    # left = data[0].get()
    left = data[0]
    log(file, 'Input 1 length: {}'.format(len(left)))
    # right = data[1].get()
    right = data[1]
    log(file, 'Input 2 length: {}'.format(len(right)))
    result = []
    while len(left) != 0 and len(right) != 0:
        if left[0] <= right[0]:
            result.append(left.pop(0))
        else:
            result.append(right.pop(0))
    if len(left) == 0:
        result += right
    else:
        result += left
    log(file, 'Completed. The length of result is {}'.format(len(result)))
except Exception as e:
    log(file, "Failed: %s" % e)
finally:
    file.close()

if __name__ == '__main__':
    try:
        length = 1000
        file = open('/root/runtime/t11_s.log', 'w')
        # pass-in arguments
        host = argv[1]
        port = int(argv[2])
        RAM = int(argv[3]) * 1024 * 1024
        log(file, "Apply memory limit: {} MB".format(RAM / 1024 / 1024))
        resource.setrlimit(resource.RLIMIT_AS, (RAM, RAM))
        sleep_time = 0
        if sleep_time != 0:
            log(file, "Start sleeping {} sec".format(sleep_time))
            sleep(sleep_time)
            log(file, "Finish sleeping")
        log(file, "Start sorting")
        input_data = [random.randint(0, 100000) for i in range(length)]
        data_to_send = sorted(input_data)
        log(file, "Sending result to {}:{}".format(host, port))
        client = JsonClient(host, port)
        client.connect()
        client.send_obj(data_to_send, client.socket)
        client.close()
    except Exception as e:
        log(file, "Failed: {}".format(e))
    finally:
        file.close()


# file_oper.py
# 将文件存入mongo, 并且取出
from pymongo import MongoClient
import bson.binary

from_img = "flower.jpg"  # 原始图片名称
to_img = "new_flower.jpg"# 生成的新图片名称

def save_img(myset):
    # 存储图片
    f = open(from_img, "rb") # 只读,二进制方式打开
    data = f.read()  # 读文件内容
    # 将data转换为mongodb的存储格式
    content = bson.binary.Binary(data)

    myset.insert({
        "filename":from_img,  # 文件名
        "data":content  # 文件数据
    })
    print("save ok")
    return

def get_img(myset): # 从mongodb读取图片
    img = myset.find_one({"filename":"flower.jpg"})
    with open(to_img, "wb") as f:#打开存入文件
        f.write(img["data"])#将data域写入文件
    print("save new img ok")
    return

# 连接数据库,取得数据库对象,取得集合对象
conn = MongoClient("localhost", 27017)
db = conn.gridfs  # 取得数据库对象
myset = db.image  # 取得集合对象

#save_img(myset)  # 存储图片函数
get_img(myset)    # 读取图片函数

conn.close()


# ===============================================================================
# Copyright 2013 Jake Ross
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============================================================================

# ============= enthought library imports =======================
from datetime import datetime
import time

from traits.api import Instance, List, on_trait_change, Bool
from pyface.tasks.action.schema import SToolBar
from pyface.tasks.task_layout import TaskLayout, PaneItem, Tabbed, VSplitter

# ============= standard library imports ========================
# ============= local library imports  ==========================
from pychron.core.helpers.ctx_managers import no_update
from pychron.dashboard.client import DashboardClient
from pychron.envisage.tasks.pane_helpers import ConsolePane
from pychron.experiment.utilities.identifier import ANALYSIS_MAPPING, SPECIAL_MAPPING
from pychron.globals import globalv
from pychron.processing.tasks.analysis_edit.panes import ControlsPane
from pychron.processing.tasks.analysis_edit.plot_editor_pane import PlotEditorPane
from pychron.processing.tasks.figures.figure_editor import FigureEditor
from pychron.processing.tasks.figures.figure_task import FigureTask
from pychron.processing.tasks.figures.panes import PlotterOptionsPane
from pychron.system_monitor.tasks.actions import AddSystemMonitorAction, ClearFigureAction, PauseAction, PlayAction, \
    NewSeriesAction
from pychron.system_monitor.tasks.connection_spec import ConnectionSpec
from pychron.system_monitor.tasks.dashboard_editor import DashboardEditor
from pychron.system_monitor.tasks.panes import ConnectionPane, AnalysisPane, DashboardPane
from pychron.system_monitor.tasks.system_monitor_editor import SystemMonitorEditor

from traitsui.api import View, Item, EnumEditor
from pychron.core.ui.preference_binding import bind_preference

class SystemMonitorTask(FigureTask):
    name = 'System Monitor'

    tool_bars = [SToolBar(AddSystemMonitorAction(),
                          NewSeriesAction(),
                          PauseAction(),
                          PlayAction()),
                 SToolBar(ClearFigureAction())]

    connection_pane = Instance(ConnectionPane)
    controls_pane = Instance(ControlsPane)
    unknowns_pane = Instance(AnalysisPane)
    console_pane = Instance(ConsolePane)

    connections = List
    connection = Instance(ConnectionSpec)

    dashboard_client = Instance(DashboardClient)
    dashboard_editor = Instance(DashboardEditor)
    dashboard_pane = Instance(DashboardPane)
    last_activated = 0

    _paused = Bool(False)
    pause_visible = Bool(True)
    play_visible = Bool(False)

    def activated(self):
        self._setup_dashboard()
        self._make_connections()

    def activate_editor(self, editor):
        if not self.last_activated or time.time() - self.last_activated > 15:
            super(SystemMonitorTask, self).activate_editor(editor)
            self.last_activated = 0

    def prepare_destroy(self):
        for e in self.editor_area.editors:
            if isinstance(e, SystemMonitorEditor):
                e.stop()
        if self.dashboard_client:
            self.dashboard_client.on_trait_change(self._handle_dashboard, 'values:value', remove=True)

        super(FigureTask, self).prepare_destroy()

    def toggle_pause(self):
        self._paused = not self._paused
        self.play_visible = not self.play_visible
        self.pause_visible = not self.pause_visible

        for ei in self.editor_area.editors:
            if self._paused:
                ei.oname = ei.name
                ei.name = '{} (Paused)'.format(ei.name)
            else:
                ei.name = ei.oname

            if isinstance(ei, SystemMonitorEditor):
                ei.pause()

    def reset_editors(self):
        for ei in self.editor_area.editors:
            if isinstance(ei, SystemMonitorEditor):
                ei.reset_editors()

    def clear_figure(self):
        ac = self.has_active_editor()
        if ac:
            ac.set_items([])

    def new_analysis_series(self):
        from pychron.system_monitor.tasks.series_search_criteria import SearchCriteria

        kw = {'mass_spectrometer': self.connection.system_name,
              'mass_spectrometers': [ci.system_name for ci in self.connections],
              'analysis_type': 'Air',
              'analysis_types': ANALYSIS_MAPPING.values()}

        if self.active_editor and hasattr(self.active_editor, 'search_tool'):
            for attr in ('weeks', 'hours', 'limit', 'days'):
                kw[attr] = getattr(self.active_editor.search_tool, attr)

        sc = SearchCriteria(**kw)
        while 1:
            info = sc.edit_traits()
            if not info.result:
                break

            at = next((k for k, v in ANALYSIS_MAPPING.iteritems() if v == sc.analysis_type))
            at = next((k for k, v in SPECIAL_MAPPING.iteritems() if v == at))

            ans = self.manager.analysis_series(sc.mass_spectrometer,
                                               analysis_type=at,
                                               weeks=sc.weeks,
                                               days=sc.days,
                                               hours=sc.hours,
                                               limit=sc.limit)
            if ans:
                self.new_series(ans)
                break
            else:
                if not self.confirmation_dialog('No analyses found given the provided search criteria. Try Again?'):
                    break

    def add_system_monitor(self):
        return self._editor_factory()

    def add_dashboard_editor(self):
        names = [v.name for v in self.dashboard_client.values]
        return self._dashboard_editor_factory(names)

    def get_connection_view(self):
        v = View(Item('connection',
                      editor=EnumEditor(name='connections')),
                 kind='livemodal',
                 buttons=['OK', 'Cancel'],
                 width=300,
                 title='Choose System')
        return v

    # private
    def _setup_dashboard(self):
        self.dashboard_client = client = self.application.get_service('pychron.dashboard.client.DashboardClient')
        if client:
            client.on_trait_change(self._handle_dashboard, 'values:value')
            bind_preference(client, 'host', 'pychron.dashboard.client.host')
            bind_preference(client, 'port', 'pychron.dashboard.client.port')

            client.connect()
            client.load_configuration()
            client.listen()

    def _refresh_plot_hook(self):
        self.last_activated = time.time()

    def _open_editor(self, editor, **kw):
        super(SystemMonitorTask, self)._open_editor(editor, **kw)
        tol = 3600
        for ei in self.editor_area.editors:
            if hasattr(ei, 'starttime'):
                dev = (datetime.now() - ei.starttime).total_seconds()
                self.debug('check editor duration. close after {}s. current dur:{}'.format(tol, dev))
                if dev > tol:
                    self.close_editor(ei)
                    # for si in self.get_editors(SystemMonitorEditor):
                    # si.close_editor(ei)

    def _editor_factory(self):
        if globalv.system_monitor_debug:
            self.connection = self.connections[0]
            result = True
        else:
            # ask user for system
            info = self.edit_traits(view='get_connection_view')
            result = info.result

        if result and self.connection:
            editor = SystemMonitorEditor(processor=self.manager,
                                         conn_spec=self.connection,
                                         task=self)
            editor.start()
            self._open_editor(editor)

            return editor

    def _dashboard_editor_factory(self, names):
        editor = DashboardEditor(processor=self.manager)
        editor.set_measurements(names)
        self._open_editor(editor)
        self.dashboard_editor = editor
        return editor

    def _active_editor_changed(self, new):
        if new:

            self.last_activated = time.time()

            if self.controls_pane:
                tool = None
                # if hasattr(new, 'tool'):
                # tool = new.tool

                if hasattr(new, 'search_tool'):
                    tool = new.search_tool

                self.controls_pane.tool = tool
            if isinstance(new, FigureEditor):
                self.plotter_options_pane.pom = new.plotter_options_manager
            if isinstance(new, SystemMonitorEditor):
                self.console_pane.name = '{} - Console'.format(new.name)
                self.console_pane.console_display = new.console_display
                self.connection_pane.conn_spec = new.conn_spec

            if self.unknowns_pane:
                if hasattr(new, 'analyses'):
                    with no_update(self.unknowns_pane, fire_update_needed=False):
                        self.unknowns_pane.trait_set(items=new.analyses)

    def _prompt_for_save(self):
        """
            dont save just close
        """
        return True

    def _make_connections(self):
        app = self.window.application
        connections = app.preferences.get('pychron.sys_mon.favorites')
        if not connections:
            self.warning_dialog('No Systems in Preferences')
            return

        cs = []
        for ci in eval(connections):
            n, sn, h, p = ci.split(',')
            cc = ConnectionSpec(system_name=sn,
                                host=h, port=int(p))
            cs.append(cc)
        self.connections = cs

    def create_dock_panes(self):
        self.connection_pane = ConnectionPane()
        self.controls_pane = ControlsPane()
        self.unknowns_pane = AnalysisPane()
        self.plotter_options_pane = PlotterOptionsPane()
        self.plot_editor_pane = PlotEditorPane()

        self.console_pane = ConsolePane()

        panes = [self.connection_pane,
                 self.controls_pane,
                 self.unknowns_pane,
                 self.plotter_options_pane,
                 self.console_pane,
                 self.plot_editor_pane]

        if self.dashboard_client:
            self.dashboard_pane = DashboardPane(model=self.dashboard_client)
            panes.append(self.dashboard_pane)

        return panes

    @on_trait_change('window:opened')
    def _opened(self):
        self.add_system_monitor()

    # # self.add_dashboard_editor()

    def _handle_dashboard(self, obj, name, old, new):
        self.debug('dashboard_client value change {} {}'.format(obj.name, new))
        if self.dashboard_editor:
            self.dashboard_editor.update_measurements(obj.name, new)

    def _default_layout_default(self):
        left = VSplitter(PaneItem('pychron.sys_mon.analyses'),
                         PaneItem('pychron.processing.controls'))
        right = Tabbed(PaneItem('pychron.processing.figures.plotter_options'),
                       PaneItem('pychron.console'))

        return TaskLayout(left=left, right=right)

# ============= EOF =============================================


from pyparsing import *

def parse_simple_info_file(file_path):
    ParserElement.setDefaultWhitespaceChars(' \t')

    DATA = []
    CURR_FILENAME = None
    CURR_TUPLE_LIST = None

    FIELD_NAME_CHARS = alphanums + "_ .-"

    def set_filename(f):
        nonlocal CURR_FILENAME
        CURR_FILENAME = f

    def append_tuple(k, v):
        nonlocal CURR_TUPLE_LIST
        if CURR_TUPLE_LIST is None:
            CURR_TUPLE_LIST = []
        CURR_TUPLE_LIST.append((k, v))

    def commit_tuple_list():
        nonlocal CURR_FILENAME
        nonlocal CURR_TUPLE_LIST
        DATA.append((CURR_FILENAME, CURR_TUPLE_LIST))
        CURR_FILENAME = None
        CURR_TUPLE_LIST = None

    EOL         = LineEnd().suppress()
    FILENAME    = restOfLine.setResultsName('filename').setParseAction(lambda s, l, t: set_filename(t.filename)) + EOL
    META_PAIR   = Combine(Word(FIELD_NAME_CHARS).setResultsName('key') + '=' + restOfLine.setResultsName('value') + EOL).setParseAction(lambda s, l, t: append_tuple(t.key.casefold(), t.value))
    SPACER      = OneOrMore(EOL).setParseAction(lambda s, l, t: commit_tuple_list())
    CHUNK       = FILENAME + ZeroOrMore(META_PAIR)
    FILE        = ZeroOrMore(EOL) + Optional(CHUNK + ZeroOrMore(SPACER + CHUNK)) + ZeroOrMore(EOL)

    with open(file_path, encoding='utf8') as info_file_handle:
        FILE.parseFile(info_file_handle, parseAll=True)
        return(DATA)

import yaml
from pprint import pprint

def parse_yaml_info_file(file_path):
    with open(file_path, encoding='utf8') as info_file_handle:
        data = yaml.load(info_file_handle)
        return data


import read_mnist
import numpy
import math

class ActivationFunction(object):

    def __init__(self, feature_size, sample_size, layer1_size, layer2_size):
        self.layer1_size = layer1_size
        self.layer2_size = layer2_size
        self.feature_size = feature_size
        self.sample_size = sample_size
        self.prev_gradient_1 = numpy.asmatrix(numpy.zeros((self.feature_size,self.layer1_size)))
        self.prev_gradient_2 = numpy.asmatrix(numpy.zeros((self.layer1_size, self.layer2_size)))

    def initialize_weights(self):
        # Xavier Initialization
        weight_1 = numpy.random.randn(self.feature_size,self.layer1_size)
        weight_2 = numpy.random.randn(self.layer1_size, self.layer2_size)
        return ((1/math.sqrt(self.feature_size)) * numpy.asmatrix(weight_1)), \
               ((1/math.sqrt(self.layer1_size)) * numpy.asmatrix(weight_2))

    def sigmoid(self, input):
        # Apply sigmoid 1/(1+exp(-x)) on given input
        return 1.0/(1.0 + numpy.exp(-input))

    def tanh(self, weight, inp):
        # Apply 1.7159 * (tanh(2/3 * x))
        dot_product = numpy.dot(inp, weight)
        tanh_output = 1.7159 * (numpy.tanh((2./3.) * dot_product))
        return tanh_output

    def tanh_gradient(self, weight, inp):
        # Gradient : 1.1439 - (0.5828*z^2)
        tanh_output = self.tanh(weight, inp)
        return 1.1439 - (0.5828 * numpy.multiply(tanh_output, tanh_output))

    def logistic_regression(self, layer1_input, weight_1):
        # Squash w.x to 1/(1+exp(-w.x)) for Logistic layer output
        dot_product = numpy.dot(layer1_input, weight_1)
        return self.sigmoid(dot_product)

    def softmax_regression(self, layer2_input, weight_2):
        dot_product = numpy.exp(numpy.dot(layer2_input, weight_2))
        tot_exp = dot_product.sum(axis=1)
        return dot_product / tot_exp

    def logistic_gradient(self, weight_1, layer1_input):
        logit_output = self.logistic_regression(layer1_input, weight_1)
        return numpy.multiply(logit_output,(1 - logit_output))

    def gradient_update(self, gradient, weight, learning_rate, input, regularize, momentum):
        # Compute gradient for the current iteration
        curr_gradient = numpy.dot(gradient.T, input).T

        if weight.shape == self.prev_gradient_1.shape:
            prev_gradient = self.prev_gradient_1
        else:
            prev_gradient = self.prev_gradient_2

        # Update weight vector
        gradient_to_update = (learning_rate*(curr_gradient+(2*regularize*weight)))
        weight += gradient_to_update + (momentum*prev_gradient)

        # Update previous gradient
        if weight.shape == self.prev_gradient_1.shape:
            self.prev_gradient_1 = gradient_to_update
        else:
            self.prev_gradient_2 = gradient_to_update
        return weight

    def threshold_output(self, my_output):
        threshold_output = my_output.copy()
        threshold_output[numpy.where(threshold_output==numpy.max(threshold_output,axis=1))] = 1
        threshold_output[numpy.where(threshold_output!=numpy.max(threshold_output,axis=1))] = 0
        return threshold_output


# coding: utf-8
# Melanie Huston
# download all health topic images from English health topic page index of all topics
# August 2017

from os.path import basename
from bs4 import BeautifulSoup
import requests
import re
import urllib.request
import io
from PIL import Image
import csv

HTindex = 'https://medlineplus.gov/all_healthtopics.html'

imagenames = []
nonstandardimages = []
processedimages = []

# Get and parse HTML from the URL
page = requests.get(HTindex)
soup = BeautifulSoup(page.content, 'html.parser')

# find all list items containing a link to a health topic
HTcrawl = soup.find_all('li', class_='item')

for li in HTcrawl:
    # get the link to the health topic and follow it
    HTurl = li.find('a', href=True)
    link_text = HTurl['href']
    htpage = requests.get(link_text)
    htsoup = BeautifulSoup(htpage.content, 'html.parser')

    # find the section on the health topic page containing the health topic image
    # and get the image url
    imagesection = htsoup.find('section', id='ht-img-section')
    link = imagesection.find("img", src=True)
    lnk = link['src']

    # if we haven't already downloaded this image, record the image name and download it
    if not basename(lnk) in imagenames:
        print(lnk)
        imagenames.append(basename(lnk))
        fd = urllib.request.urlopen(lnk)
        image_file = io.BytesIO(fd.read())
        siteimage = Image.open(image_file)
        #fd = urllib.request.urlretrieve(lnk, basename(lnk))
        #siteimage = Image.open(lnk)
        width, height = siteimage.size
        nameparts = re.match("(?P<fn>.*)\.(?P<ext>.*)$", basename(lnk))
        filename = nameparts.group('fn')
        extension = nameparts.group('ext')
        exttype = extension
        if exttype in ['jpg', 'JPG']:
            exttype = 'jpeg'

        if width == height:
            processedimages.append(basename(lnk))
            siteimage.thumbnail((100, 100), Image.LANCZOS)
            siteimage.save('images/' + filename + '_thumb.' + extension, format=exttype, subsampling=0, quality=100)
        else:
            nonstandardimages.append(basename(lnk))
            #siteimage.save('bigtest/problems/' + basename(lnk), format=exttype, subsampling=0, quality=100)
        siteimage.close()

# print the number of unique images downloaded
print("\nNumber of images converted:", len(processedimages))
print("\nNumber of non-standard images:", len(nonstandardimages))

processedimages.sort()
nonstandardimages.sort()

if len(nonstandardimages):
    for problem in nonstandardimages:
        print(problem)

with open('Images_thumbnaillist.txt', 'w') as f_new:
    for item in processedimages:
        f_new.write("%s\n" % item)
f_new.close()

with open('Images_problemlist.txt', 'w') as f_prob:
    for item in nonstandardimages:
        f_prob.write("%s\n" % item)
f_prob.close()

print('\nTwo csv files written')


#!/usr/bin/env python

'''
1. 安装 Python 3.6 https://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe
2. ctrl + r -> cmd 进命令行，输入 pip install opencv-python
3. 运行范例代码，从默认摄像头读取画面，存到 img 文件夹内
python video2image.py --source 0 --w 1024 --h 768 --shotdir img

--source 0 摄像头设备的编号，从零开始
--w 1024 画面宽度，不需要修改
--h 768 画面高度，不需要修改
--shotdir img 输出的文件夹名字，建议一类物体配一个文件夹

'''

# Python 2/3 compatibility
from __future__ import print_function

import cv2 as cv
import datetime

# built-in modules
from time import clock

def create_capture(args):
    cap = cv.VideoCapture(args.source)
    cap.set(cv.CAP_PROP_FRAME_WIDTH, args.w)
    cap.set(cv.CAP_PROP_FRAME_HEIGHT, args.h)
    if cap is None or not cap.isOpened():
        print('Warning: unable to open video source: ', args.source)
    else:
        print('created capture')
    return cap

if __name__ == '__main__':
    import sys
    import argparse
    import os

    parser = argparse.ArgumentParser(
        description='Dump image from camera device')
    parser.add_argument(
        '--source', type=int, default=0, help='Path to image or video. Skip to capture frames from camera')
    parser.add_argument('--w', type=int, default=640)
    parser.add_argument('--h', type=int, default=480)
    parser.add_argument('--shotdir')
    args = parser.parse_args()

    args.source = 'rtsp://admin:admin@192.168.1.200:8554/live'

    interactive_mode = args.shotdir is None
    needs_new_name = True

    if interactive_mode:
        print("Press SPACE in Capture window to input new folder name")

    cap = create_capture(args)
    prev_datetime = datetime.datetime.now()
    while True:
        ret, img = cap.read()
        if not ret:
            cap = create_capture(args)
            continue
        cv.imshow('Capture', img)
        ch = cv.waitKey(1)
        if ch == 32:
            # SPACE
            needs_new_name = True
        if ch == 27:
            # ESC
            break
        now = datetime.datetime.now()
        if (now - prev_datetime).microseconds > 200 * 1000:
            if interactive_mode and needs_new_name:
                args.shotdir = input("Input folder name: ")
                needs_new_name = False
            if not os.path.exists(args.shotdir):
                os.mkdir(args.shotdir)

            now_string = now.strftime("%Y-%h-%d-%H-%M-%S-%f")
            prev_datetime = now
            fn = '%s/cam_%s.jpg' % (args.shotdir, now_string)
            cv.imwrite(fn, img)
            print(fn, 'saved')
    cv.destroyAllWindows()


import collections
import os
import re
import codecs
import numpy

def read_stopwords():
    if (os.path.isfile("stopwords.txt")) != True:
        print("Stopwords file not found!")
    else:
        with open('stopwords.txt') as f:
            file = f.read().lower().replace('\n', ' ').split(' ')
        return file

def file_function(filename, path):
    handler = codecs.open(path + "\\" + filename, 'rU','latin-1')
    words = [locate.lower() for locate in re.findall('[A-Za-z0-9\']+', handler.read())]
    handler.close()
    return words

def file_finder(path):
    word_list = list()
    fileCount = 0
    for files in os.listdir(path):
        if files.endswith(".txt"):
            word_list = word_list + file_function(files, path)
            fileCount = fileCount + 1
    return word_list, fileCount

def delete_stopwords():
    for word in stopWords:
        if word in ham:
            w = 0
            ham_len=len(ham)
            while (w < ham_len):
                if (ham[w] == word):
                    ham.remove(word)
                    ham_len = ham_len - 1
                    continue
                w = w + 1
        if word in spam:
            x = 0
            spam_len=len(spam)
            while (x < spam_len):
                if (spam[x] == word):
                    spam.remove(word)
                    spam_len = spam_len - 1
                    continue
                x = x + 1
        if word in ham_test:
            y = 0
            ham_len_test=len(ham_test)
            while (y < ham_len_test):
                if (ham_test[y] == word):
                    ham_test.remove(word)
                    ham_len_test = ham_len_test - 1
                    continue
                y = y + 1
        if word in spam_test:
            z = 0
            spam_len_test=len(spam_test)
            while (z < spam_len_test):
                if (spam_test[z] == word):
                    spam_test.remove(word)
                    spam_len_test = spam_len_test - 1
                    continue
                z = z + 1

def create_mat(row, column):
    featureMatrix = [0] * row
    for i in range(row):
        featureMatrix[i] = [0] * column
    return featureMatrix

def build_mat(featureMatrix, path, listbog, rmat, classifier, fin_list):
    for fileName in os.listdir(path):
        words = file_function(fileName, path)
        temp = dict(collections.Counter(words))
        for key in temp:
            if key in listbog:
                column = listbog.index(key)
                featureMatrix[rmat][column] = temp[key]
        if (classifier == "ham"):
            fin_list[rmat] = 0
        elif (classifier == "spam"):
            fin_list[rmat] = 1
        rmat = rmat + 1
    return featureMatrix, rmat, fin_list

def sigmoid(val):
    return (1 / (1 + numpy.exp(-val)))

def sig_funct(totalFiles, totalFeatures, featureMatrix):
    global sigMoidList
    for files in range(totalFiles):
        summation = 1.0

        for features in range(totalFeatures):
            summation = summation + featureMatrix[files][features] * weightOfFeature[features]
        sigMoidList[files] = sigmoid(summation)

def calc_weight(totalFiles, numberOfFeature, featureMatrix, fin_list):
    global sigMoidList

    for feature in range(numberOfFeature):
        weight = 0
        for files in range(totalFiles):
            frequency = featureMatrix[files][feature]
            y = fin_list[files]
            sigmoidValue = sigMoidList[files]
            weight = weight + (frequency * (y - sigmoidValue))

        oldW = weightOfFeature[feature]
        weightOfFeature[feature] = weightOfFeature[feature] + ((weight * rate) - (rate * lambda_var * oldW))

    return weightOfFeature

def train_func(totalFiles, numbeOffeatures, train_mat, fin_list):
    sig_funct(totalFiles, numbeOffeatures, train_mat)
    calc_weight(totalFiles, numbeOffeatures, train_mat, fin_list)

def classification_func():
    cham, wham, cspam, wspam, i = 0, 0, 0, 0, 0
    for file in range(totalTestFiles):
        summation = 1.0
        for i in range(len(testListbog)):
            word = testListbog[i]

            if word in listbog:
                index = listbog.index(word)
                weight = weightOfFeature[index]
                wordcount = test_mat[file][i]

                summation = summation + (weight * wordcount)

        sigSum = sigmoid(summation)
        if (testfin_list[file] == 0):
            if sigSum < 0.5:
                cham = cham + 1.0
            else:
                wham = wham + 1.0
        else:
            if sigSum >= 0.5:
                cspam = cspam + 1.0
            else:
                wspam = wspam + 1.0
        i = i + 1
    tot_ham = cham + wham
    tot_spam = cspam + wspam
    total = tot_ham + tot_spam
    print("Ham Accuracy:" + str((cham / tot_ham) * 100))
    print("Spam Accuracy:" + str((cspam / tot_spam) * 100))
    print("Accuracy:" + str(((cham+cspam) / total) * 100))

train_path = input("Enter the path for the training data: ")
test_path = input("Enter the path for the testing data: ")
stopword_bool = input("Should stopwords be removed(enter 'yes' or 'no'): ").lower()
lambda_var = input("Enter a lambda value: ")
itera = input("Enter the number of desired iteras: ")
'''
train_path = "assignment3_train/train"
test_path = "assignment3_test/test"
stopword_bool = 'yes'
lambda_var = 100
itera = 25
'''
ham = list()
spam = list()
ham_count = 0
spam_count = 0
rate = 0.001

stopWords = read_stopwords()

ham_folder = train_path + '/ham'
spam_folder = train_path + '/spam'
test_ham_folder = test_path + '/ham'
spam_test_folder = test_path + '/spam'

re.compile(r'[A-Za-z0-9\']')

ham, ham_count = file_finder(ham_folder)
spam, spam_count = file_finder(spam_folder)

ham_test, counttest_ham_folder = file_finder(test_ham_folder)
spam_test, countspam_test_folder = file_finder(spam_test_folder)

if (stopword_bool == "yes"):
    delete_stopwords()

bog = ham + spam
dictbog = collections.Counter(bog)
listbog = list(dictbog.keys())
fin_list = list()
totalFiles = ham_count + spam_count

testbog = ham_test + spam_test
testDictbog = collections.Counter(testbog)
testListbog = list(testDictbog.keys())
testfin_list = list()
totalTestFiles = counttest_ham_folder + countspam_test_folder
train_mat = create_mat(totalFiles, len(listbog))
test_mat = create_mat(totalTestFiles, len(testListbog))

rmat = 0
testrmat = 0

sigMoidList = list()
for i in range(totalFiles):
    sigMoidList.append(-1)
    fin_list.append(-1)

for i in range(totalTestFiles):
    testfin_list.append(-1)

weightOfFeature = list()

for feature in range(len(listbog)):
    weightOfFeature.append(0)

train_mat, rmat, fin_list = build_mat(train_mat, ham_folder, listbog, rmat, "ham", fin_list)
train_mat, rmat, fin_list = build_mat(train_mat, spam_folder, listbog, rmat, "spam", fin_list)
test_mat, testrmat, testfin_list = build_mat(test_mat, test_ham_folder, testListbog, testrmat, "ham", testfin_list)
test_mat, testrmat, testfin_list = build_mat(test_mat, spam_test_folder, testListbog, testrmat, "spam", testfin_list)

print("Training, Please wait...")
for i in range(int(itera)):
    train_func(totalFiles, len(listbog), train_mat, fin_list)

print("Algorithm is trained, test data will now be classififed.")
classification_func()