import random
import os
import cv2
import torch
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import albumentations as A
from torch.utils.data import DataLoader
from sklearn.model_selection import train_test_split
from albumentations.pytorch import ToTensorV2

from Training.configurations import BATCH_SIZE, DATAPATH, DEVICE, SEED
from Training.path_constants import PROCESSED_DATA
from Training.dataset import ImageFolder

TRANSFORM = A.Compose([
    A.Normalize(mean = [0.485, 0.456, 0.406], std = [0.229, 0.224, 0.225], max_pixel_value = 255,),
    ToTensorV2()
])
# Apply the SEED
def seed_all(SEED_VALUE = SEED):
    random.seed(SEED_VALUE)
    os.environ['PYTHONHASHSEED'] = str(SEED_VALUE)
    np.random.seed(SEED_VALUE)
    torch.manual_seed(SEED_VALUE)
    torch.cuda.manual_seed(SEED_VALUE)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = True

def get_data_loaders(data_path = DATAPATH):
    df = pd.read_csv(data_path)
    train_data, test_data  = train_test_split(df, test_size = 0.2, random_state = SEED, stratify = df.hasTable)
    train_dataset = ImageFolder(train_data, isTrain = True, transform = None)
    test_dataset = ImageFolder(test_data, isTrain = False, transform = None)
    train_loader =  DataLoader(train_dataset, batch_size = BATCH_SIZE, shuffle = True, num_workers = 4, pin_memory = True)
    test_loader =  DataLoader(test_dataset, batch_size = 8, shuffle = False, num_workers = 4, pin_memory = True)
    return train_loader, test_loader

# Save Checkpoint
def save_checkpoint(state, filename = f"{PROCESSED_DATA}/model_checkpoint.pth.tar"):
    torch.save(state, filename)
    print("Checkpoint Saved at: ", filename)

# Load the checkpoint we saved
def load_checkpoint(checkpoint, model, optimizer = None):
    print("Loading checkpoint...")
    model.load_state_dict(checkpoint['state_dict'])
    if optimizer is not None:
        optimizer.load_state_dict(checkpoint['optimizer'])
    last_epoch = checkpoint['epoch']
    tr_metrics = checkpoint['train_metrics']
    te_metrics = checkpoint['test_metrics']
    return last_epoch, tr_metrics, te_metrics

def write_summary(writer, tr_metrics, te_metrics, epoch):
    writer.add_scalar("Table Loss/Train", tr_metrics['table_loss'], global_step = epoch)
    writer.add_scalar("Table Loss/Test", te_metrics['table_loss'], global_step = epoch)
    writer.add_scalar("Table Acc/Train", tr_metrics['table_acc'], global_step = epoch)
    writer.add_scalar("Table Acc/Test", te_metrics['table_acc'], global_step = epoch)
    writer.add_scalar("Table F1/Train", tr_metrics['table_f1'], global_step = epoch)
    writer.add_scalar("Table F1/Test", te_metrics['table_f1'], global_step = epoch)
    writer.add_scalar("Table Precision/Train", tr_metrics['table_precision'], global_step = epoch)
    writer.add_scalar("Table Precision/Test", te_metrics['table_precision'], global_step = epoch)
    writer.add_scalar("Table Recall/Train", tr_metrics['table_recall'], global_step = epoch)
    writer.add_scalar("Table Recall/Test", te_metrics['table_recall'], global_step = epoch)
    writer.add_scalar("Column Loss/Train", tr_metrics['column_loss'], global_step = epoch)
    writer.add_scalar("Column Loss/Test", te_metrics['column_loss'], global_step = epoch)
    writer.add_scalar("Column Acc/Train", tr_metrics['col_acc'], global_step = epoch)
    writer.add_scalar("Column Acc/Test", te_metrics['col_acc'], global_step = epoch)
    writer.add_scalar("Column F1/Train", tr_metrics['col_f1'], global_step = epoch)
    writer.add_scalar("Column F1/Test", te_metrics['col_f1'], global_step = epoch)
    writer.add_scalar("Column Precision/Train", tr_metrics['col_precision'], global_step = epoch)
    writer.add_scalar("Column Precision/Test", te_metrics['col_precision'], global_step = epoch)
    writer.add_scalar("Column Recall/Train", tr_metrics['col_recall'], global_step = epoch)
    writer.add_scalar("Column Recall/Test", te_metrics['col_recall'], global_step = epoch)

def display_metrics(epoch, tr_metrics, te_metrics):
    print(f"Epoch: {epoch} \n\
        Table Loss -- Train: {tr_metrics['table_loss']:.3f} Test: {te_metrics['table_loss']:.3f}\n\
        Table Acc -- Train: {tr_metrics['table_acc']:.3f} Test: {te_metrics['table_acc']:.3f}\n\
        Table F1 -- Train: {tr_metrics['table_f1']:.3f} Test: {te_metrics['table_f1']:.3f}\n\
        Table Precision -- Train: {tr_metrics['table_precision']:.3f} Test: {te_metrics['table_precision']:.3f}\n\
        Table Recall -- Train: {tr_metrics['table_recall']:.3f} Test: {te_metrics['table_recall']:.3f}\n\
        \n\
        Col Loss -- Train: {tr_metrics['column_loss']:.3f} Test: {te_metrics['column_loss']:.3f}\n\
        Col Acc -- Train: {tr_metrics['col_acc']:.3f} Test: {te_metrics['col_acc']:.3f}\n\
        Col F1 -- Train: {tr_metrics['col_f1']:.3f} Test: {te_metrics['col_f1']:.3f}\n\
        Col Precision -- Train: {tr_metrics['col_precision']:.3f} Test: {te_metrics['col_precision']:.3f}\n\
        Col Recall -- Train: {tr_metrics['col_recall']:.3f} Test: {te_metrics['col_recall']:.3f}\n"
    )

def compute_metrics(ground_truth, prediction, threshold = 0.5):
    # Ref: https://stackoverflow.com/a/56649983
    ground_truth = ground_truth.int()
    prediction = (torch.sigmoid(prediction) > threshold).int()
    TP = torch.sum(prediction[ground_truth == 1] == 1)
    TN = torch.sum(prediction[ground_truth == 0] == 0)
    FP = torch.sum(prediction[ground_truth == 1] == 0)
    FN = torch.sum(prediction[ground_truth == 0] == 1)
    acc = (TP + TN) / (TP + TN + FP+ FN)
    precision = TP / (FP + TP + 1e-4)
    recall = TP / (FN + TP + 1e-4)
    f1 = 2 * precision * recall / (precision + recall + 1e-4)
    metrics = {
        'acc': acc.item(),
        'f1': f1.item(),
        'precision':precision.item(),
        'recall': recall.item()
    }
    return metrics

def display(image, table, column, title = 'Original'):
    f, ax  = plt.subplots(1, 3, figsize = (15, 8))
    ax[0].imshow(image)
    ax[0].set_title(f'{title} Image')
    ax[1].imshow(table)
    ax[1].set_title(f'{title} Table Mask')
    ax[2].imshow(column)
    ax[2].set_title(f'{title} Column Mask')
    plt.show()

def display_prediction(image, table = None, table_image = None, no_: bool = False):
  if no_:
    f1, ax  = plt.subplots(1, 1, figsize = (7, 5))
    ax.imshow(image)
    ax.set_title('Original Image')
    f1.suptitle('No Tables Detected')
  else:
    f2, ax  = plt.subplots(1, 3, figsize = (15, 8))
    ax[0].imshow(image)
    ax[0].set_title('Original Image')
    ax[1].imshow(table)
    ax[1].set_title('Image with Predicted Table')
    ax[2].imshow(table_image)
    ax[2].set_title('Predicted Table Example')
  plt.show()

def get_TableMasks(test_image, model, transform = TRANSFORM, device = DEVICE):
    image = transform(image = test_image)["image"]
    # Get predictions
    model.eval()
    with torch.no_grad():
        image = image.to(device).unsqueeze(0)
        # With torch.cuda.amp.autocast():
        table_out, column_out  = model(image)
        table_out = torch.sigmoid(table_out)
        column_out = torch.sigmoid(column_out)
    # Remove gradients
    table_out = (table_out.cpu().detach().numpy().squeeze(0).transpose(1, 2, 0) > 0.5).astype(int)
    column_out = (column_out.cpu().detach().numpy().squeeze(0).transpose(1, 2, 0) > 0.5).astype(int)
    # Return masks
    return table_out, column_out

def fixMasks(image, table_mask, column_mask):
    """ Fix Table Bounding Box to get better OCR predictions """
    table_mask = table_mask.reshape(1024, 1024).astype(np.uint8)
    column_mask = column_mask.reshape(1024, 1024).astype(np.uint8)
    # Get contours of the mask to get number of tables
    contours, table_heirarchy = cv2.findContours(table_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
    table_contours = []
    # Ref: https://www.pyimagesearch.com/2015/02/09/removing-contours-image-using-python-opencv/
    # Remove bad contours
    for c in contours:
        # if the contour is bad, draw it on the mask
        if cv2.contourArea(c) > 2000:
            table_contours.append(c)
    if len(table_contours) == 0:
        return None
    # Ref : https://docs.opencv.org/4.5.2/da/d0c/tutorial_bounding_rects_circles.html
    # Get bounding box for the contour
    table_bound_rect = [None] * len(table_contours)
    for i, c in enumerate(table_contours):
        polygon = cv2.approxPolyDP(c, 3, True)
        table_bound_rect[i] = cv2.boundingRect(polygon)
    # Table bounding Box
    table_bound_rect.sort()
    column_bound_rects = []
    for x, y, w, h in table_bound_rect:
        column_mask_crop = column_mask[y : y + h, x : x + w]
        # Get contours of the mask to get number of tables
        contours, column_heirarchy = cv2.findContours(column_mask_crop, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        # Get bounding box for the contour
        bound_rect = [None] * len(contours)
        for i, c in enumerate(contours):
            polygon = cv2.approxPolyDP(c, 3, True)
            bound_rect[i] = cv2.boundingRect(polygon)
            # Adjusting columns as per table coordinates
            bound_rect[i] = (bound_rect[i][0] + x, bound_rect[i][1] + y, bound_rect[i][2], bound_rect[i][3])
        column_bound_rects.append(bound_rect)
    image = image[...,0].reshape(1024, 1024).astype(np.uint8)
    # Draw bounding boxes
    color = (0, 255, 0)
    thickness = 4
    for x, y, w, h in table_bound_rect:
        image = cv2.rectangle(image, (x, y),(x + w, y + h), color, thickness)
    return image, table_bound_rect, column_bound_rects


from rest_framework.decorators import action
from rest_framework.generics import RetrieveAPIView
from rest_framework import viewsets, status, generics
from rest_framework.response import Response

from .serializers import *

class CategoryView(viewsets.ModelViewSet):
    queryset = Category.objects.all()
    serializer_class = CategorySerializer

    def get_queryset(self):
        queryset = Category.objects.all()

        title = self.request.query_params.get('title')
        if title:
            queryset = queryset.filter(title__icontains=title)
        return queryset

class RegionView(viewsets.ModelViewSet):
    queryset = Region.objects.all()
    serializer_class = RegionSerializer

    def get_queryset(self):
        queryset = Region.objects.all()

        state = self.request.query_params.get('state')
        city = self.request.query_params.get('city')
        if state:
            queryset = queryset.filter(state__icontains=state)
        if city:
            queryset = queryset.filter(city__icontains=city)
        return queryset

class VacancyViewSet(viewsets.ModelViewSet):
    serializer_class = VacancySerializer
    queryset = Vacancy.objects.all()
    basename = 'vacancy'

    def get_queryset(self):
        queryset = Vacancy.objects.all()

        title = self.request.query_params.get('title')
        salary = self.request.query_params.get('salary')
        state = self.request.query_params.get('state')
        if state:
            queryset = queryset.filter(state=state)
        if salary:
            queryset = queryset.filter(salary__icontains=salary)
        if title:
            queryset = queryset.filter(title__icontains=title)
        return queryset

    @action(detail=True, methods=['get'])
    def count(self, request, pk=None):
        instance = self.get_object()
        instance.count += 1
        instance.save()
        serializer = self.get_serializer(instance)
        return Response(serializer.data, status=status.HTTP_200_OK)

class CompanyView(viewsets.ModelViewSet):
    queryset = Company.objects.all()
    serializer_class = CompanySerializer

class TopVacancyView(viewsets.ModelViewSet):
    queryset = TopVacancy.objects.all()
    serializer_class = TopVacancySerializer


# -*- coding: utf-8 -*-
"""
Created on Thu Aug 12 16:18:31 2021

@author: Jagriti
"""

"""The dataset we will be using is the CIFAR-10 dataset, which consists
 of 60,000 32x32 color images in 10 classes, with 6000 images per class.
 There are 50000 training images and 10000 test images.

 https://arxiv.org/pdf/1412.6806.pdf     """

from keras.datasets import cifar10
data= cifar10.load_data()
(X_train, y_train), (X_test, y_test) = data

"""-----------------------------------------------------------------"""
# DETERMINE DATASET CHARACTERISTICS
print('Training Images: {}'.format(X_train.shape))
print('Testing Images: {}'.format(X_test.shape))

import matplotlib.pyplot as plt

# create a grid of 3x3 images
for i in range(0,9):
    plt.subplot(330 + 1 + i)
    img = X_train[i]
    plt.imshow(img)

plt.show()

"""-----------------------------------------------------------------"""
#PREPROCESSING THE DATASET
# fix random seed for reproducibility
import numpy as np
seed = 6
np.random.seed(seed)

# normalize the inputs
X_train = X_train / X_train.max()
X_test = X_test / X_test.max()

"""-----------------------------------------------------------------"""
"""The class labels are a single integer value (0-9).
 What we really want is a one-hot vector of length ten.
 For example, the class label of 6 should be denoted as
 [0, 0, 0, 0, 0, 0, 1, 0, 0, 0].
 We can accomplish this using the np_utils.to_categorical() function """

from keras.utils import np_utils
Y_train = np_utils.to_categorical(y_train)
Y_test = np_utils.to_categorical(y_test)
num_classes = Y_test.shape[1]

"""-----------------------------------------------------------------"""
#BUILDING CNN

""" The three base networks used for classification on CIFAR-10 and CIFAR-100.

Input 32 × 32 RGB image
MODEL C

3 × 3 conv. 96 ReLU
3 × 3 conv. 96 ReLU
3 × 3 max-pooling stride 2

3 × 3 conv. 192 ReLU
3 × 3 conv. 192 ReLU
3 × 3 max-pooling stride 2

3 × 3 conv. 192 ReLU
1 × 1 conv. 192 ReLU
1 × 1 conv. 10 ReLU
global averaging over 6 × 6 spatial dimensions
10 or 100-way softmax

"""

from keras.models import Sequential
from keras.layers import Dropout, Activation, Conv2D, GlobalAveragePooling2D

def allcnn(weights=None):
    # dDEFINING SEQUENTIAL MODEL
    model = Sequential()

    # add model layers - Convolution2D, Activation, Dropout
    model.add(Conv2D(96, (3, 3), padding = 'same', input_shape=(32, 32, 3)))
    model.add(Activation('relu'))
    model.add(Conv2D(96, (3, 3), padding = 'same'))
    model.add(Activation('relu'))
    model.add(Conv2D(96, (3, 3), padding = 'same', strides = (2,2)))
    model.add(Dropout(0.5))  #TO PREVENT OVERFITTING

    model.add(Conv2D(192, (3, 3), padding = 'same'))
    model.add(Activation('relu'))
    model.add(Conv2D(192, (3, 3), padding = 'same'))
    model.add(Activation('relu'))
    model.add(Conv2D(192, (3, 3), padding = 'same', strides = (2,2)))
    model.add(Dropout(0.5))

    model.add(Conv2D(192, (3, 3), padding = 'same'))
    model.add(Activation('relu'))
    model.add(Conv2D(192, (1, 1), padding = 'valid'))
    model.add(Activation('relu'))
    model.add(Conv2D(10, (1, 1), padding = 'valid'))

    # add GlobalAveragePooling2D layer with Softmax activation
    model.add(GlobalAveragePooling2D())
    model.add(Activation('softmax'))

    # load the weights
    if weights:
        model.load_weights(weights)

    return model

"""-----------------------------------------------------------------"""
#DEFINING HYPERPARAMETERS
learning_rate = 0.01
weight_decay = 1e-6
momentum = 0.9

model = allcnn()

# DEFINING OPTIMIZER & COMPILING THE MODEL
from keras.optimizers import SGD #stochaistic gradient descent
sgd = SGD(lr=learning_rate, decay=weight_decay, momentum=momentum, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])

print (model.summary())

"""batch size simply means you're going to run 32 images.
And then after those 32 images you can update the loss or update the
parameters instead of updating them individually after every single image."""

model.fit(X_train, Y_train, validation_data=(X_test, Y_test),
          epochs=350, batch_size=32, verbose = 1)

"""-----------------------------------------------------------------"""
#PREDICTIONS

classes = range(0,10)

names = ['airplane',
        'automobile',
        'bird',
        'cat',
        'deer',
        'dog',
        'frog',
        'horse',
        'ship',
        'truck']

# zip the names and classes to make a dictionary of class_labels
class_labels = dict(zip(classes, names))

# generate batch of 9 images to predict
batch = X_test[100:109]
labels = np.argmax(Y_test[100:109],axis=-1)

# make predictions
predictions = model.predict(batch, verbose = 1)

# convert class probabilities to class labels
class_result = np.argmax(predictions,axis=-1)

# create a grid of 3x3 images
fig, axs = plt.subplots(3, 3, figsize = (15, 6))
fig.subplots_adjust(hspace = 1)
axs = axs.flatten()

for i, img in enumerate(batch):

    # determine label for each prediction, set title
    for key, value in class_labels.items():
        if class_result[i] == key:
            title = 'Prediction: {}\nActual: {}'.format(class_labels[key], class_labels[labels[i]])
            axs[i].set_title(title)
            axs[i].axes.get_xaxis().set_visible(False)
            axs[i].axes.get_yaxis().set_visible(False)

    # plot the image
    axs[i].imshow(img)

plt.show()


import numpy as np
from sklearn.base import clone
from autosklearn.classification import AutoSklearnClassifier
import pandas as pd
from autosklearn.classification import AutoSklearnClassifier

if __name__ == "__main__":

    def fit(model, X, labels):
        models = [clone(model).fit(X,labels[i]) for i in range(0,len(labels))]
        return(models)

    def predict(models, X):
        predictions = [m.predict_proba(X)[:,1] for m in models]
        return(predictions)

    # def evaluate(y, predictions):
    #     # predictions = np.where(np.array(predictions) > 0.5, 1, 0)
    #     predictions = np.array(predictions)
    #     y = np.array(y)
    #      return [roc_auc_score(y[:, i], predictions[i, :]) for i in range(y.shape[1])]

    def save_predictions(predictions, filename):
        df = pd.DataFrame(np.array(predictions).T, columns=['attribute 1', 'attribute 2', 'attribute 3', 'attribute 4', 'attribute 5'])
        df.to_csv(filename, index_label= 'Id')

    model = AutoSklearnClassifier(
      time_left_for_this_task = 45 * 60,
      per_run_time_limit = 125,
      n_jobs = -1
    )

    X_train = np.genfromtxt('protein_features_train.csv', delimiter=',')
    y_train = np.genfromtxt('protein_labels_train.csv', delimiter=',')
    X_test = np.genfromtxt('protein_features_test.csv', delimiter=',')

    labels_train = [np.copy(y_train[:,i]) for i in range(0, y_train.shape[1])]

    models = fit(model, X_train, labels_train)
    predictions = predict(models, X_test)
    save_predictions(predictions, 'predictions.csv')


"""
Unsharp masking
"""
import skimage.io as io
import numpy as np
import cv2

# read image
name = input('Enter image name : ')
filename, filetype = name.split('.')
original_image = io.imread(name)

# get image height, image width
information = original_image.shape
height = information[0]
width = information[1]

'''
Average filter
'''
n = 3
div = pow(n,2)

def avg_get_pixel(i, j):
    if (i > height-1 or j > width-1 or i < 0 or j < 0):
        return 0
    else:
        return original_image[i][j]

def avg_filter(i, j):
    sum = 0
    mask = [[i-1, j-1], [i-1, j], [i-1, j+1], [i, j-1], [i, j], [i, j+1], [i+1, j-1], [i+1, j], [i+1, j+1]]
    for pixel in mask:
        sum += avg_get_pixel(pixel[0], pixel[1])/div
    return sum

avg_image = np.zeros((height, width))
for i in range(height):
    for j in range(width):
        avg_image[i][j] = avg_filter(i, j)

io.imshow(avg_image, cmap='gray')
io.show()

io.imsave(filename + '_average_filter.' + filetype, avg_image.astype(np.uint8))

'''
Median filter
'''
def get_median(array):
    array.sort()
    l = len(array)
    if (l%2 != 0):
        position = (l+1)/2-1
        return array[int(position)]
    else:
        position1 = l/2-1
        position2 = l/2
        return ( int(array[int(position1)]) + int(array[int(position2)]) )/2

def med_get_pixel(i, j):
    if (i > height-1 or j > width-1 or i < 0 or j < 0):
        return -1
    else:
        return original_image[i][j]

def med_filter(i, j):
    values = []
    mask = [[i-1, j-1], [i-1, j], [i-1, j+1], [i, j-1], [i, j], [i, j+1], [i+1, j-1], [i+1, j], [i+1, j+1]]
    for pixel in mask:
        value = med_get_pixel(pixel[0], pixel[1])
        if (value != -1):
            values.append(value)
    return get_median(values)

med_image = np.zeros((height, width))
for i in range(height):
    for j in range(width):
        med_image[i][j] = med_filter(i,j)

io.imshow(med_image, cmap='gray')
io.show()

io.imsave(filename + '_median_filter.' + filetype, med_image.astype(np.uint8))

'''
Unsharp Masking
'''
k = 0.9

# create mask : subtract
mask_average = original_image - avg_image
mask_median = original_image - med_image

io.imshow(mask_average, cmap='gray')
io.show()
io.imshow(mask_median, cmap='gray')
io.show()

# add mask to image
unsharp_average = original_image + k*mask_average
unsharp_median = original_image + k*mask_median
io.imshow(unsharp_average, cmap='gray')
io.show()
io.imshow(unsharp_median, cmap='gray')
io.show()

# scale mask
unsharp_average = 255* ( (unsharp_average-unsharp_average.min()) / (unsharp_average.max() - unsharp_average.min()) )
unsharp_median = 255* ( (unsharp_median-unsharp_median.min()) / (unsharp_median.max() - unsharp_median.min()) )

io.imsave(filename + '_median_unsharp.' + filetype, unsharp_average.astype(np.uint8))
io.imsave(filename + '_average_unsharp.' + filetype, unsharp_median.astype(np.uint8))


from django.core.exceptions import ValidationError
from django.test import TestCase
from MaintenancePlanner.equipment.models import Equipment
from MaintenancePlanner.equipment.validators import equipment_name_validator
from tests.common.test_data import create_plant, create_department, create_equipment

class EquipmentCreationModel(TestCase):
    def setUp(self):
        self.plant = create_plant()
        self.department = create_department()

    def test_equipment_creation_with_valid_data(self):
        equipment = create_equipment()
        self.assertEqual(equipment.description, 'TestEquipment')
        self.assertEqual(equipment.type, 'Machine')
        self.assertEqual(equipment.plant.name, 'TestPlant')
        self.assertEqual(equipment.department.name, 'TestDepartment')

    def test_equipment_creation_with_invalid_data(self):
        equipment = Equipment.objects.create(
                description='TestEquipment@',
                type='Machine',
                plant=self.plant,
                department=self.department,
            )
        with self.assertRaises(ValidationError) as context:
            equipment_name_validator(equipment.description)
        message = 'Ensure this value contains only letters, numbers, and underscore.'
        self.assertEqual(message, str(context.exception.message))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='DetalleFactura',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('cantidad', models.IntegerField()),
            ],
        ),
        migrations.CreateModel(
            name='Factura',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('date', models.DateField(auto_now=True)),
            ],
        ),
        migrations.CreateModel(
            name='Producto',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=50)),
                ('price', models.DecimalField(max_digits=11, decimal_places=2)),
            ],
        ),
        migrations.AddField(
            model_name='detallefactura',
            name='factura',
            field=models.ForeignKey(to='principal.Factura'),
        ),
        migrations.AddField(
            model_name='detallefactura',
            name='producto',
            field=models.ForeignKey(to='principal.Producto'),
        ),
    ]


from selenium.webdriver import ChromeOptions, Chrome
from es_crawler.config import parse_config
from es_crawler.crawler import CrawlerProduct, CrawlerCategoryUrl, CrawlerProductUrl
from es_crawler.settings import Settings
from es_crawler.shop.hm import HM
from es_crawler.shop.lamoda import Lamoda
from es_crawler.shop.ostin import Ostin
from google.cloud import storage
import logging
import argparse
import redis

def build_driver(path_driver):
    chrome_options = ChromeOptions()
    chrome_options.add_argument("--headless")
    chrome_options.add_argument('--ignore-certificate-errors')
    chrome_options.add_argument("--disable-blink-features")
    chrome_options.add_argument('--disable-dev-shm-usage')
    chrome_options.add_argument('--no-sandbox')
    chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
    chrome_options.add_experimental_option('useAutomationExtension', False)
    chrome_options.add_argument('--allow-running-insecure-content')
    chrome_options.add_argument("--window-size=1920,1080")
    user_agent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 " \
                 "(KHTML, like Gecko) Chrome/89.0.4280.66 Safari/537.36"
    chrome_options.add_argument(f'user-agent={user_agent}')
    driver = Chrome(executable_path=path_driver, options=chrome_options)
    driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
        "source": """
              const newProto = navigator.__proto__
              delete newProto.webdriver
              navigator.__proto__ = newProto
              """
    })
    return driver

def make_shop(name):
    shops = {
        'hm': HM,
        'lamoda': Lamoda,
        'ostin': Ostin
    }
    return shops[name]()

def make_crawler(settings: Settings, step: str):
    _redis = redis.Redis(host=settings.config.redis_host, port=settings.config.redis_port, db=0)
    shop = make_shop(settings.config.shop)
    driver = build_driver(settings.config.driver)
    if step == 'category_url':
        return CrawlerCategoryUrl(
            driver=driver,
            run_id=settings.run_settings.run_id,
            shop=shop,
            r=_redis,
            num_others=settings.config.count_builder
        )
    elif step == 'product_url':
        return CrawlerProductUrl(
            driver=driver,
            run_id=settings.run_settings.run_id,
            shop=shop,
            r=_redis
        )
    elif step == 'product':
        return CrawlerProduct(
            driver=driver,
            run_id=settings.run_settings.run_id,
            shop=shop,
            r=_redis,
            bucket=storage_client.bucket(settings.config.bucket_name),
            data_dir=settings.run_settings.data_dir
        )
    else:
        raise Exception(f'Wrong step name: {step}')

def run(settings, step):
    c = make_crawler(settings, step)
    c.run()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", "--config", help="config file")
    parser.add_argument("-s", "--step", help="step crawling: category_url, product_url, product")
    args = parser.parse_args()
    config = parse_config(args.config)

    logging.basicConfig(level=config.logging_level, filename=config.log_file, filemode='a',
                        format='%(asctime)s : %(levelname)s : %(message)s')
    storage_client = storage.Client()
    run(Settings(config, config.run_id), args.step)


#Kacper Słoniec

#Opis algorytmu:
#Obliczamy czas budowy wszystkich możliwych autostrad. Czasy te sortujemy. A następnie szukamy grafu spójnego z minimalną różnicą czasów (metodą brute force).

from collections import deque
from zad8testy import runtests

def time(A, B):
    length = ((A[0]-B[0])**2 + (A[1]-B[1])**2)**0.5
    if int(length) == length: return length
    else: return int(length) + 1

def spojne(n, Tab, start, stop):
    G = [[] for _ in range(n)]
    for i in range(start, stop+1):
        G[Tab[i][1]].append(Tab[i][2])
        G[Tab[i][2]].append(Tab[i][1])

    Q = deque()
    visited = 0
    V = [False for _ in range(len(G))]

    V[0] = True
    visited += 1
    Q.append(0)

    while Q:
        u = Q.popleft()
        for v in G[u]:
            if V[v] == False:
                V[v] = True
                visited += 1
                Q.append(v)

    if visited == n:
        return True
    else:
        return False

def highway(A):
    n = len(A)

    Times = []
    for i in range(n-1):
        for j in range(i+1, n):
            Times.append((time(A[i], A[j]), i, j))

    Times.sort(key = lambda tup: tup[0])
    r = len(Times)

    minimum = float('inf')
    for start in range(r):
        for end in range(start+n-2, r):
            if spojne(n, Times, start, end):
                minimum = min(minimum, Times[end][0] - Times[start][0])

    return minimum

#A =[(10,10),(15,25),(20,20),(30,40)]
#print(highway(A))
# zmien all_tests na True zeby uruchomic wszystkie testy
runtests( highway, all_tests = True )


from django.shortcuts import render

# Create your views here.
from rest_framework.decorators import api_view, authentication_classes, permission_classes
from rest_framework import status
from rest_framework.response import Response
from rest_framework.permissions import IsAuthenticated
from rest_framework_simplejwt.authentication import JWTTokenUserAuthentication
import jwt
from django.conf import settings
from rest_framework_simplejwt.serializers import TokenObtainPairSerializer
from cerberus import Validator
import yaml
from datarepo.models import AccountRole
from users.models import User
from ..models import Industry, Mentorship
import random

@api_view(['POST'])
@authentication_classes([JWTTokenUserAuthentication])
@permission_classes([IsAuthenticated])
def mentorship(request):
    token = request.headers['Authorization']
    decoded_data = jwt.decode(token[7:], settings.JWT_SECRET_KEY, algorithms=['HS256'])
    account_role = decoded_data['account_role']
    if account_role == AccountRole.INDUSTRY_USER:
        mentor_name = request.POST.get('mentor_name', None)
        mentor_email = request.POST.get('mentor_email', None)
        mentor_phone_number = request.POST.get('mentor_phone_number', None)
        designation = request.POST.get('designation', None)
        availability = request.POST.get('availability', None)
        mode = request.POST.get('mode', None)
        linkedin_profile_url = request.POST.get('linkedin_profile_url', None)  # in rupee
        languages = request.POST.get('languages', None)

        request_schema = '''
                mentor_name:
                    type: string
                    empty: false
                    required: true
                mentor_email:
                    type: string
                    empty: false
                    required: true
                mentor_phone_number:
                    type: string
                    empty: false
                    required: true
                designation:
                    type: string
                    empty: false
                    required: true
                availability:
                    type: string
                    empty: false
                    required: true
                mode:
                    type: string
                    empty: false
                    required: true
                linkedin_profile_url:
                    type: string
                    empty: false
                    required: true
                languages:
                    type: string
                    empty: false
                    required: true
                '''
        v = Validator()
        post_data = request.POST.dict()
        schema = yaml.load(request_schema, Loader=yaml.SafeLoader)

        if v.validate(post_data, schema):
            try:
                get_industry = Industry.objects.get(user_id=request.user.id)
            except Industry.DoesNotExist:
                content = {
                    "message": "Please contact admin"
                }
                return Response(content, status.HTTP_400_BAD_REQUEST, content_type='application/json')

            new_mentorship = Mentorship.objects.create(
                industry_id=get_industry.id,
                mentor_name=mentor_name,
                mentor_email=mentor_email,
                mentor_phone_number=mentor_phone_number,
                designation=designation,
                availability=availability,
                mode=mode,
                linkedin_profile_url=linkedin_profile_url,
                languages=languages,
            )
            context = {
                "message": "Mentorship added successfully",
                'mentorship_id': new_mentorship.id
            }
            return Response(context, status.HTTP_200_OK, content_type='application/json')

        else:
            content = {
                'message': "invalid request",
                'errors': v.errors
            }
            return Response(content, status=status.HTTP_400_BAD_REQUEST, content_type='application/json')

    else:
        content = {
            "message": "You dont have the permission"
        }
        return Response(content, status.HTTP_401_UNAUTHORIZED, content_type='application/json')

@api_view(['GET'])
@authentication_classes([JWTTokenUserAuthentication])
@permission_classes([IsAuthenticated])
def mentorships(request):
    token = request.headers['Authorization']
    decoded_data = jwt.decode(token[7:], settings.JWT_SECRET_KEY, algorithms=['HS256'])
    account_role = decoded_data['account_role']
    if account_role in [AccountRole.NM_ADMIN,
                        AccountRole.INDUSTRY_ADMIN,
                        AccountRole.INDUSTRY_STAFF,
                        AccountRole.INDUSTRY_USER]:
        industry_id = request.GET.get('industry_id', None)
        page = request.GET.get('page', 0)
        limit = request.GET.get('limit', 20)
        try:
            page = int(page)
            limit = int(limit)
        except:
            page = 0
            limit = 20

        query = {}
        if industry_id:
            try:
                query['industry_id'] = int(industry_id)
            except:
                pass

        if account_role == AccountRole.INDUSTRY_USER:
            try:
                get_industry = Industry.objects.values('id').get(user_id=request.user.id)
                query['industry_id'] = get_industry['id']
            except Industry.DoesNotExist:
                content = {
                    "message": "Please contact admin"
                }
                return Response(content, status.HTTP_400_BAD_REQUEST, content_type='application/json')

        mentors_objs = Mentorship.objects.select_related('industry').filter(**query).order_by('-created')
        total_count = mentors_objs.count()
        final_mentors_list = []
        for obj in mentors_objs[(page* limit): ((page*limit) + limit)]:
            temp = {
                "industry_id": obj.industry_id,
                "industry_name": obj.industry.organisation_name if obj.industry_id else None,
                "industry_type": obj.industry.industry_type if obj.industry_id else None,

                "mentorship_id": obj.id,
                "mentor_name": obj.mentor_name,
                "mentor_email": obj.mentor_email,
                "mentor_phone_number": obj.mentor_phone_number,
                "designation": obj.designation,
                "availability": obj.availability,
                "mode": obj.mode,
                "linkedin_profile_url": obj.linkedin_profile_url,
                "languages": obj.languages,
                "created": obj.created,
            }
            final_mentors_list.append(temp)
        context = {
            "mentors": final_mentors_list,
            "filters": {
                'page': page,
                'limit': limit,
                'total_count': total_count,
                'industry_id': industry_id,
            }
        }
        return Response(context, status.HTTP_200_OK, content_type='application/json')

    else:
        content = {
            "message": "You dont have the permission"
        }
        return Response(content, status.HTTP_401_UNAUTHORIZED, content_type='application/json')


import requests, pandas as pd
import numpy as np
import math
import pprint
import matplotlib.pyplot as plt
import datetime

#Variable section for start_time and end_time:
year = 2022
month = "10"    #IMPORTANT - JAN, FEB, MARCH ...... SEP must be set to 01, 02, 03 ... 09 as a STRING
start_date = "27"  #IMPORTANT, all single digit dates must have 0 at front as STRING, Ex: 01, 02, 03
end_date = "28"    #IMPORTANT, all single digit dates must have 0 at front as STRING, Ex: 01, 02, 03
start_time = "01"  #IMPORTANT, all hours must have two digits, single digit hours must have 0 at front as a STRING
end_time = 23      #IMPORTANT, all hours must have two digits, single digit hours must have 0 at front as a STRING

data = pd.read_csv (r'C:\Users\yiyan\Downloads\ASC2022 (1).csv')
df = pd.DataFrame(data, columns= ["latitude", "longitude"])
#Read the csv file and only read the columns Latitude and Longitude
#print(df)

Lat_long_pairs = []
Lat = []
Long = []

for ind in df.index:
    Lat_long_pairs.append([df["latitude"][ind], df["longitude"][ind]])

#Create a loop to in take all the latitude and longitude pairs that we are looking to calculate the wind velocity and
#direction for and store them in a list

def get_API(lat, long): #Calls the API on the specific latitude and longitude and returns the output of the json file
    response = requests.get(
        f"https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={long}&hourly=windspeed_10m,winddirection_10m")
    #print(response.status_code)
    output = response.json()
    return(output)

def get_time(year, month, start_date, start_hour, end_date, end_hour): #Creates the time string that matches the format
    #of the json file
    #May switch to datetime later, currently datetime does not match up with the time strings in the dictionary

    #start_time = datetime.datetime(year,month, start_date, start_hour)
    #end_time = datetime.datetime(year, month, end_date, end_hour)

    start_time_ = str(year) + "-" + str(month) + "-" + str(start_date) + "T" + str(start_hour) + ":00"
    end_time_ = str(year) + "-" + str(month) + "-" + str(end_date) + "T" + str(end_hour) + ":00"
    return([start_time_, end_time_])

def get_average(lat, long, year, month, start_date, start_hour, end_date, end_hour): #Main function
    start_index = 0
    end_index = 0
    #Two variables above are meant to track which wind velocity and wind direction we want given a 7 day API wind forecast

    data_set = get_API(lat,long)
    #pprint.pprint(data_set)
    #Call the API, print function can be deleted

    time_set = get_time(year,month,start_date,start_hour,end_date,end_hour)
    for i in range(len(data_set['hourly']['time'])):
        if data_set['hourly']['time'][i] == time_set[0]:
            start_index += i
            pass
        elif data_set['hourly']['time'][i] == time_set[1]:
            end_index += i
            break

    #Find the starting wind_direction and wind_speed indeces and ending wind_direction and wind_speed indeces

    wind_speed_10m_list = (data_set['hourly']['windspeed_10m'][start_index:end_index])
    wind_direct_10m_list = (data_set['hourly']['winddirection_10m'][start_index:end_index])

    #Create two new lists consist of only the wind direction and wind speed during a specific time frame

    #print(wind_speed_10m_list)
    #print(wind_speed_10m_list)
    #print function can be deleted

    counter = 0
    speed_sum = 0
    direct_sum = 0
    for i in wind_speed_10m_list:
        counter += 1
        speed_sum += i

    for j in wind_direct_10m_list:
        try:
            direct_sum += j
        except:
            print("error occured at" + str(j))
            pass

    #Take the average of the two lists, since they have the same number of items use the same counter for both lists

    return(speed_sum/counter, direct_sum/counter)

#--------------------------------------------------

#Below is the calling of the function
average_velocity = []
average_angle = []

for i in range(len(Lat_long_pairs)):
    print(i)
    if i % 1000 == 0:
        average_velocity.append(get_average(Lat_long_pairs[i][0],Lat_long_pairs[i][1],year,month, start_date, start_time, end_date, end_time)[0])
        average_angle.append(get_average(Lat_long_pairs[i][0],Lat_long_pairs[i][1],year,month, start_date, start_time, end_date, end_time)[1])
        Lat.append(Lat_long_pairs[i][0])
        Long.append(Lat_long_pairs[i][1])
        print(average_velocity)
        print(average_angle)

print(average_angle)

u = []
for i in average_angle:
    u.append(math.cos(i))
v = []
for i in average_angle:
    v.append(math.sin(i))
North_Lat = []
North_Long = []
East_Lat = []
East_Long = []
South_Lat = []
South_Long = []
West_Lat = []
West_Long = []
for i in range(len(average_angle)):
    if (average_angle[i] >= 0 and average_angle[i] <= 45) or (average_angle[i] >= 315 and average_angle[i] <= 0):
        North_Lat.append(Lat[i])
        North_Long.append(Long[i])
    elif (average_angle[i] >= 45 and average_angle[i] <= 135):
        East_Lat.append(Lat[i])
        East_Long.append(Long[i])
    elif (average_angle[i] >= 135 and average_angle[i] <= 225):
        South_Lat.append(Lat[i])
        South_Long.append(Long[i])
    else:
        West_Lat.append(Lat[i])
        West_Long.append(Long[i])

min = min(average_velocity)
max = max(average_velocity)

plt.subplot(1,2,1)
plt.title("Average Wind Velocity")
plt.scatter(Lat, Long, s=100, c=average_velocity, cmap = "Greens", marker='o')
plt.xlabel("Latitude")
plt.ylabel("Longitude")
plt.colorbar()

plt.subplot(1,2,2)
plt.title("Average Wind Direction")

plt.scatter(North_Lat,North_Long, s=100, marker ='^')
plt.scatter(East_Lat, East_Long, s=100, marker ='>')
plt.scatter(South_Lat, South_Long, s=100, marker ='v')
plt.scatter(West_Lat, West_Long, s=100, marker ='<')

plt.show()


'''
# Module name:
    jdfundindex.py

# Prerequisite:
    Based on Python 3.4
    Need python module requests and bs4

# Purpose:
    Fund index sensor powered by JD

# Author:
    Retroposter retroposter@outlook.com

# Created:
    Sep.8th 2017

# Last Modified:
    Jan.23th 2018
'''

from datetime import datetime, timedelta
import json
import logging
import time

from bs4 import BeautifulSoup
from requests.exceptions import ConnectionError as ConnectError, HTTPError, Timeout
import requests
import re
import voluptuous as vol

from homeassistant.const import (CONF_LATITUDE, CONF_LONGITUDE, CONF_API_KEY, CONF_MONITORED_CONDITIONS, CONF_NAME, TEMP_CELSIUS, ATTR_ATTRIBUTION)
from homeassistant.components.sensor import PLATFORM_SCHEMA
from homeassistant.helpers.entity import Entity
from homeassistant.util import Throttle
import homeassistant.helpers.config_validation as cv

_LOGGER = logging.getLogger(__name__)

DEFAULT_NAME = 'Fund Index'
ATTRIBUTION = 'Powered by JD'

CONF_UPDATE_INTERVAL = 'update_interval'

PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend({
    vol.Optional(CONF_NAME, default=DEFAULT_NAME): cv.string,
    vol.Optional(CONF_UPDATE_INTERVAL, default=timedelta(hours=1)): (vol.All(cv.time_period, cv.positive_timedelta))
})

def setup_platform(hass, config, add_devices, discovery_info=None):
    """Set up the Fund sensor."""
    name = config[CONF_NAME]
    interval = config.get(CONF_UPDATE_INTERVAL)
    index_data = JdFundIndexData(interval)
    index_data.update()
    # If connection failed don't setup platform.
    if index_data.data is None:
        return False
    sensors = [JdFundIndexSensor(index_data, name)]
    add_devices(sensors, True)

class JdFundIndexSensor(Entity):
    def __init__(self, index_data, name):
        """Initialize the sensor."""
        self.index_data = index_data
        self.client_name = name
        self._state = None

    @property
    def name(self):
        """Return the name of the sensor."""
        return self.client_name

    @property
    def state(self):
        """Return the state of the sensor."""
        return self._state

    @property
    def icon(self):
        """Icon to use in the frontend, if any."""
        if self._state < 41:
            return 'mdi:emoticon-sad'
        if self._state < 50:
            return 'mdi:emoticon-neutral'
        if self._state < 61:
            return 'mdi:emoticon-happy'
        return 'mdi:emoticon-excited'

    @property
    def unit_of_measurement(self):
        """Return the unit of measurement of this entity, if any."""
        return 'Pts'

    @property
    def device_state_attributes(self):
        """Return the state attributes."""
        attrs = {}
        attrs[ATTR_ATTRIBUTION] = ATTRIBUTION
        data = self.index_data.data
        if data is None:
            return attrs
        attrs['交易日'] = data['index_date']
        attrs['整体走势'] = data['summary']
        if data['hot_list'] is not None:
            for hot in data['hot_list']:
                attrs[hot['name']] = hot['index']
        return attrs

    def update(self):
        """Get the latest data from He Weather and updates the states."""
        self.index_data.update()
        data = self.index_data.data
        if data is None:
            return
        self._state = data['index']

class JdFundIndexData(object):
    """Get the latest data from JD."""

    def __init__(self, internal):
        self.data = None
        # Apply throttling to methods using configured interval
        self.update = Throttle(internal)(self._update)

    def _update(self):
        timespan = str(time.time())
        resp = None
        try:
            resp = requests.post('https://licai.jd.com//async/financing/getJZBroadCast.action?_dc=' + timespan)
        except (ConnectError, HTTPError, Timeout, ValueError) as error:
            _LOGGER.error('Unable to connect to licai.jd.com. %s', error)
            return
        if resp is None:
            return
        json_data = json.loads(resp.text)
        self.data = {}
        self.data['index'] = json_data['heat']
        self.data['index_date'] = json_data['degreeDateStr'][0:10]
        self.data['summary'] = json_data['shareAdvise']
        self.data['hot_list'] = None

        resp = None
        try:
            resp = requests.post('https://fund.jd.com/topicRecommend.action?t=' + timespan)
        except (ConnectError, HTTPError, Timeout, ValueError) as error:
            _LOGGER.error('Unable to connect to fund.jd.com. %s', error)
            return
        if resp is None:
            return
        json_data = json.loads(resp.text)
        hot_list = []
        for industry in json_data:
            hot = {'name': industry['strategyName'], 'index': industry['degree']}
            hot_list.append(hot)
        self.data['hot_list'] = hot_list


from channels.generic.websocket import AsyncWebsocketConsumer
import json
from . import hangedman
from random import randint

class GameConsumer(AsyncWebsocketConsumer):
    games = {}

    def __getUsername(self):
        username = self.scope['user'].username
        if username == "":
            username = "Guest " +  str(self.scope['session'].session_key)[0:5]
        return username

    async def connect(self):
        # print(str(self.scope['session'].session_key))
        # Obtener el room_name a partir de la url
        self.room_name = self.scope['url_route']['kwargs']['room_name']

        # Definir nombre del Group
        self.room_group_name = 'game_%s' % self.room_name

        # Crear el Group
        await self.channel_layer.group_add(
            self.room_group_name,
            # Puntero al channel_layer que va a llegar al consumer
            self.channel_name
        )

        if self.room_group_name not in self.games:
            game = hangedman.HangedMan()
            self.games[self.room_group_name] = game

        await self.accept()

    async def disconnect(self, code):
        game = self.games[self.room_group_name]
        username = self.__getUsername()
        game.removeUsername(
            username
        )
        if len(game.usernames()) != 0:
            username = game.currentUser()
        await self.channel_layer.group_send(
                self.room_group_name,
                {
                    'type': 'send_message',
                    'state': "disconnect",
                    'message': {
                        'username': username,
                        'cUser': username,
                    }
                }
            )
        await self.channel_layer.group_discard(
            self.room_group_name,
            self.channel_name
        )
        return super().disconnect(code)

    async def receive(self, text_data):
        data_json = json.loads(text_data)

        event = data_json['event']
        message = data_json['message']
        inpt = message.pop('input', None)
        cword = None
        guessResult = None
        username = self.__getUsername()
        # Uso del Modelo
        game = self.games[self.room_group_name]
        if event == "start":
            game.addUsername(username)
            state = "init"
            message = {
                'cword' : game.cword(),
                'cUser' : game.currentUser(),
                'username': username,
            }
        else:
            canPlay = game.canPlay(username)

            if canPlay:
                if event == "letter":
                    guessResult = game.guessLetter(inpt)
                    cword = game.cword()
                    if len(inpt) != 0:
                        inpt = inpt[0]
                    else:
                        inpt = '🤐'
                elif event == "word":
                    guessResult = game.guessWord(inpt)
                    cword = game.cword()
                    if len(inpt) == 0:
                        inpt = '🤐'
                # TODO: Evento para ganar
                cUser = game.currentUser()
                gameSolved = game.checkIfSolved()
                if gameSolved:
                    state = "solved"
                else:
                    state = "unsolved"

                message = {
                    'cword': cword,
                    'cUser': cUser,
                    'input': inpt,
                    'guessResult': guessResult,
                    'username': username,
                }
            else:
                state = "invalidTurn"
                message = {
                    'username': username,
                }
        # print(game.usernames())
        await self.channel_layer.group_send(
                self.room_group_name,
                {
                    'type': 'send_message',
                    'state': state,
                    'message': message
                }
            )

    async def send_message(self, event):
        state = event['state']
        message = event['message']

        await self.send(
            text_data=json.dumps({
                'state': state,
                'message': message,
            })
        )


import gym
import numpy as np
from utils import writeData2File, DataStorage
from config import pqExp_config as configs

def run_controlGame(episode, env, Agent, DataStorage, is_storeQ=False):
    # set_memory_with_random()
    for i_episode in range(episode):
        observation = env.reset()
        if is_storeQ:
            InitState = [observation]
            FirstAction = Agent.choose_action(observation)
        totalR = 0
        while True:
            # env.render()
            action = Agent.choose_action(observation)
            observation_, reward, done, info = env.step(action)
            totalR += reward
            Agent.store_memory(observation, action, reward, observation_, done)
            # Agent.learn(costFlag=done) # costFlag = True 记录 loss
            Agent.learn()
            observation = observation_
            if done:
                print('episode: ', i_episode, ' epsilon: ', Agent.epsilon, 'total_reward:', totalR)
                if is_storeQ:
                    Q_value = Agent.brain.predict_eval_action(InitState)
                    DataStorage.store_Q(Q_value[0][FirstAction])
                DataStorage.store_reward(totalR)
                break

    # Brain.save()  # 存储神经网络

def run_Pendulum(episode, env, Agent, DataStorage, is_storeQ=False):
    ACTION_SPACE = 11
    # set_memory_with_random()
    for i_episode in range(episode):
        observation = env.reset()
        if is_storeQ:
            InitState = [observation]
            FirstAction = Agent.choose_action(observation)
        totalR = 0
        while True:
            # env.render()
            action = Agent.choose_action(observation)
            f_action = (action - (ACTION_SPACE - 1) / 2) / ((ACTION_SPACE - 1) / 4)
            observation_, reward, done, info = env.step(np.array([f_action]))
            totalR += reward
            reward /= 10
            Agent.store_memory(observation, action, reward, observation_, done)
            # Agent.learn(costFlag=done) # costFlag = True 记录 loss
            Agent.learn()
            observation = observation_
            if done:
                print('episode: ', i_episode, ' epsilon: ', Agent.epsilon, 'total_reward:', totalR)
                if is_storeQ:
                    Q_value = Agent.brain.predict_eval_action(InitState)
                    DataStorage.store_Q(Q_value[0][FirstAction])
                DataStorage.store_reward(totalR)
                break

    # Brain.save()  # 存储神经网络

def run_Game(model, env_name, episodes):
    if model == 'DQN':
        from model.mlpBrain import DQN_Brain as Brain
        from model.agent import DQN_Agent as Agent
    elif model == 'DDQN':
        from model.mlpBrain import DDQN_Brain as Brain
        from model.agent import DDQN_Agent as Agent
    elif model == 'PDQN':
        from model.mlpBrain import PDQN_Brain as Brain
        from model.agent import PDQN_Agent as Agent
    elif model == 'PDDQN':
        from model.mlpBrain import PDDQN_Brain as Brain
        from model.agent import PDDQN_Agent as Agent
    elif model == 'DQN_PER':
        from model.mlpBrain import DQN_PER_Brain as Brain
        from model.agent import DQN_PER_Agent as Agent
    elif model == 'DDQN_PER':
        from model.mlpBrain import DDQN_PER_Brain as Brain
        from model.agent import DDQN_PER_Agent as Agent
    elif model == 'DQN_InAday':
        from model.mlpBrain import DQN_InAday_Brain as Brain
        from model.agent import DQN_InAday_Agent as Agent
    elif model == 'DQN_PER_Ipm':
        from model.mlpBrain import DQN_PER_Ipm_Brain as Brain
        from model.agent import DQN_PER_Ipm_Agent as Agent
    elif model == 'DDQN_PER_Ipm':
        from model.mlpBrain import DDQN_PER_Ipm_Brain as Brain
        from model.agent import DDQN_PER_Ipm_Agent as Agent
    elif model == 'PDQN_RePER':
        from model.mlpBrain import PDQN_RePER_Brain as Brain
        from model.agent import PDQN_RePER_Agent as Agent

    env = gym.make(env_name)   # 定义使用 gym 库中的那一个环境
    # env = env.unwrapped  # 注释掉的话 每局游戏 reward之和最高200

    n_actions = 11 if env_name == 'Pendulum-v0' else env.action_space.n

    print('\nThe config:\n', configs, '\n')
    neurons_per_layer = configs['Brain']['neurons_per_layer']
    learning_rate = configs['Brain']['learning_rate']
    output_graph = configs['Brain']['output_graph']
    restore = configs['Brain']['restore']

    reward_decay = configs['Agent']['reward_decay']
    replace_target_iter = configs['Agent']['replace_target_iter']
    memory_size = configs['Agent']['memory_size']
    batch_size = configs['Agent']['batch_size']
    MAX_EPSILON = configs['Agent']['MAX_EPSILON']
    MIN_EPSILON = configs['Agent']['MIN_EPSILON']
    LAMBDA = configs['Agent']['LAMBDA']

    # learning_rate 重要
    # restore 和 MAX_EPSILON 一起调整
    brain = Brain(
        n_actions=n_actions,
        n_features=env.observation_space.shape[0],
        neurons_per_layer=neurons_per_layer,
        learning_rate=learning_rate,
        output_graph=output_graph,
        restore=restore,
        checkpoint_dir=(env_name + '_' + model + '_MLP_Net')
    )
    agent = Agent(
        brain=brain,
        n_actions=n_actions,
        observation_space_shape=env.observation_space.shape,
        reward_decay=reward_decay,
        replace_target_iter=replace_target_iter,
        memory_size=memory_size,
        batch_size=batch_size,
        MAX_EPSILON=MAX_EPSILON,
        MIN_EPSILON=MIN_EPSILON,
        LAMBDA=LAMBDA,
    )

    dataStorage = DataStorage()

    if env_name == 'Pendulum-v0':
        run_Pendulum(episodes, env, agent, False)
    else:
        run_controlGame(episodes, env, agent, dataStorage, True)  # 4-th params = True 记录 q value
    writeData2File(model, env_name, brain, agent, dataStorage)


from selenium import webdriver
import time
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import ActionChains
from selenium.webdriver.chrome.options import Options
import os
import pandas
import win32com.client
from datetime import datetime, timedelta
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities

def CHGDownloader():
    option = Options()
    option.add_experimental_option("prefs", {
        "download.default_directory": "C:\\path\\to\\Downloads",
        "download.prompt_for_download": False,
        'download.directory_upgrade': True,
        'safebrowsing.enabled': False,
        'safebrowsing.disable_download_protection': True,
    })
    option.add_argument("--disable-extensions")
    option.add_argument("--proxy-server=direct://")
    option.add_argument("--proxy-bypass-list=*")
    option.add_argument("--start-maximized")
    option.add_argument("--headless")
    option.add_argument("--disable-gpu")
    option.add_argument("--allow-insecure-localhost")
    option.add_argument("--disable-dev-shm-usage")
    option.add_argument("--no-sandbox")
    option.add_argument("--ignore-certificate-errors")

    capabilities = DesiredCapabilities.CHROME.copy()
    capabilities['acceptSslCerts'] = True
    capabilities['acceptInsecureCerts'] = True
    driver = webdriver.Chrome(
        executable_path="\\path\\to\\chromedriver.exe", options=option, desired_capabilities=capabilities)
    try:
        print("Initializing")
        driver.command_executor._commands["send_command"] = ("POST", '/session/$sessionId/chromium/send_command')
        params = {'cmd': 'Page.setDownloadBehavior',
                  'params': {'behavior': 'allow', 'downloadPath': "C:\\path\\to\\Downloads"}}
        driver.execute("send_command", params)
        driver.implicitly_wait(10)
        driver.get("https://example.com") # site
        time.sleep(10)
        driver.find_element_by_xpath("//*[@id='favorites_tab']").click()
        time.sleep(2)
        driver.find_element_by_link_text("Reports - View / Run").click()
        time.sleep(5)
        driver.switch_to.frame("gsft_main")
        driver.find_element_by_xpath("//*[@id='li_group_reports_tab']/span").click()
        time.sleep(2)
        driver.find_element_by_link_text("DailyCR_Report_DM").click()
        time.sleep(10)
        element = driver.find_element_by_xpath("/html/body/div[2]/table/tbody/tr/td/div[1]/div/span/div/div[2]/table/tbody/tr/td/div/table/thead/tr/th[3]/span/a")
        actionChains = ActionChains(driver)
        actionChains.context_click(element).perform()
        ActionChains(driver).send_keys(Keys.ARROW_DOWN).perform()
        ActionChains(driver).send_keys(Keys.ARROW_DOWN).perform()
        ActionChains(driver).send_keys(Keys.ARROW_DOWN).perform()
        ActionChains(driver).send_keys(Keys.ARROW_DOWN).perform()
        ActionChains(driver).send_keys(Keys.ARROW_DOWN).perform()
        ActionChains(driver).send_keys(Keys.ARROW_DOWN).perform()
        ActionChains(driver).send_keys(Keys.ENTER).perform()
        ActionChains(driver).send_keys(Keys.ENTER).perform()
        time.sleep(5)
        driver.find_element_by_xpath("//*[@id='download_button']").click()
        time.sleep(2)
        driver.quit()
    except:
        driver.quit()
        print("Re-initializing")
        CHGDownloader()

def emailComplete():
    outlook = win32com.client.Dispatch('Outlook.Application')
    mail_item = outlook.CreateItem(0)
    mail_item.To = 'Email Address'
    mail_item.Cc = '%s' % OnCallName
    mail_item.Subject = "Change Report (%s)" %ReqD
    mail_item.Attachments.Add(Source='C:\\path\\to\\donwloads\\change_request.xlsx')
    body = "
Hi Team,

Please find the changes scheduled today below,
"+DF+"
%s" %HTML
    mail_item.HTMLBody = (body)
    mail_item.Send()

def emailEmpty():
    outlook = win32com.client.Dispatch('Outlook.Application')
    mail_item = outlook.CreateItem(0)
    mail_item.To = 'Email Address'
    mail_item.Cc = '%s' % OnCallName
    mail_item.Subject = "Change Report (%s)" % ReqD
    mail_item.Attachments.Add(Source='C:\\path\\to\\donwloads\\change_request.xlsx')
    body = "
Hi Team,

No changes are scheduled for today.

%s" % HTML
    mail_item.HTMLBody = (body)
    mail_item.Send()

CHGDownloader()
os.chdir("C:\\path\\to\\Downloads")
Sheet = pandas.read_excel("change_request.xlsx", sheet_name=0)
Sheet = Sheet[Sheet["State"] != "Canceled"]
Sheet = Sheet.drop(columns=["Assigned to"])
Sheet = Sheet[Sheet["Short Description / Title"].str.contains(r"SFG|B2B|sfg|b2b")]
DF = Sheet.to_html(index=False)
ReqD = datetime.strftime(datetime.now(), '%m/%d/%y')
Sign = open("C:\\path\\to\\KP Sign.html")
HTML = Sign.read()
OnCall = pandas.read_excel("\\path\\to\\DM_B2B_On_Call.xlsx", sheet_name=0)
OnCallEmail = OnCall['Email'][OnCall[OnCall['Date'] == ReqD].index.tolist()].tolist()
OnCallName = str(OnCallEmail[0])
if Sheet.empty:
    emailEmpty()
else:
    emailComplete()
time.sleep(3)
try:
    os.system("taskkill /f /im  outlook.exe")
except:
    print("No PID found")
os.remove("change_request.xlsx")


import os

import dash
import dash_core_components as dcc
import dash_html_components as html
import plotly.express as px
import pandas as pd
from dash.dependencies import Input, Output
import plotly.graph_objs as go
import pickle as pkl

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

app = dash.Dash(__name__, external_stylesheets=external_stylesheets,
                update_title='Chargement...')
server = app.server

dict_prénoms = pkl.load(open('data/dict_prenoms.pkl', 'rb'))
df_nat = pd.read_feather('data/df_nat.feather')
# Suppression des prénoms trop rares :
masque_prénoms_rares = (
    df_nat.prénoms_s == '_PRENOMS_RARES - ♂') | (df_nat.prénoms_s == '_PRENOMS_RARES - ♀')
# drop des prénoms rares
df_nat = df_nat.drop(df_nat[masque_prénoms_rares].index)

app.layout = html.Div(children=[
    html.H1(children='Naissances France'),

    html.Div(id='sous-titre', children='''
        Fréquence des prénoms de naissance en France de 1900 à nos jours
    '''),
    dcc.Dropdown(
        id='dropdown-prénoms',
        options=dict_prénoms,
        value=[dict_prénoms[888]['value']],
        placeholder="Chercher un prénom",
        multi=True
    ),
    dcc.Graph(
        id='graph-fréquences',
        style={'height': '80vh'},
        config={'displayModeBar': False}
    ),
    html.Div(id='bandeau', children=""),
])

@app.callback(
    Output(component_id='bandeau', component_property='children'),
    [Input(component_id='dropdown-prénoms', component_property='value')]
)
def maj_bandeau(prénoms_selectionnés):
    if len(prénoms_selectionnés) == 0:
        return ""
    else:
        masque = df_nat.prénoms_s == prénoms_selectionnés[0]
        genre = "e" if prénoms_selectionnés[0][-1] == '♀' else ""
        return f"Il y a {df_nat[masque].nombre.sum()} {prénoms_selectionnés[0]} né{genre}s  en France depuis 1900"

@app.callback(
    Output(component_id='graph-fréquences', component_property='figure'),
    [Input(component_id='dropdown-prénoms', component_property='value')]
)
def maj_graph(prénoms_selectionnés):
    fig = go.Figure()
    for prénom_select in prénoms_selectionnés:
        masque = df_nat.prénoms_s == prénom_select
        fig.add_trace(
            go.Scatter(
                x=df_nat[masque].annais,
                y=df_nat[masque].nombre,
                name=prénom_select,
                text=df_nat[masque].rangs,
                mode='markers+lines',
                hovertemplate='Année : %{x}
Nombre : %{y}
Rang : %{text}ème'))
        fig.update_layout(
            xaxis=dict(title="Année de naissance"),
            yaxis=dict(title="Nombre de naissances"),
        )
    return fig

if __name__ == '__main__':
    app.run_server()


import json
import pandas as pd
import requests

from pathlib import Path
from tqdm import tqdm

# -------------
def ConstructRequestUrl(sciName:str) -> str:
  baseUrl = 'http://www.xeno-canto.org/api/2/recordings'
  audioType = 'song'
  country = 'Taiwan'
  return baseUrl + f'?query={sciName}%20cnt:{country}%20type:{audioType}'

def DownloadAudio(speciesCode, url, fileID, q):
  filename = f'{speciesCode}_XC{fileID}_{q}.mp3'
  filePath = Path.cwd().joinpath('audio', 'xeno-canto', f'{filename}')
  res = requests.get(url)
  with open(filePath, 'wb') as f:
    for data in res.iter_content(1024):
      f.write(data)

# -------------
if __name__ == '__main__':
  urlDF = pd.DataFrame(columns=['code', 'audio Url'])

  ## Load species Code
  codeDF = pd.read_csv(Path.cwd().joinpath('assets', 'code.csv'), header=0)
  for _, row in codeDF.iterrows():
    reqUrl = ConstructRequestUrl(row['sci Name'])
    res = requests.get(reqUrl)
    data = json.loads(res.text)

    ### Download and save audio, and save its url
    numberOfRecordings = data['numRecordings']
    recordings = data['recordings']
    for recording in tqdm(
      recordings,
      total=len(recordings),
      desc=f'{"downloading...":>16}',
      bar_format='{l_bar}{bar:32}{r_bar}{bar:-32b}'
    ):
      DownloadAudio(row['code'], recording['file'], recording['id'], recording['q'])
      urlDF = pd.concat(
        [urlDF, pd.DataFrame({
          'code': row['code'],
          'audio Url': recording['file']
        }, index=[0])],
        ignore_index=True
      )
  urlDF.to_csv(Path.cwd().joinpath('assets', 'XC_urls.csv'), header=True, index=False)


import tweepy
import config
from tweepy.auth import OAuthHandler
from time import sleep

client = tweepy.Client(config.Bearer_Token,config.API_KEY,config.API_KEY_SECRET, config.Access_Token, config.Access_Token_Select)

auth = tweepy.OAuth1UserHandler(config.API_KEY, config.API_KEY_SECRET)
auth.set_access_token(config.Access_Token, config.Access_Token_Select)
api = tweepy.API(auth)

url = ''

long_text = f""

# 將長文切成每一tweet不超過250字元的限制（中文字佔2個字元）
def cutit(chunks, url):
    chunks = [long_text[i:i+140] for i in range(0, len(long_text), 140)]
    # 先發布第一則tweet
    tweet1 = api.update_status(chunks[0])
    # 取得第一則tweet的ID
    previous_tweet_id = tweet1.id
    # 用for迴圈接續第一則tweet的下方，發布第二則tweet，以此類推
    for text in chunks[1:]:
        tweet = api.update_status(text, in_reply_to_status_id=previous_tweet_id)
        previous_tweet_id = tweet.id
        sleep(2)
    # 在最後一段tweet貼上新聞網址
    tweet = api.update_status(url,in_reply_to_status_id=previous_tweet_id)

# 手動分行tweet
def selfCut(long_text):
    tweet1=  api.update_status(long_text[0])
    previous_tweet_id = tweet1.id
    for text in long_text[1:]:
        tweet = api.update_status(text, in_reply_to_status_id=previous_tweet_id)
        previous_tweet_id = tweet.id
        sleep(2)
    tweet = api.update_status(url,in_reply_to_status_id=previous_tweet_id)

# execute:
cutit(long_text, url)

# 將每個tweet在後面加上頁數
# for i, chunk in enumerate(chunks):
#     tweet1 = api.update_status(f'{chunk}({i+1}/{len(chunks)})')
#     print(tweet1)

# 驗證twitter的API連線是否成功
# try:
#     api.verify_credentials()
#     print('yes')
# except:
#     print('failed')


import requests
from bs4 import BeautifulSoup
import pandas as pd

URL = "https://www.pepcoding.com/resources/online-java-foundation"
page = requests.get(URL)
soup = BeautifulSoup(page.content, "html.parser")

module_list = soup.find_all("li", class_="classResourceList")

question_id = []
question_name = []
topic_name = []
module_name = []

for module in module_list:
    module_div = module.find("div", class_="collapsible-header")
    mname = module_div.text.lstrip().rstrip()

    topic_list = module.find_all("li", class_="list-item")
    for topic in topic_list:
        topic_href = "https://www.pepcoding.com" + topic.find('a', href=True)['href']
        tname = topic.text.lstrip().rstrip()

        topic_page = requests.get(topic_href)
        topic_soup = BeautifulSoup(topic_page.content, "html.parser")
        question_list = topic_soup.find_all("li", attrs={"resource-type":"ojquestion"})

        count = 0
        for question in question_list:
            qname = question.find('a', href=True).text.lstrip().rstrip()
            qid = question['resource-type-id']

            question_id.append(qid)
            question_name.append(qname)
            topic_name.append(tname)
            module_name.append(mname)
            count += 1
        print(count, "->", tname)

dict = {"ID": question_id, "Question": question_name, "Topic": topic_name, "Module": module_name}
df = pd.DataFrame(dict)

df.to_csv('question_list.csv', index=False)


# -*- coding: utf-8 -*-
#
#   auth
#
# -----------------------------------------------------------------------------

import binascii
import ecdsa
import re
from web3.auto import w3

from neutral.libs.client.hash import HashType, Hasher

# -----------------------------------------------------------------------------

class EcdsaVerifier(object):
    '''Generic ECDSA verifier using python-ecdsa.'''

    def __init__(self, verifying_key):
        assert isinstance(verifying_key, ecdsa.VerifyingKey)
        self.verifying_key = verifying_key

    def verify_bytes(self, sig, data):
        assert isinstance(sig, str), sig
        sig = binascii.unhexlify(str(sig))
        if len(sig) == 65:
            sig = sig[1:]
        elif len(sig) != 64:
            raise ValueError('invalid signature length {}'.format(sig))
        self.verifying_key.verify(sig, data)

    def verify_string(self, sig, text, encoding='utf-8'):
        self.verify_bytes(sig, bytes(text, encoding=encoding))

# -----------------------------------------------------------------------------

class Web3EcdsaSigner(object):
    '''
    Web3 adds a recovery byte to the signature. We have to use this to ensure
    correct extraction of the public key in the backend (since we are not
    transmitting the public key over.
    '''

    def __init__(self, signing_key, hash_type):
        assert isinstance(signing_key, ecdsa.SigningKey)
        assert isinstance(hash_type, HashType)

        self.signing_key = signing_key
        self.private_key = signing_key.to_string()
        self.hash_type = hash_type
        self.hasher = Hasher.create(hash_type)

    def get_hash_type():
        return self.hash_type

    def sign_bytes(self, data):
        return self.sign_hash(self.hasher(data).digest())

    def sign_string(self, text, encoding='utf-8'):
        return self.sign_bytes(bytes(text, encoding=encoding))

    '''
    The backend uses (v + r + s)
    w3.eth.account.signHash returns (r + s + v)
    '''
    def sign_hash(self, data_hash):
        # print('hash ({}}: {}'.format(type(data_hash), data_hash))
        signed = w3.eth.account.signHash(data_hash, private_key=self.private_key)
        signature = re.sub(r'^0x', '', signed.signature.hex())
        signature = signature[-2:] + signature[:-2]
        return signature

# -----------------------------------------------------------------------------

class UserAuthentication(object):
    def __init__(self, signer):
        self.signer = signer
        verifying_key = signer.signing_key.get_verifying_key()
        self.verifier = EcdsaVerifier(verifying_key)

    def gen_authentication_header(self, token_str):
        signature = self.signer.sign_string(token_str)
        print('token: {}'.format(token_str))
        print('signature: {}'.format(signature))
        self.verifier.verify_string(
            signature[2:],  # strip the recovery byte
            token_str
        )
        return [
            ('session-token', token_str),
            ('user-signature', signature),
        ]

# =============================================================================


import numpy as np
import sklearn.metrics
import model.binary_performance as binary_performance

def test_accuracies():
    np.random.seed(20200218)
    vec_size = 50
    true_vals = np.random.randint(2, size=vec_size)
    pred_vals = np.random.random(vec_size)

    print("Testing accuracies...")
    pred_vals_rounded = np.round(pred_vals)
    acc, pos_acc, neg_acc = binary_performance.accuracies(
        true_vals, pred_vals_rounded
    )

    num_pos, num_neg = 0, 0
    num_pos_right, num_neg_right = 0, 0
    for i in range(vec_size):
        if true_vals[i] == 1:
            num_pos += 1
            if pred_vals[i] >= 0.5:
                num_pos_right += 1
        else:
            num_neg += 1
            if pred_vals[i] < 0.5:
                num_neg_right += 1

    print("\tSame result? %s" % all([
        acc == (num_pos_right + num_neg_right) / vec_size,
        pos_acc == num_pos_right / num_pos,
        neg_acc == num_neg_right / num_neg
    ]))

def test_corrected_precision_auprc():
    np.random.seed(20200218)
    vec_size = 10000
    neg_upsample_factor = 5
    estimate_tolerance = 0.02

    def test_single_result(true_vec, pred_vec):
        """
        Tests similarity of precision/recall/auPRC computation to `sklearn`
        library, without any downsampling of negatives
        """
        precis, recall, thresh = \
            binary_performance.estimate_imbalanced_precision_recall(
                true_vec, pred_vec, neg_upsample_factor=1
            )
        precis_score, recall_score = binary_performance.precision_recall_scores(
            precis, recall, thresh
        )
        auprc = sklearn.metrics.auc(recall, precis)
        sk_precis, sk_recall, sk_thresh = \
            sklearn.metrics.precision_recall_curve(true_vec, pred_vec)
        sk_precis_score = sklearn.metrics.precision_score(
            true_vec, np.round(pred_vec)
        )
        sk_recall_score = sklearn.metrics.recall_score(
            true_vec, np.round(pred_vec)
        )
        sk_auprc = sklearn.metrics.auc(sk_recall, sk_precis)
        print("\tSame result? %s" % all([
            np.allclose(precis, sk_precis),
            np.allclose(recall, sk_recall),
            np.allclose(thresh, sk_thresh),
            precis_score == sk_precis_score,
            recall_score == sk_recall_score,
            auprc == sk_auprc
        ]))

    def test_neg_sampling_result(true_vec, pred_vec, neg_upsample_factor):
        """
        Tests that after down-sampling negatives and re-inflating them using
        `neg_upsample_factor`, the precision/recall/auPRC are roughly the same
        as if no down-sampling had occurred.
        """
        # Get results without downsampling negatives
        precis, recall, thresh = \
            binary_performance.estimate_imbalanced_precision_recall(
                true_vec, pred_vec, neg_upsample_factor=1
            )
        precis_score, recall_score = binary_performance.precision_recall_scores(
            precis, recall, thresh
        )
        auprc = sklearn.metrics.auc(recall, precis)

        # Subsample negatives
        pos_mask = true_vals == 1
        neg_mask = true_vals == 0
        sub_mask = np.random.choice(
            [True, False], size=vec_size,
            p=[1 / neg_upsample_factor, 1 - (1 / neg_upsample_factor)]
        )
        keep_mask = pos_mask | (neg_mask & sub_mask)
        c_precis, c_recall, c_thresh = \
            binary_performance.estimate_imbalanced_precision_recall(
                true_vec[keep_mask], pred_vec[keep_mask],
                neg_upsample_factor=neg_upsample_factor
            )
        c_precis_score, c_recall_score = \
            binary_performance.precision_recall_scores(
                c_precis, c_recall, c_thresh
            )
        c_auprc = sklearn.metrics.auc(c_recall, c_precis)
        print("\tAll within %s? %s" % (
            estimate_tolerance,
            all([
                abs(precis_score - c_precis_score) < estimate_tolerance,
                abs(recall_score - c_recall_score) < estimate_tolerance,
                abs(auprc - c_auprc) < estimate_tolerance
            ])
        ))

    true_vals = np.random.choice(2, size=vec_size)

    # Random predictions
    pred_vals = np.random.random(vec_size)

    print("Testing precision/recall/auPRC on random data without correction...")
    test_single_result(true_vals, pred_vals)

    print("Testing precision/recall/auPRC on random data with correction...")
    test_neg_sampling_result(true_vals, pred_vals, 5)

    # Predictions are a bit closer to being correct; make predictions Gaussian
    # noise centered as 0 or 1, and cut off anything outside [0, 1]
    pred_vals = np.empty(vec_size)
    rand = np.random.randn(vec_size) / 2
    pos_mask = true_vals == 1
    pred_vals[pos_mask] = rand[pos_mask] + 1
    neg_mask = true_vals == 0
    pred_vals[neg_mask] = rand[neg_mask]
    pred_vals[pred_vals < 0] = 0
    pred_vals[pred_vals > 1] = 1

    print("Testing precision/recall/auPRC on good data without correction...")
    test_single_result(true_vals, pred_vals)

    print("Testing precision/recall/auPRC on good data with correction...")
    test_neg_sampling_result(true_vals, pred_vals, 5)

class FakeLogger:
    def log_scalar(self, a, b):
        pass

def test_all_metrics_on_different_predictions():
    np.random.seed(20200218)
    batch_size, num_tasks  = 100, 2
    true_vals = np.random.randint(2, size=(batch_size, num_tasks))

    _run = FakeLogger()

    # Make some "perfect" predictions, which are identical to truth
    print("Testing all metrics on some perfect predictions...")
    pred_vals = true_vals
    metrics = binary_performance.compute_performance_metrics(
        true_vals, pred_vals, 1
    )
    binary_performance.log_performance_metrics(metrics, "Perfect", _run)

    # Make some "good" predictions, which are close to truth; make predictions
    # Gaussian noise centered as 0 or 1, and cut off anything outside [0, 1]
    print("Testing all metrics on some good predictions...")
    pred_vals = np.empty((batch_size, num_tasks))
    rand = np.random.randn(batch_size, num_tasks) / 2
    pos_mask = true_vals == 1
    pred_vals[pos_mask] = rand[pos_mask] + 1
    neg_mask = true_vals == 0
    pred_vals[neg_mask] = rand[neg_mask]
    pred_vals[pred_vals < 0] = 0
    pred_vals[pred_vals > 1] = 1
    metrics = binary_performance.compute_performance_metrics(
        true_vals, pred_vals, 1
    )
    binary_performance.log_performance_metrics(metrics, "Good", _run)

    # Make some "bad" predictions, which are just random
    print("Testing all metrics on some bad predictions...")
    pred_vals = np.random.random((batch_size, num_tasks))
    metrics = binary_performance.compute_performance_metrics(
        true_vals, pred_vals, 1
    )
    binary_performance.log_performance_metrics(metrics, "Bad", _run)

if __name__ == "__main__":
    test_accuracies()
    test_corrected_precision_auprc()
    test_all_metrics_on_different_predictions()


import numpy as np

import tinytorch
from tinytorch import init, nn, math
from tinytorch.datasets import mnist, DataLoader

data_train = mnist.train('../dataset/')
data_test = mnist.test('../dataset/')

avg_data = np.average(data_train[0])
std_data = np.std(data_train[0])

data_train[0] = (data_train[0]-avg_data)/std_data
data_test[0] = (data_test[0]-avg_data)/std_data

BATCH_SIZE = 4

train = DataLoader(data_train, BATCH_SIZE, True)
#test = DataLoader(data_test, BATCH_SIZE, True)

input = tinytorch.Tensor.from_numpy(np.empty(784))
label = tinytorch.Tensor.from_numpy(np.empty(10))
seq = tinytorch.nn.Sequential([
	tinytorch.nn.Linear(784, 800, init.Initializer(init.he_normal)),
	tinytorch.nn.ReLU(),
	tinytorch.nn.Linear(800, 10, init.Initializer(init.xavier_normal))
])
loss = tinytorch.nn.SoftmaxCrossEntropyWithLogits()
weight_decay = tinytorch.nn.L2Norm()

import visdom
vis = visdom.Visdom() #for loss visualization

def get_accuracy():
	count = len(data_test[1])
	right_count = 0
	for i in range(len(data_test[1])):
		input.data = data_test[0][i].flatten()
		label = np.argmax(math.softmax(seq(input).data))
		if label == data_test[1][i]:
			right_count += 1
	return(right_count/count)

avg_loss = 0
learning_rate = 0.01
for iter, batch in enumerate(train):
	seq.zero_grad()

	out = 0
	for i in range(BATCH_SIZE):
		input.data = batch[0][i].reshape(784)
		label.data.fill(0.0)
		label.data[batch[1][i]] = 1.0

		out = loss(seq(input), label) #+ 0.01*weight_decay(seq.layer_list[0].weight) + 0.01*weight_decay(seq.layer_list[2].weight)
		out.backward()
		avg_loss += out.data

	for key, value in seq.parameters.items():
		value.data -= learning_rate*value.grad

	if iter % 10 == 0 and iter > 0:
		learning_rate = pow(learning_rate, 1.0001)

	if iter % 100 == 0 and iter > 0:
		avg_loss /= BATCH_SIZE
		avg_loss /= 100.0
		vis.line([avg_loss], [iter], 'avgloss', name='AvgLoss', update='append', opts=dict(title='AvgLoss'))
		avg_loss = 0

	if iter % 1000 == 0 and iter > 0:
		vis.line([get_accuracy()], [iter], 'accuracy', name='Accuracy', update='append', opts=dict(title='Accuracy'))


from flask import Blueprint, jsonify, request, redirect, url_for
from app.models import db, Task, Project, user

project_routes = Blueprint('projects', __name__)

@project_routes.route('/<int:id>')
def get_projects(id):
    #print('111111111111', id)
    projects = Project.query.filter_by(user_id = id)
    return {'projects': [project.project_to_dict() for project in projects]}

@project_routes.route('/tasks/<int:id>')
def get_project_tasks(id):
    tasks = Task.query.filter_by(project_id = id)
    # for task in tasks:
    #     print('--------------', task.task_to_dict(), '----------')
    return {'project_tasks': [task.task_to_dict() for task in tasks]}

@project_routes.route('/new', methods=['POST'])
def new_project():
    #print('###########')
    user_id=request.json['userId']
    color = request.json['color']
    project_name = request.json['projectName']

    project = Project(
        user_id = user_id,
        project_name = project_name,
        color = color,
    )

    db.session.add(project)
    db.session.commit()

    return project.project_to_dict()

@project_routes.route('/edit/<int:id>', methods = ['PUT'])
def edit_task(id):
    project = Project.query.get(id)

    user_id = request.json['userId']
    project_name = request.json['projectName']
    color = request.json['color']

    project.project_name = project_name
    project.color = color

    db.session.add(project)
    db.session.commit()
    return project.project_to_dict()

@project_routes.route('/delete/<int:id>', methods=['DELETE'])
def delete_project(id):
    project = Project.query.get(id)

    db.session.delete(project)
    db.session.commit()

    return project.project_to_dict()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import

from django.core.management.base import BaseCommand, CommandError

from kchart.charts.chartservice import CHART_SERVICES
from kchart.charts.models import AggregateHourlySongChart

class Command(BaseCommand):

    help = 'Updates the specified chart'

    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument('--dry-run', dest='dry_run', action='store_true',
                            help='Do not write chart updates to the database')
        parser.add_argument('--aggregate', dest='aggregate', action='store_true',
                            help='(Re)generate aggregated chart after update')
        parser.add_argument('chart', nargs='*')

    def handle(self, *args, **options):
        for chart in options['chart']:
            if chart not in CHART_SERVICES:
                raise CommandError('Unknown chart: {}'.format(chart))
            service = CHART_SERVICES[chart.lower()]()
            service.fetch_hourly(dry_run=options['dry_run'])
        if options['aggregate']:
            AggregateHourlySongChart.generate(regenerate=True)


from gi.repository import Gtk, Gdk
import git
import indicator
import settings

class GenericWindow(Gtk.Window):

    def __init__(self, builder_path, object_name, handlers, css_path='../data/styles/commit.css'):
        super().__init__()

        self._create_css_provider(css_path)
        self._init_builder(builder_path, handlers)
        window = self._builder.get_object(object_name)
        window.show_all()

    def _create_css_provider(self, css_path):
        css_provider = Gtk.CssProvider()
        css_provider.load_from_path(css_path)
        Gtk.StyleContext.add_provider_for_screen(Gdk.Screen.get_default(), css_provider, Gtk.STYLE_PROVIDER_PRIORITY_APPLICATION)

    def _init_builder(self, builder_path, handles):
        self._builder = Gtk.Builder()
        self._builder.add_from_file(builder_path)
        self._builder.connect_signals(handles)

class SettingsWindow(GenericWindow):
    BUILDER_PATH = '../data/layouts/settings_window.glade'
    OBJECT_NAME = 'SettingsWindow'

    def __init__(self):
        handlers = { 'onSaveClicked': self._save_settings }

        super().__init__(self.BUILDER_PATH, self.OBJECT_NAME, handlers)
        self._load_settings()

    def _load_settings(self):
        dir_entry = self._builder.get_object('DirEntry')
        handle_entry = self._builder.get_object('HandleEntry')
        key_entry = self._builder.get_object('KeyEntry')
        pid_entry = self._builder.get_object('PIDEntry')

        data = settings.load_settings()

        dir_entry.set_text(data['directory'])
        handle_entry.set_text(data['handle'])
        key_entry.set_text(data['key'])
        pid_entry.set_text(data['pid'])

    def _save_settings(self, button):
        dir_entry = self._builder.get_object('DirEntry')
        handle_entry = self._builder.get_object('HandleEntry')
        key_entry = self._builder.get_object('KeyEntry')
        pid_entry = self._builder.get_object('PIDEntry')

        git.set_api_key(dir_entry.get_text(), handle_entry.get_text(), key_entry.get_text())
        git.set_pid(dir_entry.get_text(), pid_entry.get_text())

        settings.save_settings(dir_entry.get_text(), handle_entry.get_text(),\
                               key_entry.get_text(), pid_entry.get_text())

class CommitWindow(GenericWindow):
    BUILDER_PATH = '../data/layouts/commit_window.glade'
    OBJECT_NAME = 'CommitWindow'

    def __init__(self, timer_item):
        handlers = { 'onCommitClicked' : self._commit }
        self._timer_item = timer_item
        super().__init__(self.BUILDER_PATH, self.OBJECT_NAME, handlers)

    def _commit(self, button):
        time_formatted = self._get_time_formatted()

        subject_entry = self._builder.get_object('SubjectEntry')
        handles_entry = self._builder.get_object('HandlesEntry')

        git.commit('/home/kuba/Work/Study/University/10_semester/vcs/python-test', \
                   subject_entry.get_text(), time_formatted, handles_entry.get_text())

    def _get_time_formatted(self):
        time = self._timer_item.current_time
        self._timer_item.reset_timer()

        minutes, _ = divmod(time, 60)
        hours, minutes = divmod(minutes, 60)

        return "%d h %d min" % (hours, minutes)

class OverviewWindow(GenericWindow):
    BUILDER_PATH = '../data/layouts/overview_window.glade'
    OBJECT_NAME = 'OverviewWindow'

    def __init__(self):
        handlers = {'onShaEntered' : self._show_time}
        super().__init__(self.BUILDER_PATH, self.OBJECT_NAME, handlers)

    def _show_time(self):
        commit_label = self._builder.get_object('CommitLabel')
        sha_entry = self._builder.get_object('ShaEntry')
        data = settings.load_settings()
        git_dir = data['directory']
        commit_text = git.show(git_dir, sha_entry.get_text())
        commit_label.set_text(commit_text)


import os
import urllib

from google.appengine.api import users
from google.appengine.ext import ndb

# For Udacity blog
from google.appengine.ext import db

import jinja2
import webapp2

PROJECT_DIR = os.path.dirname(__file__);

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(PROJECT_DIR),
    extensions=['jinja2.ext.autoescape'])

# Following Udacity handler stuff for templating
class Handler(webapp2.RequestHandler):
  def write(self, *a, **kw):
      self.response.out.write(*a, **kw)

  def render_str(self, template, **params):
      t = JINJA_ENVIRONMENT.get_template(template)
      return t.render(params)

  def render(self, template, **kw):
      self.write(self.render_str(template, **kw))

# Misc....
class index(webapp2.RequestHandler):
  def get(self):
      template = JINJA_ENVIRONMENT.get_template('index.html')
      self.response.write(template.render())

class banana(webapp2.RequestHandler):
  def get(self):
      template = JINJA_ENVIRONMENT.get_template('banana.html')
      self.response.write(template.render())

class about(webapp2.RequestHandler):
  def get(self):
      template = JINJA_ENVIRONMENT.get_template('about.html')
      self.response.write(template.render())

class resume(webapp2.RequestHandler):
  def get(self):
      template = JINJA_ENVIRONMENT.get_template('resume.html')
      self.response.write(template.render())

class contact(webapp2.RequestHandler):
  def get(self):
      template = JINJA_ENVIRONMENT.get_template('contact.html')
      self.response.write(template.render())

class error404(webapp2.RequestHandler):
  def get(self):
      template = JINJA_ENVIRONMENT.get_template('404.html')
      self.response.write(template.render())

# ------ Udacity HW3: Basic blog -------- #

class blogPostingDB(db.Model):
  # Datastore stuff
  subject = db.StringProperty(required = True)
  content = db.TextProperty(required = True)
  created = db.DateTimeProperty(auto_now_add = True) # Just a time stamp

class blog(Handler):
  # Blog front page
  def get(self):
      # Query all blog posts in database
      blogPosts = db.GqlQuery("SELECT * FROM blogPostingDB ORDER BY created DESC ")
      self.render('blog.html', blogPosts = blogPosts)

class newpost(Handler):
  # Interface to make new blog posts
  def render_newpost(self, subject="", content="", error=""):
      self.render('newpost.html', subject=subject, content=content, error=error)

  def get(self):
      self.render_newpost()

  def post(self):
      subject = self.request.get("subject")
      content = self.request.get("content")

      if subject and content:
        # Successful post
        bp = blogPostingDB(subject = subject, content = content)
        bp_key = bp.put() # Store data in database
        # Redirect to blog posting
        self.redirect('/blog/%d' % bp_key.id())
      else:
        # Error handling
        error = "Need both a subject and content"
        self.render_newpost(subject, content, error)

class permalink(blog):
  # for handling permalink to new blog posting
  def get(self, blog_id):
      s = blogPostingDB.get_by_id(int(blog_id))
      self.render("blog.html", blogPosts=[s])

# App definition stuff
app = webapp2.WSGIApplication([('/',        index),
                               ('/blog',    blog),
                               ('/blog/newpost', newpost),
                               ('/blog/(\d+)',   permalink),
                               ('/index',   index),
                               ('/home',    index),
                               ('/banana',  banana),
                               ('/about',   about),
                               ('/resume',  resume),
                               ('/contact', contact),
                               ('/404',     error404)
                              ],
                              debug=True)


from flask import Blueprint, request, make_response, current_app
from werkzeug.exceptions import BadRequest

from src.transactions.models import Transaction
from src.transactions.parser import parse_storage

import json

def create_blueprint():
    bp = Blueprint('transactions', __name__, url_prefix='/transactions')
    return bp

mod_transactions = create_blueprint()

@mod_transactions.route('/load', methods=['POST'])
def load():
    def get_transactions():
        with open('./transactions.json', 'r') as file:
            return json.loads(file.read())

    try:
        jdata = request.get_json()
        transactions = jdata['transactions']
    except (KeyError, BadRequest) as e:
        # we load the default data if the transactions were not provided.
        transactions = get_transactions()

    current_app.storage = parse_storage(transactions)

    return make_response('OK'), 200

@mod_transactions.route('/get_sequence', methods=['POST'])
def get_sequence():
    jdata = request.get_json()
    try:
        jtransaction = jdata['transaction']
    except KeyError:
        raise BadRequest("The Transaction was not provided")

    try:
        storage = current_app.storage
    except AttributeError:
        raise BadRequest("The Storage was not loaded")

    transaction = Transaction(**jtransaction)
    sequence = storage.get_sequence(transaction)

    response = make_response(str(sequence))
    response.mimetype = 'application/json'
    return response, 200


#Import Libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

#Read and understand data
d2=pd.read_csv('realest.csv')
d2.info()
d2.hist(bins=50,figsize=(20,15))
corr_matrix=d2.corr()
corr_matrix["Price"]

#Clean data
m1 = d2["Space"].median()
d2["Space"].fillna(m1, inplace=True)
m2 = d2["Lot"].median()
d2["Lot"].fillna(m2, inplace=True)
m3 = d2["Tax"].median()
d2["Tax"].fillna(m3, inplace=True)

#Create test set and training set
from sklearn.model_selection import train_test_split
tr,te=train_test_split(d2, test_size=0.2, random_state=42)

#Assign Variables
x1=np.array(tr["Space"])
x2=np.array(tr["Room"])
x3=np.array(tr["Bathroom"])
x4=np.array(tr["Garage"])
y=np.array(tr["Price"])
m=len(x1)

#Feature Normalization
x1_m=np.mean(x1)
x1_s=np.std(x1)
x2_m=np.mean(x2)
x2_s=np.std(x2)
x3_m=np.mean(x3)
x3_s=np.std(x3)
x4_m=np.mean(x4)
x4_s=np.std(x4)
y_m=np.mean(y)
y_s=np.std(y)
x1=(x1-x1_m)/x1_s
x2=(x2-x2_m)/x2_s
x3=(x3-x3_m)/x3_s
x4=(x4-x4_m)/x4_s
y=(y-y_m)/y_s

#Visualize Variables
plt.scatter(x1,y)
plt.xlabel("Space")
plt.show()
plt.scatter(x2,y)
plt.xlabel("Room")
plt.show()
plt.scatter(x3,y)
plt.xlabel("Bathroom")
plt.show()
plt.scatter(x4,y)
plt.xlabel("Garage")
plt.show()

#Apply Gradient Descent and Calculate Cost
x0=np.ones(m)
xt=np.c_[x0,x1,x2,x3,x4]
thetas=np.array([0,0,0,0,0])
h=np.dot(xt,thetas)
thetas=thetas-(0.1/m)*np.dot((h-y),xt)
cost=(np.sum((h-y)**2))/(2*m)
cost1=[]
l=[]
for i in range(15000):
 thetas=thetas-(0.01/m)*np.dot((h-y),xt)
 h = np.dot(xt,thetas)
 cost1.append((np.sum((h-y)**2))/(2*m))
 l.append(i)

#Visualize Result
plt.plot(l,cost1)
plt.show()
plt.scatter(x1,y,marker="o")
plt.scatter(x2,y,marker="o")
plt.plot(x1,h)
plt.plot(x2,h)
plt.show()

yreal=(y*y_s)+y_m
hp=np.dot(xt,thetas)
hreal=(hp*np.std(hp))+np.mean(hp)
cost=(np.sum((hreal-yreal)**2))/(2*m)
realc=np.sqrt(cost)

#using scikit-learn
from sklearn.linear_model import LinearRegression
lin_reg = LinearRegression()
lin_reg.fit(xt,y)

from sklearn.metrics import mean_squared_error
y_real=(y*y_s)+y_m
h_p = lin_reg.predict(xt)
h_real=(h_p*np.std(h_p))+np.mean(h_p)
lin_mse = mean_squared_error(y_real, h_real)
lin_rmse = np.sqrt(lin_mse)
lin_rmse

Linear Regression/Realestate Price Prediction/Real Estate Price Prediction.py


from bs4 import BeautifulSoup
import requests
from selenium import webdriver
import time

def get_list_of_songs(url):
    options = webdriver.ChromeOptions()
    driver = webdriver.Chrome(options=options)
    driver.get(url)
    time.sleep(5) # wait a second for <div id="root"> to be fully loaded
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    driver.quit()
    music_links = soup.findAll("div", {"class": "col1"})
    song_titles = []
    for link in music_links:
        song_title = link.findChild("music-link")['title']
        song_titles.append(song_title)
    return song_titles

# playlist_url = "https://music.amazon.com/playlists/B07WV4DNDR"
# get_list_of_songs(playlist_url)


""""
pip install -r requirements.txt
to run file
set FLASK_APP=app.py
flask run
"""

from flask import Flask, render_template, request
from PIL import Image

import pytesseract

UPLOAD_FOLDER = '/static/uploads/'

# allow files of a specific type
ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg'])

def read_img(img):
    # pytesseract.pytesseract.tesseract_cmd = '/app/.apt/usr/bin/tesseract'
    # /app/.apt/usr/bin/tesseract
    pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
    text = pytesseract.image_to_string(Image.open(img))
    return(text)

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

app = Flask(__name__, template_folder='templates')

@app.route('/', methods=['GET', 'POST'])
def main():
    if request.method == 'GET':
        return render_template("index.html")
    if request.method == 'POST':
        if 'file' not in request.files:
            return render_template('index.html', msg='1')
        file = request.files['file']
        # if no file is selected
        if file.filename == '':
            return render_template('index.html', msg='2')

        if file and allowed_file(file.filename):

            # call the OCR function on it
            extracted_text = read_img(file)

            # extract the text and display it
            return render_template('index.html',
                                   msg='true',
                                   extracted_text=extracted_text)

if __name__ == '__main__':
    app.run(debug=True)