import cv2
import os
import numpy as np
from random import shuffle
from tqdm import tqdm
import tflearn
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.estimator import regression
import pandas as pd
from csv import writer
import random

import tensorflow as tf

Normal_TRAIN_DIR = r'/content/drive/My Drive/DataSet/train/train/NORMAL'
Corona_TRAIN_DIR = r'/content/drive/My Drive/DataSet/train/train/COVID19 AND PNEUMONIA'
TEST_DIR = r'/content/drive/My Drive/DataSet/test/test'
IMG_SIZE = 60
LR = 0.001
MODEL_NAME = 'project_NN_CNN'

'''Helper functions for Augmentation'''

'''Labelling the dataset'''

def label_img(img):
    word_label = img

    if 'IM' in word_label:
        return [1, 0]
    else:
        return [0, 1]

def rotation(img, angle):
    angle = int(random.uniform(-angle, angle))
    h, w = img.shape[:2]
    M = cv2.getRotationMatrix2D((int(w / 2), int(h / 2)), angle, 1)
    img = cv2.warpAffine(img, M, (w, h))
    return img

'''Creating the training data and Preprocessing it'''

def create_train_data():
    training_data = []
    # Reading and preprocessing normal training dataset
    for img in tqdm(os.listdir(Normal_TRAIN_DIR)):
        # labeling the images
        label = label_img(img)

        path = os.path.join(Normal_TRAIN_DIR, img)

        img = cv2.imread(path, 0)
        img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))
        normalizedImg = np.zeros((60, 60))
        img = cv2.normalize(img, normalizedImg, 0, 255, cv2.NORM_MINMAX)

        training_data.append([np.array(img), np.array(label)])

        imgaug = cv2.flip(img, 1)
        training_data.append([np.array(imgaug), np.array(label)])

        imgaug2 = rotation(img, 30)
        training_data.append([np.array(imgaug2), np.array(label)])

    # Reading and preprocessing Covid training dataset

    for img2 in tqdm(os.listdir(Corona_TRAIN_DIR)):
        # labeling the images
        label = label_img(img2)

        path = os.path.join(Corona_TRAIN_DIR, img2)

        img2 = cv2.imread(path, 0)
        img2 = cv2.resize(img2, (IMG_SIZE, IMG_SIZE))
        normalizedImg = np.zeros((60, 60))
        img2 = cv2.normalize(img2, normalizedImg, 0, 255, cv2.NORM_MINMAX)

        training_data.append([np.array(img2), np.array(label)])

    shuffle(training_data)

    # saving Trained dataset
    np.save('train_data.npy', training_data)
    return training_data

'''Processing the given test data'''

def process_test_data():
    testing_data = []
    for img in tqdm(os.listdir(TEST_DIR)):
        path = os.path.join(TEST_DIR, img)

        img_num = img
        img = cv2.imread(path, 0)
        img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))
        normalizedImg = np.zeros((60, 60))
        img = cv2.normalize(img, normalizedImg, 0, 255, cv2.NORM_MINMAX)
        testing_data.append([np.array(img), img_num])

    # saving testing Dataset
    np.save('test_data.npy', testing_data)
    return testing_data

'''Running the training and the testing in the dataset'''
if (os.path.exists('train_data.npy')):
    train_data = np.load('train_data.npy', allow_pickle=True)

else:
    train_data = create_train_data()

if (os.path.exists('test_data.npy')):
    test_data = np.load('test_data.npy', allow_pickle=True)

else:
    test_data = process_test_data()

# Model

tf.reset_default_graph()
convnet = input_data(shape=[None, IMG_SIZE, IMG_SIZE, 1], name='input')

convnet = conv_2d(convnet, 64, 3, activation='relu')
convnet = max_pool_2d(convnet, 3)

convnet = conv_2d(convnet, 128, 3, activation='relu')
convnet = max_pool_2d(convnet, 3)

convnet = conv_2d(convnet, 256, 3, activation='relu')
convnet = max_pool_2d(convnet, 3)

convnet = conv_2d(convnet, 512, 3, activation='relu')
convnet = max_pool_2d(convnet, 3)

convnet = conv_2d(convnet, 256, 3, activation='relu')
convnet = max_pool_2d(convnet, 3)

convnet = conv_2d(convnet, 128, 3, activation='relu')
convnet = max_pool_2d(convnet, 3)

convnet = conv_2d(convnet, 64, 3, activation='relu')
convnet = max_pool_2d(convnet, 3)

convnet = fully_connected(convnet, 1024, activation='relu')
convnet = fully_connected(convnet, 2, activation='softmax')

convnet = regression(convnet, optimizer='SGD', learning_rate=LR,
                     loss='categorical_crossentropy', name='targets')

model = tflearn.DNN(convnet, tensorboard_dir='log')

train = train_data
test = test_data

# splitting test and train dataset into featues and labels
# X-Features & Y-Labels

X_train = np.array([i[0] for i in train]).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
y_train = [i[1] for i in train]

X_test = np.array([i[0] for i in test]).reshape(-1, IMG_SIZE, IMG_SIZE, 1)
y_test = [i[1] for i in test]

'''Fitting the data into our model'''

if (os.path.exists('model.tfl.meta')):
    model.load('./model.tfl')
else:

    model.fit({'input': X_train}, {'targets': y_train}, n_epoch=20,
              snapshot_step=200, show_metric=True, run_id=MODEL_NAME)
    model.save('model.tfl')

if (os.path.exists('test_data.npy')):
    test_data = np.load('test_data.npy', allow_pickle=True)
else:
    test_data = process_test_data()

for num, data in enumerate(test_data):
    # normal: [1, 0]
    # corona: [0, 1]

    # img_num is image label and image_data is image values
    img_num = data[1]
    img_data = data[0]

    data = img_data.reshape(IMG_SIZE, IMG_SIZE, 1)

    model_out = model.predict([data])[0]

    if np.argmax(model_out) == 0:
        classy = '0'
    else:
        classy = '1'

    list_of_elem = [img_num, classy]

    # writing Results

    with open('Submit.csv', 'a+', newline='') as write_obj:
        # Create a writer object from csv module
        csv_writer = writer(write_obj)
        # Add contents of list as last row in the csv file
        csv_writer.writerow(list_of_elem)
    list_of_elem.clear()

my own archeticture 0.78/tf_cnn_model.py


#!/bin/python3

import math, sympy
from sympy import *
from decimal import Decimal
from prettytable import PrettyTable

def f(x, expr):
    fx = eval(expr)
    return fx

def muller(x0, x1, exp, tolerance, iterations):
    table = PrettyTable(['Iteration','X1', 'X2', 'X3', 'Error Absoluto'])
    x2 = (x1 - x0)/2.0 # We get the third value using bisection
    fx0 = f(x0, exp)
    fx1 = f(x1, exp)
    fx2 = f(x2, exp)
    root = 0
    if fx0 == 0:
        root = x0
    elif fx1 == 0:
        root = x1
    if fx2 == 0:
        root = x2
    else:
        cont = 0
        error = tolerance + 1
        table.add_row([cont, x0, x1, x2, 'Doesnt exist'])

        while cont < iterations and error > tolerance:
            h0 = x1 - x0
            h1 = x2 - x1
            if (h0 == 0) | (h1 == 0):
                print ("h0 or h1 became 0.")
                break
            delta0 = (fx1 - fx0)/h0
            delta1 = (fx2 - fx1)/h1
            if h1 - h0 == 0:
                print ("h1 - h0 became 0.")
                break
            a = (delta1 - delta0)/(h1 - h0)
            b = a*h1 + delta1
            c = fx2
            #Solving the quadratic equation
            den = 1
            if b < 0:
                den = b - math.sqrt(b**2 - 4*a*c)
            else:
                den = b + math.sqrt(b**2 - 4*a*c)
            x3 = x2 + (-2*c)/den

            error = abs(x3 - x2) # Abs error
            cont += 1
            table.add_row([cont, x1, x2, x3, '%.2E' % Decimal(str(error))])
            x0 = x1
            x1 = x2
            x2 = x3
            fx0 = fx1
            fx1 = fx2
            fx2 = f(x3, exp)

        if error < tolerance:
            root = (x3, '%.2E' % Decimal(str(error)))
        else:
            if (cont == iterations):
                print ("The method failed.")
                root = (None, cont)
                return(2, "The method could not find a root with the required precision after the selected number of iterations."
                        + " You may want to choose a different starting value or a bigger number of iterations. You may also be finding"
                        + " a multiple root.")
            else:
                root = (x3, '%.2E' % Decimal(str(error)))

        print(table)
    return(0, table)

#print(muller(0.0, 1.0, "log(x+2)",0.5e-7, 100))

#print(stephensen(6.5, "(2*x)**(1/2)+3-x",0.5e-11, 20))
#print(muller(0.0, 1.0, "exp(-x)-x",0.5e-11, 100))
print(muller(0.5, 1.0, "log(sin(x)**2 + 1) - 1/2",0.5e-11, 100))


import torch
from torch import nn
from torchvision import models
from torchvision import transforms
from torch.nn import functional as F

from .evaluator_backend import EvaluatorBackend

class ResNet18(EvaluatorBackend):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.dev = "cuda:0"
        self._model = models.resnet18()
        self._model.fc = torch.nn.Linear(512, 1)
        self._model.load_state_dict(torch.load(str(self.trained_model),
                                               map_location='cpu'))
        self._model.eval()
        self._model.to(self.dev)

        self.img_size = 256

        try:
            self._style_model = nn.Linear(512, 1)
            # self._style_model = nn.Sequential(nn.Linear(512, 1), nn.Sigmoid())
            pth = '.'.join(str(self.trained_model).split('.')[:-1]) + '-latent.pt'
            print(f'latent model: {pth}')
            self._style_model.load_state_dict(torch.load(pth, map_location='cpu'))
            self._style_model.eval()
            self._style_model.to(self.dev)
            self._latent_model_available = True
        except:
            self._latent_model_available = False
            print('no latent model available')

        mean = [0.485, 0.456, 0.406]
        std = [0.229, 0.224, 0.225]
        self.normalize = transforms.Normalize(mean, std)
        self.multiplier = 1.
        # self.multiplier = 0.25

        for mod in self._model.modules():
            if hasattr(mod, 'inplace'):
                mod.inplace = False

    def rate_image_batch(self, image_batch: torch.Tensor, enable_grad: bool = False) -> torch.Tensor:
        torch.cuda.empty_cache()
        if enable_grad:
            image_batch = F.interpolate(image_batch, size=self.img_size)
            return self._model(self.normalize(image_batch))
        else:
            with torch.no_grad():
                image_batch = F.interpolate(image_batch, size=self.img_size)
                return torch.tanh(self.multiplier * self._model(self.normalize(image_batch)))

    def rate_style_batch(self, style_batch: torch.Tensor, enable_grad: bool = False) -> torch.Tensor:
        if not self._latent_model_available:
            return None
        if enable_grad:
            return self._style_model(style_batch)
        else:
            with torch.no_grad():
                return self._style_model(style_batch)


from django import forms

from inventory.models import *

class ProductForm(forms.ModelForm):
    class Meta:
        model = Product
        fields = '__all__'

class SupplierForm(forms.ModelForm):
    class Meta:
        model = Supplier
        fields = '__all__'

class TransactionForm(forms.ModelForm):
    class Meta:
        model = Transaction
        fields = '__all__'

class QueryForm(forms.Form):
    query = forms.CharField(required=False)
    query_by = forms.CharField()
    query_limit = forms.IntegerField(initial=100, required=False)
    order_by = forms.CharField(empty_value='pk')
    order_type = forms.ChoiceField(choices=[
        ('desc', 'desc'),
        ('ascn', 'ascn')
    ], required=False)

class TransactionQueryForm(forms.Form):
    query_by = forms.CharField()
    query_by_suffix = forms.CharField(required=False, empty_value='icontains')
    query = forms.CharField(required=False)
    query_limit = forms.IntegerField(initial=100, required=False)
    order_by = forms.CharField(empty_value='created_on')
    order_type = forms.ChoiceField(choices=[
        ('desc', 'desc'),
        ('ascn', 'ascn')
    ], required=False)


from pyspark.sql import SparkSession

spark = SparkSession \
    .builder \
    .appName("Python Spark regression example") \
    .config("spark.some.config.option", "some-value") \
    .getOrCreate()

# from PySparkAudit import dtypes_class, hist_plot, bar_plot, freq_items,feature_len
# from PySparkAudit import dataset_summary, rates, trend_plot

# path = '/home/feng/Desktop'

from AutoFeatures import AutoFeatures

# load dataset
data = spark.read.csv(path='../data/credit_example.csv',
                      sep=',', encoding='UTF-8', comment=None, header=True, inferSchema=True)
data = data.fillna(0)

print(data.toPandas().head(5))

indexCol = ['SK_ID_CURR']
labelCol = 'TARGET'

task = 'classification'

Fs = AutoFeatures()

# correlation selector
to_drop = Fs.corr_selector(data, index_col=indexCol, label_col=labelCol,
                           corr_thold=0.9, method="pearson", rotation=True,
                           display=False, tracking=False, cat_num=2)
print('corr_selector::{}'.format(to_drop))

# essential selector (included: missing selector, unique selector, correlation selector)
to_drop = Fs.essential_drop(data, index_col=indexCol, label_col=labelCol,
                            missing_thold=0.6, corr_thold=0.9, method="pearson", rotation=True,
                            display=True, tracking=True, cat_num=2)
print('essential_drop::{}'.format(to_drop))

# ensemble selector (ensemble selector is based on essential selector.)
to_drop = Fs.ensemble_drop(data, index_col=indexCol, label_col=labelCol, task=task, tracking=True)
print('ensemble_drop::{}'.format(to_drop))


import os
import logging

from urllib import request

from fades import helpers

logger = logging.getLogger(__name__)

PIP_INSTALLER = "https://bootstrap.pypa.io/get-pip.py"

class PipManager():
    """A manager for all PIP related actions."""
    def __init__(self, env_bin_path, pip_installed=False):
        self.env_bin_path = env_bin_path
        self.pip_installed = pip_installed
        self.pip_exe = os.path.join(self.env_bin_path, "pip")
        basedir = helpers.get_basedir()
        self.pip_installer_fname = os.path.join(basedir, "get-pip.py")

    def install(self, dependency):
        """Install a new dependency."""
        if not self.pip_installed:
            logger.info("Need to install a dependency with pip, but no builtin, "
                        "doing it manually (just wait a little, all should go well)")
            self._brute_force_install_pip()

        str_dep = str(dependency)
        args = [self.pip_exe, "install", str_dep]
        logger.info("Installing dependency: %s", str_dep)
        try:
            helpers.logged_exec(args)
        except Exception as error:
            logger.exception("Error installing %s: %s", str_dep, error)
            exit()

    def get_version(self, dependency):
        """Returns the installed version parsing the output of 'pip show'."""
        logger.debug("getting installed version for %s", dependency)
        stdout = helpers.logged_exec([self.pip_exe, "show", dependency])
        version = [line for line in stdout if line.startswith('Version:')]
        if len(version) == 1:
            version = version[0].strip().split()[1]
            logger.debug("Installed version of %s is: %s", dependency, version)
            return version
        else:
            logger.error('Fades is having problems getting the installed version. '
                         'Run with -v or check the logs for details')
            return ''

    def _brute_force_install_pip(self):
        """A brute force install of pip itself."""
        if os.path.exists(self.pip_installer_fname):
            logger.debug("Using pip installer from %r", self.pip_installer_fname)
        else:
            logger.debug(
                "Installer for pip not found in %r, downloading it", self.pip_installer_fname)
            u = request.urlopen(PIP_INSTALLER)
            with open(self.pip_installer_fname, 'wb') as fh:
                fh.write(u.read())

        logger.debug("Installing PIP manually in the virtualenv")
        python_exe = os.path.join(self.env_bin_path, "python")
        helpers.logged_exec([python_exe, self.pip_installer_fname])
        self.pip_installed = True


#!/usr/bin/python3
import json
import redis
import time

class RempliApp():
    TMT=5

    def get_mod_var_bool(self,mname,vname):
        k='wiio.%s.%s' % (mname,vname)
        val=self.r.get(k)
        if val==None or val!='1':
            return False
        else:
            return True

    def set_mod_var_bool(self,mname,vname,val,tmt=TMT):
        k='wiio.%s.%s' % (mname,vname)
        val=self.r.set(k,1 if val==True else 0)
        if tmt!=None:
            self.r.expire(k,tmt)

    def get_rempli_var(self,rname,vname):
        val=self.r.get('remplis.%s.%s' % (rname,vname))
        return val

    def get_rempli_bool(self,rname,vname):
        val=self.get_rempli_var(rname,vname)
        if val==None or val!='1':
            return False
        else:
            return True

    def get_rempli_int(self,rname,vname):
        val=self.get_rempli_var(rname,vname)
        if val!=None and val.isnumeric():
            return int(val,10)
        else:
            return None

    def set_rempli_var(self,rname,vname,val,tmt=TMT):
        k='remplis.%s.%s' % (rname,vname)
        self.r.set(k,val)
        if tmt!=None:
            self.r.expire(k,tmt)

    def set_rempli_bool(self,rname,vname,val,tmt=TMT):
        self.set_rempli_var(rname,vname,1 if val==True else 0,tmt)

    def add_remplissage(self,name,pump,dst,src):
        self.set_rempli_var(name,'name',name,None)
        self.set_rempli_var(name,'src',src,None)
        self.set_rempli_var(name,'dst',dst,None)
        self.set_rempli_var(name,'pmp',pump,None)

        self.set_rempli_bool(name,'on',False,None)
        self.set_rempli_var(name,'cons_src',3,None)
        self.set_rempli_var(name,'cons_dst',0,None)

    def __init__(self,ip='192.168.3.200',port=6379):
        self.r = redis.Redis(host=ip, port=port, db=0, decode_responses=True)

        # On efface tout et on recree selon initial
        ks=self.r.keys('remplis.*')
        p=self.r.pipeline()
        for k in ks:
            p.delete(k)
        p.execute()

        self.add_remplissage("main_reduit"  , "main"  , "reduit", "main")
        self.add_remplissage("main_paul"    , "paul"  , "paul"  , "main")
        self.add_remplissage("reduit_barbec", "reduit", "barbec", "reduit")

    def tolvl(self,n1,n2,n3):
        if n3==True:
            return 3
        elif n2==True:
            return 2
        elif n1==True:
            return 1
        return 0

    def loop(self):
        ks=self.r.keys('remplis.*.name')
        for k in ks:
            name=self.r.get(k)
            print(name)

            src=self.get_rempli_var(name,'src')
            dst=self.get_rempli_var(name,'dst')
            pmp=self.get_rempli_var(name,'pmp')

            on=self.get_rempli_bool(name,'on')

            cons_src=self.get_rempli_int(name,'cons_src')
            cons_dst=self.get_rempli_int(name,'cons_dst')

            lvl_src=None
            lvl_dst=None

            if on==True:
                cmd=False

                # Si le niveau de la source est donne, on arrete qd trop bas !
                src_too_low=False
                if src!=None:
                    if self.get_mod_var_bool(src,'valid')==False:
                        src_too_low=True
                    else:
                        n1=self.get_mod_var_bool(src,'n1')
                        n2=self.get_mod_var_bool(src,'n2')
                        n3=self.get_mod_var_bool(src,'n3')
                        lvl_src=self.tolvl(n1,n2,n3)

                        if cons_src==1:
                            src_too_low=not n1
                        elif cons_src==2:
                            src_too_low=not n2
                        elif cons_src==3:
                            src_too_low=not n3

                # Sinon, on cherche a atteindre la consigne
                if src_too_low==False:
                    if dst!=None:
                        if self.get_mod_var_bool(dst,'valid')==False:
                            cmd=False
                        else:
                            n1=self.get_mod_var_bool(dst,'n1')
                            n2=self.get_mod_var_bool(dst,'n2')
                            n3=self.get_mod_var_bool(dst,'n3')
                            lvl_dst=self.tolvl(n1,n2,n3)
                            print('%s %s %s' % (n1,n2,n3) )

                            if cons_dst==1:
                                cmd=not n1
                            elif cons_dst==2:
                                cmd=not n2
                            elif cons_dst==3:
                                cmd=not n3
                            else:
                                cmd=False
                    else:
                        cmd=True

                else:
                    cmd=False

                if pmp!=None:
                    #pass
                    self.set_mod_var_bool(pmp,'to_cmd',cmd)

                txt='%-20s: on=%-6s, %s[%s] -> %s[%s], pmp=%s' % (name,on,lvl_src,cons_src,lvl_dst,cons_dst,cmd)
                print(txt)

if __name__=='__main__':
    print('Démarrage de Remplissage...')
    app=RempliApp()

    while True:
        print('_'*40)
        app.loop()
        time.sleep(2)


"""update project model

Revision ID: 4044ab2fae9b
Revises: 1b954f2c765
Create Date: 2015-03-14 14:56:42.488263

"""

# revision identifiers, used by Alembic.
revision = '4044ab2fae9b'
down_revision = '1b954f2c765'

from alembic import op
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('project', sa.Column('budget', sa.String(length=80), nullable=True))
    op.add_column('project', sa.Column('email', sa.String(length=80), nullable=True))
    op.drop_constraint(u'project_project_category_id_fkey', 'project', type_='foreignkey')
    op.drop_constraint(u'project_user_id_fkey', 'project', type_='foreignkey')
    op.drop_constraint(u'project_project_budget_id_fkey', 'project', type_='foreignkey')
    op.drop_column('project', 'skills')
    op.drop_column('project', 'project_category_id')
    op.drop_column('project', 'user_id')
    op.drop_column('project', 'project_budget_id')
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('project', sa.Column('project_budget_id', sa.INTEGER(), autoincrement=False, nullable=True))
    op.add_column('project', sa.Column('user_id', sa.INTEGER(), autoincrement=False, nullable=True))
    op.add_column('project', sa.Column('project_category_id', sa.INTEGER(), autoincrement=False, nullable=True))
    op.add_column('project', sa.Column('skills', sa.VARCHAR(length=80), autoincrement=False, nullable=True))
    op.create_foreign_key(u'project_project_budget_id_fkey', 'project', 'project_budget', ['project_budget_id'], ['id'])
    op.create_foreign_key(u'project_user_id_fkey', 'project', 'user', ['user_id'], ['id'])
    op.create_foreign_key(u'project_project_category_id_fkey', 'project', 'project_category', ['project_category_id'], ['id'])
    op.drop_column('project', 'email')
    op.drop_column('project', 'budget')
    ### end Alembic commands ###


import requests
#from selenium import webdriver
from lxml import html
import json
import time
import csv
def numb_only(text):
    #find styles: $2.0,$2.0kg,$ 2,$ 2.0
    strlist = text.replace("$ ","$").split(' ')
    for s in strlist:
        if ('$' in s and len(s) > 1) or '.' in s or '%' in s:
            return ''.join(list(filter(lambda ch: ch in '0123456789.',s)))

def PLfromJson(dictctgr,jsonList,product_list):
    #list
    # All in Json Type
    n = 0
    jsondata = json.loads(jsonList)
    pr_list = jsondata["Bundles"]
    #get info
    m=len(pr_list)
    categoryl1 = dictctgr['l1name']
    categoryl1id = dictctgr['l1id']
    categoryl2 = dictctgr['l2name']
    categoryl2id = dictctgr['l2id']

    for bk in pr_list:
        pName = bk['Name']
        #Mostly Under Product
        ProductL = bk['Products']
        for ProductNode in ProductL:
            n = n + 1
            try:
                #Name
                pInStock = ProductNode['IsInStock'] #check Stock,directly pass
                if pInStock == False:
                    #print('record {0},{1} No Stock,(╯‵□′)╯︵┻━┻'.format(n,pName))
                    continue

                pPartID = str(ProductNode['Stockcode'])
                pItemID = ProductNode['Barcode']
                pImgAddr = ProductNode['LargeImageFile'] #in addr,switch small,medium,large
                pTitle  = ProductNode['Description']#Brand+name+size,often has

                pTitle = pTitle.replace('
',' ')
                pTitle = pTitle.replace('...',' ')
                pLink = 'https://www.woolworths.com.au/shop/productdetails/{0}/'.format(pPartID)+ProductNode['UrlFriendlyName'] #not sure its working
                pBrand = ProductNode['Brand']
                pSize = ProductNode['PackageSize']
                pUnitPrice = ProductNode['CupString']
                pBrand = ProductNode['Brand']
                #price and disount
                pDscPrice = ProductNode['Price']
                pOrgPrice = ProductNode['WasPrice']

                #find special
                pDscType = "None"
                pDscText = ""
                pPrcSav = "0.00%"

                #Save directly
                if (pDscPrice != pOrgPrice):
                    pPrcSav = '{:.2%}'.format(1-float(pDscPrice)/float(pOrgPrice))

                    if (ProductNode['IsOnSpecial'] == True) and (ProductNode['InstoreIsOnSpecial'] == False):
                        pDscType = "SaveX OL"#Online Only
                        pDscText = pPrcSav+' off(Online Only)'
                    if (ProductNode['IsOnSpecial'] == True) and (ProductNode['InstoreIsOnSpecial'] == True):
                        pDscType = "SaveX"#
                        pDscText = pPrcSav+' off'
                pDscPrice = str(pDscPrice)
                pOrgPrice = str(pOrgPrice)
                #check centre tage
                centretag = ProductNode['CentreTag']['TagContent']
                if centretag != None:
                    selector =  html.fromstring(centretag)
                    pDscText = selector.xpath('//span/text()')[0].strip()
                    #Buy N for X  Example:4 for $20.00
                    if "for" in pDscText:
                        pDscType = "buyNforN"
                        words = pDscText.split(' ')
                        tDscNumber = int(words[0])
                        tDscSum = float(numb_only(pDscText))
                        pDscPrice = str(tDscSum/tDscNumber)
                        pPrcSav = '{:.2%}'.format(1-float(pDscPrice)/float(pOrgPrice))
                    #X% off for N Example:25% off any 6 or more bottles
                    elif "off any" in pDscText:
                        pDscType = "XoffforN"
                        tDscpercent = float(numb_only(pDscText))/100.0
                        pDscPrice = str(float(pDscPrice)*(1.0-tDscpercent))
                        pPrcSav = '{:.2%}'.format(1.0-float(pDscPrice)/float(pOrgPrice))
                    #Dropped Example:Was $19.99 04/08/16
                    elif "Was" in pDscText:
                        pDscType = "Dropped"
                        pOrgPrice = numb_only(pDscText)
                        if float(pOrgPrice)+0.1 > float(pDscPrice):
                            pPrcSav = '{:.2%}'.format(1.0-float(pDscPrice)/float(pOrgPrice))
                        else:
                            #CupPrice reason
                            tCup = ProductNode['CupPrice']
                            pOrgPrice = float(pOrgPrice)*float(pDscPrice)/float(tCup)
                            pPrcSav = '{:.2%}'.format(1.0-float(pDscPrice)/float(pOrgPrice))
                    else:
                        #As known:
                        knowntag = "Variety may vary,\
                                Designs may vary,\
                                No further discounts,\
                                Offer Details Here,\
                                Limit of 3 per order"

                        if pDscText not in knowntag:
                            print("Unexpected tag:{0}--{1}".format(pName,pDscText))
                        pDscText = ""

                    if float(pDscPrice) > float(pOrgPrice):
                        print("Unexpected save:{0}--{1}--{2}".format(pName,pDscPrice,pOrgPrice))

                dictitem = {'l1name': categoryl1,
                            'l1id':categoryl1id,
                            'l2name':categoryl2,
                            'l2id':categoryl2id,
                            'ImgAddr':pImgAddr,
                            'Title':pTitle,
                            'PartID':pPartID,
                            'ItemID':pItemID,
                            'Link':pLink,
                            'Brand':pBrand,
                            'Name':pName,
                            'OrgPrice':pOrgPrice,
                            'Size':pSize,
                            'UnitPrice':pUnitPrice,
                            'DscType':pDscType,
                            'DscPrice':pDscPrice,
                            'DscText':pDscText,
                            'PrcSav':pPrcSav}

                product_list.append(dictitem)
            except Exception as e:
                print(pName,pDscText)
                print('record {0},error:{1},(╯‵□′)╯︵┻━┻'.format(n,e))

            print(' '*80,end='\r')
            print(categoryl2,':','☺'*n,end='\r')
    #print('----Done List')
    return n

def EntryByPage():

    totaltimestart = time.time()
    totalitem = 0

    urllines = []
    with open('wwsall\\wwsurll2.csv', 'r', newline='') as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:
            urllines.append(row)
    #start seesion
    urlbase = "https://www.woolworths.com.au"

    timestr = time.strftime("%y%m%d", time.localtime())
    itemfilename = 'wwsall\\wwsitem_{0}.csv'.format(timestr)
    with open(itemfilename, 'w', newline='',encoding='utf-8') as csvfile:
        fieldnames = ['l1name','l1id','l2name','l2id',
                    'ImgAddr','Title','PartID','ItemID',
                    'Link','Brand','Name','OrgPrice','Size','UnitPrice',
                    'DscType','DscPrice','DscText','PrcSav']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()

    for row in urllines:
        productlist = []
        mSession = requests.Session()
        urll2 = urlbase+row["l2url"]
        #into page
        res1 = mSession.get(urll2)
        if res1.status_code != 200 :
            print('Get Main page fail---RES.CODE：{0}'.format(res1.status_code))
            return
        #print("front page pass")
        '''
        #pie category (no essential)
        CategoryUrl = 'https://www.woolworths.com.au/apis/ui/PiesCategoriesWithSpecials/'
        res2 = mSession.get(CategoryUrl)

        if res2.status_code != 200 :
            print('Get Category fail RES.CODE：{0}'.format(res2.status_code))
            return
        print("pie pass")
        #3 SeoMetatags(not essential)
        params = {
            "FormatObject": "{}",
            "IsSpecial": False,
            "PageType": "Home",
            "UrlPath": row["l2url"],
        }

        res3 = mSession.post('https://www.woolworths.com.au/apis/ui/SeoMetatags',data=params)

        if res3.status_code != 200 :
            print('Get Seo fail---RES.CODE：{0}'.format(res3.status_code))
            return
        print("Seo pass")
        '''
        #products get product. First Page for total number
        #payroll Example
        #post https://www.woolworths.com.au/apis/ui/browse/category
        '''
        categoryId:"1_22C1314"
        filters:[]
        formatObject:"{"name":"Chips & Wedges"}"
        isBundle:false
        isMobile:false
        isSpecial:false
        location:"/shop/browse/freezer/chips-wedges"
        pageNumber:2
        pageSize:36
        sortType:"TraderRelevance"
        url:"/shop/browse/freezer/chips-wedges?pageNumber=2"
        '''
        #SEARCH OPTION
        oPageSize = 36 #max 36 in test, wws use 24 as default

        params = {"categoryId":row["l2id"],
            "pageNumber":1,
            "pageSize":oPageSize,
            "sortType":"TraderRelevance",
            "url":row["l2url"],
            "location":row["l2url"],
            "formatObject":"{\"name\":\""+row["l2name"]+"\"}",
            "isSpecial":False,
            "isBundle":False,
            "isMobile":False,
            "filters":None
        }

        res4 = mSession.post('https://www.woolworths.com.au/apis/ui/browse/category',data=params)
        if res4.status_code != 200 :
            print('Get page1 fail---RES.CODE：{0}'.format(res4.status_code))
            return
        #print('Get page1 done')
        hdoc = res4.text
        #load Json Count
        jsonCount = json.loads(hdoc)
        ctCount = jsonCount["TotalRecordCount"]
        if ctCount <= 0:
            continue
        #print('Count result---{0} items'.format(ctCount))
        pageMax = int((ctCount-1)/oPageSize) + 1
        #Parse Productlist
        totalitem += PLfromJson(row,hdoc,productlist)

        for pageNum in range(2,pageMax+1):# link and parse
            #linking
            params = {"categoryId":row["l2id"],
                "pageNumber":pageNum,
                "pageSize":oPageSize,
                "sortType":"TraderRelevance",
                "url":row["l2url"]+"?pageNumber={0}".format(pageNum),
                "location":row["l2url"],
                "formatObject":"{\"name\":\""+row["l2name"]+"\"}",
                "isSpecial":False,
                "isBundle":False,
                "isMobile":False,
                "filters":None
            }
            res5 = mSession.post('https://www.woolworths.com.au/apis/ui/browse/category',data=params)
            if res5.status_code != 200 :
                print('Get {0}list {1} fail RES.CODE：{2}'.format(row["l2name"],pageNum,res5.status_code))
                continue
            #print('List {0}: Done'.format(pageNum))
            hdoc = res5.text
            #Parse
            totalitem += PLfromJson(row,hdoc,productlist)
        #print('List {0}: Done'.format(row["l2name"]))
        mSession.close()
        with open(itemfilename, 'a+', newline='',encoding='utf-8') as csvfile:
            fieldnames = ['l1name','l1id','l2name','l2id',
                    'ImgAddr','Title','PartID','ItemID',
                    'Link','Brand','Name','OrgPrice','Size','UnitPrice',
                    'DscType','DscPrice','DscText','PrcSav']
            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
            writer.writerows(productlist)

    totaltimeend = time.time()
    costtime = totaltimeend - totaltimestart
    print('SUM ITEM:{0},SPEND：{1}s'.format(totalitem,costtime))
    return

if __name__ == '__main__':
    EntryByPage()
    #input('wait for quit:')


'''
Description: 请填写文件简介
Version: 0.0
Autor: jlx
Date: 2022-08-25 16:00:24
LastEditors: jlx
LastEditTime: 2022-08-25 17:39:57
'''
import re
import json
import requests
import time
from bs4 import BeautifulSoup

# 推荐文章接口
url = "https://api.juejin.cn/recommend_api/v1/article/recommend_all_feed"
# 抓包 获取的参数
info = {"id_type": 2, "client_type": 2608, "sort_type": 200, "cursor": "0", "limit": 20}
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36',
}
#  标记是否有更多数据
has_more = True

# 利用set 取一下重
dataSet = set()
# 存储数据的列表
dataList = []
# 需要的数据 数量
num = int(input("需要多少条文章数据："))

# 根据文章id获取文章的content
def get_content(aid):
    childUrl = f"https://juejin.cn/post/{aid}"
    childResp = requests.get(childUrl, headers=headers)
    childResp.encoding = "utf-8"
    content = childResp.text
    # print(content)

    res = re.findall('mark_content:"(.*?)",display_count:d', content)[0]
    res = re.sub(r'(\\u[\s\S]{4})', lambda x: x.group(1).encode("utf-8").decode("unicode-escape"), res)
    res = res.strip('"')
    return res

# 获取文章数据
def get_data():
    global has_more,num
    resp = requests.post(url, json=info, headers=headers)  # data自动使用json方式发送
    data = json.loads(resp.text)
    info["cursor"] = data["cursor"]
    has_more = data["has_more"]
    for i in data["data"]:
        item = i["item_info"]
        try:  # 去除广告推广内容的报错
            if item["article_id"] and len(dataSet) < num:
                if item["article_id"] in dataSet:
                    continue
                dataSet.add(item["article_id"])
                dic = {}
                content = get_content(item["article_id"])
                # id 封面 标题 简介 创建时间 修改事件 作者 头像 文章内容
                dic["article_id"] = item["article_id"]
                dic["cover_image"] = item["article_info"]["cover_image"]
                dic["title"] = item["article_info"]["title"]
                dic["brief"] = item["article_info"]["brief_content"]
                dic["ctime"] = item["article_info"]["ctime"]
                dic["mtime"] = item["article_info"]["mtime"]
                dic["user_name"] = item["author_user_info"]["user_name"]
                dic["avatar"] = item["author_user_info"]["avatar_large"]
                dic["content"] = content
                dataList.append(dic)

        except:
            continue
    print(f"当前{len(dataSet)}条数据已整理完毕！")
    time.sleep(4)

while has_more and len(dataSet) < num:
    get_data()

with open("article.json", "w", encoding="utf-8") as f:
    json.dump(dataList, f, indent=4, ensure_ascii=False)
f.close()

# 完成提示
print("over!!!")

# 最后结束提示
# 使用前需要进行 replace \\n  => \n 否则会出现乱码


from app_template import *
from dormListPanel import DormListPanel
import wx.lib.scrolledpanel
import csv

# User: user main system

# Get dorm icon path
imgDir = 'img/'
icon = '/icon.jpg'
fullIconPath = ''

class User(wx.App):
    def __init__(self):
        super().__init__()
        self.frame = MyFrame(parent=None, title='RENT-A-DORM', size=(1080, 720))
        self.frame.Center()
        self.frame.Show()

class MyFrame(wx.Frame):
    def __init__(self, parent, title, size):
        super().__init__(parent=parent, title=title, size=size)
        self.SetMinSize(size)
        self.SetMaxSize(size)

        self.panel = MyPanel(parent=self)

# Set panel as default template
class MyPanel(Template):
    def __init__(self, parent):
        super().__init__(parent=parent)
        self.SetBackgroundColour('#f3bad6')
        self.title_panel.SetBackgroundColour('#e05297')

        # Display data
        self.data_panel = DataPanel(parent=self)

        self.hpanel_box = wx.BoxSizer(wx.HORIZONTAL)
        self.hpanel_box.AddSpacer(60)

        self.vpanel_box.Add(self.data_panel)
        self.vpanel_box.Add(self.hpanel_box)

class DataPanel(wx.Panel):
    def __init__(self, parent):
        super().__init__(parent=parent)

        # Color, Font, and Style
        self.SetBackgroundColour('#f3bad6')

        # Layout
        self.vsizer = wx.BoxSizer(wx.VERTICAL)
        self.vsizer.AddSpacer(20)
        self.top_sizer = wx.BoxSizer(wx.HORIZONTAL)
        self.top_sizer.AddSpacer(60)

        # Log Out button
        self.logOut_button = wx.Button(self, label='Log Out', size=(200, 33))
        self.top_sizer.Add(self.logOut_button, wx.ALL)
        self.vsizer.Add(self.top_sizer)
        self.vsizer.AddSpacer(30)
        # EVT
        self.logOut_button.Bind(wx.EVT_BUTTON, self.logOutClicked)

        # Dorm List scrolled panel
        self.hsizer = wx.BoxSizer(wx.HORIZONTAL)
        self.hsizer.AddSpacer(60)

        self.scroll_panel = wx.lib.scrolledpanel.ScrolledPanel(self, size=(955, 500), style=wx.SIMPLE_BORDER)
        self.scroll_panel.SetupScrolling()
        self.scroll_panel.SetBackgroundColour('#fceef5')

        self.scroll_vsizer = wx.BoxSizer(wx.VERTICAL)
        self.scroll_vsizer.AddSpacer(20)

        # List of dorm information
        self.dorm_list = []
        # Read every row from csv file and store in a list
        with open('assets/dorm.csv', newline='') as f:
            reader = csv.reader(f)
            dormList = reader
            for row in dormList:
                self.dorm_list.append(row)

        tempSet = set()
        # Match dorm name with directory
        self.dormName = []
        # Change dorm name to all lower case
        for i in self.dorm_list:
            tempSet.add(i[1])
            self.dormName.append(i[0].lower())

        tempList = []
        # Dictionary to store path of each icon
        self.iconPath = {}
        # Replace all space bar with _
        for i in range(len(self.dormName)):
            temp = self.dormName[i].replace(' ', '_')
            tempList.append(temp)
            self.iconPath[self.dorm_list[i][0]] = temp
        self.dormName = tempList

        # Add dorm into dorm lists
        for i in range(len(self.dorm_list)):
            # Get icon path
            full_iconPath = imgDir + self.iconPath.get(self.dorm_list[i][0]) + icon
            dorm_list = DormListPanel(self.scroll_panel, self.dorm_list[i], full_iconPath)
            self.scroll_vsizer.Add(dorm_list, flag=wx.ALIGN_CENTER | wx.ALL, border=10)

        self.scroll_panel.SetSizer(self.scroll_vsizer)
        self.hsizer.Add(self.scroll_panel)

        # Initialize
        self.vsizer.Add(self.hsizer)
        self.SetSizer(self.vsizer)

    # EVT Functions
    def logOutClicked(self, event):
        app = self.GetParent().GetParent()
        app.Destroy()
        wx.Exit()

if __name__ == "__main__":
    user = User()
    user.MainLoop()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations

class Migration(migrations.Migration):
    dependencies = [
        ('registration_history', '0003_switch_to_managed_mode'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='oldregistrationprofile',
            options={
                'managed': True,
                'verbose_name': 'registration profile',
                'verbose_name_plural': 'registration profiles'
            },
        ),
        migrations.RemoveField(
            model_name='oldregistrationprofile',
            name='needs',
        ),
        migrations.DeleteModel(
            name='OldRegistrationProfileNeed',
        ),
    ]


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models

class Migration(migrations.Migration):

    dependencies = [
        ('orders', '0001_initial'),
    ]

    operations = [
        migrations.AlterField(
            model_name='orderinfo',
            name='trade_no',
            field=models.CharField(null=True, blank=True, default='', max_length=100, verbose_name='支付编号'),
        ),
    ]


import os
from torch.utils.data import DataLoader, Dataset
import torch
import cv2
import numpy as np
import lib.transform_cv2 as T
import time
from PIL import Image

# Potsdam Dataset Information
# 0: Impervious surfaces (RGB: 255, 255, 255)
# 1: Building (RGB: 0, 0, 255)
# 2: Low vegetation (RGB: 0, 255, 255)
# 3: Tree (RGB: 0, 255, 0)
# 4: Car (RGB: 255, 255, 0)
# 5: Clutter/background (RGB: 255, 0, 0)
class Potsdam(Dataset):
    def __init__(self, data_root, split_file_path, trans_func, mode='train'):
        super(Potsdam, self).__init__()
        time_start_init = time.time()
        # initializing parameters
        assert mode in ('train', 'test', 'val'), "mode wrong with: {}".format(mode)
        self.mode = mode
        self.lb_map = {
            (255, 255, 255): 0,
            (0, 0, 255): 1,
            (0, 255, 255): 2,
            (0, 255, 0): 3,
            (255, 255, 0): 4,
            (255, 0, 0): 5
        }
        self.trans_func = trans_func  # trans_func is in get_dataloader.py
        self.to_tensor = T.ToTensor(
            mean=(0.3392992, 0.36275864, 0.33674471),
            std=(0.14043275, 0.1387484, 0.1442598)
        )
        self.n_cats = 6
        self.lb_ignore = 255
        # reading the corresponding split file (train, test, val)
        assert os.path.isfile(split_file_path), "split file path wrong: {}".format(split_file_path)
        with open(split_file_path, 'r') as file:
            split_list = file.readlines()
        self.image_label_list = []
        image_root = os.path.join(data_root, '2_Ortho_RGB_split_1024')
        label_root = os.path.join(data_root, '5_Labels_all_split_1024_fine')
        for i in split_list:
            i = i.rstrip('\n')
            assert '\n' not in i, "\\n in {}".format(i)
            i_list = i.split(',')  # i_list = [image name, label name]
            image_path = os.path.join(image_root, i_list[0])
            label_path = os.path.join(label_root, i_list[1])
            assert os.path.exists(image_path), "image path not exists: {}".format(image_path)
            assert os.path.exists(label_path), "label path not exists: {}".format(label_path)
            self.image_label_list.append((image_path, label_path))
        # checking
        # now image_label_list contains the path of all images and labels corresponding to the specific split file
        assert len(self.image_label_list) == len(split_list)
        time_end_init = time.time()
        print("Dataset init time: {}s".format(time_end_init - time_start_init))

    def convert_labels(self, image):
        image_shape = image.shape
        new = np.zeros((image_shape[0], image_shape[1]), dtype=np.int64)
        for i in range(image_shape[0]):
            for j in range(image_shape[1]):
                tmp = tuple(image[i, j])
                if tmp in self.lb_map:
                    new[i, j] = self.lb_map[tmp]
                else:
                    new[i, j] = 255
        return new

    def __getitem__(self, index):
        image_path = self.image_label_list[index][0]
        label_path = self.image_label_list[index][1]
        # check again
        os.path.isfile(image_path), "image path not exists: {}".format(image_path)
        os.path.isfile(label_path), "label path not exists: {}".format(label_path)
        # reading image and label
        # image = cv2.imread(image_path)[:, :, ::-1]  # transform the channel from "BGR" to "RGB"
        # label = cv2.imread(label_path)[:, :, ::-1]
        image = np.array(Image.open(image_path))
        label = np.array(Image.open(label_path))
        # transform the label
        # label = self.convert_labels(label)
        im_lb = dict(im=image, lb=label)
        if self.trans_func is not None:
            im_lb = self.trans_func(im_lb)
        im_lb = self.to_tensor(im_lb)
        image, label = im_lb['im'], im_lb['lb']
        return image.detach(), label.detach()

    def __len__(self):
        return len(self.image_label_list)

if __name__ == '__main__':
    from tqdm import tqdm
    from torch.utils.data import DataLoader

    dataset = Potsdam('/root/bisenet-1024/datasets/potsdam', '/root/bisenet-1024/datasets/potsdam/split_1024/val.txt',
                      trans_func=None, mode='val')
    dataloader = DataLoader(dataset, batch_size=4, shuffle=True, num_workers=4, drop_last=True)
    for imgs, label in dataloader:
        print(len(imgs))
        for el in imgs:
            print(el.size())
        break
    # for index, (image, label) in enumerate(tqdm(dataloader)):
    #     assert image.shape == (3, 1024, 1024), "image shape wrong {}, {}".format(index, image.shape)
    #     assert label.shape == (1024, 1024), "label shape wrong {}, {}".format(index, label.shape)
    print("DONE")


# -*- coding: utf-8 -*-
from __future__ import print_function, division

import torch
import torch.nn as nn

class NLLLoss(nn.Module):
    def __init__(self, params):
        super(NLLLoss, self).__init__()
        self.nll_loss = nn.NLLLoss()

    def forward(self, loss_input_dict):
        predict = loss_input_dict['prediction']
        labels  = loss_input_dict['ground_truth']
        logsoft = nn.LogSoftmax(dim = 1)
        predict = logsoft(predict)
        loss = self.nll_loss(predict, labels)
        return loss


from flask import (Blueprint,
                   render_template,
                   redirect,
                   url_for,
                   request,
                   session)
from models.reviews import Review

main = Blueprint('index', __name__)

@main.route('/')
def index():
    return render_template('index.html')

@main.route('/submit/<int:id>', methods=['POST', 'GET'])
def submit(id):
    if request.method == 'POST':
        form = request.form
        review = Review.new(form)
        review.get_classification()
        review.save()
        return redirect(url_for('index.submit', review=review, id=review.id))
    review = Review.find(id=id)
    return render_template('result.html', review=review)

@main.route('/return_right', methods=['POST'])
def return_right():
    return redirect(url_for('index.thanks'))

@main.route('/return_wrong', methods=['POST'])
def return_wrong():
    id = request.args.get('id')
    review = Review.find(id=int(id))
    print(id, review)
    if review.classification == 'Positive':
        review.classification = 'Negative'
    else:
        review.classification = 'Positive'
    review.save()
    return redirect(url_for('index.thanks'))

@main.route('/thanks')
def thanks():
    return render_template('thanks.html')


from bs4 import BeautifulSoup
from datetime import date
import requests
import csv

source = requests.get('http://www.sjsu.edu/').text

soup = BeautifulSoup(source,'lxml')

output_file = open(f'sjsu_event_scrape_{date.today()}.csv','w')

writer = csv.writer(output_file)
writer.writerow(['Name','Date','Time','Location','Link'])

for event in soup.find_all('a',class_='o-calendar__event'):

	event_title = event.find('p',class_='o-calendar__event-name u-bar u-bar--light u-bar--sjsu-gold').text
	print(event_title)

	event_month = event.find('span',class_='o-calendar__month').text
	event_day = event.find('span',class_='o-calendar__day').text
	event_date = f'{event_month} {event_day}'

	event_time = event.find('span',class_='o-calendar__detail u-icon u-icon--clock u-color--sjsu-gold').text
	event_time = event_time.replace('Time:','').strip()

	print(event_date, event_time)

	event_location = event.find('span',class_='o-calendar__detail u-icon u-icon--location u-color--sjsu-gold').text
	event_location = event_location.replace('Location:','').strip()
	print(event_location)

	try:
		event_link = event['href']
	except Exception as e:
		event_link = None

	print(event_link)
	print()

	writer.writerow([event_title,event_date,event_time,event_location,event_link])


# 49 그룹 애너그램
#실패 (시간 제한 초과)
from typing import List

class Solution:
    def groupAnagrams(self, strs: List[str]) -> List[List[str]]:
        sorted_strs = []
        for word in strs: # ["eat","tea","tan","ate","nat","bat"]
            sorted_strs.append(''.join(sorted(word)))   # 각각의 문자열을 sort하여 재배열
            # ['aet', 'aet', 'ant', 'aet', 'ant', 'abt']

        strs_set = set(sorted_strs) # 집합으로 만들어서 중복된 단어 제거 ['aet', 'ant', 'abt']

        result = []
        for j in range(len(strs_set)): # ['aet', 'ant', 'abt']
            anagrams = []
            for i in range(len(strs)):
                if ''.join(sorted(strs[i])) == strs_set[j]: # strs에 포함된 문자열의 sort가 집합의 요소와 같다면
                    anagrams.append(strs[i])            # anagrams에 추가
            result.append(anagrams) # result에 모든 anagrams묶음을 추가
        return result

strs = ["eat","tea","tan","ate","nat","bat"]
sol = Solution()
print(sol.groupAnagrams(strs))


import os

from flask import Flask

from flask_maple.lazy import LazyExtension
from forums.admin.urls import admin

from .app import register_app
from .filters import register_jinja2
from .logs import register_logging

def create_app(config):
    templates = os.path.abspath(
        os.path.join(os.path.dirname(__file__), os.pardir, 'templates'))
    static = os.path.abspath(
        os.path.join(os.path.dirname(__file__), os.pardir, 'static'))

    app = Flask(__name__, template_folder=templates, static_folder=static)
    app.config.from_object(config)
    if app.config['SUBDOMAIN']['forums']:
        app.url_map._rules.clear()
        app.url_map._rules_by_endpoint.clear()
        app.url_map.default_subdomain = 'forums'
        app.add_url_rule(
            app.static_url_path + '/<path:filename>',
            endpoint='static',
            view_func=app.send_static_file,
            subdomain='forums')
    register(app)
    return app

def register(app):
    register_extension(app)
    register_router(app)
    register_logging(app)
    register_jinja2(app)
    register_app(app)

def register_router(app):
    from forums.api.urls import api_routers
    from forums.docs.urls import docs_routers
    api_routers(app)
    docs_routers(app)

def register_extension(app):
    extension = LazyExtension(
        module='forums.extension.',
        extension=['db', 'avatar', 'cache', 'csrf', 'bootstrap', 'captcha',
                   'error', 'redis_data', 'principal', 'babel',
                   'login_manager', 'maple_app', 'mail', 'middleware',
                   'search'])
    extension.init_app(app)
    admin.init_app(app)


# I have created this file Abhi

from django.http import HttpResponse
from django.shortcuts import render

def index(request):
    return render(request,'index.html')

def analyze(request):
    djtext = request.POST.get('text', 'default')
    removepunc = request.POST.get('removepunc', 'off')
    capfirst = request.POST.get('capfirst', 'off')
    linerem = request.POST.get('newlinerv', 'off')
    sprem = request.POST.get('sprem', 'off')
    charcount = request.POST.get('charcount', 'off')

    if removepunc=="on":
        punctuations = '''!()-[]{};:'"\,<>./?@#$%^&*_~'''
        analyzed = ""
        for char in djtext:
            if char not in punctuations:
                analyzed = analyzed + char
        params = {'purpose': 'Remove Punctuations', 'analyzed_text': analyzed}
        djtext = analyzed

    if (linerem == "on"):
        analyzed = ""
        for char in djtext:
            if char != '\n' and char != '\r':
                analyzed = analyzed + char
        params = {'purpose': 'New Line Remover', 'analyzed_text': analyzed}
        djtext = analyzed

    if (sprem == "on"):
        analyzed = ""
        for index,char in enumerate(djtext):
            if not(djtext[index]==" " and djtext[index+1]==" "):
                analyzed = analyzed + char
        params = {'purpose': 'Extra Soace Remover', 'analyzed_text': analyzed}
        djtext = analyzed

    if (charcount == "on"):
        count = 0
        for char in djtext:
            if not(char == " "):
                count +=1
        params = {'purpose': 'Number of Character', 'analyzed_text': count}

    if(capfirst=="on"):
        analyzed = ""
        for char in djtext:
            analyzed = analyzed + char.upper()
        params = {'purpose': 'Changed to Uppercase', 'analyzed_text': analyzed}

    if (removepunc != "on" and linerem != "on" and sprem != "on" and capfirst != "on" and charcount != "on"):
        return HttpResponse("please select any operation and try again")

    return render(request, 'analyze.html', params)


from django.test import TestCase
from usuariosInfo import service
from usuariosInfo.classes import UsuariosInfo
import json

from core.exception import AppException #que hace esto?

class UsuarioServiceTest(TestCase):
    #esto no toco
    def test_usuario_save_and_get(self):

        id = service.save(UsuariosInfo(
            string='asd',
            integer=12,
            decimal=1.5 #TENGO Q PREGUNTAR A LOBI
        ))

        usuario = service.get(id)
        usuario_none = service.get(123)

        self.assertEqual(usuario.id, id)
        self.assertEqual(usuario_none, None)

    def test_usuario_list(self):

        list_empty = service.list()

        service.save(UsuariosInfo(
            string='asd',
            integer=12,
            decimal=1.5
        ))

        list_not_empty = service.list()

        self.assertEqual(list_empty, [])
        self.assertEqual(len(list_not_empty), 1)

    def test_usuario_delete(self):

        id = service.save(UsuariosInfo(
            string='asd',
            integer=12,
            decimal=1.5
        ))

        service.delete(id)
        list_empty = service.list()

        self.assertEqual(list_empty, [])
        self.assertRaises(AppException, service.delete, 123)

class UsuariosClassTest(TestCase):

    def test_equals_class(self):

        usuario_1 = UsuariosInfo(
            string='asd',
            integer=1,
            decimal=1.5,
            id=1
        )

        usuario_2 = UsuariosInfo(
            string='qwe',
            integer=2,
            decimal=2.5,
            id=1
        )

        self.assertEquals(usuario_1, usuario_2)

class UsuarioViewTest(TestCase):

    def test_list(self):

        response = self.client.get('/api/v1/usuariosInfo/', follow=True)

        self.assertEquals(response.status_code, 200)

    def test_get(self):

        id = service.save(UsuariosInfo(
            string='asd',
            integer=12,
            decimal=1.5
        ))

        response = self.client.get(f'/api/v1/usuariosInfo/{id}', follow=True)
        response_not_exist = self.client.get(
            f'/api/v1/usuariosInfo/123', follow=True)

        self.assertEquals(response.status_code, 200)
        self.assertEquals(response.json()['id'], id)
        self.assertEquals(response_not_exist.status_code, 204)

    def test_post(self):

        body = {
            'string': 'asd',
            'integer': 12,
            'decimal': 1.5,
            'date': '2020-12-12T12:12:12.000'
        }
        response = self.client.post(
            f'/api/v1/usuariosInfo/', data=body, follow=True, content_type='application/json')

        self.assertEquals(response.status_code, 201)
        self.assertEquals(response.json()['id'], 1)

    def test_delete(self):

        id = service.save(UsuariosInfo(
            string='asd',
            integer=12,
            decimal=1.5
        ))

        response = self.client.delete(f'/api/v1/usuariosInfo/{id}', follow=True)
        response_error = self.client.delete(f'/api/v1/usuariosInfo/123', follow=True)

        self.assertEquals(response.status_code, 204)
        self.assertEquals(response_error.status_code, 409)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import numpy as np
import pickle as pkl
from scipy.ndimage import gaussian_filter as gf
from numpy import ma
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA, FastICA
from .hyperheader import HyperHeader
from .config import HYSS_ENVIRON

class HyperCube(object):

    def __init__(self, fname=None, fpath=None, hname=None, hpath=None,
                 fac=None, dim=None):

        # -- get the defaults
        self.config = HYSS_ENVIRON

        fname  = fname if fname else HYSS_ENVIRON['HYSS_DNAME']
        fpath  = fpath if fpath else HYSS_ENVIRON['HYSS_DPATH']
        hname  = hname if hname else HYSS_ENVIRON['HYSS_HNAME']
        hpath  = hpath if hpath else HYSS_ENVIRON['HYSS_HPATH']
        fac    = fac if fac else HYSS_ENVIRON['HYSS_FAC']

        # -- set the input file
        infile = os.path.join(fpath,fname)
        ftree  = os.path.split(infile) # directory tree pointing to the file

        self.fpath  = fpath
        self.fname  = fname

        # -- read the header information
        if hname:
            self.hdr    = HyperHeader(hname,hpath)
            self.nwav   = len(self.hdr.wavelength)
            self.nrow   = self.hdr.samples/fac
            self.ncol   = self.hdr.lines/fac
        elif dim==None:
            print("HYPERCUBE: No associated header file, must set "
                  "dim=[nwav,nrow,ncol]!!!")
            return
        else:
            self.nwav = dim[0]
            self.nrow = dim[1]
            self.ncol = dim[2]

        self.fac    = fac
        self.ind    = np.zeros([self.nrow,self.ncol],dtype=bool)
        self.thresh = 0.5
        self.nthr   = 1

        # -- read in the data
        print("HYPERCUBE: reading {0}".format(self.fpath))
        print("HYPERCUBE:   {0}".format(self.fname))
        self.data = np.zeros([self.nwav,self.nrow,self.ncol])

        try:
            fopen            = open(infile,'rb')
            self.data[:,:,:] = np.fromfile(fopen).reshape(self.nwav,self.nrow,
                                                          self.ncol)
            fopen.close()
        except:
            print("HYPERCUBE: Error reading raw data file, is binning factor "
                  "set?")
            return

        # -- set the wavelengths
        if hname:
            self.wavelength = self.hdr.wavelength
            self.indexing   = False
        else:
            self.wavelength = np.zeros(self.nwav)
            self.indexing   = True

        # -- construct the total luminosity image
        self.img_L = self.data.mean(0)

        return

    def threshold(self,thresh=None,nthr=None,luminosity=True):
        """
        Set active indices as pixels for which at least nthr points are
        greater than the threshold.
        """

        # -- defaults
        if thresh:
            self.thresh = thresh
        if nthr:
            self.nthr = nthr

        # -- threshold the array
        self.ind = self.img_L>self.thresh if luminosity else \
            (self.data>self.thresh).sum(0)>self.nthr

        return

    def remove_disconnected(self):
        """
        Remove active pixels that are disconnected from any neighbors.
        """

        # -- check active pixels above, below, left, and right
        self.ind = self.ind & (np.roll(self.ind,1,0) | np.roll(self.ind,-1,0) |
                               np.roll(self.ind,1,1) | np.roll(self.ind,-1,1))

        return

    def binarize(self, sigma=None, smooth=None):
        """
        Convert spectra to boolean values at each wavelength.

        The procedure estimates the noise by taking the standard deviation
        of the derivative spectrum and dividing by sqrt(2).  The
        zero-point offset for each spectrum is estimated as the mean of
        the first 10 wavelengths (empirically seen to be "flat" for most
        spectra) and is removed.  Resultant points >5sigma [default] are
        given a value of True.

        Parameters
        ----------
        sigma : float, optional
            Sets the threshold, above which the wavelength is considered
            to have flux.
        """

        # -- smooth if desired
        dat = self.data if not smooth else gf(self.data,[smooth,0,0])

        if sigma:
            # -- estimate the noise and zero point for each spectrum
            print("BINARIZE: estimating noise level and zero-point...")
            sig = (dat[1:]-dat[:-1])[-100:].std(0)/np.sqrt(2.0)
            zer = dat[:10].mean(0)

            # -- converting to binary
            print("BINARIZE: converting spectra to boolean...")
            self.bdata = (dat-zer)>(sigma*sig)
        else:
            # -- careful about diffraction spikes which look like absoportion
            mn_tot = dat.mean(0)
            mn_end = dat[-100:].mean(0)
            index  = mn_tot > mn_end
            mn     = mn_tot*index + mn_end*~index

            # -- binarize by comparison with mean
            self.bdata = dat>mn

        return

    def image(self, lam_lo=None, lam_hi=None):
        """
        Generate an average image of the scene over the specified wavelength
        range.

        Parameters
        ----------
        lam_lo : float, optional
            The lower limit (inclusive) of the wavelength range.

        lam_hi : float, optional
            The upper limit (exclusive) of the wavelength range.

        Returns
        -------
        mimage : ndarray, nrow x ncol float
            The data cube averaged over the defined wavelength region.
        """

        # -- return pre-computed full range
        if not (lam_lo or lam_hi):
            return self.img_L

        # -- if the above isn't true, lam_lo must be specified
        lam_hi = lam_hi if lam_hi else self.wavelength[1]

        return self.data[(self.wavelength>=lam_lo) &
                         (self.wavelength<lam_hi)].mean(0)

    def regress(self,templates):
        """
        Regress the input templates against the active pixels.

        The input templates must be at the same wavelengths as the data.
        """

        data  = self.data[:,self.ind].copy()
        data -= data.mean(0)
        data /= data.std(0)

        P  = templates.T.copy()
        P -= P.mean(0)
        P /= P.std(0)

        PtP    = np.dot(P.T,P)
        Ptdata = np.dot(P.T,data)
        PtPinv = np.linalg.pinv(PtP)
        avec   = np.dot(PtPinv,Ptdata)

        return avec

    def correlate(self,templates,select=False):
        """
        Correlate the input templates against the active pixels.

        The input templates must be at the same wavelengths as the data.

        Parameters
        ----------
        templates : an NtemplatesxNwavelength numpy array
            The templates with which to correlate.

        select : bool, optional
            If True, return the index of the highest correlation coefficient,
            if False, return all of the correlation coefficients.

        Returns
        -------
        corr : NtemplatesxNpix numpy array
            The Ntempaltes correlation coefficients for each active pixel.
        """

        # -- make copies of the data
        temps = templates.T.copy()
        specs = self.data[:,self.ind].copy()

        # -- normalize
        temps -= temps.mean(0)
        temps /= temps.std(0)
        specs -= specs.mean(0)
        specs /= specs.std(0)

        # -- calculate the correlation coefficient
        corr = np.dot(temps.T,specs)/float(temps.shape[0])

        return corr if not select else np.argmax(corr[0])

    def kmeans(self,n_clusters=15,**kwargs):

        # -- give an alert
        npix = self.ind.sum()

        if npix==0:
            print("HYPERCUBE: active pixels have not been set.")
            print("HYPERCUBE: thresholding with current values...")
            self.threshold()
            npix = self.ind.sum()

        # -- properly format the data
        lgt  = self.data[:,self.ind].reshape(self.nwav,npix)
        lgt -= lgt.mean(0)
        lgt /= lgt.std(0)

        # -- run K-Means
        print("HYPERCUBE: running K-Means with {0} ".format(n_clusters) +
              "clusters and {0} points...".format(npix))

        self.km = KMeans(n_clusters=n_clusters,**kwargs)
        self.km.fit(lgt.T)

        return

    def write_kmeans(self, kmname=None, kmpath=None):
        """
        Write the K-Means solution (and important attributes) to a file.
        """
        # -- defaults
        kmpath = kmpath if kmpath else HYSS_ENVIRON['HYSS_WRITE']

        # -- define the file
        kmname = kmname if kmname else self.fname.split('.')[0] + '_km.pkl'

        # -- write to file
        fopen = open(os.path.join(kmpath,kmname),'wb')

        pkl.dump(self.km,fopen)
        pkl.dump(self.config,fopen)
        pkl.dump(self.fac,fopen)
        pkl.dump(self.nrow,fopen)
        pkl.dump(self.ncol,fopen)
        pkl.dump(self.thresh,fopen)
        pkl.dump(self.nthr,fopen)
        pkl.dump(self.ind,fopen)
        pkl.dump(self.img_L,fopen)

        fopen.close()

        return

    def read_kmeans(self, fname, fpath=None):
        """
        Read the pickled output from write_kmeans.

        Parameters
        ----------
        fname : str
            The file name of the K-Means output.

        fpath : str, optional
            The path to the K-Means output (defaults to
            HYSS_ENVIRON['HYSS_WRITE']).
        """

        # -- read the file
        fpath  = fpath if fpath else HYSS_ENVIRON['HYSS_WRITE']
        kmfile = os.path.join(fpath,fname)
        fopen  = open(kmfile,'rb')

        self.km     = pkl.load(fopen)
        self.config = pkl.load(fopen)
        self.fac    = pkl.load(fopen)
        self.nrow   = pkl.load(fopen)
        self.ncol   = pkl.load(fopen)
        self.thresh = pkl.load(fopen)
        self.nthr   = pkl.load(fopen)
        self.ind    = pkl.load(fopen)
        self.img_L  = pkl.load(fopen)

        fopen.close()

        return

    def run_pca(self, **kwargs):
        """
        Run principle component decomposition on the active spectra.

        This wrapper accepts the same keywords as the scikit-learn
        implementation of PCA.
        """

        # -- prepare the data vector
        print("HYPERCUBE: normalizing spectra...")
        norm  = self.data[:,self.ind]
        norm -= norm.min(0)
        norm /= norm.sum(0)
        norm -= norm.mean(0)

        # -- run PCA
        print("HYPERCUBE: running PCA...")
        self.pca = PCA(**kwargs)
        self.pca.fit(norm.T)

        return

    def run_ica(self, **kwargs):
        """
        Run independent component analysis on the active spectra.

        This wrapper accepts the same keywords as the scikit-learn
        implementation FastICA.
        """

        # -- prepare the data vector
        print("HYPERCUBE: normalizing spectra...")
        norm  = self.data[:,self.ind]
        norm -= norm.min(0)
        norm /= norm.sum(0)
        norm -= norm.mean(0)

        # -- run PCA
        print("HYPERCUBE: running FastICA...")
        self.ica = FastICA(**kwargs)
        self.ica.fit(norm.T)

        return


import numpy as np
import torch

def fbeta_numpy(targets, preds, beta=0.5, smooth=1e-5):
    """
    https://www.kaggle.com/competitions/vesuvius-challenge-ink-detection/discussion/397288
    """
    y_true_count = targets.sum()
    ctp = preds[targets == 1].sum()
    cfp = preds[targets == 0].sum()
    beta_squared = beta * beta

    c_precision = ctp / (ctp + cfp + smooth)
    c_recall = ctp / (y_true_count + smooth)
    dice = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall + smooth)

    return dice

def dice_coef_torch(targets, preds, beta=0.5, smooth=1e-5):

    # flatten label and prediction tensors
    preds = preds.contiguous().view(-1).float()
    targets = targets.contiguous().view(-1).float()

    y_true_count = targets.sum()
    ctp = preds[targets==1].sum()
    cfp = preds[targets==0].sum()
    beta_squared = beta * beta

    c_precision = ctp / (ctp + cfp + smooth)
    c_recall = ctp / (y_true_count + smooth)
    dice = (1 + beta_squared) * (c_precision * c_recall) / (beta_squared * c_precision + c_recall + smooth)

    return dice

def calc_fbeta(mask, mask_pred, th):
    dice = dice_coef_torch(mask.long(), (mask_pred >= th).long(), beta=0.5)

    return dice

def calc_cv(mask_gts, mask_preds, orig_sizes, logger):
    best_th = 0
    best_dice = 0
    ths = np.array(range(30, 70 + 1, 5)) / 100
    orig_h, orig_w = orig_sizes
    mask_gt = mask_gts[:orig_h, :orig_w]
    mask_pred = mask_preds[:orig_h, :orig_w]
    #mask_pred = torch.sqrt(mask_pred)
    mask_gt = torch.from_numpy(mask_gt).to("cuda")

    for th in ths:
        dice = calc_fbeta(mask_gt, mask_pred, th)
        print(f"th: {th} dice: {dice}")

        if dice > best_dice:
            best_dice = dice
            best_th = th

    logger.info(f'best_th: {best_th}, fbeta: {best_dice}')

    return best_dice, best_th