import os
import numpy as np
import cv2
from PIL import Image
from torch.utils import data
from torchvision import transforms
from torchvision.transforms.functional import to_tensor
from config.parameters import *
import torch
import random
from .rec_img_aug import warp

nums = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
lower_char = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']
upper_char = ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z']

# 将str标签转化为数值label
def StrtoLabel(Str):
    # print(Str)
    label = []
    for i in range(0, charNumber):
        if Str[i] >= '0' and Str[i] <= '9':
            # ord函数返回ASCII码值
            label.append(ord(Str[i]) - ord('0'))
        elif Str[i] >= 'a' and Str[i] <= 'z':
            label.append(ord(str(Str[i])) - ord("a") + 10)
        else:
            label.append(ord(str(Str[i])) - ord("A") + 36)
    return label

# 将label转化为标签
def LabeltoStr(Label):
    Str = ""
    for i in Label:
        if i <= 9:
            Str += chr(ord('0') + i)
        elif i <= 35:
            Str += chr(ord('a') + i - 10)
        else:
            Str += chr(ord('A') + i - 36)
    return Str

# 加载数据集
class Captcha(data.Dataset):
    # 输入已经读取的CSV表格, 图片地址, 是否测试集
    def __init__(self, csv, img_path, data_mode='train'):
        # 表格数据
        self.data = csv
        # 图片地址
        self.img_path = img_path
        # 是否是测试集
        self.data_mode = data_mode
        # 图像处理
        self.transform = transforms.Compose([
            transforms.Resize((ImageHeight, ImageWidth)),
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.843, 0.846, 0.842], std=[0.196, 0.193, 0.198])
        ])

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        # 获取图片名称及label
        ImgName, label = self.data.iloc[index, :]
        # 获取某一张图片地址
        imgPath = os.path.join(self.img_path, ImgName)
        # 打开图片 (有的图片4通道)
        # image = Image.open(imgPath).convert('RGB')
        image = cv2.cvtColor(cv2.imread(imgPath), cv2.COLOR_BGR2RGB)

        if self.data_mode == 'train':
            # 训练集要做数据增强
            image = warp(image)
            # resize + Norm归一化
            image = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
            image = self.transform(image)
            # image = to_tensor(image)
            labelTensor = torch.Tensor(StrtoLabel(label))
            return image, labelTensor
        elif self.data_mode == 'val':
            # 验证集要有标签
            image = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
            image = self.transform(image)
            # image = to_tensor(image)
            # 将标签转化为数值label
            labelTensor = torch.Tensor(StrtoLabel(label))
            return image, labelTensor
        elif self.data_mode == 'test':
            # 测试集正常处理
            image = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
            image = self.transform(image)
            # image = to_tensor(image)
            return image, ImgName


import collections

import numpy as np
import webcolors

from moldesign import units as u

class VolumetricGrid(object):
    """
    Helper object for preparing gaussian CUBE files
    """
    UNITS = u.angstrom
    def __init__(self, positions, padding=2.5*u.angstrom, npoints=25):
        mins = positions.min(axis=0) - padding
        maxes = positions.max(axis=0) + padding
        self.npoints = npoints
        self.xr = (mins[0], maxes[0])
        self.yr = (mins[1], maxes[1])
        self.zr = (mins[2], maxes[2])
        self._origin = mins.value_in(self.UNITS)
        self.dx = (self.xr[1] - self.xr[0]).value_in(self.UNITS) / (float(npoints) - 1)
        self.dy = (self.yr[1] - self.yr[0]).value_in(self.UNITS) / (float(npoints) - 1)
        self.dz = (self.zr[1] - self.zr[0]).value_in(self.UNITS) / (float(npoints) - 1)
        self.fxyz = None

    def xyzlist(self):
        stride = self.npoints * 1j
        grids = np.mgrid[self.xr[0]:self.xr[1]:stride,
                self.yr[0]:self.yr[1]:stride,
                self.zr[0]:self.zr[1]:stride]
        return grids * self.UNITS

    def origin(self):
        return tuple(self._origin)

def get_all_atoms(*objects):
    """ Given Atoms, AtomContainers, lists of Atoms, and lists of AtomContainers,
    return a flat list of all atoms contained therein.

    A given atom is only returned once, even if it's found more than once.

    Args:
        *objects (moldesign.Atom OR moldesign.AtomContainer OR List[moldesign.Atom] OR
            List[moldesign.AtomContainer]): objects to take atoms from

    """
    from moldesign import molecules

    atoms = collections.OrderedDict()

    for obj in objects:
        if isinstance(obj, molecules.Atom):
            atoms[obj] = None
        elif hasattr(obj, 'atoms'):
            atoms.update((x,None) for x in obj.atoms)
        else:
            for item in obj:
                if isinstance(item, molecules.Atom):
                    atoms[item] = None
                elif hasattr(item, 'atoms'):
                    atoms.update((x, None) for x in item.atoms)

    return molecules.AtomList(atoms.iterkeys())

def kinetic_energy(momenta, masses):
    return 0.5 * (momenta*momenta/masses).sum()

def kinetic_temperature(ke, dof):
    from moldesign.units import k_b
    t = (2.0*ke)/(k_b*dof)
    return t.defunits()

# def get_residues(obj, **queries):
#     """
#
#     Args:
#         obj ():
#         **queries ():
#
#     Returns:
#
#     """
#     for residue in obj.residues:
#         pass
#

DEF_CATEGORICAL = 'Paired'
DEF_SEQUENTIAL = None  # should be inferno, but that's only MPL >1.5

def colormap(cats, mplmap='auto'):
    # should make it easy to choose one for:
    #  categorical data
    #  sequential (low, high important)
    #  diverging data (low, mid, high important)
    # Can deal with numerical and categorical data
    # we'll treat ints as categories for now
    global DEF_SEQUENTIAL
    from matplotlib import cm

    if hasattr(cm, 'inferno'):
        DEF_SEQUENTIAL = 'inferno'
    else:
        DEF_SEQUENTIAL = 'BrBG'

    # strip units
    units = None
    if hasattr(cats[0], 'magnitude'):
        arr = u.array(cats)
        units = arr.units
        cats = arr.magnitude

    if not isinstance(cats, np.ndarray) and not isinstance(cats[0], float):  # treat as
        # categorical
        values = np.zeros(len(cats), dtype='float')
        to_int = collections.OrderedDict()
        for i, item in enumerate(cats):
            if item not in to_int:
                to_int[item] = len(to_int)
            values[i] = to_int[item]
        if mplmap == 'auto':
            mplmap = DEF_CATEGORICAL
    else:  # it's numerical
        values = np.array(cats, dtype='float')
        if mplmap == 'auto':
            mplmap = DEF_SEQUENTIAL

    cmap = getattr(cm, mplmap)
    mx = values.max()
    mn = values.min()
    r = (values - mn) / (mx - mn)  # rescale to [0.0,1.0]
    rgb = cmap(r)
    hexcolors = [webcolors.rgb_to_hex(np.array(r[:3]) * 256) for r in rgb]
    return hexcolors


import numpy as np
import matplotlib.pyplot as plt

#===============================Carregar Arquivo===================================
#Usando csv.reader
#with open('Advertising.csv') as f:
#    dados = list(csv.reader(f, delimiter=','))

#Usando NumPy
dados = np.genfromtxt("Adversiting.csv", delimiter=',', skip_header=1)
print(dados)
X = dados[:,1:4]
y = dados[:,4]
#===================================================================================

#=========================Método de regressão simples===========================================================
class SampleLinearRegressor():
  #construtor
  def __init__(self):
    pass

  #Calculo de B1 e B0
  def fit(self, X, y):
    xMean = np.mean(X)
    self.yMean = np.mean(y)

    sumUpB1 = 0
    sumDownB1 = 0
    i = 0
    while(i < X.shape[0] ):
        sumUpB1 += (X[i] - xMean) * (y[i] - self.yMean)
        sumDownB1 += (X[i] - xMean) ** 2
        self.B1 = sumUpB1/sumDownB1
        self.B0 = self.yMean - (self.B1 * xMean)
        i = i+1

  #Aplicar a equação no X que se recebe como entrada
  def predictEq(self, x):
      return (self.B0 + (self.B1 * x))

  #Aplica o predict em cada item do vetor da entrada
  def predict(self, x):
    return self.B0 + self.B1*x

   #===================================================

   #==============Medição de Erros======================
  def RSS(self, X, y):
       error = 0
       i = 0
       while i < len(X):
           error = error + ((y[i] - self.predictEq(X[i])) ** 2)
           i = i + 1
       return error

  def RSE(self, X, y):
      return (self.RSS(X, y)/len(X)) ** 0.5

  def TSS(self, X, y):
      error = 0
      i = 0
      while(i < len(X)):
          error += ((y[i] - self.yMean) ** 2)
          i += 1
      return error

  def R2(self, X, y):
      return (1 - (self.RSS(X, y)/self.TSS(X, y)))
   #====================================================

#======================================================================================

#=============================Metodo de Regressão Multipla=============================
class MultipleLinearRegressor():
  #construtor
  def __init__(self):
    pass

  #Calculo de B1 e B0
  def fit(self, X, y):
      bias = np.ones((X.shape[0], 1))
      X = np.hstack((bias,X))
      inv = np.linalg.inv(np.matmul(np.transpose(X),X))
      self.b = np.matmul(np.matmul(inv,np.transpose(X)), y)

  #Aplicar a equação no X que se recebe como entrada
  def predict(self, X):
      vetorResult = [self.b[0]]
      i = 1
      while i < self.b.shape[0]:
          vetorResult += (self.b[i]*X[:,i-1])
          i = i+1
      return vetorResult

  def RSS(self, X):
        totalSum = 0
        for i in range(0, X.shape[0]):
           roundMinus = y[i]
           for j in range(0, self.b.shape[0]-1):
               roundMinus = roundMinus - (self.b[j]*X[i,j])
           totalSum = totalSum + (roundMinus ** 2)
        return totalSum

  def RSE(self, X):
      return (self.RSS(X)/len(X)) ** 0.5

  def TSS(self, X):
      error = 0
      i = 0
      while(i < len(X)):
          error += ((y[i] - np.mean(y)) ** 2)
          i += 1
      return error

  def R2(self, X):
      return (1 - (self.RSS(X)/self.TSS(X)))

   #===================================================

#==========================================================================================

# Simplifica chamada do objeto
slr = SampleLinearRegressor()
mlr = MultipleLinearRegressor()

print(dados)
#================================Front===============================================================================
print("Regressão Linear")
option = input("Opções: 0 - TV x Sales |  1 - Radio x Sales  |  2 - Newspaper x Sales  |  3 - Investimentos x Sales (Regressao Multipla)\
    ")

# Treinar meu modelo usando fit de acordo com a opção
if option == '0' or option == '1' or option == '2':
    slr.fit(X[:,int(option)], y)

    #Plotagem=======================================================================

    #De acordo com a opcao selecionada, produz o plot
    if option == '0':
        plt.title("TVs X Sales - B0: " + str(slr.B0) + " - B1: " + str(slr.B1))
        plt.xlabel("TV")
        plt.ylabel("Sales")
        plt.plot(dados[:,1], y, 'go')
        plt.plot(slr.predict(np.array(range(1,300))))

        # Medir erros
        print("RSS: " + str(slr.RSS(dados[:,1], y)))
        print("RSE: " + str(slr.RSE(dados[:,1], y)))
        print("R2: " + str(slr.R2(dados[:,1], y)))
        print("TSS: " + str(slr.TSS(dados[:,1], y)))

    elif option == '1':
        plt.title("Radio X Sales - B0: " + str(slr.B0) + " - B1: " + str(slr.B1))
        plt.xlabel("Radio")
        plt.ylabel("Sales")
        plt.plot(dados[:,2], y, 'go')
        plt.plot(slr.predict(np.array(range(1,50))))

        # Medir erros
        print("RSS: " + str(slr.RSS(dados[:,2], y)))
        print("RSE: " + str(slr.RSE(dados[:,2], y)))
        print("R2: " + str(slr.R2(dados[:,2], y)))
        print("TSS: " + str(slr.TSS(dados[:,2], y)))

    elif option == '2':
        plt.title("Newspaper X Sales - B0: " + str(slr.B0) + " - B1: " + str(slr.B1))
        plt.xlabel("Newspaper")
        plt.ylabel("Sales")
        plt.plot(dados[:,3], y, 'go')
        plt.plot(slr.predict(np.array(range(1,120))))

        # Medir erros
        print("RSS: " + str(slr.RSS(X, y)))
        print("RSE: " + str(slr.RSE(X, y)))
        print("R2: " + str(slr.R2(X, y)))
        print("TSS: " + str(slr.TSS(X, y)))

    # B1 e B0
    print("B1: " + str(slr.B1))
    print("B0: " + str(slr.B0))

    # Plotar na Tela
    plt.show()

elif option == '3':
    mlr.fit(X, y)
    plt.title("Regressão Linear Múltipla")
    plt.xlabel("Investimentos")
    plt.ylabel("Sales")
    plt.plot(mlr.predict(X), 'go')
    plt.show()

    print(mlr.predict(X))

    print("B3: " + str(mlr.b[3]))
    print("B2: " + str(mlr.b[2]))
    print("B1: " + str(mlr.b[1]))
    print("B0: " + str(mlr.b[0]))

    print("RSS: " + str(mlr.RSS(X)))
    print("RSE: " + str(mlr.RSE(X)))
    print("TSS: " + str(mlr.TSS(X)))
    print("R2: " + str(mlr.R2(X)))

else:
    print("nao foi")

#=====================================================================================


from gtts import gTTS
from subprocess import *
import time
from mutagen.mp3 import MP3

from datetime import date, datetime
today = date.today()
print("Today's date:", today)

goodmorning = "Good morning, the date is: " + datetime.now().strftime("%A-%b-%d (%I:%M%p)")
mp3File = '/Users/davidskinner/Desktop/hello.mp3'

tts = gTTS(goodmorning, 'en')
tts.save(mp3File)

audio = MP3(mp3File)
audioLength = audio.info.length
print("run length is: " + str(audioLength))

p = Popen(["vlc", '-I dummy', mp3File])
time.sleep(audioLength)
print('p = ', p.pid)
Popen.kill(p)


# Convert BST To Greater TreeQ

"""
Given the "root" of a Binary Search TreeQ(BST), convert it to Greater TreeQ
such that every key of the original BST is changed to the original
key plus the sum of keys greater than the original key in BST.

As a reminder, a binary search three is a tree that satisfies these constraints:
    The left subtree of a node contains only nodes with keys less than the node's key.
    The right subtree of a node contains only with keys greater than the node's key.
    Both the left and right subtrees must also be binary trees.

Example1:

                4 -30

     1 -36                 6-21

0-36      2-35       5-26       7 -15

            3-33                       8-8

Input : root = [4,1,6,0,2,5,7, null, null,null,3 null, null, null, 8]
Output : [30,36,21,3 6,35,26,15,null,null,nukk,33,null,null,null,8]

"""
from typing import Optional

import null as null

class TreeNode:

    def __init__(self, value):
        self.value = value
        self.left = None
        self.right = None

class Solution:
    def convertBST(selfself, root: Optional[TreeNode]) -> Optional[TreeNode]:

        sumOfValues = 0

        def traverse(node):
            if not node:
                return

            nonlocal sumOfValues
            if node.right is not None:  # node.right is not None -->ile aynısı
                traverse(node.right)

            # node.velue = 8, sumOfvalues= 0
            # temp = 8, node.value = 8, sumofValues = 8
            # node.velue = 7, sumOfvalues= 8
            # temp = 7, node.value = 15, sumofValues = 15
            # node.value = 6, sumOfvalues= 15
            # temp = 6, node.value = 21, sumofValues = 21
            # node.value = 5, sumOfvalues= 15
            # temp = 6, node.value = 21, sumofValues = 21
            temp = node.value
            node.value += sumOfValues
            sumOfValues += temp

            if node.left:
                traverse(node.left)
        traverse(root)
        return root

tree = TreeNode(5)
sol = Solution()

sol.convertBST(5)

"""
        4
    1       6
0     2   5     7
       3           8
"""


import datetime

from flask_script import Command, Option
from sqlalchemy.orm import load_only
from sqlalchemy import and_

from app import db
from app.resources.accounts.models import Account
from app.resources.account_stats.models import AccountStats
from app.resources.users.models import User

class UpdateAccountStats(Command):
    """
    Command to update stats of account

    :arg verbose:
        print progress
    :arg dry:
        dry run
    """

    option_list = [
        Option('--verbose', '-v', dest='verbose', action='store_true',
               default=False),
        Option('--dry', '-dry', dest='dry', action='store_true',
               default=False),
    ]

    def run(self, verbose, dry):
        if verbose:
            print('---' + str(datetime.datetime.utcnow()) + '---')
            print('updating account stats')

        try:
            account_ids = [a.row_id for a in Account.query.options(
                load_only('row_id')).all()]
            if account_ids:
                for account_id in account_ids:
                    user_count = User.query.filter(and_(
                        User.account_id == account_id,
                        User.deleted.is_(False))).count()
                    db.session.add(AccountStats(
                        account_id=account_id, total_users=user_count))
            db.session.commit()
        except Exception as e:
            db.session.rollback()
            print(e)
            exit(1)

        print('---' + str(datetime.datetime.utcnow()) + '---')
        print('Done')


from builtins import str
from builtins import map
from pyramid.view import view_config
from pyramid.httpexceptions import HTTPFound, HTTPMethodNotAllowed, HTTPNotFound,\
    HTTPBadRequest
from pyramid.security import forget
from pyramid.threadlocal import get_current_registry
from pyramid.response import FileResponse

from deform import ValidationFailure

from gecoscc import messages
from gecoscc.socks import is_websockets_enabled
from gecoscc.forms import (AdminUserAddForm, AdminUserEditForm, AdminUserVariablesForm,
    MaintenanceForm, UpdateForm, PermissionsForm)
from gecoscc.i18n import gettext as _
from gecoscc.models import (AdminUser, AdminUserVariables, Maintenance, UpdateModel, AdminUserOUPerms,
    AdminUserOUPerm, Permissions)
from gecoscc.pagination import create_pagination_mongo_collection
from gecoscc.utils import delete_chef_admin_user, get_chef_api, toChefUsername, getNextUpdateSeq, get_filter_nodes_belonging_ou,\
    SERIALIZED_UPDATE_PATTERN
from gecoscc.tasks import script_runner
from gecoscc.views.reports import get_complete_path

import os
import pymongo
import time
import pickle
import json
import collections
import logging
import re
import shutil
import sys
import traceback
import zipfile
logger = logging.getLogger(__name__)

from chef.exceptions import ChefServerError, ChefServerNotFoundError
from bson import ObjectId

@view_config(route_name='admins', renderer='templates/admins/list.jinja2',
             permission='is_superuser')
def admins(context, request):
    filters = {}
    q = request.GET.get('q', None)
    if q:
        filters = {'username': {'$regex': '.*%s.*' % q,
                                '$options': '-i'}}
    count = request.userdb.count_users(filters)
    admin_users = request.userdb.list_users(filters).sort('username')
    page = create_pagination_mongo_collection(request, admin_users, count)
    return {'admin_users': admin_users,
            'page': page}

@view_config(route_name='updates', renderer='templates/admins/updates.jinja2',
             permission='is_superuser')
def updates(context, request):
    filters = {}
    q = request.GET.get('q', None)
    if q:
        filters = {'name': {'$regex': '.*%s.*' % q,
                            '$options': '-i'}}

    # Order
    order = request.GET.get('order', None)
    if order:
        if order == "desc":
            ordering = pymongo.DESCENDING
        elif order == "asc":
            ordering = pymongo.ASCENDING
    logger.debug("admins.py ::: updates - order = {}".format(order))

    # Orderby
    sorting = ('timestamp', -1) # default
    s = request.GET.get('orderby', None)
    if s:
        if s == '_id':
            sorting = ('_id', ordering)
        elif s == 'name':
            sorting = ('name', ordering)
        elif s == 'log':
            sorting = ('timestamp', ordering)

    logger.debug("admins.py ::: updates - sorting = {}".format(sorting))
    count = request.db.updates.count_documents(filters)
    updates = request.db.updates.find(filters).sort([sorting])

    # "format" filter in jinja2 only admits "%s", not "{0}"
    settings = get_current_registry().settings
    controlfile = settings['updates.control'].replace('{0}','%s')

    latest =  "%04d" % (int(getNextUpdateSeq(request.db))-1)
    page = create_pagination_mongo_collection(request, updates, count)

    return {'updates': updates,
            'latest': latest,
            'controlfile': controlfile,
            'page': page}

@view_config(route_name='updates_log', permission='is_superuser')
def updates_log(_context, request):
    sequence = request.matchdict['sequence']
    rollback = request.matchdict['rollback']
    settings = get_current_registry().settings
    logfile = settings['updates.rollback'].format(sequence)
    if not rollback:
        logfile = settings['updates.log'].format(sequence)

    if not os.path.isfile(logfile):
        raise HTTPNotFound()

    response = FileResponse(
        logfile,
        request=request,
        content_type='text/plain'
    )
    headers = response.headers
    headers['Content-Type'] = 'application/download'
    headers['Content-Disposition'] = "attachment; filename=\"" + \
        os.path.basename(logfile) + "\"; filename*=UTF-8''"+ \
        str(sequence + "_" + os.path.basename(logfile))

    return response

@view_config(route_name='updates_download', permission='is_superuser')
def updates_download(_context, request):
    settings = get_current_registry().settings
    _id = request.matchdict['id']

    update = request.db.updates.find_one({'_id': _id})
    if update is None:
        raise HTTPNotFound()

    update_file = os.path.join(update['path'],
                               update['name'])
    if not os.path.isfile(update_file):
        raise HTTPNotFound()

    response = FileResponse(
        update_file,
        request=request,
        content_type='application/zip'
    )
    headers = response.headers
    headers['Content-Type'] = 'application/download'
    headers['Content-Disposition'] = 'attachment;filename=' + \
        str(update['name'])

    return response

@view_config(route_name='admins_ou_manage', renderer='templates/admins/ou_manage.jinja2',
             permission='is_superuser')
def admins_ou_manage(context, request):
    ou_choices = [(ou['_id'], ou['name']) for ou in request.db.nodes.find({'type': 'ou', 'path': 'root'})]
    ou_choices = [('', 'Select an Organisational Unit')] + ou_choices
    username = request.matchdict['username']
    schema = Permissions().bind(ou_choices=ou_choices)
    form = PermissionsForm(schema=schema,
                           collection=request.db['adminusers'],
                           username=username,
                           request=request)
    controls = {}
    instance = request.userdb.get_user(username)
    if '_submit' in request.POST:
        controls = list(request.POST.items())
        # Removing blanks OUs (not filling up by user)
        data = [tup for tup in controls if not (tup[0] == 'ou_selected' and tup[1] == '')]
        logger.debug("admins_ou_manage ::: data = {}".format(data))

        try:
            deserialized = form.validate(data)
            form.save(deserialized['perms'])
            return HTTPFound(location=get_url_redirect(request))
        except ValidationFailure as e:
            logger.error("admins_ou_manage ::: ValidationFailure = {}".format(e))
            form = e

    if instance and not controls:
        # instance = {'perms': [
        #     {'permission': [u'READONLY', u'LINK'], 'ou_selected': [u'562f7adee488e3664c6264e5']},
        #     {'permission': [u'REMOTE'], 'ou_selected': [u'5821789be488e34fcd2cf61c']},
        #     {'permission': [u'MANAGE'], 'ou_selected': [u'5526358508d70a63c6024794']}
        # ]}

        # Mapping model to appstruct
        instance['perms'] = []
        ou_managed = instance.get('ou_managed', [])
        ou_availables = instance.get('ou_availables', [])
        ou_remote = instance.get('ou_remote', [])
        ou_readonly = instance.get('ou_readonly', [])

        ous = set(ou_managed + ou_availables + ou_remote + ou_readonly)

        for ou in ous:

            perm = {'permission': [], 'ou_selected': [ou]}

            if ou in ou_managed:
                perm['permission'].append('MANAGE')
            if ou in ou_availables:
                perm['permission'].append('LINK')
            if ou in ou_remote:
                perm['permission'].append('REMOTE')
            if ou in ou_readonly:
                perm['permission'].append('READONLY')

            instance['perms'].append(perm)

        form_render = form.render(instance)
    else:
        form_render = form.render()

    return {
        'ou_manage_form': form_render,
        'username': username
    }

@view_config(route_name='admins_add', renderer='templates/admins/add.jinja2',
             permission='is_superuser')
def admin_add(context, request):
    return _admin_edit(request, AdminUserAddForm)

@view_config(route_name='admins_edit', renderer='templates/admins/edit.jinja2',
             permission='is_superuser_or_my_profile')
def admin_edit(context, request):
    return _admin_edit(request, AdminUserEditForm,
                       username=request.matchdict['username'])

@view_config(route_name='admins_set_variables', renderer='templates/admins/variables.jinja2',
             permission='is_superuser_or_my_profile')
def admins_set_variables(context, request):
    username = request.matchdict['username']
    user = request.db.adminusers.find_one({'username':username})

    # Ous managed by admin (user)
    if not user.get('is_superuser'):
        if 'ou_managed' in user:
            admin_ous = list(map(ObjectId, user['ou_managed']))
            ou_managed = [(ou['_id'], ou['name']) for ou in request.db.nodes.find({'_id': {'$in': admin_ous}})]
        else: # Recently new admin created
            ou_managed = []
    else: # Superuser
        ou_managed = [(ou['_id'], ou['name']) for ou in request.db.nodes.find({'type':'ou'})]

    ou_managed = [('', 'Select an Organisational Unit')] + ou_managed

    schema = AdminUserVariables().bind(ou_choices=ou_managed)
    form = AdminUserVariablesForm(schema=schema,
                                  collection=request.db['adminusers'],
                                  username=username,
                                  request=request)
    data = {}
    instance = request.userdb.get_user(username).get('variables', None)
    if '_submit' in request.POST:
        data = list(request.POST.items())
        try:
            variables = form.validate(data)
            form.save(variables)
            return HTTPFound(location=get_url_redirect(request))
        except ValidationFailure as e:
            logger.error('admins_set_variables - form validation error = '
                         + '{}'.format(e.error))
            form = e
    if instance and not data:
        form_render = form.render(instance)
    else:
        form_render = form.render()
    return {'variables_form': form_render,
            'username': username}

@view_config(route_name='admin_delete', permission='is_superuser_or_my_profile',  xhr=True, renderer='json')
def admin_delete(context, request):
    if request.method != 'DELETE':
        raise HTTPMethodNotAllowed("Only delete method is accepted")
    username = request.GET.get('username')
    if request.session['auth.userid'] == username:
        forget(request)
    settings = get_current_registry().settings
    api = get_chef_api(settings, request.user)
    success_remove_chef = delete_chef_admin_user(api, username)
    if not success_remove_chef:
        messages.created_msg(request, _('User deleted unsuccessfully from chef'), 'danger')
    request.userdb.delete_user({'username': username})
    messages.created_msg(request, _('User deleted successfully'), 'success')
    return {'ok': 'ok'}

@view_config(route_name='updates_add', renderer='templates/admins/updates_add.jinja2',
             permission='is_superuser')
def updates_add(context, request):
    schema = UpdateModel()
    form = UpdateForm(schema=schema,
                      request=request)

    instance = controls = {}
    if '_submit' in request.POST:
        controls = list(request.POST.items())
        logger.info('admin_updates - controls = %s' % controls)
        try:
            params = form.validate(controls)
            logger.info('admin_updates - params = %s' % params)
            form.save(params)
            return HTTPFound(location='')
        except ValidationFailure as e:
            logger.error('admin_updates - form validation error = {}'.format(e))
            form = e

    if instance and not controls:
        form_render = form.render(instance)
    else:
        form_render = form.render()

    return {
        'update_form': form_render,
    }

@view_config(route_name='updates_repeat', permission='is_superuser',
             renderer='templates/admins/tail.jinja2')
def updates_repeat(_context, request):
    settings = get_current_registry().settings
    sequence = request.matchdict.get('sequence')
    logger.debug('admins.py ::: updates_repeat - sequence = %s' % sequence)

    # Check sequence number
    to_repeat = request.db.updates.find_one({'_id': sequence})
    if to_repeat is None:
        logging.error('admins.py ::: updates_repeat: ' + \
                      'update not found with sequence=%s'%(sequence))
        raise HTTPNotFound()

    updatefile = os.path.join(to_repeat['path'], to_repeat['name'])
    if not os.path.isfile(updatefile):
        raise HTTPNotFound()

    if re.match(SERIALIZED_UPDATE_PATTERN, to_repeat['name']) is not None:
        logging.error('admins.py ::: updates_repeat: ' + \
                      'update %s is a serialized update!'%(sequence))
        raise HTTPBadRequest()

    if 'repetition' in to_repeat:
        logging.error('admins.py ::: updates_repeat: ' + \
                      'update %s is repetition!'%(sequence))
        raise HTTPBadRequest()

    try:
        # Calculate the next repetition number for this update
        count = request.db.updates.count_documents(
            {'name': to_repeat['name'], 'repetition': {'$exists': True}})

        nrep = 0
        if count > 0:
            cursor = request.db.updates.find(
                {'name': to_repeat['name'], 'repetition': {'$exists': True}},
                {'_id':1, 'repetition': 1}).sort('repetition',-1).limit(1)

            latest = int(cursor.next().get('repetition'))
            nrep = (latest+1)

        logger.debug('admins.py ::: updates_repeat - repetition = %s' % nrep)

        # Copy the zip file
        separator = '_'
        rsequence = '%s%s%s'%(sequence, separator, nrep)
        updatesdir = settings['updates.dir'] + rsequence
        while os.path.exists(updatesdir):
            separator = separator+ '_'
            rsequence = '%s%s%s'%(sequence, separator, nrep)
            updatesdir = settings['updates.dir'] + rsequence

        os.mkdir(updatesdir)
        shutil.copy(updatefile, os.path.join(updatesdir, to_repeat['name']))

        # Decompress zip file
        zip_ref = zipfile.ZipFile(updatefile,'r')
        zip_ref.extractall(updatesdir)
        zip_ref.close()

        # Decompress cookbook zipfile
        cookbookdir = settings['updates.cookbook'].format(rsequence)
        logger.debug('admins.py ::: updates_repeat - ' + \
                     ' cookbookdir = %s' % cookbookdir)
        for cookbook in os.listdir(cookbookdir):
            cookbook = cookbookdir + os.sep + cookbook
            logger.debug("admins.py ::: updates_repeat - cookbook = %s" % cookbook)
            if zipfile.is_zipfile(cookbook):
                zip_ref = zipfile.ZipFile(cookbook,'r')
                zip_ref.extractall(
                    os.path.join(cookbookdir, settings['chef.cookbook_name']))
                zip_ref.close()

        # Insert update register
        request.db.updates.insert_one({
            '_id': rsequence,
            'name': to_repeat['name'],
            'path': updatesdir,
            'timestamp': int(time.time()),
            'rollback':0,
            'user': request.user['username'],
            'repetition': nrep})

        # Launching task for script execution
        script_runner.delay(request.user, rsequence)

        messages.created_msg(request, _('The update has been repeated!'), 'success')

    except:
        e = sys.exc_info()[0]
        logger.error("admins.py ::: updates_repeat - " + \
            "error repeating update: %s"%(str(e)))
        logger.error("Traceback: %s"%(traceback.format_exc()))
        messages.created_msg(request,
            _('There was an error trying to repeat an update'), 'danger')

    return HTTPFound(request.route_url('updates'))

@view_config(route_name='updates_tail', permission='is_superuser', renderer='templates/admins/tail.jinja2')
def updates_tail(_context, request):
    logger.info("Tailing log file ...")
    sequence = request.matchdict.get('sequence')
    rollback = request.matchdict.get('rollback', '')
    logger.debug('admins.py ::: updates_tail - sequence = %s' % sequence)
    logger.debug('admins.py ::: updates_tail - rollback = %s' % rollback)

    if rollback == 'rollback' and request.db.updates.find_one({'_id': sequence}).get('rollback', 0) == 0:
        # Update mongo document
        request.db.updates.update_one({'_id':sequence},{'$set':{
            'rollback':1, 'timestamp_rollback': int(time.time()),
            'rolluser': request.user['username']}})

        # Celery task
        script_runner.delay(request.user, sequence, rollback=True)

    return {
        'websockets_enabled': json.dumps(is_websockets_enabled()),
        'request': request,
        'sequence': sequence,
        'rollback': rollback
    }

@view_config(route_name='admin_maintenance', permission='is_superuser', renderer="templates/admins/maintenance.jinja2")
def admin_maintenance(_context, request):

    schemaMaintenance = Maintenance()
    form = MaintenanceForm(schema=schemaMaintenance,
                           request=request)

    data = {}
    settings = get_current_registry().settings
    instance = request.db.settings.find_one({'key':'maintenance_message'})
    if '_submit' in request.POST:
        data = list(request.POST.items())
        try:
            postdata = form.validate(data)
            logger.debug("admins_log ::: admin_maintenance  = %s" % (postdata))
            form.save(postdata)
            return HTTPFound(location='')
        except ValidationFailure as e:
            form = e

    if instance and not data:
        instance['maintenance_message'] = instance.get('value')
        form_render = form.render(instance)
    else:
        form_render = form.render()

    # Query String: maintenance mode ON/OFF
    mode = request.GET.get('mode', None)
    logger.info("admin_maintenance ::: mode = %s" % mode)
    obj = request.db.settings.find_one({'key':'maintenance_mode'})
    if obj is None:
        obj = {'key':'maintenance_mode', 'value': mode == 'true', 'type':'string'}
        request.db.settings.insert_one(obj)

    else:
        if mode is not None:
            request.db.settings.update_one(
                {'key':'maintenance_mode'},{'$set':{ 'value': mode == 'true' }})
            logger.info("admin_maintenance ::: obj = %s" % (obj))
            if mode == 'false':
                request.db.settings.delete_one({'key':'maintenance_message'})

    # Active users
    sessions = [pickle.loads(session['value']) for session in request.db.backer_cache.find({},{'_id':0, 'value':1})]
    logger.info("admin_maintenance ::: sessions = %s" % sessions)
    last_action = int(time.time()) - int(settings.get('idle_time',15*60))
    logger.info("admin_maintenance ::: last_action = %s" % last_action)
    active_users = [ session['auth.userid'] for session in sessions if session.get('auth.userid',None) and int(session['_accessed_time']) > last_action ]
    logger.info("admin_maintenance ::: active_users = %s" % active_users)

    filters = {'username':{'$in': active_users }}
    count = request.userdb.count_users(filters)
    admin_users = request.userdb.list_users(filters).sort('username')
    page = create_pagination_mongo_collection(request, admin_users, count)

    return {
       'admin_users': admin_users,
       'page': page,
       'maintenance': obj['value'],
       'form_maintenance': form_render
    }

@view_config(route_name='statistics', permission='edit', renderer="templates/admins/statistics.jinja2")
def statistics(context, request):

    object_counters = []
    policy_counters = []
    ous = {}
    ous_visibles = []
    settings = get_current_registry().settings
    policyname = "name_{}".format(settings['pyramid.default_locale_name'])

    is_superuser = request.user.get('is_superuser', False)
    ou_id = request.GET.get('ou_id', None)
    logger.debug("admins.py ::: statistics - ou_id = {}".format(ou_id))

    if is_superuser:
        ous_visibles = request.db.nodes.find(
            {"type": "ou"},
            {"_id": 1, "name": 1, "path": 1}
        )
    else:
        # Get managed ous for admin
        oids = request.user.get('ou_managed', []) + request.user.get('ou_readonly', [])
        ous_visibles = request.db.nodes.find(
            {"_id": { "$in": list(map(ObjectId, oids)) }},
            {"_id": 1, "name": 1, "path": 1}
        )
    for ou in ous_visibles:
        path = ou['path'] + ',' + str(ou['_id'])
        ous.update({str(ou['_id']): get_complete_path(request.db, path)})

    sorted_ous = collections.OrderedDict(sorted(list(ous.items()), key=lambda kv: len(kv[1])))
    logger.debug("admins.py ::: statistics - sorted_ous = {}".format(sorted_ous))

    # Defaults
    if not ou_id and list(sorted_ous.items()) is not None and len(list(sorted_ous.items()))>0:
        ou_id = str(list(sorted_ous.items())[0][0])

    logger.debug("admins.py ::: statistics - ou_id = {}".format(ou_id))

    # Objects
    object_counters = request.db.nodes.aggregate([
        {"$match" : { "path": get_filter_nodes_belonging_ou(ou_id)}},
        {"$group" : { "_id" : "$type", "count": {"$sum":1}}}
    ], cursor={})

    logger.debug("admins.py ::: statistics - object_counters = {}".format(object_counters))

    # Policies
    for pol in request.db.policies.find().sort("name"):
        c = request.db.nodes.count_documents({
            "$or": [{"path": get_filter_nodes_belonging_ou(ou_id)}, {"_id":ObjectId(ou_id)}],
            "policies." + str(pol['_id']): {'$exists': True}
        })
        try:
            policy_counters.append([pol[policyname],c])
        except KeyError:
            policy_counters.append([pol['name'],c])

    logger.debug("admins.py ::: statistics - policy_counters = {}".format(policy_counters))

    return {
       'policy_counters': policy_counters,
       'object_counters': object_counters,
       'ou_managed': sorted_ous,
       'ou_selected': ou_id,
       'is_superuser': is_superuser
    }

def _check_if_user_belongs_to_admin_group(request, organization, username):
    chefusername = toChefUsername(username)
    settings = get_current_registry().settings
    api = get_chef_api(settings, request.user)

    admins_group = api['/organizations/%s/groups/admins'%(organization)]
    if not chefusername in admins_group:
        # Check if exists an association request for this user
        assoc_requests = None
        try:
            assoc_requests = api['/organizations/%s/association_requests'%(organization)]
        except ChefServerNotFoundError:
            pass

        association_id = None
        for req in assoc_requests:
            if req["username"] == chefusername:
                association_id = req["id"]

        if association_id is None:
            # Set an association request for the user in that organization
            try:
                data = {"user": chefusername}
                response = api.api_request('POST', '/organizations/%s/association_requests'%(organization), data=data)
                association_id = response["uri"].split("/")[-1]
            except ChefServerError:
                # Association already exists?
                pass

        if association_id is not None:
            # Accept the association request
            logger.info('Adding %s user to default organization'%(username))
            api.api_request('PUT', '/users/%s/association_requests/%s'%(chefusername, association_id),  data={ "response": 'accept' })

        # Add the user to the group
        logger.info('Adding %s user to admins group'%(username))
        admins_group['users'].append(chefusername)
        api.api_request('PUT', '/organizations/%s/groups/admins'%(organization), data={ "groupname": admins_group["groupname"],
            "actors": {
                "users": admins_group['users'],
                "groups": admins_group["groups"]
            }
            })

def _admin_edit(request, form_class, username=None):
    admin_user_schema = AdminUser()
    admin_user_form = form_class(schema=admin_user_schema,
                                 collection=request.db['adminusers'],
                                 username=username,
                                 request=request)
    instance = data = {}
    settings = get_current_registry().settings
    if username:
        instance = request.userdb.get_user(username)
    if '_submit' in request.POST:
        data = list(request.POST.items())
        if username:
            data.append(('username', username))
        try:
            admin_user = admin_user_form.validate(data)
            if username is None:
                username = admin_user['username']
            logger.info('Save %s data in GECOS database'%(username))
            success = admin_user_form.save(admin_user)
            if success:
                # At this moment all GECOS domains are in the "default" Chef organization.
                # So, all the administrator users must belong to the "default" organization's "admins" group
                if int(settings.get('chef.version').split('.')[0]) >= 12 and username is not None:
                    _check_if_user_belongs_to_admin_group(request, 'default', username)

                return HTTPFound(location=get_url_redirect(request))
        except ValidationFailure as e:
            admin_user_form = e
    if instance and not data:
        form_render = admin_user_form.render(instance)
    else:
        form_render = admin_user_form.render()

    # At this moment all GECOS domains are in the "default" Chef organization.
    # So, all the administrator users must belong to the "default" organization's "admins" group
    if int(settings.get('chef.version').split('.')[0]) >= 12 and username is not None:
        _check_if_user_belongs_to_admin_group(request, 'default', username)

    return {'admin_user_form': form_render,
            'username': username,
            'instance': instance,
            'registry': get_current_registry()}

def get_url_redirect(request):
    user = request.user
    if user.get('is_superuser'):
        redirect_view = 'admins'
    else:
        redirect_view = 'home'
    return request.route_url(redirect_view)


from fastapi import FastAPI
from pydantic import BaseModel
from typing import List

import regras_de_uso

app = FastAPI()

@app.get("/")
async def bem_vinda():
    site = "Seja bem vinda"
    return site.replace('\n', '')

###################################################################################
# Classes
# Classe representando os dados do usuário
class Usuario(BaseModel):
    #def __init__(self, id, nome, email, senha):
    id: int
    nome: str
    email: str
    senha: str

# Classe representando os dados do cliente
class Endereco(BaseModel):
    rua: str
    cep: str
    cidade: str
    estado: str

# Classe representando a lista de endereços de um cliente (Não usei ainda..)
class ListaDeEnderecosDoUsuario(BaseModel):
    usuario: Usuario
    enderecos: List[Endereco] = []

# Classe representando os dados do produto
class Produto(BaseModel):
    id: int
    nome: str
    descricao: str
    preco: float

# Classe representando o carrinho de compras de um cliente com uma lista de produtos
class CarrinhoDeCompras(BaseModel):
    id_usuario: int
    id_produtos: List[Produto] = ()
    preco_total: float
    quantidade_de_produtos: int

###################################################################################
#ok Criar um usuário,
#ok se tiver outro usuário com o mesmo ID retornar falha,
#ok se o email não tiver o @ retornar falha,
#ok senha tem que ser maior ou igual a 3 caracteres,
#ok senão retornar OK
@app.post("/usuario/")
async def criar_usuario(usuario: Usuario):
    print(usuario.dict())
    novo_user = regras_de_uso.regras_usuario_cadastrar(usuario.dict())

    return novo_user

###################################################################################
# Se o id do usuário existir, retornar os dados do usuário
# senão retornar falha
@app.get("/usuario/{id}")
async def retornar_usuario(id: int):
    dados_usuario = regras_de_uso.regras_id_usuario(id)

    return dados_usuario

##################################################################################
# Se existir um usuário com exatamente o mesmo nome, retornar os dados do usuário
# senão retornar falha
@app.get("/usuario/nome/{nome}")
async def retornar_usuario_com_nome(nome: str):
    print(nome)
    mesmo_usuario = regras_de_uso.regras_nome_usuario(nome)
    print(mesmo_usuario)

    return mesmo_usuario

#################################################################################
# Se o id do usuário existir, deletar o usuário e retornar OK
# senão retornar falha
# ao deletar o usuário, deletar também endereços e carrinhos vinculados a ele
@app.delete("/usuario/{id}")
async def deletar_usuario(id: int):
    dados_usuario_apaga = regras_de_uso.regras_id_usuario_delete(id)

    return dados_usuario_apaga

#################################################################################
# Se não existir usuário com o id_usuario retornar falha,
# senão retornar uma lista de todos os endereços vinculados ao usuário
# caso o usuário não possua nenhum endereço vinculado a ele, retornar
# uma lista vazia
### Estudar sobre Path Params (https://fastapi.tiangolo.com/tutorial/path-params/)
@app.get("/usuario/{id_usuario}/endereços/")
async def retornar_enderecos_do_usuario(id_usuario: int):
    endereço_usuario = regras_de_uso.regras_endereco_usuario(id_usuario)

    return endereço_usuario

#################################################################################
# Retornar todos os emails que possuem o mesmo domínio
# (domínio do email é tudo que vêm depois do @)
# senão retornar falha
@app.get("/usuarios/emails/{dominio}")
async def retornar_emails(dominio: str):
    email_database = regras_de_uso.regras_email_database(dominio)

    return email_database

##################################################################################
# Se não existir usuário com o id_usuario retornar falha,
# senão cria um endereço, vincula ao usuário e retornar OK
@app.post("/endereco/{id_usuario}")
async def criar_endereco(endereco: Endereco, id_usuario: int):
    cria_endereco_id_usuario = \
        regras_de_uso.regras_cria_endereco_usuario(endereco.dict(), id_usuario)

    return cria_endereco_id_usuario

##################################################################################
# Se não existir endereço com o id_endereco retornar falha,
# senão deleta endereço correspondente ao id_endereco e retornar OK
# (lembrar de desvincular o endereço ao usuário)
@app.delete("/endereco/{id_endereco}")
async def deletar_endereco(id_endereco: int):
    dados_endereco_apaga = regras_de_uso.regras_id_endereco_delete(id_endereco)

    return dados_endereco_apaga

#################################################################################
# Se tiver outro produto com o mesmo ID retornar falha,
# senão cria um produto e retornar OK
@app.post("/produto/")
async def criar_produto(produto: Produto):
    cria_produto = regras_de_uso.regras_cria_produto(produto.dict())
    return cria_produto

#################################################################################
# Se não existir produto com o id_produto retornar falha,
# senão deleta produto correspondente ao id_produto e retornar OK
# (lembrar de desvincular o produto dos carrinhos do usuário) TODO entender vinculação
@app.delete("/produto/{id_produto}/")
async def deletar_produto(id_produto: int):
    dados_produto_apaga = regras_de_uso.regras_produto_apaga(id_produto)

    return dados_produto_apaga

#################################################################################
# Se não existir usuário com o id_usuario ou id_produto retornar falha,
# se não existir um carrinho vinculado ao usuário, crie o carrinho
# e retornar OK
# senão adiciona produto ao carrinho e retornar OK
@app.post("/carrinho/{id_usuario}/{id_produto}/")
async def adicionar_carrinho(carrinho: CarrinhoDeCompras, id_usuario: int, id_produto: int):
    criar_carrinho = regras_de_uso.regras_cria_carrinho(id_usuario, id_produto, carrinho.dict())
    return criar_carrinho

#################################################################################
# Se não existir carrinho com o id_usuario retornar falha,
# senão retorna o carrinho de compras.
@app.get("/carrinho/{id_user}")
async def retornar_carrinho(id_user: int):
    dados_carrinho = regras_de_uso.regras_retornar_carrinho(id_user)
    return dados_carrinho

#################################################################################
# Se não existir carrinho com o id_usuario retornar falha,
# senão retorna o o número de itens e o valor total do carrinho de compras.
@app.get("/carrinho/{id_usuario}/checkout")
async def retornar_total_carrinho(id_usuario: int):
    informação_carrinho = regras_de_uso.regras_itens_valor_total(id_usuario)
    if informação_carrinho is list:
        numero_itens = informação_carrinho[0]
        valor_total = informação_carrinho[1]

        return numero_itens, valor_total
    else:
        return informação_carrinho

##################################################################################
# Se não existir usuário(?? Não seria carrinho?) com o id_usuario retornar falha,
# senão deleta o carrinho correspondente ao id_usuario e retornar OK
@app.delete("/carrinho/{id_usuario}/")
async def deletar_carrinho(id_usuario: int):
    deleta_carrinho_user = regras_de_uso.regras_deleta_carrinho(id_usuario)

    return deleta_carrinho_user


import os
import pyforms

import harvestlist
import hlpickle

from   pyforms          import BaseWidget
from   pyforms.Controls import ControlText
from   pyforms.Controls import ControlButton
from   pyforms.Controls import ControlLabel
from   pyforms.Controls import ControlList
from   pyforms.Controls import ControlEmptyWidget
from   pyforms.Controls import ControlDir
from   pyforms.Controls import ControlFile
from   pyforms.Controls import ControlSaveFile
from PyQt5 import QtCore

class Main:
    # Main window
    root = None
    # Stores the data of the harvest list
    harvest_list = None

    # The HarvestList class
    hl = harvestlist.HarvestList()

    # Stores the object to be pickled
    hl_pickle = hlpickle.HLPickle()

    output_dir = None
    output_path = None

class mainWindow(BaseWidget):
    def __init__(self):

        super().__init__('Oogstlijst Manager')
        self._panel = ControlEmptyWidget()
        self._panel.value = Home()

        Main.root = self

class Home(BaseWidget):
    def __init__(self):
        super().__init__('Home')

        self.button_new = ControlButton('Nieuwe oogstlijst')
        self.button_open = ControlButton('Open bestaande oogstlijst')
        self.formset = ['button_new', 'button_open']

        self.button_new.value = self._new
        self.button_open.value = self._open

    def _new(self):
        window = NewHarvest()
        Main.root._panel.value = window

    def _open(self):
        window = OpenHarvest()
        Main.root._panel.value = window

class OpenHarvest(BaseWidget):
    def __init__(self):
        super().__init__('Open oogstlijst')
        self._file = ControlFile('Project')
        self._file.filter = "OLM files (*.olm)"
        self._button_continue = ControlButton('Kies')

        self._button_continue.value = self.open_file

    def open_file(self):
        try:
            Main.hl_pickle = Main.hl.load_pickle(self._file.value)
        except FileNotFoundError:
            message = Message('Het opgegeven bestand kon niet worden geopend.')
            message.parent = self
            message.show()
        else:
            Main.hl.harvest_list = Main.hl_pickle.harvest_list
            Main.hl.name = Main.hl_pickle.name

            window = EditHarvest()
            Main.root._panel.value = window

class NewHarvest(BaseWidget):
    def __init__(self):
        super().__init__('Nieuwe oogstlijst')

        self._nameField = ControlText('Naam')
        self._button_continue= ControlButton('Begin')
        self._button_continue.value = self._continue

        self.formset = ['_nameField', '_button_continue']

    def _continue(self):
        name = self._nameField.value
        if not name:
            return

        Main.hl_pickle = hlpickle.HLPickle(name)
        Main.hl.name = name

        window = EditHarvest()
        Main.root._panel.value = window

class EditHarvest(BaseWidget):
    def __init__(self):
        super().__init__('Oogstlijst Manager')
        reference_list = Main.hl.reference_data

        self._list_left = ControlList('Stamlijst')
        self._list_left.readonly = True

        self._list_right = ControlList('Oogstlijst')
        self._list_right.select_entire_row = True
        self._list_right.horizontal_headers = [
            'Oogst',
            'Hoeveelheid (kg)',
        ]

        self._save_button = ControlButton('Verder')
        self._save_button.value = self._continue

        self.formset = ['_list_left', '_list_right', '_save_button']

        def onclick_left(row, column):
            r = self._list_left.get_currentrow_value()
            self._list_right.__add__(r + [0])

            # cell = self._list_right.get_cell(0, self._list_right.rows_count - 1)
            # cell.setFlags(QtCore.Qt.ItemIsEditable)
            # cell = self._list_right.get_cell(1, self._list_right.rows_count - 1)
            # cell.setTextAlignment(QtCore.Qt.AlignCenter)

        def onclick_right(row, column):
            # # print(dir(self._list_right))
            if column == 0:
                index = self._list_right.selected_row_index
                print(index)
                self._list_right.__sub__(index)
            # # r = self._list_right.get_currentrow_value()
            # self._list_right.__sub__(self._list_right.selected_row_index())
            # #cell = self._list_right.get_cell(1, self._list_right.rows_count - 1)
            # #cell.setTextAlignment(QtCore.Qt.AlignCenter)

        self._list_left.cell_double_clicked_event = onclick_left
        self._list_right.cell_double_clicked_event = onclick_right

        for r in reference_list:
            self._list_left.__add__([' '.join(r[1:3])])

        if Main.hl.harvest_list:
            for r in Main.hl.harvest_list:
                self._list_right.__add__([r[0], r[1]])

    def _continue(self):
        user_input = self._list_right.value
        if not user_input:
            return

        harvest_list = []

        # Combine user input with reference data
        for row in user_input:
            reference = [ ref for ref in Main.hl.reference_data if\
                 ref[1] and ref[1] in row[0] and ref[2] and ref[2] in row[0] ][0]
            if not reference:
                raise Exception('Reference data not found')

            row += [reference[0]] # add prod nr
            row += [reference[2]] # add part
            row += [reference[3]] # add date
            harvest_list.append(row)

        Main.hl.harvest_list = Main.hl_pickle.harvest_list = harvest_list

        window = Rapport()
        Main.root._panel.value = window

class Rapport(BaseWidget):
    def __init__(self):
        super().__init__('Oogstlijst Manager')
        self._plan_button = ControlButton('Genereer planning')
        self._harvest_list_button = ControlButton('Genereer oogstlijst')
        self._save = ControlButton('Opslaan')

        self._plan_button.value = self.generate_plan
        self._harvest_list_button.value = self.generate_list
        self._save.value = self.save_pickle

        self.formset = ['_plan_button', '_harvest_list_button', '_save']

    def generate_plan(self):
        window = SelectFile()
        window.parent = self

        def callback():
            path = os.path.join(Main.output_dir ,Main.hl_pickle.name +\
                              '_planning.xlsx')
            Main.hl.save_plan(path)
            success = Message('Planning gemaakt in {}'.format(path))
            success.parent = self
            success.show()

        window.callback = callback
        window.show()

    def generate_list(self):
        window = SelectFile()
        window.parent = self

        def callback():
            path = os.path.join(Main.output_dir ,Main.hl_pickle.name +\
                              '_oogstlijst.xlsx')
            Main.hl.save_list(path)
            success = Message('Oogstlijst gemaakt in {}'.format(path))
            success.parent = self
            success.show()

        window.callback = callback
        window.show()

    def save_pickle(self):
        #window = SelectFile()
        window = SaveFile()
        window.parent = self

        def callback():
            path = os.path.abspath(Main.output_path)
            #path = os.path.join(Main.output_dir ,Main.hl_pickle.name +\
            #                  '.olm')
            Main.hl.save_pickle(Main.hl_pickle, path)
            success = Message('Project opgeslagen in {}'.format(path))
            success.parent = self
            success.show()

        window.callback = callback
        window.show()

class SelectFile(BaseWidget):
    def __init__(self):
        super().__init__('Oogstlijst Manager')
        self._label = ControlLabel('Kies de map waarin je het bestand wil opslaan')
        self._dir = ControlDir('Map')
        self._select_button = ControlButton('Kies')
        self._select_button.value = self.select

    def select(self):
        directory = self._dir.value
        if not directory:
            return
        Main.output_dir = directory
        self.close()

        if self.callback:
            self.callback()

class SaveFile(BaseWidget):
    def __init__(self):
        super().__init__('Oogstlijst Manager')
        self._file = ControlSaveFile('Opslaan')
        self._file.filter = 'Oogstlijst Manager bestanden (*.olm)'
        self._select_button = ControlButton('Verder')
        self._select_button.value = self.select

        self.formset = ['_file', '_select_button']

    def select(self):
        path = self._file.value
        if not path:
            return

        # extension
        filename, extension = os.path.splitext(path)
        if extension.lower() != '.olm':
            path += '.olm'

        Main.output_path = path
        self.close()

        if self.callback:
            self.callback()

class Message(BaseWidget):
    def __init__(self, message):
        super().__init__('Oogstlijst Manager')
        self._label = ControlLabel(message)

def main():
    pyforms.start_app(mainWindow)
    #pyforms.start_app(EditHarvest)

if __name__ == "__main__":
    main()


from django.urls import path
from django.views.static import serve
from django.conf.urls import url
from django.conf import settings
from .views import (
    AboutUs,
    Accessories,
    CardPaymentView,
    Clothes,
    ItemDetailView,
    CheckoutView,
    HomeView,
    OrderSuccessful,
    OrderSummaryView,
    PhoneView,
    PrivacyPolicy,
    SearchView,
    TermsConditions,
    add_to_cart,
    remove_from_cart,
    remove_single_item_from_cart,
    PaymentView,
    AddCouponView,
    RequestRefundView
)

app_name = 'rope'

urlpatterns = [
    path('', HomeView.as_view(), name='home'),
    path('accessories', Accessories.as_view(), name='accessories'),
    path('search', SearchView.as_view(), name='search'),
    path('phones', PhoneView.as_view(), name='phones'),
    path('checkout/', CheckoutView.as_view(), name='checkout'),
    path('order-summary/', OrderSummaryView.as_view(), name='order-summary'),
    path('product/<slug>/', ItemDetailView.as_view(), name='product'),
    path('add-to-cart/<slug>/', add_to_cart, name='add-to-cart'),
    path('add-coupon/', AddCouponView.as_view(), name='add-coupon'),
    path('remove-from-cart/<slug>/', remove_from_cart, name='remove-from-cart'),
    path('remove-item-from-cart/<slug>/', remove_single_item_from_cart,
         name='remove-single-item-from-cart'),
    path('payment/<payment_option>/', PaymentView.as_view(), name='payment'),
    path('cardpayment/<payment_option>/', CardPaymentView.as_view(), name='cardpayment'),

    path('ordersuccessful', OrderSuccessful.as_view(), name='ordersuccessful'),
    path('clothes', Clothes.as_view(), name='clothes'),
    path('about_us', AboutUs.as_view(), name='about_us'),
    path('privacy-policy', PrivacyPolicy.as_view(), name='privacy-policy'),
    path('termsandconditions', TermsConditions.as_view(), name='termsandconditions'),

    path('request-refund/', RequestRefundView.as_view(), name='request-refund'),
    url(r'^media/(?P<path>.*)$', serve,{'document_root': settings.MEDIA_ROOT}),
    url(r'^static/(?P<path>.*)$', serve,{'document_root': settings.STATIC_ROOT}),
]


import io
from django.http import FileResponse
from reportlab.pdfgen import canvas
from reportlab.lib.units import mm, cm
from .models import Order, OrderItem
from customers.models import Customers
from products.models import Product

def some_view(request, id):

    data_customer = get_customer_data(id)
    data_items = get_items_order(id)

    business_details = (
        u"Naty's Juane",
        u"Facebook: Naty's Juane",
        u'Instagram: @natysjuane',
        u'WhastApp: 957 906 764'
    )
    buffer = io.BytesIO()
    pagesize = (57 * mm, 228 * mm)
    p = canvas.Canvas(buffer, pagesize=pagesize)
    h_header = 85
    v_header =630

    #DRAW HEADER
    for i in range(0, len(business_details)):
        if i == 0:
            p.setFont("Helvetica-Bold",9)
        else:
            p.setFont("Helvetica",8)

        p.drawCentredString(h_header , v_header, business_details[i])
        v_header-=10

    p.drawRightString(155,585, '================================')

    # DRAW CUSTOMER INFO
    p.drawCentredString(h_header, 575, "B001-"+str(id))
    h_customer=10

    p.drawString(h_customer, 565, "Nombre: "+data_customer['name'])
    p.drawString(h_customer, 555, "Celular: "+data_customer['phone'])
    p.drawString(h_customer, 545, "Direccion: "+data_customer['direction'])
    p.drawString(h_customer, 535, "Distrito: "+data_customer['distric'])
    p.drawString(h_customer, 525, "Referencia: "+data_customer['reference'])
    p.drawRightString(155,510, '================================')

    # DRAW ORDER ITEMS
    h_items = 5
    v_items = 495
    p.drawString(h_items, v_items, 'Producto')
    p.drawString(h_items+40, v_items, 'Cantidad')
    p.drawString(h_items+80, v_items, 'P.Unit')
    p.drawString(h_items+110, v_items, 'SUBTOTAL')
    p.drawString(h_items, v_items-10, '---------------------------------------------------------')
    total_order=0
    v_items2 = 475
    for key, value in data_items.items():
        p.drawString(h_items, v_items2,value['product'])
        p.drawString(h_items+50, v_items2-10,str(value['quantity']))
        p.drawString(h_items+80, v_items2-10,str(value['price']))
        p.drawString(h_items+130, v_items2-10,str(value['total_item']))
        total_order+=value['total_item']
        v_items2-=20

    gravada = float(total_order)/1.18
    igv = gravada*0.18
    p.drawString(h_items, v_items2, '---------------------------------------------------------')
    p.drawRightString(h_items+100, v_items2-10, 'OP.GRAVADA' )
    p.drawRightString(h_items+110, v_items2-10, 'S/' )
    p.drawString(h_items+130, v_items2-10, str(round(gravada,2)))

    p.drawRightString(h_items+100, v_items2-20, 'IGV' )
    p.drawRightString(h_items+110, v_items2-20, 'S/' )
    p.drawString(h_items+130, v_items2-20, str(round(igv,2)))

    p.setFont("Times-Bold",10)
    p.drawRightString(h_items+100, v_items2-30, 'TOTAL' )
    p.drawRightString(h_items+110, v_items2-30, 'S/' )
    p.drawString(h_items+125, v_items2-30, str(total_order))
    # DRWA FOOT
    p.drawString(h_items+10, v_items2-70, '*** Gracias por su compra ***')
    p.showPage()
    p.save()
    buffer.seek(0)

    return FileResponse(buffer, as_attachment=True, filename=f'ticket{id}.pdf')

def get_customer_data(id):
    query_order = Order.objects.filter(id=id).values_list()
    id_customer = query_order[0][1]
    query_customer = Customers.objects.filter(id=id_customer).values_list()
    customer_data = {
        'name' : query_customer[0][1],
        'phone' : query_customer[0][2],
        'direction' : query_customer[0][3],
        'distric' : query_customer[0][4],
        'reference' : query_customer[0][5]
    }
    return customer_data

def get_items_order(id):
    query_order_item = OrderItem.objects.filter(order=id).values_list()
    items = {}
    key_item = 0
    for item in query_order_item:
        name_item = Product.objects.filter(id=item[2]).values()
        items[key_item] = {
            'product': name_item[0]['name'],
            'price' : item[4],
            'quantity' : item[3],
            'total_item': item[3]*item[4]
        }
        key_item +=1
    return items


import warnings
warnings.filterwarnings("ignore")
import numpy as np
import pandas as pd
from sklearn import model_selection
from sklearn.preprocessing import StandardScaler, OneHotEncoder, LabelEncoder
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from datetime import date

def load_data(path='customer_data', test_size=0.3, encoding=True) -> tuple:
    """
    train, test, validation 데이터를 불러오는 함수
    """

    if path == 'original_data':
        preprocess_data()
        path = 'customer_data'

    train_data = pd.read_csv(f'{path}/train.csv')
    train_label = np.array(train_data[['target']])
    test_data = pd.read_csv(f'{path}/test.csv')
    validation_set = tuple()

    if test_size > 0:
        validation_set = make_validation_set(train_data, test_size)

    if encoding:
        train_data, test_data, validation_set = feature_encoding(train_data, test_data, validation_set)

    return train_data, test_data, train_label, validation_set

###########################################################################
############################# Preprocessing ###############################
###########################################################################

def preprocess_data():
    """
    train, test 데이터를 전처리하고 저장하는 함수
    """

    train = pd.read_csv('original_data/train.csv')
    test = pd.read_csv('original_data/test.csv')

    train, test = label_encoding(train, test)
    train, test = make_extra_variables(train, test)
    train, test = remove_outliers(train, test)
    train, test = reindex_data(train, test)

    train.to_csv('customer_data/train.csv', index=False)
    test.to_csv('customer_data/test.csv', index=False)

def label_encoding(train: pd.DataFrame, test: pd.DataFrame) -> tuple:
    """
    문자열 범주형 데이터에 Label Encoder를 적용하는 함수
    """

    train, test = train.copy(), test.copy()

    single_list = ['Alone','YOLO','Absurd']
    train['Marital_Status'] = train['Marital_Status'].apply(lambda x: 'Single' if x in single_list else x)
    test['Marital_Status'] = test['Marital_Status'].apply(lambda x: 'Single' if x in single_list else x)

    label_features = ['Education','Marital_Status']
    for feature in label_features:
        label_encoder = LabelEncoder()
        train[feature] = label_encoder.fit_transform(train[feature])
        test[feature] = label_encoder.transform(test[feature])

    return (train, test)

def make_extra_variables(train: pd.DataFrame, test: pd.DataFrame) -> tuple:
    """
    파생변수를 생성하는 함수
    """

    train, test = train.copy(), test.copy()

    for df in [train, test]:
        df['Age'] = 2022-df['Year_Birth']
        df['Age_Range'] = (df['Age']//10).replace({2:3,8:7})

        df['Dt_Customer'] = df['Dt_Customer'].apply(lambda x: list(map(int,x.split('-'))))
        df['Dt_Customer'] = df['Dt_Customer'].apply(lambda x: date(x[2],x[1],x[0]))
        df['Days_Customer'] = df['Dt_Customer'].apply(lambda x: (date(2022,1,1)-x).days)

        max_income = df[['Age_Range','Income']].groupby('Age_Range').max()
        df['Income_Level'] = [row['Income']/max_income.loc[row['Age_Range']][0] for _,row in df.iterrows()]
        df['Income_Per'] = df['Income']/(df['Kidhome']+1)

        purchase_list = [f'Num{cat}Purchases' for cat in['Web','Catalog','Store']]
        df['NumPurchases'] = [sum(row) for _,row in df[purchase_list].iterrows()]
        purchase_dict = {cat: i for i, cat in enumerate(purchase_list)}
        df['Perferred_Purchase'] = [purchase_dict[row.index[row.argmax()]] for _,row in df[purchase_list].iterrows()]

        campains = [f'AcceptedCmp{i}' for i in range(1,6)]+['Response']
        df['NumAcceptedCmp'] = sum([df[campain] for campain in campains])

    return (train, test)

def remove_outliers(train: pd.DataFrame, test: pd.DataFrame) -> tuple:
    """
    이상치를 제거하는 함수
    """

    train, test = train.copy(), test.copy()

    columns = ['Year_Birth','Income']
    for column, outlier in zip(columns,[0,2]):
        cutted_data = pd.cut(train[column],bins=3,labels=[0,1,2])
        train = train[cutted_data != outlier]

    # num_cat = [f'Num{cat}Purchases' for cat in['Deals','Web','Catalog','Store']]+['NumWebVisitsMonth']
    for df in [train, test]:
    #     for column in num_cat:
    #         df[column] = df[column].apply(lambda x: 13 if x > 13 else x)
        df.drop(['Year_Birth','Dt_Customer'], axis=1, inplace=True)

    return (train, test)

def reindex_data(train: pd.DataFrame, test: pd.DataFrame) -> tuple:
    """
    열을 정렬하는 함수
    """

    train, test = train.copy(), test.copy()

    columns = train.drop(['id','target'], axis=1).columns.tolist()
    train = train.reindex(columns=['id']+sorted(columns)+['target'])
    test = test.reindex(columns=['id']+sorted(columns))

    return (train, test)

###########################################################################
############################ Train/Test Split #############################
###########################################################################

def make_validation_set(train: pd.DataFrame, test_size: int) -> tuple:
    """
    train 데이터에 Train/Test Split을 적용하는 함수
    """

    y_train = np.array(train[['target']])
    x_train = train.drop(['id','target'], axis=1)

    x_train, x_test, y_train, y_test = \
        model_selection.train_test_split(x_train, y_train, test_size=test_size, random_state=0)

    return (x_train, x_test, y_train, y_test)

###########################################################################
############################ Feature Encoding #############################
###########################################################################

def feature_encoding(train: pd.DataFrame, test: pd.DataFrame, validation=tuple()) -> tuple:
    """
    전체 데이터에 StandardScalor와 OneHotEncoder를 적용하는 함수
    """

    if validation:
        x_train, x_test, y_train, y_test = validation

    numerical_transformer = StandardScaler()
    numerical_features = ['Age','Age_Range','Days_Customer','Income','Income_Level','Income_Per',
                            'NumAcceptedCmp','NumCatalogPurchases','NumDealsPurchases',
                            'NumStorePurchases','NumWebPurchases','NumWebVisitsMonth',
                            'NumPurchases','Recency']

    categorical_transformer = OneHotEncoder(categories='auto', handle_unknown='ignore')
    categorical_features = ['AcceptedCmp1','AcceptedCmp2','AcceptedCmp3','AcceptedCmp4',
                            'AcceptedCmp5','Complain','Education','Kidhome',
                            'Marital_Status','Perferred_Purchase','Response','Teenhome']

    preprocessor = ColumnTransformer(
        transformers=[
            ('num', numerical_transformer, numerical_features),
            ('cat', categorical_transformer, categorical_features)])

    pipe = Pipeline(steps=[('preprocessor', preprocessor)])
    train_data = pipe.fit_transform(train.drop(['id','target'], axis=1))
    test_data = pipe.transform(test.drop(['id'], axis=1))

    if validation:
        validation = (pipe.transform(x_train), pipe.transform(x_test), y_train, y_test)

    return train_data, test_data, validation


from sanic.app import Sanic
from sanic.response import html

app = Sanic()

@app.route('/', methods=['GET'])
async def index(request):
    return html("
This is index
")

if __name__ == '__main__':
    app.run()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
uniswap-python examples
@author: mpascariu
"""

# Setup
import os
import json
import sys
from web3 import Web3
from uniswap import Uniswap

# Here I have a file containing API keys, wallet and smart contract addresses
sys.path.append("/Volumes/DEIFX/py.defi")
import _init

# Set up the API connection
w3 = Web3(Web3.HTTPProvider(_init.polygon_url_alchemy))
print(w3.is_connected())

# Uniswap-python

uniswap = Uniswap(
    address     = _init.w_cologne,
    private_key = _init.pk_cologne,
    provider    = _init.polygon_url_alchemy,
    version     = 3
    )

wmatic = uniswap.get_weth_address()
dai   = '0x8f3Cf7ad23Cd3CaDbD9735AFf958023239c6A063'
eure  = '0x18ec0A6E18E5bc3784fDd3a3634b31245ab704F6'
usdt  = '0xc2132D05D31c914a87C6611C10748AEb04B58e8F'
usdc  = '0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174'

def value(x):
    out = w3.from_wei(x,  'ether')
    out = round(out, 4)
    return print(out)

# sell 1000 tokens for DAI
# the amount of DAI received is:
amount     = 1000
decimals   = 18
amount_wei = amount * (10**decimals)

value(uniswap.get_price_output(wmatic, dai, amount_wei))
value(uniswap.get_price_output(eure, dai, amount_wei))
uniswap.get_raw_price(token_in = dai, token_out = eure) * 1000

# spend 1000 DAI for tokens
# the amount of tokens received is:
value(uniswap.get_price_input(dai, wmatic, amount_wei))
value(uniswap.get_price_input(dai, eure, amount_wei))
uniswap.get_raw_price(token_in = dai, token_out = eure) * 1000
uniswap.get_raw_price(token_in = usdt, token_out = eure) * 1000

# check wallet ballance
value(uniswap.get_token_balance(eure))
value(uniswap.get_token_balance(dai))


#!/usr/bin/env python

# this script can extract time frame into jpg images

import csv
import cv2
import glob
import os
import pandas as pd
import math
from subprocess import call
import settings
import function_list as ff
cg = settings.Experiment()

def extract_timeframes(main_path,movie_path,excel_file):
    """After we have all of our videos split between train and test, and
    all nested within folders representing their classes, we need to
    make a data file that we can reference when training our RNN(s).
    This will let us keep track of image sequences and other parts
    of the training process.

    We'll first need to extract images from each of the videos. We'll
    need to record the following data in the file:

    [train|test] or batch, class, filename, nb frames

    Extracting can be done with ffmpeg:
    `ffmpeg -i video.mpg image-%04d.jpg`
    """
    data =[]

    # create image folder
    image_folder = os.path.join(main_path,'images')
    ff.make_folder([image_folder])

    # find all the movies
    excel_file = pd.read_excel(excel_file)
    movie_list = excel_file['video_name']

    # extract time frames from each movie:
    for i in range(0,excel_file.shape[0]):
        case = excel_file.iloc[i]
        print(i, case['video_name'])

        # set the file name for images
        file_name = case['video_name']
        file_name_sep = file_name.split('.') # remove .avi
        file_name_no_ext = file_name_sep[0]
        if len(file_name_sep) > 1:
            for ii in range(1,len(file_name_sep)-1):
                file_name_no_ext += '.'
                file_name_no_ext += file_name_sep[ii]
        save_folder = os.path.join(image_folder,file_name_no_ext)
        ff.make_folder([save_folder])

        src = os.path.join(movie_path,file_name)
        if os.path.isfile(src) == 0:
            ValueError('no movie file')

        if os.path.isfile(os.path.join(save_folder,file_name_no_ext+'-0001.jpg')) == 0:
            cap = cv2.VideoCapture(src)
            count = 1
            frameRate = 1
            while(cap.isOpened()):
                frameId = cap.get(1) # current frame number
                ret, frame = cap.read()

                if (ret != True):
                    break
                if (frameId % math.floor(frameRate) == 0):
                    if count < 10:
                        number = '000'+str(count)
                    if count >=10:
                        number = '00'+str(count)

                dest = os.path.join(save_folder,file_name_no_ext+'-'+number+'.jpg')
                cv2.imwrite(dest,frame)
                count += 1
            cap.release()
            # call(["ffmpeg", "-i", src, dest]) % this will cause some error (not extract exact 20 frames) in some avis

        # Now get how many frames it is.
        nb_frames = len(ff.find_all_target_files(['*.jpg'],save_folder))
        data.append([case['video_name'],file_name_no_ext,nb_frames])

    print('done extraction')
    data_df = pd.DataFrame(data,columns = ['video_name','video_name_no_ext','nb_frames'])
    data_file = pd.merge(excel_file,data_df,on = "video_name")
    data_file.to_excel(os.path.join(cg.nas_main_dir,'data_file.xlsx'),index=False)

def main():
    """
    Extract images from videos and build a new file that we
    can use as our data input file. It can have format:

    [train|test], class, filename, nb frames
    """
    main_path = cg.local_dir
    movie_path = os.path.join(main_path,'original_movie')
    excel_file = os.path.join(cg.nas_main_dir,'movie_list_w_classes.xlsx')
    extract_timeframes(main_path,movie_path,excel_file)

if __name__ == '__main__':
    main()


#!/usr/bin/python3
import requests
from PIL import Image
from io import BytesIO

matches = {
    (237, 28, 36, 255) : "apple",
    (253, 215, 0, 255) : "banana",
    (47, 21, 62, 255) : "epic coin",
    (255, 204, 0, 255) : "golden fish",
    (255, 242, 0, 255) : "golden coin",
    (217, 17, 27, 255) : "life potion",
    (0, 198, 255, 255) : "normie fish",
    (255, 21, 21, 255) : "ruby",
    (255, 130, 170, 255) : "unicorn horn",
    (46, 46, 52, 255) : "wolf skin",
    (77, 98, 11, 255) : "zombie eye"
}

def handle_guard(message):

    r = requests.get(message.attachments[0].url)
    img = Image.open(BytesIO(r.content))
    pixel_data = set(img.getdata())
    for k, v in matches.items():
        if k in pixel_data:
            return matches[k]

    return "none found"


import filetype
import os
import subprocess
import multiprocessing

folders = os.listdir('/home/cosmadrian/Desktop/iMaterialist Challenge/data')
base_dir = '/home/cosmadrian/Desktop/iMaterialist Challenge/data/'
def delete_files(folder):
    files = [f for f in os.listdir(base_dir + folder)]
    for_deletion = [
        base_dir + folder + '/' + img
        for img in files
            if filetype.guess(base_dir + folder + '/' + img)is None or
                filetype.guess(base_dir + folder + '/' + img).mime.split('/', 1)[0] != 'image'
    ]
    print(folder, len(for_deletion))

    for file in for_deletion:
        os.remove(file)

for folder in folders:
    delete_files(folder)


#!/usr/bin/env python
"""Driver
This loads data into Cockroach DB.
Script expects Cockroach IP is passed.
"""
import sys
import subprocess
import time
import psycopg2

if len(sys.argv) == 1:
    print('Must pass IP from Cockroach server')
    sys.exit()
else:
    print('Host IP: ' + str(sys.argv[1]))

subprocess.call('cp ~/data-science/jupyter_home/kickstart.csv.xz  /tmp/', shell=True)
time.sleep(5)
subprocess.call('xz -d /tmp/kickstart.csv.xz', shell=True)
time.sleep(5)
subprocess.call('hdfs dfs -put /tmp/kickstart.csv /', shell=True)

CONN = psycopg2.connect(
    database='tx',
    user='test_user',
    port='26257',
    host=str(sys.argv[1])
)
CONN.set_session(autocommit=True)
CUR = CONN.cursor()

CAT = subprocess.Popen(["hadoop", "fs", "-cat", "/kickstart.csv"], stdout=subprocess.PIPE)

i = 0
for line in CAT.stdout:
    i = i + 1
    l = str(line).strip('[b"\\n]').split(',')
    for k in range(0, 8):
        l[k] = l[k].strip('\'')

    if l[0] != 'category':
        sql = 'INSERT INTO tx.kickstart VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s)'
        CUR.execute(sql, l)

    if i%1000 == 0:
        print(str(i) + ' records loaded')


import argparse
import cv2
import numpy as np
import mediapipe as mp

if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("-p", "--person", required = True, help = "path to image file of person")
    ap.add_argument("-b", "--background", required = True, help = "path to background image")
    args = vars(ap.parse_args())

    image = cv2.imread(args["person"])
    background_img = cv2.imread(args["background"])

    resized_bg_img = cv2.resize(background_img, (image.shape[1], image.shape[0]))

    mp_selfie_segmentation = mp.solutions.selfie_segmentation
    segment = mp_selfie_segmentation.SelfieSegmentation(model_selection = 0)

    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

    results = segment.process(image)

    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

    image_seg_mask = results.segmentation_mask

    threshold = 0.5
    binary_mask = image_seg_mask > threshold

    mask3d = np.dstack((binary_mask, binary_mask, binary_mask))

    replaced_img = np.where(mask3d, image, resized_bg_img)

    cv2.imwrite("result.jpg", replaced_img)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('ablog', '0002_tag'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='entry',
            options={'ordering': ['-created'], 'verbose_name': 'Blog Entry', 'verbose_name_plural': 'Blog Entries'},
        ),
        migrations.AddField(
            model_name='entry',
            name='tags',
            field=models.ManyToManyField(to='ablog.Tag'),
            preserve_default=True,
        ),
    ]


from typing import Any
from unittest import mock

from pyesg import stochastic_process
from esg.regime_switching_brownian_motion import RegimeSwitching
import pytest
import numpy as np
import pandas as pd
from unittest.mock import MagicMock, Mock, call, patch
import unittest

class NumpyArg:

    def __init__(self, arg: Any) -> None:
        self.arg = arg

    def __eq__(self, __value: object) -> bool:
        return np.array_equiv(self.arg, __value)

    def __repr__(self) -> str:
        return f'numpy arg: {self.arg}'

class ReturnValueWithArgs:

    def __init__(self, return_value, *args, **kwargs) -> None:
        self.args = args
        self.return_value = return_value

    def __call__(self, *args: Any, **kwds: Any) -> Any:
        if(self.args == args):
            return self.return_value
        else:
            raise ValueError(f'No return value for args: {args}, but for {self.args}')

#transition matrix for two regimes
k2_tm = [[0.65, 0.35],
         [0.06, 0.94]]

#transition matrix for three regimes
k3_tm = [[0.65, 0.25, 0.1],
         [0.06, 0.74, 0.2],
         [0.11, 0.24, 0.65]]

@pytest.fixture()
def transition_matrix():
    def create_transition_matrix(n_regimes):
        if (n_regimes == 2):
            return k2_tm
        if (n_regimes == 3):
            return k3_tm
        if(n_regimes != 2 or n_regimes != 3):
            raise ValueError(f'Returns transition matrix for 2 and 3 regimes, you passed: {n_regimes}')

    return create_transition_matrix

class TestRegimeSwithing():

    def setup_method(cls):
      cls.model = RegimeSwitching.example()

    @pytest.mark.parametrize('number_of_regimes,expected', [
        (2, [0,1]),
        (3, [0,1,2])
        ])
    def test_should_return_regimes_ids(self, number_of_regimes, expected, transition_matrix):

        self.model.probs = transition_matrix(number_of_regimes)

        actual = self.model.regimes_idx

        np.testing.assert_array_equal(actual, expected)

    @pytest.mark.parametrize('number_of_scenarios,current_regimes,expected_calls',[
        (2, 1, [call(n_scenarios=1,probs=k2_tm[1])]),
        (2, [0,0,0], [call(n_scenarios=3,probs=k2_tm[0])]),
        (2,[0,1,0], [call(n_scenarios=2,probs=k2_tm[0]),call(n_scenarios=1,probs=k2_tm[1])]),
        (3,[0,1,0,2], [call(n_scenarios=2,probs=k3_tm[0]),call(n_scenarios=1,probs=k3_tm[1]),call(n_scenarios=1,probs=k3_tm[2])])
    ])
    def test_when_next_regime_called_should_call_random_regime(self, number_of_scenarios, current_regimes, expected_calls,transition_matrix):

        self.model.probs = transition_matrix(number_of_scenarios)

        self.model.random_regimes = Mock(return_value=[0])

        self.model.next_regime(current_regimes)

        self.model.random_regimes.assert_has_calls(expected_calls)

    @pytest.mark.parametrize('number_of_regimes,current_regimes,random_regimes,expected_regimes',[
        (2, 1, [np.array([1])], [1]),
        (2, [0,0], [np.array([0,1])], [0,1]),
        (2, [0,1,0], [np.array([0,1]), np.array([1])], [0,1,1]),
        (3, [0,1,0,2], [np.array([1,2]), np.array([0]), np.array([1])], [1,0,2,1])
    ])
    def test_when_next_regime_called_should_return_next_regimes(self, number_of_regimes, current_regimes, random_regimes, expected_regimes,transition_matrix):

        self.model.probs = transition_matrix(number_of_regimes)

        self.model.random_regimes = Mock(side_effect=random_regimes)

        result = self.model.next_regime(current_regimes)

        np.testing.assert_equal(result,expected_regimes)

    @pytest.mark.parametrize('n_regimes, n_scenarios,expected_shape',[
        (2, 1,(1,)),
        (2, 5,(5,)),
        (3, 5,(5,))
    ])
    def test_random_regimes_should_return_shape(self, n_regimes, n_scenarios,expected_shape, transition_matrix):

        self.model.probs = transition_matrix(n_regimes)
        result = self.model.random_regimes(n_scenarios=n_scenarios,probs=self.model.probs[0])

        assert result.shape == expected_shape
        assert all(item in self.model.regimes_idx for item in result)

    @pytest.mark.parametrize('regime,expected_value',[
        ([1], 11.3),
        ([0], 10.9),
    ])
    @patch('pyesg.stochastic_process.StochasticProcess')
    @patch('pyesg.stochastic_process.StochasticProcess')
    def test_call_step_should_return_value_for_single_initial_value(self,  stochastic_model1, stochastic_model2, regime,expected_value):

        model0 = stochastic_model2()
        model0.step.return_value = 10.9
        model1 = stochastic_model1()
        model1.step.return_value = 11.3

        regime_switching_model = RegimeSwitching(models= [model0, model1],probs=k2_tm)
        regime_switching_model.next_regime = Mock(side_effect = ReturnValueWithArgs(regime,current_regimes=0))

        result = regime_switching_model.step(x0=10,current_regimes=0)

        assert result == expected_value

    @pytest.mark.parametrize('next_regimes,initial_values0,initial_values1,model0_return, model1_return,expected_value',[
        ([0,0,0], [10,15,20], None, [10.9, 15.6, 20.8], None, [10.9, 15.6,20.8]),
        ([1,0,1], [15], [10,20], [11.6], [15.3,20.3], [15.3, 11.6, 20.3]),
        ([0,1,0], [10,20], [15], [10.9,20.8], [15.6], [10.9, 15.6, 20.8]),
    ])
    @patch('pyesg.stochastic_process.StochasticProcess')
    @patch('pyesg.stochastic_process.StochasticProcess')
    def test_call_step_should_return_value_for_multiple_initial_value(self,
                                                                      stochastic_model1,
                                                                      stochastic_model2,
                                                                      next_regimes,
                                                                      initial_values0,
                                                                      initial_values1,
                                                                      model0_return,
                                                                      model1_return,
                                                                      expected_value):

        model0 = stochastic_model2()
        model0.step.side_effect = ReturnValueWithArgs(model0_return,NumpyArg(initial_values0),1)
        model1 = stochastic_model1()
        model1.step.side_effect = ReturnValueWithArgs(model1_return,NumpyArg([initial_values1]),1)

        initial_values= [10,15,20]

        regime_switching_model = RegimeSwitching(models= [model0, model1],probs=k2_tm)
        regime_switching_model.next_regime = Mock(side_effect = ReturnValueWithArgs(next_regimes,current_regimes=[0,0,1]))

        result = regime_switching_model.step(x0=initial_values,current_regimes=[0,0,1],dt=1)

        np.testing.assert_array_equal(result, expected_value)

    def test_scenarios_regimes_multiple_initial_values(self):

        n_scenarios = 3
        n_steps = 5
        current_regime = [0, 1, 0]

        result = self.model.scenarios_regimes(current_regime,n_steps)
        assert result.shape == (n_steps+1, n_scenarios)

    def test_scenarios_with_multiple_initial_value(self):

        n_steps = 3
        n_scenarios = 2
        initial_values = [10, 10]
        current_regimes = [0, 0]
        dt = 1/4

        result = self.model.scenarios(initial_values, current_regimes, dt, n_steps, n_scenarios)

        assert result.shape == (n_scenarios,n_steps+1)

    def test_scenarios_with_single_initial_value(self):

        n_steps = 3
        n_scenarios = 2
        initial_values = 10
        current_regimes = 0
        dt = 1/4

        result = self.model.scenarios(initial_values, current_regimes, dt, n_steps, n_scenarios)

        assert result.shape == (n_scenarios,n_steps+1)


"""
Make a route-level df from trips and shapes.

Must include the route's line geom and service hours and
district.
"""
import datetime
import geopandas as gpd
import intake
import pandas as pd
import sys

from loguru import logger
from typing import Literal

from calitp_data_analysis import geography_utils, utils
from shared_utils import portfolio_utils
from bus_service_utils import create_parallel_corridors
from segment_speed_utils import helpers, gtfs_schedule_wrangling
from update_vars import (BUS_SERVICE_GCS, COMPILED_CACHED_GCS,
                         ANALYSIS_DATE,
                        )

catalog = intake.open_catalog(
    "../_shared_utils/shared_utils/shared_data_catalog.yml")

def shape_geom_to_route_geom(
    analysis_date: str
) -> gpd.GeoDataFrame:
    """
    Merge routes and trips to get line geometry.
    """
    cols_to_keep = ["route_id", "shape_id", "geometry"]
    operator_cols = ["feed_key", "name"]

    df = gtfs_schedule_wrangling.get_trips_with_geom(
        analysis_date,
        trip_cols = operator_cols + [
                     "trip_id",
                     "shape_id","shape_array_key",
                     "route_id", "route_type"
                    ],
        exclude_me = ["Amtrak Schedule", "*Flex"],
        crs = geography_utils.CA_StatePlane
        # make sure units are in feet
    )

    # Merge trips with the shape's line geom, and get it to shape_id level
    df = (df[operator_cols + cols_to_keep + ["route_type"]]
        .drop_duplicates(subset=operator_cols + ["shape_id"])
        .reset_index(drop=True)
    )

    # Now, get it from shape_id level to route_id level
    routes = create_parallel_corridors.process_transit_routes(
        df, operator_cols = operator_cols
    )

    # Add gtfs_dataset_name back
    feed_gtfs_data_crosswalk = helpers.import_scheduled_trips(
        analysis_date,
        columns = ["feed_key", "gtfs_dataset_key"],
        get_pandas = True
    ).rename(columns = {"schedule_gtfs_dataset_key": "gtfs_dataset_key"})

    routes2 = pd.merge(
        routes,
        feed_gtfs_data_crosswalk,
        on = "feed_key",
        how = "inner"
    )

    return routes2

def aggregate_trip_service_to_route_level(
    analysis_date: str,
    route_cols: list
) -> pd.DataFrame:
    """
    Aggregate trip-level service hours to the route-level.
    """
    trips = helpers.import_scheduled_trips(
        analysis_date,
        columns = route_cols + ["trip_id", "service_hours"],
        get_pandas = True
    ).rename(columns = {"schedule_gtfs_dataset_key": "gtfs_dataset_key"})

    route_service_hours = portfolio_utils.aggregate_by_geography(
        trips,
        group_cols = route_cols,
        sum_cols = ["service_hours"]

    )

    return route_service_hours

def add_district(route_df: gpd.GeoDataFrame,
                 route_cols: list) -> gpd.GeoDataFrame:
    """
    Merge in district info (only 1 district per route_id)
    """
    districts = (catalog.caltrans_districts.read()
                 [["DISTRICT", "geometry"]]
                 .rename(columns = {"DISTRICT": "district"})
                ).to_crs(route_df.crs)

    # Find overlay and calculate overlay length
    # Keep the district with longest overlay for that route_id
    overlay_results = gpd.overlay(
        route_df[route_cols + ["geometry"]].drop_duplicates(),
        districts,
        how = "intersection",
        keep_geom_type = True
    )

    overlay_results = overlay_results.assign(
        overlay_length = overlay_results.geometry.to_crs(
            geography_utils.CA_StatePlane).length
    )

    longest_overlay = (overlay_results
                       .sort_values(route_cols + ["overlay_length"])
                       .drop_duplicates(subset=route_cols, keep="last")
                       .reset_index(drop=True)
    )

    # Only keep 1 district per route
    gdf = pd.merge(
        route_df,
        longest_overlay[route_cols + ["district"]],
        on = route_cols,
        how = "left"
    )

    return gdf

if __name__ == "__main__":

    logger.add("./logs/quarterly_performance_pipeline.log", retention="6 months")
    logger.add(sys.stderr,
               format="{time:YYYY-MM-DD at HH:mm:ss} | {level} | {message}",
               level="INFO")

    start = datetime.datetime.now()

    route_cols = ["gtfs_dataset_key", "feed_key", "name", "route_id"]

    # Merge to get shape_level geometry, then pare down to route-level geometry
    route_geom = shape_geom_to_route_geom(
        ANALYSIS_DATE)

    # Add district
    route_geom_with_district = add_district(route_geom, route_cols)

    # Get route-level service
    route_service = aggregate_trip_service_to_route_level(ANALYSIS_DATE, route_cols)

    # Merge service hours with route-level geometry and district
    gdf = pd.merge(
        route_geom_with_district,
        route_service,
        on = route_cols,
        how = "left",
        validate = "1:1"
    )

    utils.geoparquet_gcs_export(
        gdf,
        BUS_SERVICE_GCS,
        f"routes_{ANALYSIS_DATE}.parquet"
    )

    end = datetime.datetime.now()
    logger.info(f"route level service: {ANALYSIS_DATE} {end - start}")


import torch
import re
import os
import pickle
import random
from tqdm import tqdm

"""
common utils
"""
def collect_all_data(path):
    random.seed(999)

    files = os.listdir(path)
    all_data = []
    for file in files:
        with open(path + file, 'rb') as f:
            data = pickle.load(f)
        all_data.append(data)
    all_data = sum(all_data, [])

    random.shuffle(all_data)

    with open(path + 'all_data.pkl', 'wb') as f:
        pickle.dump(all_data, f)

    return all_data

def make_vocab_file(data, data_save_path, special_token_save_path):
    all_s = []
    special_tok = set()
    for d in tqdm(data, desc='making vocab train file...'):
        for s in d:
            # find special token e.g. #@이름#
            compile = re.compile(r'(?<=#@)(.*?)(?=#)')
            special_tok.update(compile.findall(s))
            s = ' '.join(s.strip().split())
            all_s.append(s + '\n')

    with open(data_save_path, 'w') as f:
        f.writelines(all_s)

    all_special_tokens = []
    for tok in list(special_tok):
        all_special_tokens.append('#@' + tok + '#\n')

    with open(special_token_save_path, 'w') as f:
        f.writelines(all_special_tokens)

def load_dataset(path):
    with open(path, 'rb') as f:
        data = pickle.load(f)
    return data

def save_checkpoint(file, model, optimizer, scheduler=None):
    if scheduler == None:
        state = {'model': model.state_dict(), 'optimizer': optimizer.state_dict()}
    else:
        state = {'model': model.state_dict(), 'optimizer': optimizer.state_dict(), 'scheduler': scheduler.state_dict()}
    torch.save(state, file)
    print('model pt file is being saved\n')

def tensor2list(tensor, tokenizer):
    tensor = tensor.detach().cpu().tolist()
    tensor = [tokenizer.decode(t[1:]) for t in tensor]
    return tensor

def isInTopk(gt_id, selected_id, topk):
    selected_id = selected_id.tolist()
    if len(selected_id) < topk:
        raise IndexError("topk is larger than selected id")

    if gt_id in selected_id[:topk]:
        return 1
    return 0