# -*- coding: utf-8 -*-

# Python Forces Path Exporter by Aaron Foster Wallace
# Translated from 3Dmax Sript  PathExporter.ms  by Radfordhound
# at same time Built on LWPathImporter by arukibree
# end of the chain

from collections import namedtuple
import struct
import enum
import bpy
import io
from mathutils import Vector,Quaternion

GLOBAL_UP = [0,1,0]
yzflip = True
meterScale = 10
VERSION = 1 # 0: wii U / 1: PC (only pc is tested)
BE=">"
LE="<"

LONGPAD=b'\x00'*4
LONGLONGPAD=b'\x00'*8
RELATIVE=1

def  midPoint(v1,v2):
   x1,y1,z1=v1
   x2,y2,z2=v2
   xm=x1+ ((x2-x1)*0.5)
   ym=y1+ ((y2-y1)*0.5)
   zm=z1+ ((z2-z1)*0.5)
   return (xm,ym,zm)

"""
TODO:
I use this funtions because i test outside
blender i plan to use the Vector class of blender
"""
def cross(a, b):
    return [a[1]*b[2] - a[2]*b[1],
         a[2]*b[0] - a[0]*b[2],
         a[0]*b[1] - a[1]*b[0]]

def norm(a):
    return (a[0]**2+a[1]**2+a[2]**2)**0.5

def normalize(a):
    N=norm(a)
    return [a[0]/N,a[1]/N,a[2]/N]

def  directionTo(v1,v2):
    return normalize([
            v2[0] - v1[0],
            v2[1] - v1[1],
            v2[2] - v1[2]
            ])

def  distance(v1,v2):
    return norm([
            v2[0] - v1[0],
            v2[1] - v1[1],
            v2[2] - v1[2]
            ])

########################################
#file utils
#Fill a file to a byte size that is multiplier of num

def writePad(f,num):
    while (f.tell()% num ) != 0:
        f.write(b'\x00')

def  writePoint3(f,edian,P3):
    f.write(struct.pack(edian+"fff",*P3))

StringEntry=namedtuple('StringEntry' , 'st addresses')

def addString( f,stringEntries,st):
    #The string Exists
    unique = True
    for s in stringEntries:
        if s.st == st:
            s.addresses.append(f.tell())
            unique=False
            break
    if unique:
        #Is a new String
        entry = StringEntry(st,[f.tell()])
        stringEntries.append(entry)

    f.write(LONGLONGPAD)

def  writeTable(f,edian,stringEntries) :
    for s in stringEntries:
        offset = f.tell()
        for a in s.addresses:
            f.seek(a)
            f.write(struct.pack(edian+"L",offset - 0x40))
        f.seek(offset)
        f.write(s.st.encode())
        f.write(b'\x00')

########################################
class ForcesCurveType(enum.Enum):
   Object = 0
   SideView = 1
   GrindRail = 2
   #print (ForcesCurveType(2)) acces by value
   #print (ForcesCurveType['GrindRail']) acces by name

class ForcesCurveIO:

    def __init__(self,name,splines):
        self.name = name
        self.splines = splines
        self.ptype=0#object the default
        self.BBMin=[999999999, 999999999, 999999999]
        self.BBMax=[-999999999, -999999999, -999999999]
        self.mid_spline=[]
        self.upvecs=[]
        self.forwardvecs=[]
        self.errors=[]
        self.pathLength = 0.0
        self.distances=[]
        self.uuid=0

    def getName(self):#for sorting ls.sort(key=ForcesCurveIO.getName)
        return self.name

    def validate(self):
        slen=len(self.splines)
        self.errors=[]
        if slen<1:
            self.errors.append("The curve object {} cannot be exported because it has no splines".format(self.name))
        if slen>2:
            self.errors.append("The curve object{} cannot be exported because has more than two splines, have {} splines".format(self.name,slen))

        if len(self.splines[0])<2:
            self.errors.append("The curve object{} cannot be exported because it  has 1st spline with less than 2 knots  or is not a bezier curve ".format(self.name))

        if slen==2:
            if len(self.splines[1])<2:
                self.errors.append("The curve object{} cannot be exported because it has a 2nd spline with less than 2 knots or is not a bezier curve".format(self.name))
            if len(self.splines[0])!=len(self.splines[1]):
                self.errors.append("The curve object{} cannot be exported because it has  2 splines of diferent  knots count".format(self.name))
        return len(self.errors)==0

    def exportPrepare(self):
        if not self.validate():
            return
        #Default _type = ForcesCurveType.Object.value
        if (self.name[-3:] in ["_SV","_SL"]):
           self.ptype = ForcesCurveType.SideView.value
        elif self.name[-3:] == "_GR":
           self.ptype = ForcesCurveType.GrindRail.value

        #BoundingBOx
        for s in self.splines:
            for k in s:
                for i in range(3):
                    self.BBMin[i]=min(k[i],self.BBMin[i])
                    self.BBMax[i]=max(k[i],self.BBMax[i])

        #Midpoints
        if len(self.splines)==2:
            self.mid_spline=[midPoint(*v)  for v in zip(self.splines[0],self.splines[1])]

        #distances
        self.pathLength = 0
        prevk=self.splines[0][0]#first node
        self.distances=[0.0]
        for k in self.splines[0][1:]:
            self.pathLength += distance(k,prevk)
            prevk=k
            self.distances.append(self.pathLength)

        #TODO: In case of doble splines Check if parralel curves order maters , maybe up vector directions matters
        #When have 2 splines the right vector is more consistent, points to the 2nd  self
        splinezip=list(zip(*self.splines))# tuples of one or two parrallel points
        last = GLOBAL_UP
        for i in range(len(splinezip)-1):
            k=splinezip[i][0]
            nextk=splinezip[i+1][0]
            forward = directionTo(k,nextk)
            if len(self.splines)==2:
                kB=splinezip[i][1] # 2nd parrallel spline point
                right = directionTo(k,kB)
            else:
               right = normalize(cross(forward,last))
            up = normalize(cross(right,forward))
            last=up
            self.upvecs.append(up)
            self.forwardvecs.append(forward)

        #Last points
        self.upvecs.append(self.upvecs[-1])
        self.forwardvecs.append(self.forwardvecs[-1])

        #uuid
        #self.uuid=
        self.uuid=0
        pts=self.name.split("_")
        pts.reverse()
        for u in pts:
            if u.isdigit():
                self.uuid=int(u)
                break
    """
    @f output file
    @edian '<'  or '>' for use it with struct.pack
    @path number for the header offset in file
    """
    def writeToFile(self,f,edian,pathnumber,stringEntries,offset_array):
        klen=len(self.splines[0])
        # bool array
        #unknown,but setting everything to 0 has no noticeable effect
        boolsOffset = f.tell()
        f.write(b'\x00'*klen)

        writePad(f,4)	#fill to be 4 multiplier
        # dist array
        distOffset = f.tell()
        #writeFloatarray
        f.write(struct.pack(edian+"f"*klen,*self.distances))

        # knot array
        knotsOffset = f.tell()

        knots=self.splines[0]
        if len(self.splines)==2:
            knots=self.mid_spline

        for k in knots:
            writePoint3(f,edian,k)

        # Up vector array
        upVecsOffset = f.tell()

        for u  in self.upvecs:
                writePoint3(f,edian,u)

        # forward vector array
        forwardVecsOffset = f.tell()
        for fv in self.forwardvecs:
                writePoint3(f,edian,fv)

        # double spline knot array
        doubleSplineOffset = f.tell()

        if len(self.splines)==2:
            for k1,k2 in zip(*self.splines):
                writePoint3(f,edian,k1)
                writePoint3(f,edian,k2)
        else:
            doubleSplineOffset = 0

        # metadata
        writePad(f,8)
        #TODO is shiffted by 8

        metaOffset = f.tell()
        metaCount = 0

        # write metadata
        # "type" field present for all types
        offset_array.append(f.tell())
        addString(f,stringEntries,"type")
        f.write(LONGLONGPAD)
        f.write(struct.pack(edian+"Q",self.ptype))
        metaCount += 1

        # "uid" field present for type Object or SV path UUIDs
        if self.ptype in [0,1]:
            offset_array.append(f.tell())
            addString(f,stringEntries,"uid")
            f.write(LONGLONGPAD)
            f.write(struct.pack(edian+"Q",self.uuid))
            metaCount += 1

        # KD tree
        # the exact structure of this part is unknown, but luckily we can cheat our way through this part.
        # there are no noticeable ingame issues from doing it this way; it's likely only a very small performance hit
        KDTreeOffset = f.tell()

        numLineSegments=klen*len(self.splines)-len(self.splines)

        f.write(struct.pack(edian+"L",0))
        f.write(struct.pack(edian+"L",2))
        offset_array.append(f.tell())
        f.write(struct.pack(edian+"Q",KDTreeOffset + 0x30 - 0x40))

        f.write(struct.pack(edian+"Q",1))
        offset_array.append(f.tell())
        f.write(struct.pack(edian+"Q",KDTreeOffset + 0x40 - 0x40))

        f.write(struct.pack(edian+"Q",numLineSegments))
        offset_array.append(f.tell())
        f.write(struct.pack(edian+"Q",KDTreeOffset + 0x48 - 0x40))

        # the first data section is the unknown bit
        f.write(struct.pack(edian+"L",0))
        f.write(struct.pack(edian+"L",0))
        f.write(struct.pack(edian+"L",3))
        f.write(struct.pack(edian+"L",0))

        # the second bit assigns line segments to... something.
        # [count] int pairs - first is count, second is starting index.
        # eg "add 6 segments staring from segment 7"
        f.write(struct.pack(edian+"Q",numLineSegments))

        # the third bit seems to be just line segment indices/IDs
        for i in range(numLineSegments):
            f.write(struct.pack(edian+"L",i))

        writePad(f,8)

        # done with path data - now backtrack and write path header
        pathDataEnd = f.tell()

        headerOffset = 0x58 + 0x80 * pathnumber
        f.seek(headerOffset)

        # path header
        f.write(LONGLONGPAD) # this is the name offset; will be filled in by the writeTable() call later
        f.write(b'\x01') # unknown value, always 0x1
        f.write(b'\x00') # unknown value, always 0x0
        f.write(struct.pack(edian+"H",klen))
        f.write(struct.pack(edian+"f",self.pathLength))
        f.write(struct.pack(edian+"Q",boolsOffset - 0x40))
        f.write(struct.pack(edian+"Q",distOffset - 0x40))
        f.write(struct.pack(edian+"Q",knotsOffset - 0x40))
        f.write(struct.pack(edian+"Q",upVecsOffset - 0x40))
        f.write(struct.pack(edian+"Q",forwardVecsOffset - 0x40))
        f.write(struct.pack(edian+"Q",klen*2 if len(self.splines)==2 else 0))

        if doubleSplineOffset != 0:
            f.write(struct.pack(edian+"Q",(doubleSplineOffset - 0x40)))
        else:
            f.write(LONGLONGPAD)

        writePoint3(f,edian,self.BBMin)
        writePoint3(f,edian,self.BBMax)
        f.write(struct.pack(edian+"Q",metaCount))
        f.write(struct.pack(edian+"Q",(metaOffset - 0x40)))
        f.write(LONGLONGPAD) # unknown value, always 0
        f.write(struct.pack(edian+"Q",(KDTreeOffset - 0x40)))
        f.seek(pathDataEnd)

def exportCurvesToFile(path_file,curves):
    f = open(path_file,"wb")
    edian=""
    if (VERSION == 0):
        edian=BE
    elif (VERSION == 1):
        edian=LE
    offset_array = []
    stringEntries = []
    pathCount = len(curves)
    # start writing
    # BINA header
    f.write(b'BINA') # BINA
    f.write(b'210L') # 210L
    f.write(LONGPAD) # file size (fill in later) Long
    f.write(struct.pack(edian+"L",0x1))
    f.write(b'DATA') # DATA
    f.write(LONGPAD) # DATA size (fill in later) Long
    f.write(LONGPAD) # string table offset (fill in later)Long
    f.write(LONGPAD) # string table length (fill in later)Long
    f.write(LONGPAD) # final table length (fill in later)Long

    f.write(struct.pack(edian+"L",0x18))
    f.write(LONGLONGPAD)
    f.write(LONGLONGPAD)
    f.write(LONGLONGPAD)

    # header
    f.write(b'HTAP' if edian==LE else b'PATH') # "PATH" magic
    f.write(struct.pack(edian+"H",0x200))
    f.write(struct.pack(edian+"H",0))
    f.write(struct.pack(edian+"Q",pathCount))

    offset_array.append(f.tell())

    f.write(struct.pack(edian+"Q",0x18))
    # path headers - first pass, enter junk. We'll write this later. the only thing we'll do now is add strings/offsets.

    for cur in curves:
        start = f.tell()
        offset_array.append(f.tell())
        addString(f,stringEntries,cur.name)

        offset_array.append(start + 0x10)
        offset_array.append(start + 0x18)
        offset_array.append(start + 0x20)
        offset_array.append(start + 0x28)
        offset_array.append(start + 0x30)
        if len(cur.splines)==2:
            offset_array.append(start + 0x40)
        offset_array.append(start + 0x68)
        offset_array.append(start + 0x78)

        # write temp junk (space for 30 Longs)
        f.write(LONGPAD*30)

    pathnumber=0
    for cur in curves:
        cur.writeToFile(f,edian,pathnumber,stringEntries,offset_array)
        pathnumber+=1

    # write string and offset table
    stringTablePos = f.tell()
    writeTable(f,edian,stringEntries)
    writePad(f,4)
    footerStartPos = f.tell()
    lastOffsetPos = 0x40

    #Seems that the following bitwise operations
    #are for write a number that is in an unknow format
    #warning: only works on little edian i thoug
    for offset  in offset_array:
        d = (offset - lastOffsetPos) >> 2
        if d<=0x3F:
            n=0x40 | d
            print(n,d)
            f.write(struct.pack("B",n))
        elif d <= 0x3FFF:
            n=(0x80 << 8) | d
            f.write(struct.pack(BE+"H",n))
        else:
            n=(0xC0 << 24) | d
            f.write(struct.pack(BE+"L",n))
        lastOffsetPos = offset

    # fix padding
    writePad(f,4)
    fileSize = f.tell()

    # fill-In header values
    f.seek(0x8)
    f.write(struct.pack(edian+"L",fileSize)) # file size
    f.seek(0x8,RELATIVE)
    f.write(struct.pack(edian+"L",(fileSize - 0x10))) # DATA size

    f.write(struct.pack(edian+"L",(stringTablePos - 0x40))) # string table offset
    f.write(struct.pack(edian+"L",(footerStartPos - stringTablePos))) # string table size
    f.write(struct.pack(edian+"L",(fileSize - footerStartPos))) # offset table size

    # finished
    f.close()


"""
Assignment 4: Neural Network
Part 2

@author: Divyam Jain
         17HS20047
"""
#Importing some essential libraries

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import string
import math

#Importing the dataset

dataset = pd.read_csv('Assignment_4_data.txt', header=None, delimiter='\t')

"""------Preprocessing------"""

#Removing punctuations

dataset2 = pd.DataFrame(index=range(len(dataset)),columns=[0,1])
dataset2[0]=dataset[0]
for i in range(len(dataset)):
    dataset2[1][i]=dataset[1][i].translate(str.maketrans('', '', string.punctuation))

#Convert into lowercase

dataset2[1] = dataset2[1].apply(lambda x: " ".join(x.lower() for x in x.split()))

#Removing stopwords

stop = np.loadtxt("NLTK's list of english stopwords", dtype='str')
dataset2[1] = dataset2[1].apply(lambda x: " ".join(x for x in x.split() if x not in stop))

#Removing stop words spelt in another way
freq = pd.Series(' '.join(dataset2[1]).split()).value_counts()[:500]
freq_common = ['u', '2', 'im', 'ur', '4', 'dont', 'ü', 'r', 'da', 'd', 'wat', 'n', 'cant', 'thats', 'didnt', 'ive','dun', 'youre', 'v' ]

dataset2[1] = dataset2[1].apply(lambda x: " ".join(x for x in x.split() if x not in freq_common))

#Applying Porter Stemming

from nltk.stem import PorterStemmer

st=PorterStemmer()
dataset2[1] = dataset2[1].apply(lambda x: " ".join([st.stem(word) for word in x.split()]))

#Removing extremely rare words (that only appear once)
freq_rare = pd.Series(' '.join(dataset2[1]).split()).value_counts()
freq_rare = freq_rare[freq_rare==1]
freq_rare = list(freq_rare.index)
dataset2[1] = dataset2[1].apply(lambda x: " ".join(x for x in x.split() if x not in freq_rare))

#Tokenization

dataset3=dataset2.copy()
for i in range(len(dataset)):
    dataset3[1][i]=dataset2[1][i].split()

#Splitting into training set and test set

from sklearn.model_selection import train_test_split

X_train, X_test, Y_train, Y_test = train_test_split(dataset3[1].values, dataset3[0].values, test_size=0.2)

#Making the vector V

V = np.unique(np.array(' '.join(dataset2[1]).split()))

#Create an input and output representation

   #For training
rep_in = pd.DataFrame(0,index=V, columns = range(len(X_train)))

for i in range(len(X_train)):
    for j in X_train[i]:
        rep_in[i][j]=1

rep_out = np.ndarray((len(Y_train),1))
for i in range(len(Y_train)):
    if Y_train[i]=='spam':
        rep_out[i][0]=1
    else:
        rep_out[i][0]=0

   #For testing
reptest_in = pd.DataFrame(0,index=V, columns = range(len(X_test)))

for i in range(len(X_test)):
    for j in X_test[i]:
        reptest_in[i][j]=1

reptest_out = np.ndarray((len(Y_test),1))
for i in range(len(Y_test)):
    if Y_test[i]=='spam':
        reptest_out[i][0]=1
    else:
        reptest_out[i][0]=0

#Building the neural network

n = 3   # n = number of layers

X = []
xx = np.ndarray(shape=(len(V)+1,1))
X.append(xx)
hl_1 = int(input('How many neurons do you want in the first hidden layer?'))
xx = np.ndarray(shape=(hl_1+1,1))
X.append(xx)
hl_2 = int(input('How many neurons do you want in the second hidden layer?'))
xx = np.ndarray(shape=(hl_2+1,1))
X.append(xx)
xx = np.ndarray(shape=(2,1))
X.append(xx)

#Weight initialiser

W = []
for i in range(1,n):
    W.append(np.random.randn(len(X[i-1]),len(X[i])-1)/100)
W.append(np.random.randn(len(X[n-1]),len(X[n]))/100)

#Forward Pass

def forward(zero_layer):

    for i in range(0,n):
        X[i][0][0]=1

    X[0][1:,0]=zero_layer

    for i in range(1,n):
        for j in range(1,len(X[i])):
            X[i][j][0]=1/(1+math.exp(-1*np.matmul(X[i-1].T,W[i-1][:,j-1])[0]))

    for j in range(0,len(X[n])):
        #print('cccccccccccc')
        #print(np.matmul(X[n-1].T,W[n-1][:,j])[0])
        X[n][j][0]=math.exp(np.matmul(X[n-1].T,W[n-1][:,j])[0])

    total = np.sum(X[n])

    for j in range(0,len(X[n])):
        X[n][j][0]=X[n][j][0]/total

#Back Propogation

def backward(actual):
    delta = [None]*n
    #print('Holoooooo')
    #print(X[n][0][0])
    if (X[n][0][0]==1 and actual==1) or (X[n][1][0]==1 and actual==0):
        error = 0
    else:
        error = -(actual*math.log(X[n][0][0])+(1-actual)*math.log(X[n][1][0]))

    delta[n-1]=np.ndarray(shape=(2,1))

    delta[n-1][0][0]=(X[n][0][0]-actual)
    delta[n-1][1][0]=(X[n][1][0]-(1-actual))

    for i in range(n-1,0,-1):
        delta[i-1]=np.ndarray(shape=(len(X[i])-1,1))
        for j in range(1,len(X[i])):
            delta[i-1][j-1][0]=0
            for k in range(0,len(delta[i])):
                delta[i-1][j-1][0]+=delta[i][k][0]*W[i][j][k]
            delta[i-1][j-1][0]=X[i][j][0]*(1-X[i][j][0])*delta[i-1][j-1][0]

    alpha = 0.1 #learning rate
    for i in range(0,len(W)):
        for j in range(0,len(W[i])):
            for k in range(0,len(delta[i])):
                W[i][j][k]=W[i][j][k]-alpha*X[i][j][0]*delta[i][k][0]

#Training

def training (epochs):

    for i in range(epochs):
        for j in range(len(X_train)):
            print("Epoch:",i,"Sample:",j)
            forward(rep_in[j])
            backward(rep_out[j][0])
        training_accuracy[i], test_accuracy[i] = accuracy(i)

training_accuracy = np.zeros(10)
test_accuracy = np.zeros(10)
predicted_train = [[],[],[],[],[],[],[],[],[],[]]
predicted_test = [[],[],[],[],[],[],[],[],[],[]]

def accuracy(index):

    #Training Accuracy
    for i in range(len(X_train)):
            forward(rep_in[i])
            if X[n][0][0]>X[n][1][0]:
                predicted_train[index].append(1)
            else:
                predicted_train[index].append(0)

    count = 0
    for i in range(len(predicted_train[index])):
        if predicted_train[index][i]==rep_out[i][0]:
            count+=1
    training_accuracy = count/len(predicted_train[index])*100

    #Test Accuracy

    for i in range(len(X_test)):
            forward(reptest_in[i])
            if X[n][0][0]>X[n][1][0]:
                predicted_test[index].append(1)
            else:
                predicted_test[index].append(0)

    count = 0
    for i in range(len(predicted_test[index])):
        if predicted_test[index][i]==reptest_out[i][0]:
            count+=1
    test_accuracy = count/len(predicted_test[index])*100

    return training_accuracy,test_accuracy

#Running the network

training(10)

#Plotting the graph

plt.plot([1,2,3,4,5,6,7,8,9,10], training_accuracy, color = 'red')
plt.plot([1,2,3,4,5,6,7,8,9,10], test_accuracy, color = 'blue')
plt.title('Training accuracy and Test accuracy for epoch 1 to 10')
plt.xlabel('Number of epochs')
plt.ylabel('Accuracy')
plt.legend(labels=('Training accuracy', 'Test accuracy'))
plt.show()

#To calculate training and test set errors
'''
pred_train = []
pred_test = []
def acc():

    #Training Accuracy
    for i in range(len(X_train)):
            forward(rep_in[i])
            pred_train.append(X[n][0][0])
    error=0
    for i in range(len(pred_train)):
        if (pred_train[i]==1 and rep_out[i][0]==1 ) or (pred_train[i]==0 and rep_out[i][0]==0):
            error+=0
        else:
            error += -(rep_out[i][0]*math.log(pred_train[i])+(1-rep_out[i][0])*math.log(1-pred_train[i]))

    train_error = error/len(X_train)

    #Test Accuracy

    for i in range(len(X_test)):
            forward(reptest_in[i])
            pred_test.append(X[n][0][0])

    error=0
    for i in range(len(pred_test)):
        if (pred_test[i]==1 and reptest_out[i][0]==1 ) or (pred_test[i]==0 and reptest_out[i][0]==0):
            error+=0
        else:
            error += -(reptest_out[i][0]*math.log(pred_test[i])+(1-reptest_out[i][0])*math.log(1-pred_test[i]))

    test_error = error/len(X_test)

    return train_error,test_error

train_error,test_error = acc()
'''

Assignment 4 - Neural Networks/Part2_A4.py


from django.shortcuts import render, redirect, reverse, get_object_or_404
from products.models import Product
from .forms import ReviewForm
from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django import forms

@login_required
def add_review(request, product_id):
    product = get_object_or_404(Product, pk=product_id)
    if request.method == 'POST':
        form = ReviewForm(request.POST, request.FILES)
        form.instance.user = request.user
        if form.is_valid():
            form.save()
            messages.success(request, 'Successfully added review!')
            return redirect(reverse('product_detail', args=[product.id]))
        else:
            messages.error(
                request, 'Review not added. Please check your form.')
    else:
        initial = {'product': product}
        form = ReviewForm(initial=initial)
        form.fields['product'].widget = forms.HiddenInput()

    template = 'review/add_review.html'
    context = {
        'form': form,
        'product': product,
    }
    return render(request, template, context)


import os
import logging
from datetime import datetime, timezone
from ibm_botocore.credentials import DefaultTokenManager

from lithops.utils import is_lithops_function
from lithops.config import CACHE_DIR, load_yaml_config, dump_yaml_config

logger = logging.getLogger(__name__)

class IBMIAMAPIKeyManager:
    def __init__(self, component_name, iam_api_key, token=None, token_expiry_time=None):
        self.component_name = component_name
        self.iam_api_key = iam_api_key

        self._token_manager = DefaultTokenManager(api_key_id=self.iam_api_key)
        self._token_filename = os.path.join(CACHE_DIR, self.component_name, 'iam_token')

        if token:
            logger.debug("Using IBM IAM API Key - Reusing Token from config")
            self._token_manager._token = token
            self._token_manager._expiry_time = datetime.strptime(token_expiry_time, '%Y-%m-%d %H:%M:%S.%f%z')
            logger.debug("Token expiry time: {} - Minutes left: {}".format(self._token_manager._expiry_time, self._get_token_minutes_diff()))
        elif os.path.exists(self._token_filename):
            logger.debug("Using IBM IAM API Key - Reusing Token from local cache")
            token_data = load_yaml_config(self._token_filename)
            self._token_manager._token = token_data['token']
            self._token_manager._expiry_time = datetime.strptime(token_data['token_expiry_time'], '%Y-%m-%d %H:%M:%S.%f%z')
            logger.debug("Token expiry time: {} - Minutes left: {}".format(self._token_manager._expiry_time, self._get_token_minutes_diff()))

    def _get_token_minutes_diff(self):
        return int((self._token_manager._expiry_time - datetime.now(timezone.utc)).total_seconds() / 60.0)

    def _generate_new_token(self):
        self._token_manager._token = None
        self._token_manager.get_token()
        token_data = {}
        token_data['token'] = self._token_manager._token
        token_data['token_expiry_time'] = self._token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z')
        dump_yaml_config(self._token_filename, token_data)

    def get_token(self):
        if (self._token_manager._is_expired() or self._get_token_minutes_diff() < 11) and not is_lithops_function():
            logger.debug("Using IBM IAM API Key - Token expired. Requesting new token")
            self._generate_new_token()

        token = self._token_manager._token
        token_expiry_time = self._token_manager._expiry_time.strftime('%Y-%m-%d %H:%M:%S.%f%z')

        return token, token_expiry_time


from markdown.extensions import Extension
from markdown.preprocessors import Preprocessor
from markdown.treeprocessors import Treeprocessor
from markdown.postprocessors import Postprocessor
from markdown.inlinepatterns import Pattern
from markdown.util import etree

from pybtex.database.input import bibtex
from pybtex.style.template import FieldIsMissing

from collections import OrderedDict
import re

from .styles import APAStyle

BRACKET_RE = re.compile(r'\[([^\[]+)\]')
CITE_RE    = re.compile(r'@(\w+)')
DEF_RE     = re.compile(r'\A {0,3}\[@(\w+)\]:\s*(.*)')
INDENT_RE  = re.compile(r'\A\t| {4}(.*)')

CITATION_RE = r'\[([^@^\[]*)(@[a-zA-Z0-9;@\s]+)\]'
INNER_CITATION_RE = re.compile(r'@(\w+)')

class Bibliography(object):
    """ Keep track of document references and citations for exporting """

    def __init__(self, extension, bibtex_file, order):
        self.extension = extension
        self.order = order

        self.citations = OrderedDict()
        self.references = dict()
        self.style = APAStyle()

        if bibtex_file:
            try:
                parser = bibtex.Parser()
                self.bibsource = parser.parse_file(bibtex_file).entries
            except:
                print("Error loading bibtex file")
                self.bibsource = dict()
        else:
            self.bibsource = dict()

    def addCitation(self, citekey):
        self.citations[citekey] = self.citations.get(citekey, 0) + 1

    def setReference(self, citekey, reference):
        self.references[citekey] = reference

    def citationID(self, citekey):
        return "cite-" + citekey

    def referenceID(self, citekey):
        return "ref-" + citekey

    def formatAuthor(self, author):
        out = "%s %s."%(author.last()[0], author.first()[0][0])
        if author.middle():
            out += "%s."%(author.middle()[0][0])
        return out

    def formatReference(self, key, entry):
        try:
            return self.style.format_entry(key, entry, self.bibsource).text.render_as('html')
        except FieldIsMissing as error:
            return self.formatError(key, error)

    def formatError(self, key, error):
        return '%s: %s' % (key, error)

    def makeBibliography(self, root):
        if self.order == 'alphabetical':
            raise(NotImplementedError)

        div = etree.Element("div")
        div.set("class", "references")

        if not self.citations:
            return div

        for id in sorted(self.citations):

            ref_txt = etree.SubElement(div, "div")
            ref_txt.set("id", self.referenceID(id))
            ref_txt.set("class", "bibentry")
            if id in self.references:
                self.extension.parser.parseChunk(ref_txt, self.references[id])
            elif id in self.bibsource:
                par = etree.SubElement(ref_txt, "p")
                par.text = self.formatReference(id, self.bibsource[id])
            else:
                par = etree.SubElement(ref_txt, "p")
                par.text = self.formatError(id, 'Missing Citation')

        return div

class CitationsPreprocessor(Preprocessor):
    """ Gather reference definitions and citation keys """

    def __init__(self, bibliography):
        self.bib = bibliography

    def subsequentIndents(self, lines, i):
        """ Concatenate consecutive indented lines """
        linesOut = []
        while i < len(lines):
            m = INDENT_RE.match(lines[i])
            if m:
                linesOut.append(m.group(1))
                i += 1
            else:
                break
        return " ".join(linesOut), i

    def run(self, lines):
        linesOut = []
        i = 0

        while i < len(lines):
            # Check to see if the line starts a reference definition
            m = DEF_RE.match(lines[i])
            if m:
                key = m.group(1)
                reference = m.group(2)
                indents, i = self.subsequentIndents(lines, i+1)
                reference += ' ' + indents

                self.bib.setReference(key, reference)
                continue

            # Look for all @citekey patterns inside hard brackets
            for bracket in BRACKET_RE.findall(lines[i]):
                for c in CITE_RE.findall(bracket):
                    self.bib.addCitation(c)
            linesOut.append(lines[i])
            i += 1

        return linesOut

class CitationsPattern(Pattern):
    """ Handles converting citations keys into links """

    def __init__(self, pattern, bibliography):
        super(CitationsPattern, self).__init__(pattern)
        self.bib = bibliography

    def handleMatch(self, m):

        intro = m.group(2)
        citations = m.group(3)

        if intro:
            if len(intro) == 1:
                intro = ''
            template_et_al = '%s et al. (%s)'
            template = '%s (%s)'
            prefix = ''
            suffix = ''
        else:
            template_et_al = '%s et al., %s'
            template = '%s, %s'
            prefix = '('
            suffix = ')'

        container = etree.Element('span')
        container.text = intro
        citations = [m.group(1) for m in INNER_CITATION_RE.finditer(citations)]

        for i, citation in enumerate(citations):

            if citation in self.bib.citations:

                a = etree.SubElement(container, "a")
                a.set('id', self.bib.citationID(citation))
                a.set('href', '#' + self.bib.referenceID(citation))
                a.set('class', 'citation')

                if citation in self.bib.bibsource:

                    ref = self.bib.bibsource[citation]
                    authors = ref.persons['author']
                    year = ref.fields.get('year', '?')
                    if len(authors) > 1:
                        a.text = prefix + template_et_al % (authors[0].last()[0], year)
                    elif authors:
                        a.text = prefix + template % (authors[0].last()[0], year)
                    else:
                        a.text = prefix + citation
                else:

                    a.text = prefix + citation

                if i < len(citations)-1:
                    a.tail = '; '
                    prefix = ''
                else:
                    a.text += suffix

        return container

class CitationsTreeprocessor(Treeprocessor):
    """ Add a bibliography/reference section to the end of the document """

    def __init__(self, bibliography):
        self.bib = bibliography

    def run(self, root):
        citations = self.bib.makeBibliography(root)
        root.append(citations)

class CitationsExtension(Extension):

    def __init__(self, *args, **kwargs):

        self.config = {
            "bibtex_file": ["",
                            "Bibtex file path"],
            'order': [
                "unsorted",
                "Order of the references (unsorted, alphabetical)"]
        }
        super(CitationsExtension, self).__init__(*args, **kwargs)
        self.bib = Bibliography(
            self,
            self.getConfig('bibtex_file'),
            self.getConfig('order'),
        )

    def extendMarkdown(self, md, md_globals):
        md.registerExtension(self)
        self.parser = md.parser
        self.md = md

        md.preprocessors.add("mdx_bib",  CitationsPreprocessor(self.bib), "<reference")
        md.inlinePatterns.add("mdx_bib", CitationsPattern(CITATION_RE, self.bib), "<reference")
        md.treeprocessors.add("mdx_bib", CitationsTreeprocessor(self.bib), "_begin")

def makeExtension(*args, **kwargs):
    return CitationsExtension(*args, **kwargs)


import d2bot.visualizer as visualizer
import d2bot.core.game_env as game_env
import d2bot.simulator as simulator

from d2bot.torch.a3c.ActorCritic import ActorCritic

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable

import random
import time
import sys

from tensorboardX import SummaryWriter
writer = SummaryWriter()

'''
    model from https://github.com/ikostrikov/pytorch-a3c
'''

class Spliter(torch.nn.Module):

    def __init__(self, num_inputs, num_hidden):
        super(Spliter, self).__init__()

        self.num_hidden = num_hidden
        self.num_inputs = num_inputs
        self.num_outs = 2

        self.lstm_layer = nn.LSTM(num_inputs, self.num_hidden)

        self.linear = nn.Linear(self.num_hidden, self.num_outs)
        self.train()

    def forward(self, inputs):
        t = torch.FloatTensor(inputs)
        t = t.view(len(inputs),1,-1)

        lstm_outs,_ = self.lstm_layer(t)
        lstm_out = lstm_outs[-1]
        lstm_out = F.tanh(lstm_out)
        out = self.linear(lstm_out)
        return out

class SpliterWarpper:
    def __init__(self,num_inputs, num_hidden):
        self.nn_spliter = Spliter(num_inputs, num_hidden)
        self.optimizer = optim.SGD(self.nn_spliter.parameters(), lr=0.01)
        self.outs = []
        self.nll_loss_fn = nn.NLLLoss()

    def reset(self):
        self.outs.clear()

    def step(self, inputs):
        out = F.log_softmax(self.nn_spliter(inputs))
        self.outs.append(out)

    def train(self, labels):
        assert(len(labels) == len(self.outs))
        labels = labels.type(torch.LongTensor)
        loss = self.nll_loss_fn(torch.cat(self.outs),labels) / len(labels)
        pred = torch.cat(self.outs)
        score = torch.argmax(pred,1) == labels
        print(loss,
            float(sum(score).numpy()) / len(labels),
            float(sum(labels).numpy()) / len(labels))

        writer.add_scalar("train/spliter_loss",loss.item())
        writer.add_scalar("train/spliter_score",sum(score).item() / len(labels))
        writer.add_scalar("train/acc",sum(score).item() / len(labels))

        loss.backward()
        self.reset()

    def update(self):
        self.optimizer.step()
        self.optimizer.zero_grad()
        self.nn_spliter.zero_grad()

class A3CSpliterEnv(game_env.GameEnv):
    def __init__(self, *args, **kwargs):
        super(A3CSpliterEnv,self).__init__(*args, **kwargs)

        random.seed(time.time())

        self.game_no = 0

        self.out_classes = 9

        self.a3c_model = ActorCritic(5, self.out_classes, 128)
        self.a3c_model.load_state_dict(torch.load('F:\\Dota2BotStepByStep\\tmp\\model_3023_24392'))
        self.optimizer = optim.SGD(self.a3c_model.parameters(), lr=0.01)

        self.reset()

        self.gamma = 0.99
        self.tau = 1.0
        self.entropy_coef = 0.01

        self.batch_size = 256
        self.buffer_size = 1000
        self.i = 0

        self.nll_loss_fn = nn.NLLLoss()

        self.sw = SpliterWarpper(5, 128)

    def reset(self):
        self.entropies = []
        self.values = []
        self.rewards = []
        self.log_probs = []
        self.raw_log_probs = []
        self.raw_probs = []
        self.predefined_steps = []

    def update(self):
        print("Game %d updating"%self.game_no)
        self.optimizer.step()
        self.a3c_model.zero_grad()
        self.optimizer.zero_grad()
        self.sw.update()

    def train(self):
        #just make it simple

        R = torch.zeros(1, 1)
        gae = torch.zeros(1, 1)

        self.values.append(R)

        policy_loss = 0
        value_loss = 0
        teacher_loss = 0

        for i in reversed(range(len(self.rewards))):
            R = self.gamma * R + self.rewards[i]
            adv = R - self.values[i]
            value_loss = value_loss + 0.5 * adv.pow(2)

            #GAE
            delta_t = self.rewards[i] + self.gamma * self.values[i + 1].data - self.values[i].data
            gae = gae * self.gamma * self.tau + delta_t

            policy_loss = policy_loss - self.log_probs[i] * Variable(gae) - self.entropy_coef * self.entropies[i]

            #teacher_loss = teacher_loss + self.nll_loss_fn(self.raw_log_probs[i], self.predefined_steps[i])

        teacher_loss = torch.cat(self.raw_probs)
        teacher_loss = teacher_loss - torch.zeros(len(self.predefined_steps),self.out_classes).scatter_(1, torch.cat(self.predefined_steps).view(-1,1), 1)
        teacher_loss = torch.sum(teacher_loss ** 2)

        total_loss = (policy_loss + 0.5 * value_loss + teacher_loss)
        total_loss = total_loss / len(self.raw_probs)
        #print(total_loss)
        total_loss.backward()
        torch.nn.utils.clip_grad_norm(self.a3c_model.parameters(), 10)

        print("reward %f"%sum(self.rewards))

        writer.add_scalar("train/loss",total_loss.item())
        writer.add_scalar("train/policy_loss",policy_loss.item() / len(self.raw_probs))
        writer.add_scalar("train/policy_loss",value_loss.item() / len(self.raw_probs))
        writer.add_scalar("train/teacher_loss",teacher_loss.item() / len(self.raw_probs))
        writer.add_scalar("train/rewards",sum(self.rewards))
        writer.add_scalar("train/values",sum(self.values).item() / len(self.values))
        writer.add_scalar("train/entropy",sum(self.entropies).item() / len(self.entropies))

        acts = torch.cat(self.raw_log_probs)
        pd = torch.tensor(self.predefined_steps)
        self.sw.train(torch.argmax(acts,1) == pd)

    def _generator_run(self, input_):
        self.game_no = self.game_no + 1
        self.init_fn(input_)

        self.engine = simulator.Simulator(feature_name='unit_test1',
            actionspace_name='lattice1',
            canvas=self.canvas)

        self.reset()
        self.sw.reset()

        while self.engine.get_time() < 300:
            self.i = self.i + 1

            #print(dire_predefine_step)
            dire_state = self.engine.get_state_tup("Dire", 0)

            dire_predefine_step = self.engine.predefined_step("Dire",0)
            predefine_move = torch.LongTensor([dire_predefine_step[1]])

            is_end = dire_state[2]
            if is_end:
                break

            self.predefined_steps.append(predefine_move)
            action_out, value_out = self.a3c_model(dire_state[0])

            self.sw.step(dire_state[0])

            prob = F.softmax(action_out)
            self.raw_probs.append(prob)
            log_prob = F.log_softmax(action_out)
            self.raw_log_probs.append(log_prob)

            entropy = - (log_prob * prob).sum(1, keepdim=True)
            self.entropies.append(entropy)

            #action = prob.multinomial(num_samples=1).data
            action = torch.argmax(log_prob, 1).data.view(-1,1)
            log_prob = log_prob.gather(1,Variable(action))

            self.engine.set_order("Dire", 0, (1,action))

            self.engine.loop()

            reward = dire_state[1]
            self.rewards.append(reward)
            self.values.append(value_out)
            self.log_probs.append(log_prob)

            yield

        self.train()
        self.update()

def test():
    v = visualizer.Visualizer(A3CSpliterEnv)
    v.visualize()

def test_without_gui():
    env = A3CSpliterEnv()
    gen = env.run(True)
    gen.send(None)
    while True:
        try:
            gen.send((None,))
        except StopIteration:
            gen = env.run(True)
            gen.send(None)

if __name__ == '__main__':
    if len(sys.argv) == 2:
        if sys.argv[1] == "visible":
            test()
    test_without_gui()


"""
Author: Johan Lundstroem
Email: lundstrom.se at gmail.com
Date: 12/9 2011
License: GPLv2
"""

import __init__
import subprocess
from datetime import datetime
from datetime import timedelta
import settings
import logging
import time

class Testcase(object):

  def init(self, thread=None, log_name=None):
    self.log_name = log_name
    if not thread == None:
      self.name = self.name+"_thread:"+str(thread)
    if self.log == None:
      if thread == None:
        # Creating a log file
        self.log_name = caller = self.name
        self.file_path = settings.log_file_path
        time = datetime.now()
        time = str(time)
        time = time.replace(' ','')
        time = time.replace(':','-')
        self.file_name = "%s_%s_%s_%s.log" % (caller, caller, caller, time)
        self.file = self.file_path+self.file_name
        f = open(self.file, "w")
        f.close()
        logging.basicConfig(filename=self.file, level=logging.DEBUG,format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
      log = logging.getLogger(self.log_name)
      log.setLevel(logging.INFO)
      # create console handler and set level to debug
      ch = logging.StreamHandler()
      ch.setLevel(logging.INFO)
      # create formatter
      formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
      # add formatter to ch
      ch.setFormatter(formatter)
      # add ch to logger
      log.addHandler(ch)
      self.log=log
    self.log.info("%s initiated" % (self.name))

  def run(self):
    pass

  def exe(self, what, max_time=None):
    if max_time == None: max_time = timedelta(seconds=10, microseconds=0)
    start = datetime.now()
    #self.log.info("start %s..." % (inspect.stack()[1][3]))
    self.log.info("%s start %s..." % (self.name, what.__name__))
    what()
    #self.log.info("end %s..." % (inspect.stack()[1][3]))
    #self.log.info("end %s..." % (what.__name__))
    stop = datetime.now()
    exe_time = stop-start
    if exe_time > max_time:
      self.log.info('The maximum response time was exseeded. %s' % (str(exe_time)))
      # Handle timeout here
    #self.log.info("%s duration: %s" % (inspect.stack()[1][3], str(stop-start)))
    self.log.info("%s end %s duration: %s" % (self.name, what.__name__, str(stop-start)))

# Manual execution
#if __name__ == "__main__":
#  t = testcase()
#  t.do_test()


# iaalib/logic.py
# Part of "IAA IoT Alarm and Automation"
#
# Gianluigi Cosari - gianluigi@cosari.it
# logic Ver 1.0 July 2017
# Licensed under MIT

from transitions import Machine

#Define door status values (MQTT Wi-Fi module)
DOOR_CLOSED = 0
DOOR_OPEN = 1

#Define I/O output module values
IO_OFF = 1
IO_ON = 0

#Define I/O output relays duty
IO_SIREN = 9            # connected to siren (energized -> siren playing)
IO_SYSTEM_ARMED = 10    # notice for sistem armed (energized -> system armed)
IO_PEDESTRIAN_GATE = 11 # operated by RFID reader (energized for 1 second -> gate opens)
IO_CAR_GATE = 12        # operated by RFID reader (energized for 1 second -> gate opens)
IO_HOME_DOOR = 13       # operated by RFID reader (energized for 1 second -> door opens)
IO_SECURITY_LIGHTS = 14 # exterior perimeter lights (energized -> lights turn on)
IO_AUX1 = 15            # free
IO_AUX2 = 16            # free

# Finite state machine model implementing the IAA logic
class IAAmodel(object):

	#ricordarsi di gestire il tamper e la manutenzione

	states = ['idle', 'watching', 'armed', 'alert']

	transitions = [
		{ 'trigger': 'node_signal', 'source': 'idle', 'dest': 'idle', 'prepare': 'set_door_state', 'conditions': ['door_changed_state'], 'after': 'send_notification' },
		{ 'trigger': 'node_signal', 'source': 'watching', 'dest': 'watching', 'prepare': 'set_door_state', 'conditions': ['door_opened'], 'after': 'send_notification' },
		{ 'trigger': 'node_signal', 'source': 'armed', 'dest': 'alert', 'prepare': 'set_door_state', 'conditions': ['door_opened'], 'after': 'send_notification' },
		{ 'trigger': 'node_signal', 'source': 'alert', 'dest': 'alert', 'prepare': 'set_door_state', 'conditions': ['door_changed_state'], 'after': 'send_notification' },
		{ 'trigger': 'arm', 'source': ['idle', 'watching'], 'dest': 'armed', },
		{ 'trigger': 'disarm', 'source': ['watching', 'armed', 'alert'], 'dest': 'idle' },
		{ 'trigger': 'watch', 'source': 'idle', 'dest': 'watching', 'after': 'send_notification' }
	]

	def __init__(self, logger, messenger, i2c_q):

		#Initialize shared parameters
		self.logger = logger
		self.messenger = messenger
		self.i2c_q = i2c_q

		#Initialize node parameters
		self.node_id = None
		self.node_name = None
		self.door_state = None
		self.door_state_changed = False
		self.tamper_status = None

		#Initialize the state machine
		self.machine = Machine(model = self, states = IAAmodel.states, transitions = IAAmodel.transitions, initial='idle')

	#This method receives node data from main queue processor
	def set_door_state(self, node_data = None):
		self.node_id, self.node_name, self.door_state, self.door_state_changed, self.tamper_status = node_data
		self.logger.debug('FSM: node "%s" state set' % self.node_name)

	def door_changed_state(self, *args): #True when door status changed from closed to open OR from open to closed
		return self.door_state_changed

	def door_opened(self, *args): #True when door status changed from closed to open
		return self.door_state_changed and (self.door_state == DOOR_OPEN)

	def send_notification(self, *args):
		door_status_desc = 'stato non disponibile'
		if self.door_state == DOOR_OPEN: door_status_desc = 'aperta'
		if self.door_state == DOOR_CLOSED: door_status_desc = 'chiusa'
		self.logger.debug('FSM: sending door "%s" status to Telegram : ' % door_status_desc)
		self.messenger.put(self.node_name + ' ' + door_status_desc)

	def on_enter_idle(self, *args):
		self.logger.info('FSM: System entered status IDLE')

		#de-energize system armed notice relay
		item = (IO_SYSTEM_ARMED, IO_OFF)
		self.logger.debug("FSM: item {}".format(item))
		self.i2c_q.put(item)

		#de-energize siren relay
		item = (IO_SIREN, IO_OFF)
		self.logger.debug("FSM: item {}".format(item))
		self.i2c_q.put(item)

		self.messenger.put('Sistema DISARMATO')

	def on_enter_armed(self, *args):
		self.logger.info('FSM: System entered status ARMED')

		#energize system armed notice relay
		item = (IO_SYSTEM_ARMED, IO_ON)
		self.logger.debug("FSM: item {}".format(item))
		self.i2c_q.put(item)

		self.messenger.put('Sistema ARMATO')

	def on_enter_alert(self, *args):
		self.logger.info('FSM: System entered status ALERT')

		#energize siren relay
		item = (IO_SIREN, IO_ON)
		self.logger.debug("FSM: item {}".format(item))
		self.i2c_q.put(item)

		self.messenger.put('Sistema IN ALLARME!')


from flask import Flask
from flask_sockets import Sockets

import numpy as np
import torch
from torchvision import transforms
from torchvision.datasets import MNIST
from torchvision.datasets import ImageFolder
from tqdm import tqdm

# from utils import AddGaussianNoise  # pylint: disable=import-error

KEEP_LABELS_DICT = {
    "alice": [0, 1, 2, 3],
    "bob": [4, 5, 6],
    "charlie": [7, 8, 9],
    None: list(range(10)),
}

def create_app(node_id, debug=False, database_url=None, data_dir: str = None):
    """ Create / Configure flask socket application instance.

        Args:
            node_id (str) : ID of Grid Node.
            debug (bool) : debug flag.
            test_config (bool) : Mock database environment.
        Returns:
            app : Flask application instance.
    """
    app = Flask(__name__)
    app.debug = debug

    app.config["SECRET_KEY"] = "justasecretkeythatishouldputhere"

    # Enable persistent mode
    # Overwrite syft.object_storage methods to work in a persistent way
    # Persist models / tensors
    if database_url:
        app.config["REDISCLOUD_URL"] = database_url
        from .main.persistence import database, object_storage

        db_instance = database.set_db_instance(database_url)
        object_storage.set_persistent_mode(db_instance)

    from .main import html, ws, hook, local_worker, auth

    # Global socket handler
    sockets = Sockets(app)

    # set_node_id(id)
    local_worker.id = node_id
    hook.local_worker._known_workers[node_id] = local_worker
    local_worker.add_worker(hook.local_worker)

    # add data
    if data_dir:
        print("register data")
        if "mnist" in data_dir.lower():
            dataset = MNIST(
                root="./data",
                train=True,
                download=True,
                transform=transforms.Compose(
                    [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
                ),
            )
            if node_id in KEEP_LABELS_DICT:
                indices = np.isin(dataset.targets, KEEP_LABELS_DICT[node_id]).astype(
                    "uint8"
                )
                selected_data = (
                    torch.native_masked_select(  # pylint:disable=no-member
                        dataset.data.transpose(0, 2),
                        torch.tensor(indices),  # pylint:disable=not-callable
                    )
                    .view(28, 28, -1)
                    .transpose(2, 0)
                )
                selected_targets = torch.native_masked_select(  # pylint:disable=no-member
                    dataset.targets,
                    torch.tensor(indices),  # pylint:disable=not-callable
                )
            """ dataset = sy.BaseDataset(
                    data=selected_data,
                    targets=selected_targets,
                    transform=dataset.transform,
                )
            dataset_name = "mnist"
            """
        else:

            train_tf = [
                transforms.RandomVerticalFlip(p=0.5),
                transforms.RandomAffine(
                    degrees=30,
                    translate=(0, 0),
                    scale=(0.85, 1.15),
                    shear=10,
                    #    fillcolor=0.0,
                ),
                transforms.Resize(224),
                transforms.RandomCrop(224),
                transforms.ToTensor(),
                transforms.Normalize((0.57282609,), (0.17427578,)),
                # transforms.RandomApply([AddGaussianNoise(mean=0.0, std=0.05)], p=0.5),
            ]
            """train_tf.append(
                transforms.Lambda(
                    lambda x: torch.repeat_interleave(  # pylint: disable=no-member
                        x, 3, dim=0
                    )
                )
            )"""
            target_dict_pneumonia = {0: 1, 1: 0, 2: 2}
            dataset = ImageFolder(
                data_dir,
                transform=transforms.Compose(train_tf),
                target_transform=lambda x: target_dict_pneumonia[x],
            )
            data, targets = [], []
            for d, t in tqdm(dataset, total=len(dataset)):
                data.append(d)
                targets.append(t)
            selected_data = torch.stack(data)  # pylint:disable=no-member
            selected_targets = torch.from_numpy(np.array(targets))  # pylint:disable=no-member
            #dataset = sy.BaseDataset(data=data, targets=targets)
            """dataset = PPPP(
                "data/Labels.csv",
                train=True,
                transform=transforms.Compose(train_tf),
                seed=1
            )"""
            dataset_name = "pneumonia"

        local_worker.register_obj(selected_data, "data")
        local_worker.register_obj(selected_targets, "targets")

        print("registered data")

    # Register app blueprints
    app.register_blueprint(html, url_prefix=r"/")
    sockets.register_blueprint(ws, url_prefix=r"/")

    # Set Authentication configs
    app = auth.set_auth_configs(app)

    return app


import datetime
from explorer.api import util
from flask import request
from explorer.services.message import TipsetGasService, TipsetService, MessageService
from base.response import response_json

def sync_tipset_gas():
    """
    同步单个区块的全网gas汇总
    :return:
    """

    end_index = request.form.get('end_index')
    start_index = request.form.get('start_index')
    TipsetGasService.sync_overview_tipset_gas()
    return response_json(True)

def sync_messages_stat():
    """

    :return:
    """
    TipsetGasService.sync_messages_stat()
    return response_json(True)

def get_gas_trends():
    """
    获取gas费用趋势图
    :return:
    """
    search_type = request.form.get('search_type', 'day')
    start_date = request.form.get('start_date')
    end_date = request.form.get('end_date')
    step, start_date, end_date = util.trends_parm(search_type, start_date, end_date)
    data = TipsetService.get_gas_trends(start_date=start_date, end_date=end_date, step=step)
    return response_json(data)

def get_gas_stat_all():
    """
    获取gas24小时统计
    :return:
    """
    data = TipsetService.get_gas_stat_all()
    return response_json(data)

def get_message_list():
    miner_id = request.form.get('miner_id')
    block_hash = request.form.get('block_hash')
    msg_method = request.form.get('msg_method')
    msgrct_exit_code = request.form.get('msgrct_exit_code')
    page_index = int(request.form.get('page_index', 1))
    page_size = min(int(request.form.get('page_size', 20)), 50)
    data = MessageService.get_message_list(miner_id, block_hash, msg_method, msgrct_exit_code,
                                           page_index=page_index, page_size=page_size)
    return response_json(data)

def get_message_method_list():
    """
    获取方法详情
    :return:
    """
    miner_id = request.form.get('miner_id')
    block_hash = request.form.get('block_hash')
    data = MessageService.get_message_method_list(miner_id, block_hash)
    return response_json(data)

def get_message_info():
    """
    获取消息详情
    :return:
    """
    msg_cid = request.form.get('msg_cid')
    height = int(request.form.get('height'))
    result = MessageService.get_message_info(msg_cid, height)
    return response_json(result)

def get_mpool_list():
    key_words = request.form.get('key_words')
    msg_method = request.form.get('msg_method')
    page_index = int(request.form.get('page_index', 1))
    page_size = min(int(request.form.get('page_size', 10)), 50)
    data = MessageService.get_mpool_list(key_words, msg_method, page_index=page_index, page_size=page_size)
    return response_json(data)

def get_mpool_info():
    cid = request.form.get('cid')
    result = MessageService.get_mpool_info(cid)
    return response_json(result)

def get_transfer_list():
    """
    获取转账列表
    :return:
    """
    msg_method = request.form.get('msg_method')
    miner_id = request.form.get('miner_id')
    start_date = request.form.get('start_date')
    end_date = request.form.get('end_date')
    page_size = int(request.form.get('page_size', 20))
    page_index = int(request.form.get('page_index', 1))
    data = MessageService.get_message_list(miner_id, msg_method=msg_method, start_date=start_date, end_date=end_date,
                                           is_transfer=True, page_index=page_index, page_size=page_size)
    return response_json(data)

def get_transfer_method_list():
    """
    获取转账列表方法详情
    :return:
    """
    miner_id = request.form.get('miner_id')
    start_date = request.form.get('start_date')
    end_date = request.form.get('end_date')
    data = MessageService.get_message_method_list(miner_id, start_date=start_date, end_date=end_date,
                                                  is_transfer=True)
    return response_json(data)


import requests
import string

# lua types
#'any',
#'string',
#'number',
#'integer',
#'boolean',
#'table',
#'function',
#'nil',
#'userdata',
#'thread',
#

typeTranslationTable = {
    "float": "number",
    "int": "number",
    "bool": "boolean",
    "BOOL": "boolean",
    "Any": "any",
    "char": "string",
    "Vector3": "vector3"
}

# Lua keywords
blacklistedParams = ["and", "end", "in", "repeat", "break", "false", "local", "return", "do", "for", "nil", "then", "else", "function", "not", "true", "elseif", "if", "or", "until", "while",]

docs = ["https://runtime.fivem.net/doc/natives.json", "https://runtime.fivem.net/doc/natives_cfx.json"]
for doc in docs:
    print(doc)
    resp = requests.get(doc)
    respBody = resp.json()
    for nativeNamespace in respBody:
        output = open("stubs/natives_{}.lua".format(nativeNamespace.lower()), "w", encoding="utf-8")

        linesWritten = 0
        fileCounter = 0
        for nativeHash in respBody[nativeNamespace]:
            # Split bigger files into smaller files - https://github.com/sumneko/lua-language-server/issues/64
            if linesWritten >= 1000:
                output.close()
                linesWritten = 0
                fileCounter += 1
                output = open("stubs/natives_{}_{}.lua".format(nativeNamespace.lower(), fileCounter), "w", encoding="utf-8")

            nativeFunc = respBody[nativeNamespace][nativeHash]
            if not "name" in nativeFunc:
                continue
            nativeUnfriendlyName = nativeFunc['name']
            nativeParams = nativeFunc['params']
            nativeDesc = nativeFunc['description'].replace("`", "").replace("\n", "\n--- ")
            if len(nativeDesc) > 1 and not nativeDesc.startswith("---"):
                nativeDesc = "--- " + nativeDesc

            nativeExamples = nativeFunc['description']
            nativeReturnType = nativeFunc['results'].replace("*", "")

            #if nativeUnfriendlyName != "REGISTER_COMMAND":
            #    continue

            #print(nativeFunc)
            # _LOCK_RADIO_STATION -> LockRadioStation
            #env.addFilter(new Twing.TwingFilter('makenative', input => {
            #    return input.toLowerCase().replace('0x', 'n_0x')
            #        .replace(/_([a-z])/g, (sub, bit) => bit.toUpperCase())
            #        .replace(/^([a-z])/, (sub, bit) => bit.toUpperCase());
            #}));

            #N_0x8db8cffd58b62552
            nativeName = nativeUnfriendlyName.lower().replace("0x", "N_0x")
            nativeName = nativeName.replace("_", " ")
            nativeName = string.capwords(nativeName)
            nativeName = nativeName.replace(" ", "")
            output.write("{}\n".format(nativeDesc))

            functionDefParams = ""
            for i, nativeParam in enumerate(nativeParams):
                paramName = nativeParam['name']
                if paramName in blacklistedParams:
                    paramName = "_b_{}".format(paramName)

                paramType = nativeParam['type'].replace("*", "")

                # Convert type to Lua type
                if paramType in typeTranslationTable:
                    paramType = typeTranslationTable[paramType]

                if i == len(nativeParams) - 1:
                    functionDefParams += paramName
                else:
                    functionDefParams += paramName + ", "
                # TODO: Convert types to Lua types

                output.write("--- @param {} {}\n".format(paramName, paramType))

            if nativeReturnType != "void":
                if nativeReturnType in typeTranslationTable:
                    nativeReturnType = typeTranslationTable[nativeReturnType]

                output.write("--- @return {}\n".format(nativeReturnType))
            output.write("function {}({}) end\n\n".format(nativeName, functionDefParams))

            linesWritten += (3 + len(nativeParams) + nativeDesc.count('\n'))

        output.close()


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /home/fraoustin/Téléchargements/ocglances/tmp/ocglances/plugins/glances_system.py
# Compiled at: 2017-02-11 10:25:25
"""System plugin."""
import os, platform, re
from io import open
from ocglances.compat import iteritems
from ocglances.plugins.glances_plugin import GlancesPlugin
snmp_oid = {'default': {'hostname': '1.3.6.1.2.1.1.5.0', 'system_name': '1.3.6.1.2.1.1.1.0'},
   'netapp': {'hostname': '1.3.6.1.2.1.1.5.0', 'system_name': '1.3.6.1.2.1.1.1.0',
              'platform': '1.3.6.1.4.1.789.1.1.5.0'}}
snmp_to_human = {'windows': {'Windows Version 10.0': 'Windows 10 or Server 2016', 'Windows Version 6.3': 'Windows 8.1 or Server 2012R2',
               'Windows Version 6.2': 'Windows 8 or Server 2012',
               'Windows Version 6.1': 'Windows 7 or Server 2008R2',
               'Windows Version 6.0': 'Windows Vista or Server 2008',
               'Windows Version 5.2': 'Windows XP 64bits or 2003 server',
               'Windows Version 5.1': 'Windows XP',
               'Windows Version 5.0': 'Windows 2000'}}

def _linux_os_release():
    """Try to determine the name of a Linux distribution.

    This function checks for the /etc/os-release file.
    It takes the name from the 'NAME' field and the version from 'VERSION_ID'.
    An empty string is returned if the above values cannot be determined.
    """
    pretty_name = ''
    ashtray = {}
    keys = ['NAME', 'VERSION_ID']
    try:
        with open(os.path.join('/etc', 'os-release')) as (f):
            for line in f:
                for key in keys:
                    if line.startswith(key):
                        ashtray[key] = re.sub('^"|"$', '', line.strip().split('=')[1])

    except (OSError, IOError):
        return pretty_name

    if ashtray:
        if 'NAME' in ashtray:
            pretty_name = ashtray['NAME']
        if 'VERSION_ID' in ashtray:
            pretty_name += (' {}').format(ashtray['VERSION_ID'])
    return pretty_name

class Plugin(GlancesPlugin):
    """Glances' host/system plugin.

    stats is a dict
    """

    def __init__(self, args=None):
        """Init the plugin."""
        super(Plugin, self).__init__(args=args)
        self.display_curse = True
        self.reset()

    def reset(self):
        """Reset/init the stats."""
        self.stats = {}

    @GlancesPlugin._check_decorator
    @GlancesPlugin._log_result_decorator
    def update(self):
        """Update the host/system info using the input method.

        Return the stats (dict)
        """
        self.reset()
        if self.input_method == 'local':
            self.stats['os_name'] = platform.system()
            self.stats['hostname'] = platform.node()
            self.stats['platform'] = platform.architecture()[0]
            if self.stats['os_name'] == 'Linux':
                try:
                    linux_distro = platform.linux_distribution()
                except AttributeError:
                    self.stats['linux_distro'] = _linux_os_release()

                if linux_distro[0] == '':
                    self.stats['linux_distro'] = _linux_os_release()
                else:
                    self.stats['linux_distro'] = (' ').join(linux_distro[:2])
                self.stats['os_version'] = platform.release()
            elif self.stats['os_name'].endswith('BSD') or self.stats['os_name'] == 'SunOS':
                self.stats['os_version'] = platform.release()
            elif self.stats['os_name'] == 'Darwin':
                self.stats['os_version'] = platform.mac_ver()[0]
            elif self.stats['os_name'] == 'Windows':
                os_version = platform.win32_ver()
                self.stats['os_version'] = (' ').join(os_version[::2])
                if self.stats['platform'] == '32bit' and 'PROCESSOR_ARCHITEW6432' in os.environ:
                    self.stats['platform'] = '64bit'
            else:
                self.stats['os_version'] = ''
            if self.stats['os_name'] == 'Linux':
                self.stats['hr_name'] = self.stats['linux_distro']
            else:
                self.stats['hr_name'] = ('{} {}').format(self.stats['os_name'], self.stats['os_version'])
            self.stats['hr_name'] += (' {}').format(self.stats['platform'])
        elif self.input_method == 'snmp':
            try:
                self.stats = self.get_stats_snmp(snmp_oid=snmp_oid[self.short_system_name])
            except KeyError:
                self.stats = self.get_stats_snmp(snmp_oid=snmp_oid['default'])

            self.stats['os_name'] = self.stats['system_name']
            if self.short_system_name == 'windows':
                for r, v in iteritems(snmp_to_human['windows']):
                    if re.search(r, self.stats['system_name']):
                        self.stats['os_name'] = v
                        break

            self.stats['hr_name'] = self.stats['os_name']
        return self.stats

    def msg_curse(self, args=None):
        """Return the string to display in the curse interface."""
        ret = []
        if args.client:
            if args.cs_status.lower() == 'connected':
                msg = 'Connected to '
                ret.append(self.curse_add_line(msg, 'OK'))
            elif args.cs_status.lower() == 'snmp':
                msg = 'SNMP from '
                ret.append(self.curse_add_line(msg, 'OK'))
            elif args.cs_status.lower() == 'disconnected':
                msg = 'Disconnected from '
                ret.append(self.curse_add_line(msg, 'CRITICAL'))
        msg = self.stats['hostname']
        ret.append(self.curse_add_line(msg, 'TITLE'))
        if self.stats['os_name'] == 'Linux' and self.stats['linux_distro']:
            msg = (' ({} {} / {} {})').format(self.stats['linux_distro'], self.stats['platform'], self.stats['os_name'], self.stats['os_version'])
        else:
            try:
                msg = (' ({} {} {})').format(self.stats['os_name'], self.stats['os_version'], self.stats['platform'])
            except Exception:
                msg = (' ({})').format(self.stats['os_name'])

        ret.append(self.curse_add_line(msg, optional=True))
        return ret


# -*- coding: utf-8 -*-
import os
import json
import fido
import time
import warnings
import pandas as pd
import plotly.io as pio

try:
    from tqdm.notebook import tqdm
except ImportError:
    from tqdm import tqdm

from hashlib import md5
from copy import deepcopy
from urllib.parse import urlparse
from pyisemail import is_email
from collections import defaultdict
from ratelimit import limits, sleep_and_retry
from pyisemail.diagnosis import BaseDiagnosis
from swagger_spec_validator.common import SwaggerValidationError
from bravado_core.formatter import SwaggerFormat
from bravado.client import SwaggerClient
from bravado.fido_client import FidoClient  # async
from bravado.http_future import HttpFuture
from bravado.swagger_model import Loader
from bravado.config import bravado_config_from_config_dict
from bravado_core.spec import Spec
from json2html import Json2Html
from IPython.display import display, HTML
from boltons.iterutils import remap
from pymatgen.core import Structure
from concurrent.futures import as_completed
from requests_futures.sessions import FuturesSession

MAX_WORKERS = 10
DEFAULT_HOST = "contribs-api.materialsproject.org"
BULMA = "is-narrow is-fullwidth has-background-light"
PROVIDERS = {"github", "google", "facebook", "microsoft", "amazon"}
VALID_URLS = {f"http://{h}:{p}" for h in ["localhost", "contribs-api"] for p in [5000, 5002, 5003]}
VALID_URLS |= {f"https://{n}-api.materialsproject.org" for n in ["contribs", "lightsources", "ml"]}

j2h = Json2Html()
pd.options.plotting.backend = "plotly"
pio.templates.default = "simple_white"
warnings.formatwarning = lambda msg, *args, **kwargs: f"{msg}\n"
warnings.filterwarnings("default", category=DeprecationWarning, module=__name__)

def get_md5(d):
    s = json.dumps(d, sort_keys=True).encode("utf-8")
    return md5(s).hexdigest()

def validate_email(email_string):
    if email_string.count(":") != 1:
        raise SwaggerValidationError(f"{email_string} not of format <provider>:<email>.")

    provider, email = email_string.split(":", 1)
    if provider not in PROVIDERS:
        raise SwaggerValidationError(f"{provider} is not a valid provider.")

    d = is_email(email, diagnose=True)
    if d > BaseDiagnosis.CATEGORIES["VALID"]:
        raise SwaggerValidationError(f"{email} {d.message}")

email_format = SwaggerFormat(
    format="email",
    to_wire=str,
    to_python=str,
    validate=validate_email,
    description="e-mail address including provider",
)

def validate_url(url_string, qualifying=("scheme", "netloc")):
    tokens = urlparse(url_string)
    if not all([getattr(tokens, qual_attr) for qual_attr in qualifying]):
        raise SwaggerValidationError(f"{url_string} invalid")

url_format = SwaggerFormat(
    format="url", to_wire=str, to_python=str, validate=validate_url, description="URL",
)

def chunks(lst, n=250):
    if isinstance(lst, set):
        lst = list(lst)
    elif not isinstance(lst, list):
        raise ValueError("chunks needs list or set as input")

    n = max(1, n)
    for i in range(0, len(lst), n):
        to = i + n
        yield lst[i:to]

class FidoClientGlobalHeaders(FidoClient):
    def __init__(self, headers=None):
        super().__init__()
        self.headers = headers or {}

    def request(self, request_params, operation=None, request_config=None):
        request_for_twisted = self.prepare_request_for_twisted(request_params)
        request_for_twisted["headers"].update(self.headers)
        future_adapter = self.future_adapter_class(fido.fetch(**request_for_twisted))
        return HttpFuture(
            future_adapter, self.response_adapter_class, operation, request_config
        )

def visit(path, key, value):
    if isinstance(value, dict) and "display" in value:
        return key, value["display"]
    return True

class Dict(dict):
    def pretty(self, attrs=f'class="table {BULMA}"'):
        return display(
            HTML(j2h.convert(json=remap(self, visit=visit), table_attributes=attrs))
        )

def load_client(apikey=None, headers=None, host=None):
    warnings.warn(
        "load_client(...) is deprecated, use Client(...) instead", DeprecationWarning
    )

class Client(SwaggerClient):
    """client to connect to MPContribs API

    We only want to load the swagger spec from the remote server when needed and not everytime the
    client is initialized. Hence using the Borg design nonpattern (instead of Singleton): Since the
    __dict__ of any instance can be re-bound, Borg rebinds it in its __init__ to a class-attribute
    dictionary. Now, any reference or binding of an instance attribute will actually affect all
    instances equally.
    """

    _shared_state = {}

    def __init__(self, apikey=None, headers=None, host=None):
        # - Kong forwards consumer headers when api-key used for auth
        # - forward consumer headers when connecting through localhost
        self.__dict__ = self._shared_state

        if not host:
            host = os.environ.get("MPCONTRIBS_API_HOST", DEFAULT_HOST)

        if not apikey:
            apikey = os.environ.get("MPCONTRIBS_API_KEY")

        if apikey and headers is not None:
            apikey = None
            print("headers set => ignoring apikey!")

        self.apikey = apikey
        self.headers = {"x-api-key": apikey} if apikey else headers
        self.host = host
        self.protocol = "https" if host.endswith(".materialsproject.org") else "http"
        self.url = f"{self.protocol}://{self.host}"

        if self.url not in VALID_URLS:
            raise ValueError(f"{self.url} not a valid URL (one of {VALID_URLS})")

        if "swagger_spec" not in self.__dict__ or (
            self.swagger_spec.http_client.headers != self.headers
        ):
            self.load()

    def load(self):
        http_client = FidoClientGlobalHeaders(headers=self.headers)
        loader = Loader(http_client)
        origin_url = f"{self.url}/apispec.json"
        spec_dict = loader.load_spec(origin_url)
        spec_dict["host"] = self.host
        spec_dict["schemes"] = [self.protocol]

        config = {
            "validate_responses": False,
            "use_models": False,
            "include_missing_properties": False,
            "formats": [email_format, url_format],
        }
        bravado_config = bravado_config_from_config_dict(config)
        for key in set(bravado_config._fields).intersection(set(config)):
            del config[key]
        config["bravado"] = bravado_config

        swagger_spec = Spec.from_dict(spec_dict, origin_url, http_client, config)
        super().__init__(
            swagger_spec, also_return_response=bravado_config.also_return_response
        )

        # expand regex-based query parameters for `data` columns
        try:
            resp = self.projects.get_entries(_fields=["columns"]).result()
        except AttributeError:
            # skip in tests
            return

        columns = {"text": [], "number": []}
        for project in resp["data"]:
            for column in project["columns"]:
                if column["path"].startswith("data."):
                    col = column["path"].replace(".", "__")
                    if column["unit"] == "NaN":
                        columns["text"].append(col)
                    else:
                        col = f"{col}__value"
                        columns["number"].append(col)

        operators = {"text": ["contains"], "number": ["gte", "lte"]}
        for path, d in spec_dict["paths"].items():
            for verb in ["get", "put", "post", "delete"]:
                if verb in d:
                    old_params = deepcopy(d[verb].pop("parameters"))
                    new_params, param_names = [], set()

                    while old_params:
                        param = old_params.pop()
                        if param["name"].startswith("^data__"):
                            op = param["name"].rsplit("__", 1)[1]
                            for typ, ops in operators.items():
                                if op in ops:
                                    for column in columns[typ]:
                                        new_param = deepcopy(param)
                                        param_name = f"{column}__{op}"
                                        if param_name not in param_names:
                                            new_param["name"] = param_name
                                            desc = f"filter {column} via ${op}"
                                            new_param["description"] = desc
                                            new_params.append(new_param)
                                            param_names.add(param_name)
                        else:
                            new_params.append(param)

                    d[verb]["parameters"] = new_params

        swagger_spec = Spec.from_dict(spec_dict, origin_url, http_client, config)
        super().__init__(
            swagger_spec, also_return_response=bravado_config.also_return_response
        )

    def get_project(self, project):
        """Convenience function to get full project entry and display as HTML table"""
        return Dict(self.projects.get_entry(pk=project, _fields=["_all"]).result())

    def get_contribution(self, cid):
        """Convenience function to get full contribution entry and display as HTML table"""
        fields = list(
            self.swagger_spec.definitions.get("ContributionsSchema")._properties.keys()
        )  # don't return dynamic fields (card_*)
        fields.remove("notebook")
        return Dict(self.contributions.get_entry(pk=cid, _fields=fields).result())

    def get_table(self, tid_or_md5):
        """Convenience function to get full Pandas DataFrame for a table.

        Args:
            tid_or_md5 (str): ObjectId or MD5 hash digest for table

        Returns:
            pd.DataFrame: pandas DataFrame containing table data
        """
        str_len = len(tid_or_md5)
        if str_len not in {24, 32}:
            raise ValueError(f"'{tid_or_md5}' is not a valid table id or md5 hash digest!")

        if str_len == 32:
            tables = self.tables.get_entries(md5=tid_or_md5, _fields=["id"]).result()
            if not tables:
                raise ValueError(f"table for md5 '{tid_or_md5}' not found!")
            tid = tables["data"][0]["id"]
        else:
            tid = tid_or_md5

        table = {"data": []}
        page, pages = 1, None

        while pages is None or page <= pages:
            resp = self.tables.get_entry(
                pk=tid, _fields=["_all"], data_page=page, data_per_page=1000
            ).result()
            table["data"].extend(resp["data"])
            if pages is None:
                pages = resp["total_data_pages"]
                table["attrs"] = resp["attrs"]
                table["index"] = resp["index"]
                table["columns"] = resp["columns"]
            page += 1

        df = pd.DataFrame.from_records(
            table["data"], columns=table["columns"], index=table["index"]
        )
        df.attrs = table["attrs"]
        labels = table["attrs"].get("labels", {})

        if "index" in labels:
            df.index.name = labels["index"]
        if "variable" in labels:
            df.columns.name = labels["variable"]

        return df

    def get_structure(self, sid):
        """Convenience function to get pymatgen structure."""
        return Structure.from_dict(
            self.structures.get_entry(
                pk=sid, _fields=["lattice", "sites", "charge"]
            ).result()
        )

    def init_columns(self, project, columns):
        """initialize columns to set their order and desired units"""
        self.projects.update_entry(pk=project, project={"columns": []}).result()
        cols = []

        for path, unit in columns.items():
            col = {"path": f"data.{path}"}
            if unit is not None:
                col["unit"] = unit

            cols.append(col)

        return self.projects.update_entry(
            pk=project, project={"columns": cols}
        ).result()

    def delete_contributions(self, project, per_page=100, max_workers=5):
        """Convenience function to remove all contributions for a project"""
        tic = time.perf_counter()

        if max_workers > MAX_WORKERS:
            max_workers = MAX_WORKERS
            print(f"max_workers reset to max {MAX_WORKERS}")

        cids = self.get_contributions(project)["ids"]
        total = len(cids)

        if cids:
            with FuturesSession(max_workers=max_workers) as session:
                while cids:
                    futures = [
                        session.delete(
                            f"{self.url}/contributions/",
                            headers=self.headers,
                            params={
                                "project": project,
                                "id__in": ",".join(chunk),
                                "per_page": per_page,
                            },
                        )
                        for chunk in chunks(cids, n=per_page)
                    ]

                    self._run_futures(futures, total=len(cids))
                    cids = self.get_contributions(project)["ids"]

                self.load()

        # reset columns to be save (sometimes not all are reset BUGFIX?)
        self.projects.update_entry(pk=project, project={"columns": []}).result()
        toc = time.perf_counter()
        dt = (toc - tic) / 60
        print(f"It took {dt:.1f}min to delete {total} contributions.")

    # TODO ratelimit support in bravado (wrapped around get_entry..., monkey-patch?)
    @sleep_and_retry
    @limits(calls=175, period=60)
    def get_unique_identifiers_flag(self, name):
        return self.projects.get_entry(
            pk=name, _fields=["unique_identifiers"]
        ).result()["unique_identifiers"]

    @sleep_and_retry
    @limits(calls=175, period=60)
    def get_total_pages(self, name, per_page):
        return self.contributions.get_entries(
            project=name, per_page=per_page, _fields=["id"],
        ).result()["total_pages"]

    def get_contributions(self, name):
        """get list of existing contributions"""
        ret = defaultdict(set)
        ret["unique_identifiers"] = self.get_unique_identifiers_flag(name)
        pages = self.get_total_pages(name, 250)

        @sleep_and_retry
        @limits(calls=175, period=60)
        def get_future(page):
            future = session.get(
                f"{self.url}/contributions/",
                headers=self.headers,
                params={
                    "project": name,
                    "page": page,
                    "per_page": 250,
                    "_fields": "id,identifier,structures,tables",
                },
            )
            setattr(future, "track_id", page)
            return future

        with FuturesSession(max_workers=MAX_WORKERS) as session:
            # bravado future doesn't work with concurrent.futures
            futures = [get_future(page + 1) for page in range(pages)]

            while futures:
                responses = self._run_futures(futures)

                for resp in responses.values():
                    for contrib in resp["data"]:
                        ret["ids"].add(contrib["id"])
                        ret["identifiers"].add(contrib["identifier"])

                        for component in ["structures", "tables"]:
                            md5s = set(d["md5"] for d in contrib[component])
                            ret[component] |= md5s

                futures = [
                    future
                    for future in futures
                    if future.track_id not in responses.keys()
                ]

        return ret

    def submit_contributions(
        self,
        contributions,
        skip_dupe_check=False,
        ignore_dupes=False,
        retry=False,
        per_page=100,
        max_workers=3,
    ):
        """Convenience function to submit a list of contributions

        Args:
            contributions (list): list of contribution dicts to submit
            skip_dupe_check (bool): skip check for duplicates of identifiers and components
            ignore_dupes (bool): force duplicate components to be submitted
            retry (bool): keep trying until all contributions successfully submitted
            per_page (int): number of contributions to submit in each chunk/request
            max_workers (int): number of parallel requests to use to submit chunk
        """
        tic = time.perf_counter()

        if max_workers > MAX_WORKERS:
            max_workers = MAX_WORKERS
            print(f"max_workers reset to max {MAX_WORKERS}")

        # get existing contributions
        print("get existing contributions ...")
        existing = defaultdict(set)
        existing["unique_identifiers"] = True
        project_name = contributions[0]["project"]

        if not skip_dupe_check:
            existing = self.get_contributions(project_name)

        # prepare contributions
        print("prepare contributions ...")
        contribs = []
        digests = defaultdict(set)
        components = {"structures", "tables"}
        fields = [
            comp
            for comp in self.swagger_spec.definitions.get(
                "ContributionsSchema"
            )._properties.keys()
            if comp not in components
        ]

        # TODO parallelize?
        for contrib in tqdm(contributions, leave=False):
            if (
                existing["unique_identifiers"]
                and contrib["identifier"] in existing["identifiers"]
            ):
                continue

            contribs.append({k: deepcopy(contrib[k]) for k in fields if k in contrib})

            for component in components:
                contribs[-1][component] = []
                elements = contrib.get(component, [])
                nelems = len(elements)

                for idx, element in enumerate(elements):
                    is_structure = isinstance(element, Structure)
                    if component == "structures" and not is_structure:
                        raise ValueError("Only accepting pymatgen Structure!")
                    elif component == "tables" and not isinstance(
                        element, pd.DataFrame
                    ):
                        raise ValueError("Only accepting pandas DataFrame!")

                    if is_structure:
                        dct = element.as_dict()
                        del dct["@module"]
                        del dct["@class"]

                        if not dct.get("charge"):
                            del dct["charge"]
                    else:
                        element.index = element.index.astype(str)
                        for col in element.columns:
                            element[col] = element[col].astype(str)
                        dct = element.to_dict(orient="split")

                    digest = get_md5(dct)

                    if is_structure:
                        dct["name"] = getattr(element, "name", None)
                        if not dct["name"]:
                            c = element.composition
                            comp = c.get_integer_formula_and_factor()
                            dct["name"] = f"{comp[0]}-{idx}" if nelems > 1 else comp[0]
                    else:
                        name = f"table-{idx}" if nelems > 1 else "table"
                        dct["name"] = element.attrs.get("name", name)
                        title = element.attrs.get("title", dct["name"])
                        labels = element.attrs.get("labels", {})
                        index = element.index.name
                        variable = element.columns.name

                        if index and "index" not in labels:
                            labels["index"] = index
                        if variable and "variable" not in labels:
                            labels["variable"] = variable

                        dct["attrs"] = {"title": title, "labels": labels}

                    dupe = digest in digests[component] or digest in existing[component]

                    if not ignore_dupes and dupe:
                        msg = f"Duplicate: {dct['name']}!"
                        raise ValueError(msg)

                    if not dupe:
                        digests[component].add(digest)
                        contribs[-1][component].append(dct)

        # submit contributions
        if contribs:
            print("submit contributions ...")
            with FuturesSession(max_workers=max_workers) as session:
                # bravado future doesn't work with concurrent.futures
                ncontribs = len(contribs)
                headers = {"Content-Type": "application/json"}
                headers.update(self.headers)

                @sleep_and_retry
                @limits(calls=175, period=60)
                def post_future(chunk):
                    return session.post(
                        f"{self.url}/contributions/",
                        headers=headers,
                        data=json.dumps(chunk).encode("utf-8"),
                    )

                while contribs:
                    futures = [
                        post_future(chunk) for chunk in chunks(contribs, n=per_page)
                    ]

                    self._run_futures(futures, total=len(contribs))

                    if existing["unique_identifiers"] and retry:
                        existing = self.get_contributions(project_name)
                        contribs = [
                            c
                            for c in contribs
                            if c["identifier"] not in existing["identifiers"]
                        ]
                    else:
                        contribs = []  # abort retrying
                        if not existing["unique_identifiers"] and retry:
                            print("Please resubmit failed contributions manually.")

                self.load()
                toc = time.perf_counter()
                dt = (toc - tic) / 60
                print(f"It took {dt:.1f}min to submit {ncontribs} contributions.")
        else:
            print("Nothing to submit.")

    def _run_futures(self, futures, total=None):
        """helper to run futures/requests"""
        responses = {}

        with tqdm(leave=False, total=total if total else len(futures)) as pbar:
            for future in as_completed(futures):
                response = future.result()
                status = response.status_code

                if status in {200, 201, 400, 401, 404, 500, 502}:
                    resp = response.json()
                    if status in {200, 201}:
                        if total:
                            cnt = len(resp["data"]) if "data" in resp else resp["count"]
                        else:
                            cnt = 1
                        pbar.update(cnt)
                        if hasattr(future, "track_id"):
                            responses[future.track_id] = resp
                        if "warning" in resp:
                            print(resp["warning"])
                    elif status != 502:
                        warnings.warn(resp["error"])
                elif status not in {503, 504}:
                    warnings.warn(response.content.decode("utf-8"))

        return responses


import pytest

import pandas as pd
from sklearn.datasets import load_digits
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss

from gama import GamaClassifier

@pytest.fixture
def gamaclassifier():
    return GamaClassifier(random_state=0, max_total_time=60)

def _gama_on_digits(gama):
    X, y = load_digits(return_X_y=True)
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, stratify=y, random_state=0
    )

    # Add checks on individuals (reproducibility)
    gama.fit(X_train, y_train)

    # Add checks
    _ = gama.predict(X_test)
    y_proba = gama.predict_proba(X_test)

    assert log_loss(y_test, y_proba) == gama.score(X_test, y_test)
    assert log_loss(y_test, y_proba) == gama.score(X_test, pd.Series(y_test))
    assert log_loss(y_test, y_proba) == gama.score(
        X_test, LabelEncoder().fit_transform(y_test.reshape(-1, 1))
    )

def test_full_system_single_core(gamaclassifier):
    from gama.utilities.generic.async_evaluator import AsyncEvaluator

    AsyncEvaluator.n_jobs = 1
    _gama_on_digits(gamaclassifier)

def test_full_system_multi_core(gamaclassifier):
    from gama.utilities.generic.async_evaluator import AsyncEvaluator

    AsyncEvaluator.n_jobs = 2
    _gama_on_digits(gamaclassifier)


# -*- coding: utf-8 -*-
# pp/utils/test_tasksort.py

from datetime import datetime
from dateutil.parser import parse as dt
from pprint import pprint

import pytest
import mock

from pp.utils.optionlines import OptionLines

@pytest.mark.xfail
def test_sort_tasks_by_urgency123():
    source_text = """\
* [ ] Sometime 2
**[ ] Today 1
* [ ] Sometime 1
  [ ] This week 2
**[ ] Today 2
  [ ] This week 1
* [ ] Sometime 3
"""
    option_lines = OptionLines(source_text)
    assert len(option_lines) == 7
    assert 0 == 56