#!/usr/bin/python
from __future__ import print_function
from __future__ import division
import os
import sys
from numpy import *
import numpy
import struct
import matplotlib
import matplotlib.pyplot as plt
import functions as fn
import multiprocessing as MP
import copy

for arg in sys.argv:
    if arg.startswith('params='):
        arg=arg.lstrip('params=')
        par = __import__(arg)
        del sys.argv[sys.argv.index(arg)]
else:
    import params as par

########## Temp Variables ##########

directory_in = par.directory_base+par.directory_in
directory_out= par.directory_base+par.directory_out

main_block = 'mass'
snapshot_number = -1

if len(os.sys.argv)>1:                          # the number identifing the single snapshot, -1 for all
    snapshot_number = int(os.sys.argv[1])
    if len(os.sys.argv)>2:
        main_block = os.sys.argv[2]

BOX=MP.Value('d',0.0)
Z=MP.Value('d',0.0)

search_region=[0.5,0.5,0.5,1.0,1.0,1.0]

R_MIN=0.0005
R_MAX=0.2
BINS =256

SO_VEL_DISPERSION = True

select_type = 1

write_files = True

group_type=[('num',int),('length',int),('sub_length',int),('mass',float),('sub_mass',float),
            ('x',float),('y',float),('z',float),('R200',float),('M200',float)]

########## Modified output class that flush when killed (for multiprocessing purposes) ############

class buffered_stdout():
    def __init__(self):
        self.buf=[]
        self.old=sys.stdout
    def write(self,text):
        self.buf.append(text)
    def flush(self):
        if self.buf!=[]:
            sys.__stdout__.write(''.join(self.buf))
    def __del__(self):
        del self.buf

########## start of single snapshot sector #########

def single_snapshot_reader(in_dir,snap,temp_exists,lock):

    f=open(in_dir+snap,'rb')

    if sys.flags.debug==True:
        print ('\n',"Reading ",snap)

    header=fn.header_reader(f)

    snap_num = int(snap[snap.find('.')+1:])

    Npart=header.Npart
    Nall=header.Nall
    Massarr=header.Massarr
    Boxsize=header.Boxsize
    Redshift=header.Redshift

    BOX.value=Boxsize
    Z.value=Redshift

    if header.Time!=1./(1.+Redshift):
        Z.value=round(1./header.Time-1.,2)

    if sys.flags.debug==True:
        print ("at redshift z = ",Z.value)

    B={}

    Nmass  = array([ Npart[i] * (Massarr[i]==0) * (Nall[i]!=0) for i in range(6) ])

    flag, mask_n = fn.block_reader(f,header.Npart,B,'pos','pos',lambda x : fn.slice_mask(x,header.Boxsize,search_region) )

    flag = fn.block_reader(f,Npart,B,'vel',main_block)
    fn.blocks_present['vel'] = True

    flag = fn.block_reader(f,Npart,B,'ID',main_block)
    fn.blocks_present['ID'] = True

    if Nmass.sum()!=0:
        flag = fn.block_reader(f,Nmass,B,'mass','mass')
        for q in range(6):
            if Npart[q]!=0 and Nmass[q]==0:
                B['mass'][q] = ones(Npart[q]) * Massarr[q]
    else:
        B['mass'] = [ ones(Npart[i]) * Massarr[i] * (Npart[i]!=0) * (Nall[i]!=0) if Npart[i]!=0 else None for i in range(6) ]
        if sys.flags.debug==True:
            print("no mass block found ...")

    fn.blocks_present['mass'] = True

    if main_block not in ['pos','mass','vel','ID']:

        flag = 0

        if par.guess_blocks == True:

            while flag != 1:
                flag = fn.block_reader(f,Npart,B,'guess',main_block)

        else:

            for b in fn.blocks_ordered[4:]:

                num = Npart * [ int(i) for i in reversed(bin(fn.blocks_part[b])[2:].zfill(6)) ]

                if num.sum() == 0 or fn.blocks_present[b] == False:
                    continue

                flag = fn.block_reader(f,num,B,b,main_block)

                if flag == -1:
                    break

    f.close()

    if par.guess_blocks==True:

        for i in fn.blocks_present:
            if fn.blocks_present[i] == None:
                fn.blocks_present[i] = False

        if fn.blocks_present[main_block] == False:
            print(fn.blocks_present)
            print("\nMAIN BLOCK NOT FOUND! \n")
            MP.current_process().terminate()

    for q in range(6):

        if main_block=='acc':

            if Npart[q]==0 or B['acc']==None or B['dqp']==None:
                continue

            rad_n=copy.deepcopy(B['pos'][q])
            rad_n[:,0]=rad_n[:,0]-par.x_center*Boxsize
            rad_n[:,1]=rad_n[:,1]-par.y_center*Boxsize
            rad_n[:,2]=rad_n[:,2]-par.z_center*Boxsize

            B['acc'][q]=subtract(B['acc'][q],B['dqp'][q])

            #B['acc'][q]=mod(B['acc'][q])
            #B['dqp'][q]=mod(B['dqp'][q])

            # 1d
            #B['acc'][q]=B['acc'][q][:,0]

            # 3d
            B['acc'][q]=(B['acc'][q]*rad_n).sum(axis=1) / linalg.norm(rad_n,axis=1)

        if main_block=='drho':

            if Npart[q]==0 or B['drho']==None:
                continue

            rad_n=copy.deepcopy(B['pos'][q])
            rad_n[:,0]=rad_n[:,0]-par.x_center*Boxsize
            rad_n[:,1]=rad_n[:,1]-par.y_center*Boxsize
            rad_n[:,2]=rad_n[:,2]-par.z_center*Boxsize

            B['drho'][q]=(B['drho'][q]*rad_n).sum(axis=1) / linalg.norm(rad_n,axis=1)        # drho * dr

        if main_block=='dqp':

            if Npart[q]==0 or B['dqp']==None:
                continue

            # 1d
            #B['dqp'][q]=B['dqp'][q][:,0]
            #continue

            B['dqp'][q] = linalg.norm(B['dqp'][q],axis=1)  # |dqp|^2
            continue

            rad_n=copy.deepcopy(B['pos'][q])
            rad_n[:,0]=rad_n[:,0]-par.x_center*Boxsize
            rad_n[:,1]=rad_n[:,1]-par.y_center*Boxsize
            rad_n[:,2]=rad_n[:,2]-par.z_center*Boxsize

            #B['dqp'][q]=(B['dqp'][q]*rad_n).sum(axis=1) / linalg.norm(rad_n,axis=1)        # dqp * dr
            #B['dqp'][q] = B['dqp'][q]*B['dqp'][q]  # |dqp*dr|^2
            #B['dqp'][q] = (B['dqp'][q]*B['dqp'][q]).sum(axis=1)  # |dqp|^2
            #B['dqp'][q] = 0.5 * B['dqp'][q] + 2.0 * B['qp'][q]   # dqp = 2 * qp - |dqp|^2 / 2
            #B['dqp'][q] = 2.5 * B['dqp'][q] + 2.0 * B['qp'][q]   # dqp = 2 * qp - |dqp|^2 / 2

    weight_n=B[main_block]

    temp_arr=[[] for i in range(6)]

    particle_selected=zeros(6).astype(int)

    for q in range(6):

        if Npart[q] * int(list(reversed(bin(fn.blocks_part[main_block])[2:].zfill(6)))[q]) * int(fn.blocks_present[main_block]) == 0:
            continue

        if len(mask_n[q])==0:
            B['pos'][q]=array([],dtype=B['pos'][q].dtype)
            weight_n[q]=array([],dtype=weight_n[q].dtype)

        particle_selected[q] = len(mask_n[q])

        temp_arr[q]=array(zeros(particle_selected[q]),dtype=fn.part_dtype).view(recarray)

        #if len(mask_n[q])==0:
        #    continue

        temp_arr[q].x=B['pos'][q][:,0]
        temp_arr[q].y=B['pos'][q][:,1]
        temp_arr[q].z=B['pos'][q][:,2]

        temp_arr[q].m=weight_n[q]

    perc = [100*p/n if n!=0 else 0 for (p,n) in zip(particle_selected,Npart)]
    perc = [''+str(round(i,2))+'%' for i in perc]

    if particle_selected.sum()!=0:
        print('\nParticles belonging to the slice in',snap,' ( z = ',Redshift,')')
        print("type:    "+''.join(['%16s']*(particle_selected!=0).sum()) %tuple(compress(particle_selected!=0,arange(6))))
        print("---------"+''.join(['----------------']*(particle_selected!=0).sum()))
        print("select:  "+''.join(['%16s']*(particle_selected!=0).sum()) %tuple(compress(particle_selected!=0,Npart)))
        print("total:   "+''.join(['%16s']*(particle_selected!=0).sum()) %tuple(compress(particle_selected!=0,particle_selected)))

    sys.stdout.flush()

    return temp_arr

########## group reader ##########

def single_group_reader(in_dir,group):

    f=open(in_dir+group,'rb')

    if sys.flags.debug==True:
        print ('\n',"Reading ",group)

    Ngroups, TOTNgroups, NIDs = struct.unpack('3i',f.read(3*4))
    TOTNIDs = struct.unpack('l',f.read(8))
    NTasks  = struct.unpack('i',f.read(4))

    GLength = array(struct.unpack(str(Ngroups)+'i',f.read(Ngroups*4)))
    GOffset = array(struct.unpack(str(Ngroups)+'i',f.read(Ngroups*4)))
    GMass   = array(struct.unpack(str(Ngroups)+'f',f.read(Ngroups*4)))
    GCenter = array(struct.unpack(str(3*Ngroups)+'f',f.read(3*Ngroups*4))).reshape((Ngroups,3))

    if sys.flags.debug==True:
        print ("Found ",Ngroups,"in",group)

    return GCenter

########## subhalo reader ##########

def single_subgroup_reader(in_dir,subgroup):

    f=open(in_dir+subgroup,'rb')

    if sys.flags.debug==True:
        print ('\n',"Reading ",subgroup)

    Ngroups, TOTNgroups, NIDs = struct.unpack('3i',f.read(3*4))
    TOTNIDs = struct.unpack('l',f.read(8))
    NTasks  = struct.unpack('i',f.read(4))
    Nsubgroups, TOTNsubgroups = struct.unpack('2i',f.read(2*4))

    # group properties #

    if Ngroups > 0:

        GLength = array(struct.unpack(str(Ngroups)+'i',f.read(Ngroups*4)))
        GOffset = array(struct.unpack(str(Ngroups)+'i',f.read(Ngroups*4)))
        GMass   = array(struct.unpack(str(Ngroups)+'f',f.read(Ngroups*4)))
        GCenter = array(struct.unpack(str(3*Ngroups)+'f',f.read(3*Ngroups*4))).reshape((Ngroups,3))

        M200_mean = array(struct.unpack(str(Ngroups)+'f',f.read(Ngroups*4)))
        R200_mean = array(struct.unpack(str(Ngroups)+'f',f.read(Ngroups*4)))

        M200_crit = array(struct.unpack(str(Ngroups)+'f',f.read(Ngroups*4)))
        R200_crit = array(struct.unpack(str(Ngroups)+'f',f.read(Ngroups*4)))

        M200_that = array(struct.unpack(str(Ngroups)+'f',f.read(Ngroups*4)))
        R200_that = array(struct.unpack(str(Ngroups)+'f',f.read(Ngroups*4)))

        if SO_VEL_DISPERSION == True:
            S200_mean = array(struct.unpack(str(Ngroups)+'f',f.read(Ngroups*4)))
            S200_crit = array(struct.unpack(str(Ngroups)+'f',f.read(Ngroups*4)))
            S200_that = array(struct.unpack(str(Ngroups)+'f',f.read(Ngroups*4)))

        Contamination_N = array(struct.unpack(str(Ngroups)+'i',f.read(Ngroups*4)))
        Contamination_M = array(struct.unpack(str(Ngroups)+'f',f.read(Ngroups*4)))

        Nsubstruct      = array(struct.unpack(str(Ngroups)+'i',f.read(Ngroups*4)))
        First_substruct = array(struct.unpack(str(Ngroups)+'i',f.read(Ngroups*4)))

    # group_type=[('num',int),('length',int),('sub_length',int),('mass',float),('sub_mass',float),
    #             ('x',float),('y',float),('z',float),('R200',float),('M200',float)]

    temp_arr = zeros(Ngroups,dtype=group_type).view(recarray)

    temp_arr.num = First_substruct
    temp_arr.length = GLength
    temp_arr.mass = GMass

    temp_arr.x = GCenter[:,0]
    temp_arr.y = GCenter[:,1]
    temp_arr.z = GCenter[:,2]

    temp_arr.M200 = M200_crit
    temp_arr.R200 = R200_crit

    # subgroup properties #

    if Nsubgroups > 0:

        SGLength = array(struct.unpack(str(Nsubgroups)+'i',f.read(Nsubgroups*4)))
        SGOffset = array(struct.unpack(str(Nsubgroups)+'i',f.read(Nsubgroups*4)))
        SGParent = array(struct.unpack(str(Nsubgroups)+'i',f.read(Nsubgroups*4)))
        SGMass   = array(struct.unpack(str(Nsubgroups)+'f',f.read(Nsubgroups*4)))
        SGCenter = array(struct.unpack(str(3*Nsubgroups)+'f',f.read(3*Nsubgroups*4))).reshape((Nsubgroups,3))

    if sys.flags.debug==True:
        print ("Found ",Ngroups,"groups with",Nsubgroups," subgroups in",subgroup)

    subtemp_arr = zeros(Nsubgroups,dtype=[('sub_length',int),('sub_mass',float)
                                          ,('x',float),('y',float),('z',float)]).view(recarray)

    subtemp_arr.sub_length = SGLength
    subtemp_arr.sub_mass = SGMass
    subtemp_arr.x = SGCenter[:,0]
    subtemp_arr.y = SGCenter[:,1]
    subtemp_arr.z = SGCenter[:,2]

    """
    for j in range(Ngroups):
        print(GLength[j],First_substruct[j],Nsubstruct[j])
        for i in range(First_substruct[j],First_substruct[j]+Nsubstruct[j]):
            print('\t',SGLength[i],SGMass[i],SGCenter[i])
    """

    #print(temp_arr[1],subtemp_arr[1])

    return (temp_arr, subtemp_arr)

########## start of main ##########

fn.init_parser()

file_list=fn.snapshot_hunter(snapshot_number)
file_list=file_list[argsort(file_list[:,1])[::-1]]

for files,in_dir,out_dir,temp_exists in file_list:

    lock=MP.Lock()

    manager=MP.Manager()

    P_N=[[] for i in range(6)]

    def single_snap(snap):
        #print(MP.current_process().pid,"deals with",snap)
        return single_snapshot_reader(in_dir,snap,temp_exists,lock)

    if __name__=='__main__':

        proc = len(files)
        if par.MaxProcesses!=0:
            proc = par.MaxProcesses

        proc = min( proc, MP.cpu_count() )

        print("\n Taking care of",in_dir,", let's split up in",proc,"processes!")

        #sys.stdout = buffered_stdout()

        pool = MP.Pool(processes=proc)

        res = pool.map(single_snap,files)

        pool.close()
        pool.join()

        sys.stdout = sys.__stdout__

        print ("\n Enough with reading!")

        res=array(res)

        for q in range(6):

            P_N[q] = concatenate(res[:,q]).view(recarray)

        del res

    ########## start of snapshot directory sector #########

    mask = array([len(p) for p in P_N])
    P_N  = [p if len(p)>1 else None for p in P_N]

    if Z.value<0:
        Z.value=0.0

    print("\n The total number of particles in",directory_in,"is:")

    print("type:    "+''.join(['%16s']*(mask!=0).sum()) %tuple(compress(mask!=0,arange(6))))
    print("---------"+''.join(['----------------']*(mask!=0).sum()))
    print("         "+''.join(['%16s']*(mask!=0).sum()) %tuple(compress(mask!=0,mask)))

########## Look Groups ##########

group_list = fn.snapshot_hunter(snapshot_number=snapshot_number,snapshot_basename='subhalo_tab')

for files,in_dir,out_dir,temp_exists in group_list:

    def single_group(group):
        #print(MP.current_process().pid,"deals with",group)
        return single_group_reader(in_dir,group)

    def single_subgroup(group):
        #print(MP.current_process().pid,"deals with",group)
        return single_subgroup_reader(in_dir,group)

    if __name__=='__main__':

        proc = len(files)
        if par.MaxProcesses!=0:
            proc = par.MaxProcesses

        proc = min( proc, MP.cpu_count() )

        print("\n Taking care of",in_dir,", let's split up in",proc,"processes!")

        #sys.stdout = buffered_stdout()

        pool = MP.Pool(processes=proc)

        data = pool.map(single_subgroup,files)

        pool.close()
        pool.join()

        group_data = [0]*len(data)
        subgroup_data = [0]*len(data)

        for i in range(len(data)):

            group_data[i], subgroup_data[i] = data[i]

        sys.stdout = sys.__stdout__

        group_data = array(group_data)
        group_data = concatenate(group_data[:]).view(recarray)

        subgroup_data = array(subgroup_data)
        subgroup_data = concatenate(subgroup_data[:]).view(recarray)

        for g in range(len(data)):

            #print(g,group_data[g].num)

            group_data[g].sub_length = subgroup_data[group_data[g].num].sub_length
            group_data[g].sub_mass = subgroup_data[group_data[g].num].sub_mass
            group_data[g].x = subgroup_data[group_data[g].num].x
            group_data[g].y = subgroup_data[group_data[g].num].y
            group_data[g].z = subgroup_data[group_data[g].num].z

            group_data[g].num = g

        print ("\nFound ",len(group_data),"groups!")

        del subgroup_data
        del data

    # plot #

    distance = logspace(log10(R_MIN),log10(R_MAX),BINS+1)

    path = par.directory_base + par.directory_in + 'profiles_mass/'

    os.system('mkdir -p '+path)

    # group File "%s/profiles/groups_properties_bin_%01d_%03d.txt"
    # [group_num, GroupLength, SubGroupLength, GroupMass, SubGroupMass, x, y, z, R_200, M_200]

    if write_files == True:
        print("Writing",path+"groups_properties_bin_{}.txt".format(in_dir[-4:-1]))
        savetxt(path+"groups_properties_bin_{}.txt".format(in_dir[-4:-1]),group_data,fmt='%4.6g')

    for q in range(6):

        if P_N[q] == None:
            continue

        if select_type != None and q != select_type:
            continue

        search_region[3:] = [2.0 * R_MAX / BOX.value] * 3

        count = 0

        for gc in group_data[['x','y','z']]:

            gc = array(list(gc))

            #if count == 1:
            #    gc[1] = 49.620
            #    gc[0] = 49.225
            #    gc[2] = 49.634
            #    print('magic!')

            if ( gc == [0.0,0.0,0.0] ).prod() == 1:
                print("zero center, I think we're done!")
                break

            search_region[0:3] = gc / BOX.value

            print("\nSearch particles of group number",count,"(",len(group_data),")"," at center",gc,"within",R_MIN,"and",R_MAX)

            r, mask = fn.slice_mask(P_N[q][['x','y','z']],BOX.value,search_region=search_region)

            print("\t...found",len(mask),"particles in it!")

            if len(mask) == 0:
                print("no particle, skip!")
                continue

            r = r[mask]

            r = sqrt( (r.x - gc[0])**2 + (r.y - gc[1])**2 + (r.z - gc[2])**2 )

            m = P_N[q].m[mask]

            rho, b = histogram(r,weights=m,bins=distance)

            #n, b = histogram(r,bins=distance)
            #rho /= n.clip(min=1)

            if main_block == 'mass':
                vol = 4.0 / 3.0 * pi * ( b[1:]**3 - b[:-1]**3 )
                rho = rho / vol
            elif main_block == 'rho':
                n = histogram(r,bins=distance)[0]
                rho /= n.clip(min=1)

            b = sqrt( b[1:] * b[:-1] )
            mass = rho.cumsum()

            # single group file "%s/profiles/snap_%03d_group_%03d_density_type_1.txt"
            # [rbin_mean,rho,rbin_right,mass_cum,rbin_min]

            if write_files==True:
                print("Writing",path+'snap_{}_group_{}_density_type_{}.txt'.format(in_dir[-4:-1],str(count).zfill(3),q))
                savetxt(path+'snap_{}_group_{}_density_type_{}.txt'.format(in_dir[-4:-1],str(count).zfill(3),q),
                        array([ b, rho, distance[1:], mass, [R_MIN] * BINS ]).T )

            """

            plt.plot(b,rho,c='b')

            plt.xlim(R_MIN,R_MAX)
            plt.xscale('log')
            plt.yscale('log')

            """

            count = count + 1

            #if count % 10 == 0:
            #plt.show()


"""
https://leetcode.com/problems/construct-the-rectangle/#/description

For a web developer, it is very important to know how to design a web page's size. So, given a specific rectangular web page’s area, your job by now is to design a rectangular web page, whose length L and width W satisfy the following requirements:

1. The area of the rectangular web page you designed must equal to the given target area.

2. The width W should not be larger than the length L, which means L >= W.

3. The difference between length L and width W should be as small as possible.
You need to output the length L and the width W of the web page you designed in sequence.
Example:
Input: 4
Output: [2, 2]
Explanation: The target area is 4, and all the possible ways to construct it are [1,4], [2,2], [4,1].
But according to requirement 2, [1,4] is illegal; according to requirement 3,  [4,1] is not optimal compared to [2,2].
So the length L is 2, and the width W is 2.
Note:
The given area won't exceed 10,000,000 and is a positive integer
The web page's width and length you designed must be positive integers.
"""

class Solution(object):
    def constructRectangle(self, area):
        """
        :type area: int
        :rtype: List[int]
        """
        from math import sqrt, floor
        root = int(floor(sqrt(area)))
        factors = []
        for x in xrange(1, root):
            if area % x == 0:
                factors.append(x)
        if not factors:
            return [area, 1]
        length = width = root
        while length <= area and width >= 1:
            if length * width == area:
                return [length, width]
            elif length * width < area:
                length += 1
            else:
                width -= 1

    def optimalConstructRectangle(self, area):
        import math
        mid = int(math.sqrt(area))
        while mid > 0:
            if area % mid == 0:
                return [int(area / mid), int(mid)]
            mid -= 1


"""Cart-related context processors."""
from .utils import get_cart_from_request

cart_skip_views = ('search:search',)

def cart_counter(request):
    """Expose the number of items in cart."""
    if request.resolver_match is None or request.resolver_match.view_name not in cart_skip_views:
        cart = get_cart_from_request(request)
        num_items = cart.quantity
    else:
        num_items = 0
    return {'cart_counter': num_items}


# -*- coding: utf-8 -*-
"""
将散点图与条形图结合
"""
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.font_manager as fm
myfont = fm.FontProperties(fname=u'C:\Windows\Fonts\simsun.ttc',size=16)
plt.style.use('ggplot')
np.random.seed(0)

x = np.random.randn(200)
y = x + np.random.randn(200) * 0.5

margin_border = 0.1
width  = 0.6
margin_between = 0.02 # 要小于0.1
height = 0.2

left_s = margin_border
bottom_s= margin_border
height_s = width
width_s = width

left_x = margin_border
bottom_x = margin_border + width + margin_between
height_x = height
width_x = width

left_y = margin_border + width + margin_between
bottom_y = margin_border
height_y = width
width_y = height

fig = plt.figure(1,figsize=(8,8))

rect_s = [left_s,bottom_s,width_s,height_s]
rect_x = [left_x,bottom_x,width_x,height_x]
rect_y = [left_y,bottom_y,width_y,height_y]

axScatter = fig.add_axes(rect_s)
axHisX = fig.add_axes(rect_x)
axHisX.set_xticks([])
axHisY = fig.add_axes(rect_y)
axHisY.set_yticks([])

axScatter.scatter(x,y)

bin_width = 0.15
xymax = np.max( [ np.max(np.fabs(x)),
                 np.max(np.fabs(y)) ] ) * bin_width
lim = int(xymax / bin_width +1)
axScatter.set_xlim(-lim,lim)
axScatter.set_ylim(-lim,lim)

axHisX.hist(x,bins=np.arange(-lim,lim+bin_width,bin_width))
axHisY.hist(y,bins=np.arange(-lim,lim+bin_width,bin_width) ,orientation='horizontal')

axHisX.set_xlim(axScatter.get_xlim())
axHisY.set_ylim(axScatter.get_ylim())

fig.text(0.5,0.95,'散点图与条形图结合',font_properties= myfont)
plt.show()

散点条形图_对象形式_中文标题.py


# -*- coding: utf-8 -*-
# @Time    : 2020/9/9 19:09
# @Author  : xiaolu
# @FileName: get_train_data.py
# @Software: PyCharm
from transformers import BertTokenizer
import random
import numpy as np
import collections
import json
from tqdm import tqdm

MaskedLmInstance = collections.namedtuple("MaskedLmInstance", ["index", "label"])

def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens):
    """Truncates a pair of sequences to a maximum sequence length."""
    while True:
        total_length = len(tokens_a) + len(tokens_b)
        if total_length <= max_num_tokens:
            break
        trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b
        assert len(trunc_tokens) >= 1
        # We want to sometimes truncate from the front and sometimes from the
        # back to add more randomness and avoid biases.
        if random.random() < 0.5:
            del trunc_tokens[0]
        else:
            trunc_tokens.pop()

def create_masked_lm_predictions(tokens, max_ngram, masked_lm_prob, max_predictions_per_seq, vocab_list):
    """Creates the predictions for the masked LM objective. This is mostly copied from the Google BERT repo, but
    with several refactors to clean it up and remove a lot of unnecessary variables."""

    # n-gram masking Albert
    ngrams = np.arange(1, max_ngram + 1, dtype=np.int64)
    pvals = 1. / np.arange(1, max_ngram + 1)
    pvals /= pvals.sum(keepdims=True)  # p(n) = 1/n / sigma(1/k)
    cand_indices = []
    for (i, token) in enumerate(tokens):
        if token == "[CLS]" or token == "[SEP]":
            continue
        # Whole Word Masking means that if we mask all of the wordpieces
        # corresponding to an original word. When a word has been split into
        # WordPieces, the first token does not have any marker and any subsequence
        # tokens are prefixed with ##. So whenever we see the ## token, we
        # append it to the previous set of word indexes.
        #
        # Note that Whole Word Masking does *not* change the training code
        # at all -- we still predict each WordPiece independently, softmaxed
        # over the entire vocabulary.
        cand_indices.append(i)
    num_to_mask = min(max_predictions_per_seq, max(1, int(round(len(tokens) * masked_lm_prob))))
    random.shuffle(cand_indices)
    masked_token_labels = []
    covered_indices = set()
    for index in cand_indices:
        n = np.random.choice(ngrams, p=pvals)
        if len(masked_token_labels) >= num_to_mask:
            break
        if index in covered_indices:
            continue
        if index < len(cand_indices) - (n - 1):
            for i in range(n):
                ind = index + i
                if ind in covered_indices:
                    continue
                covered_indices.add(ind)
                # 80% of the time, replace with [MASK]
                if random.random() < 0.8:
                    masked_token = "[MASK]"
                else:
                    # 10% of the time, keep original
                    if random.random() < 0.5:
                        masked_token = tokens[ind]
                    # 10% of the time, replace with random word
                    else:
                        masked_token = random.choice(vocab_list)
                masked_token_labels.append(MaskedLmInstance(index=ind, label=tokens[ind]))
                tokens[ind] = masked_token
    # assert len(masked_token_labels) <= num_to_mask
    masked_token_labels = sorted(masked_token_labels, key=lambda x: x.index)
    mask_indices = [p.index for p in masked_token_labels]
    masked_labels = [p.label for p in masked_token_labels]
    return tokens, mask_indices, masked_labels

def create_instances_from_document(all_documents, document_index, max_seq_length, short_seq_prob,
                                   max_ngram, masked_lm_prob, max_predictions_per_seq, vocab_words):
    # 将单文档处理成训练数据
    document = all_documents[document_index]  # 得到一个文档

    # Account for [CLS], [SEP], [SEP]
    max_num_tokens = max_seq_length - 3   # max_seq_length 最大序列长度 最长设置到512, 可根据训练数据集进行调整

    target_seq_length = max_num_tokens

    # 有一定的比例，如10%的概率，我们使用比较短的序列长度，以缓解预训练的长序列和调优阶段（可能的）短序列的不一致情况
    if random.random() < short_seq_prob:    # 有10%选用短文本
        target_seq_length = random.randint(2, max_num_tokens)

    # 设法使用实际的句子，而不是任意的截断句子，从而更好的构造句子连贯性预测的任务
    instances = []
    current_chunk = []  # 当前处理的文本段，包含多个句子
    current_length = 0
    i = 0
    # print("###document:",document) # 一个document可以是一整篇文章、新闻、词条等.
    while i < len(document):  # 从文档的开始看起  一句话一句话的看
        segment = document[i]   # segment是列表，代表的是按字分开的一个完整句子

        # segment = get_new_segment(segment)  # whole word mask for chinese: 结合分词的中文的whole mask设置即在需要的地方加上“##”
        current_chunk.append(segment)  # 将一个独立的句子加入到当前的文本块中
        current_length += len(segment)  # 累计到为止位置接触到句子的总长度
        if i == len(document) - 1 or current_length >= target_seq_length:
            if current_chunk:  # 如果当前块不空
                a_end = 1
                if len(current_chunk) >= 2:   # 当前块，如果包含超过两个句子，取当前块的一部分作为“A[SEP]B“中的A部分
                    a_end = random.randint(1, len(current_chunk) - 1)
                # 将当前文本段中选取出来的前半部分，赋值给A即tokens_a
                tokens_a = []
                for j in range(a_end):
                    tokens_a.extend(current_chunk[j])

                # 构造“A[SEP]B“中的B部分(有一部分是正常的当前文档中的后半部;在原BERT的实现中一部分是随机的从另一个文档中选取的，）
                tokens_b = []
                for j in range(a_end, len(current_chunk)):
                    tokens_b.extend(current_chunk[j])

                # 有百分之50%的概率交换一下tokens_a和tokens_b的位置
                # print("tokens_a length1:",len(tokens_a))
                # print("tokens_b length1:",len(tokens_b)) # len(tokens_b) = 0
                if len(tokens_a) == 0 or len(tokens_b) == 0:
                    continue
                if random.random() < 0.5:  # 交换一下tokens_a和tokens_b
                    is_random_next = True
                    temp = tokens_a
                    tokens_a = tokens_b
                    tokens_b = temp
                else:
                    is_random_next = False

                truncate_seq_pair(tokens_a, tokens_b, max_num_tokens)
                assert len(tokens_a) >= 1
                assert len(tokens_b) >= 1

                # 把tokens_a & tokens_b加入到按照bert的风格，即以[CLS]tokens_a[SEP]tokens_b[SEP]的形式，结合到一起，作为最终的tokens;
                # 也带上segment_ids，前面部分segment_ids的值是0，后面部分的值是1.
                tokens = ["[CLS]"] + tokens_a + ["[SEP]"] + tokens_b + ["[SEP]"]
                # The segment IDs are 0 for the [CLS] token, the A tokens and the first [SEP]
                # They are 1 for the B tokens and the final [SEP]
                segment_ids = [0 for _ in range(len(tokens_a) + 2)] + [1 for _ in range(len(tokens_b) + 1)]

                # 创建masked LM的任务的数据
                tokens, masked_lm_positions, masked_lm_labels = create_masked_lm_predictions(
                    tokens, max_ngram, masked_lm_prob, max_predictions_per_seq, vocab_words)
                instance = {
                    "tokens": tokens,
                    "segment_ids": segment_ids,
                    "is_random_next": is_random_next,
                    "masked_lm_positions": masked_lm_positions,
                    "masked_lm_labels": masked_lm_labels
                }
                '''
                {'tokens': ['[CLS]', '肥', '市', '庐', '[MASK]', '[MASK]', '中', '铁', '国', '际', '城', '湿', '地', '公', '园', '散', '步', '时', '[MASK]', '[MASK]', '[MASK]', '湿', '地', '公', '园', '临', '近', '出', '口', '（', '出', '口', '位', '[MASK]', '东', '西', '走', '向', '的', '清', '源', '路', '）', '的', '内', '部', '道', '路', '时', '，', '被', '中', 'x5', '物', '业', '管', '理', '公', '司', '合', '肥', '##棗', '公', '司', '[SEP]', '人', '陈', '述', '和', '002', '##座', '[MASK]', '确', '认', '的', '证', '据', '[MASK]', '本', '院', '认', '定', '事', '实', '如', '下', '：', '韩', '家', '麒', '[MASK]', '公', '民', '身', '[MASK]', '号', '码', '##ute', '户', '[MASK]', '登', '记', '类', '别', '为', '非', '农', '业', '家', '庭', '户', '）', '与', '孔', 'x', '##0', '系', '夫', '妻', '关', '系', '[MASK]', '[MASK]', '[MASK]', '生', '育', '有', '[SEP]'],
                 'segment_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                 'is_random_next': True,
                 'masked_lm_positions': [4, 5, 18, 19, 20, 33, 55, 61, 69, 70, 71, 77, 90, 94, 97, 99, 121, 122, 123],
                 'masked_lm_labels': ['阳', '区', '，', '行', '至', '于', '管', '分', '经', '审', '查', '，', '（', '份', '，', '籍', '，', '两', '人']}
                '''
                instances.append(instance)
            current_chunk = []  # 清空当前块
            current_length = 0  # 重置当前文本块的长度
        i += 1  # 接着文档中的内容往后看
    return instances

def create_training_instances(input_file, tokenizer, max_seq_len, short_seq_prob,
                              max_ngram, masked_lm_prob, max_predictions_per_seq):
    all_documents = [[]]
    f = open(input_file, 'r', encoding='utf8')
    lines = f.readlines()

    for line_cnt, line in tqdm(enumerate(lines)):
        line = line.strip()
        # Empty lines are used as document delimiters   # 文档直接使用空行进行分开的
        if not line:
            all_documents.append([])
        tokens = tokenizer.tokenize(line)
        if tokens:
            all_documents[-1].append(tokens)
    # print(all_documents[0])   # [[[文章1的句子1分词形式 是列表], [文章1的句子2分词形式]...], [[文章2的句子1分词形式], 文章2的句子2分词形式...]] ... ]
    # print(len(all_documents))
    all_documents = [x for x in all_documents if x]   # 移除空文本

    random.shuffle(all_documents)   # 将文章进行shuffle

    vocab_words = list(tokenizer.vocab.keys())   # 这里直接认为我们的词表就是bert-base的词表

    instances = []
    print(len(all_documents))

    for document_index in tqdm(range(len(all_documents))):
        instances.extend(
            create_instances_from_document(
                all_documents, document_index, max_seq_len, short_seq_prob,
                max_ngram, masked_lm_prob, max_predictions_per_seq, vocab_words)
        )
    random.shuffle(instances)
    return instances

if __name__ == '__main__':
    # input_file = './corpus/pro_data.txt'
    tokenizer = BertTokenizer(vocab_file='./bert_base_pretrain/vocab.txt', do_lower_case=True)
    max_seq_len = 512
    short_seq_prob = 0.3
    max_ngram = 3
    masked_lm_prob = 0.15
    max_predictions_per_seq = 20

    file_list = ['./data/train.data']
    for i, input_file in enumerate(file_list):
        print('处理第{}个文件的数据'.format(i))
        with open('./data/processed_data{}.json'.format(i), 'w', encoding='utf8') as f:
            file_examples = create_training_instances(input_file, tokenizer, max_seq_len, short_seq_prob,
                                                      max_ngram, masked_lm_prob, max_predictions_per_seq)

            file_examples = [json.dumps(instance) for instance in file_examples]
            for instance in file_examples:
                f.write(instance + '\n')


#!/usr/bin/env python3
import seaborn as sns
import argparse
import numpy as np
import pandas as pd
from numpy import median
import matplotlib.pyplot as plt
import pathlib
import pathlib
import kube_env
import kube_util as util

def graph(title, csv_path, output):
    old_to_new_names = {}
    df = pd.read_csv(csv_path)
    for col in df.columns:
        if col != "Time":
            start_pod_name = col.find("pod_name")
            start_pod = col[start_pod_name:].find(":")+start_pod_name+1
            end_pod = col[start_pod:].find(".")
            pod_name = col[start_pod:start_pod+end_pod]

            start_container_name = col.find("container_name")
            start_container = col[start_container_name:].find(":")+start_container_name+1
            end_container = col[start_container:].find(".")
            container_name = col[start_container: start_container+end_container]

            new_column_name = pod_name + "_" + container_name
            old_to_new_names[col] = new_column_name

    df.rename(columns=old_to_new_names, inplace=True)
    df["Time"] = df["Time"].str[3:]
    df["Time"] = df["Time"].str[:21]
    df["Time"] = pd.to_datetime(df["Time"], infer_datetime_format=True)

    # get different columns
    app_cols = df.columns[df.columns.str.contains('_server|_storage-upstream')]
    istio_cols = df.columns[df.columns.str.contains('istio-proxy')]
    fluentbit_cols = df.columns[df.columns.str.contains('fluentbit')]
    gke_cols = df.columns[df.columns.str.contains('gke-metrics')]
    kube_cols = df.columns[df.columns.str.contains('kube-dns|kube-proxy')]
    pdcsi_cols = df.columns[df.columns.str.contains('pdcsi')]
    time_col = df.columns[df.columns == 'Time']
    other_cols = df.columns[~df.columns.isin(app_cols|istio_cols|fluentbit_cols|gke_cols|kube_cols|pdcsi_cols|time_col)]
    assert df.columns.shape[0] == app_cols.shape[0] + istio_cols.shape[0] + fluentbit_cols.shape[0] + gke_cols.shape[0] + kube_cols.shape[0] + pdcsi_cols.shape[0] + other_cols.shape[0] + 1

    subtitle_fontsize = 12

    # app
    fig = plt.gcf()
    fig.set_size_inches(12, 5)
    plt.subplots_adjust(right=0.7)

    for col in app_cols:
        df[col] = df[col].replace("undefined", "nan")
        df[col] = df[col].astype(float, errors='ignore')
        dplt = sns.lineplot(data=df, x="Time", y=col)
        dplt.legend(labels=app_cols, bbox_to_anchor=(1.01, 1.01), loc='upper left', ncol=1, mode="expand", prop={'size': 8})
        dplt.set(ylabel = "Seconds of CPU time used per second")
        dplt.set_title("CPU Usage By Application Container", fontsize=subtitle_fontsize)
        axes = plt.gca()
        axes.set_ylim([0,0.1])

    print("Saving to: ", f"{output}_app.png")
    plt.savefig(f"{output}_app.png")
    plt.clf()
    plt.cla()

    # istio
    fig = plt.gcf()
    fig.set_size_inches(12, 5)
    plt.subplots_adjust(right=0.7)

    for col in istio_cols:
        df[col] = df[col].replace("undefined", "nan")
        df[col] = df[col].astype(float, errors='ignore')
        dplt = sns.lineplot(data=df, x="Time", y=col)
        dplt.legend(labels=app_cols, bbox_to_anchor=(1.01, 1.01), loc='upper left', ncol=1, mode="expand", prop={'size': 8})
        dplt.set(ylabel = "Seconds of CPU time used per second")
        dplt.set_title("CPU Usage By Istio Container", fontsize=subtitle_fontsize)

        axes = plt.gca()
        axes.set_ylim([0,0.1])

    print("Saving to: ", f"{output}_istio.png")
    plt.savefig(f"{output}_istio.png")
    plt.clf()
    plt.cla()

    # fluentbit
    fig = plt.gcf()
    fig.set_size_inches(12, 5)
    plt.subplots_adjust(right=0.7)

    for col in fluentbit_cols:
        df[col] = df[col].replace("undefined", "nan")
        df[col] = df[col].astype(float, errors='ignore')
        dplt = sns.lineplot(data=df, x="Time", y=col)
        dplt.legend(labels=fluentbit_cols, bbox_to_anchor=(1.01, 1.01), loc='upper left', ncol=1, mode="expand", prop={'size': 8})
        dplt.set(ylabel = "Seconds of CPU time used per second")
        dplt.set_title("CPU Usage By Fluentbit Container", fontsize=subtitle_fontsize)
        axes = plt.gca()
        axes.set_ylim([0,0.1])

    print("Saving to: ", f"{output}_fluentbit.png")
    plt.savefig(f"{output}_fluentbit.png")
    plt.clf()
    plt.cla()

    # kube
    fig = plt.gcf()
    fig.set_size_inches(12, 5)
    plt.subplots_adjust(right=0.7)

    for col in kube_cols:
        df[col] = df[col].replace("undefined", "nan")
        df[col] = df[col].astype(float, errors='ignore')
        dplt = sns.lineplot(data=df, x="Time", y=col)
        dplt.legend(labels=kube_cols, bbox_to_anchor=(1.01, 1.01), loc='upper left', ncol=1, mode="expand", prop={'size': 8})
        dplt.set(ylabel = "Seconds of CPU time used per second")
        dplt.set_title("CPU Usage By Kubernetes Container", fontsize=subtitle_fontsize)
        axes = plt.gca()
        axes.set_ylim([0,0.1])

    print("Saving to: ", f"{output}_kube.png")
    plt.savefig(f"{output}_kube.png")
    plt.clf()
    plt.cla()

    # gke
    fig = plt.gcf()
    fig.set_size_inches(12, 5)
    plt.subplots_adjust(right=0.7)

    for col in gke_cols:
        df[col] = df[col].replace("undefined", "nan")
        df[col] = df[col].astype(float, errors='ignore')
        dplt = sns.lineplot(data=df, x="Time", y=col)
        dplt.legend(labels=gke_cols, bbox_to_anchor=(1.01, 1.01), loc='upper left', ncol=1, mode="expand", prop={'size': 8})
        dplt.set(ylabel = "Seconds of CPU time used per second")
        dplt.set_title("CPU Usage By GKE Metrics Container", fontsize=subtitle_fontsize)
        axes = plt.gca()
        axes.set_ylim([0,0.1])

    print("Saving to: ", f"{output}_gke.png")
    plt.savefig(f"{output}_gke.png")
    plt.clf()
    plt.cla()

    # pdcsi
    fig = plt.gcf()
    fig.set_size_inches(12, 5)
    plt.subplots_adjust(right=0.7)

    for col in pdcsi_cols:
        df[col] = df[col].replace("undefined", "nan")
        df[col] = df[col].astype(float, errors='ignore')
        dplt = sns.lineplot(data=df, x="Time", y=col)
        dplt.legend(labels=pdcsi_cols, bbox_to_anchor=(1.01, 1.01), loc='upper left', ncol=1, mode="expand", prop={'size': 8})
        dplt.set(ylabel = "Seconds of CPU time used per second")
        dplt.set_title("CPU Usage By PDCSI Container", fontsize=subtitle_fontsize)
        axes = plt.gca()
        axes.set_ylim([0,0.1])

    print("Saving to: ", f"{output}_pdcsi.png")
    plt.savefig(f"{output}_pdcsi.png")
    plt.clf()
    plt.cla()

    # other
    fig = plt.gcf()
    fig.set_size_inches(12, 5)
    plt.subplots_adjust(right=0.65)

    for col in other_cols:
        df[col] = df[col].replace("undefined", "nan")
        df[col] = df[col].astype(float, errors='ignore')
        dplt = sns.lineplot(data=df, x="Time", y=col)
        dplt.legend(labels=other_cols, bbox_to_anchor=(1.01, 1.01), loc='upper left', ncol=1, mode="expand", prop={'size': 8})
        dplt.set(ylabel = "Seconds of CPU time used per second")
        dplt.set_title("CPU Usage By Miscellaneous Other Containers (e.g., No Other Category)", fontsize=subtitle_fontsize)
        axes = plt.gca()
        axes.set_ylim([0,0.1])

    print("Saving to: ", f"{output}_other_.png")
    plt.savefig(f"{output}_other.png")
    plt.clf()
    plt.cla()

    cols = []
    for col in df.columns:
        if col != "Time":
            cols.append(col)

    fig = plt.gcf()
    fig.set_size_inches(12, 5)
    plt.subplots_adjust(right=0.7)

    df["cpu_all"] = df[cols].sum(axis=1)
    df["cpu_all"] = df["cpu_all"].replace("undefined", "nan")
    df["cpu_all"] = df["cpu_all"].astype(float, errors='ignore')
    hplt = sns.lineplot(data=df, x="Time", y="cpu_all")
    hplt.set(ylabel="Seconds of CPU time used per second")
    hplt.set_title("Sum of CPU Usage Of All Containers")

    print("Saving to: ", f"{output}_aggregated.png")
    plt.savefig(f"{output}_aggregated.png")
    plt.clf()
    plt.cla()

    return (df["Time"], df["cpu_all"])

def compare_cpu_usages():
    users = ["100", "200", "300", "400", "500"]
    filter_types = ["no_filter", "empty_filter", "snicket_filter", "snicket_filter_distributed"]
    # everything should be in columns labeled:  cpu usage, users, filter type
    all_data = pd.DataFrame()
    for user in users:
        for filter_type in filter_types:
            csv_path = f"CPU_usage_time_{user}_users_{filter_type}.csv"
            df = pd.read_csv(csv_path)
            # 1. preprocess time
            df["Time"] = df["Time"].str[3:]
            df["Time"] = df["Time"].str[:21]
            df["Time"] = pd.to_datetime(df["Time"], infer_datetime_format=True)
            df["Time"] = df["Time"] - df["Time"][0] # normalize time

            # 2. get cpu_all, with startup data discarded
            cols = []
            for col in df.columns:
                if col != "Time":
                    cols.append(col)

            df["cpu_all"] = df[cols].sum(axis=1)
            df["cpu_all"] = df["cpu_all"].replace("undefined", "nan")
            df["cpu_all"] = df["cpu_all"].astype(float, errors='ignore')
            # here we remove the first few entries because locust was starting up
            # for 100-300, we remove 6 entries, which is equivalent to 60 sec
            # because the size of the spawn rate was 5
            # for 400 we remove 8, and for 500 we remove 10
            if user == '100' or user == '200' or user == '300':
                df["cpu_all"] = df["cpu_all"][5:]
            elif user == '400':
                df["cpu_all"] = df["cpu_all"][10:]
            else:
                df["cpu_all"] = df["cpu_all"][15:]

            df["cpu_all"] = df["cpu_all"].replace("undefined", "nan")
            df["cpu_all"] = df["cpu_all"].astype(float, errors='ignore')

            # 3. Save in a way that makes this plot-able later
            #    this means that you have three columns:  Value, Users, Filter
            # new_df = df["cpu_all"].copy()
            new_df = df.filter(['cpu_all'], axis=1)
            new_df.insert(1, 'users', user)
            new_df.insert(2, 'filter_type', filter_type)
            all_data = pd.concat([all_data, new_df])

    all_data = all_data.dropna(axis=0)
    ax = sns.barplot(data=all_data, x='users', y='cpu_all', hue='filter_type', estimator=median, ci="sd")
    plt.savefig(f"cpu_plot.png")

def main(args):
    #return graph(args.title, args.source, args.output)
    return compare_cpu_usages()

if __name__ == '__main__':
    path = pathlib.Path().absolute()
    parser = argparse.ArgumentParser()
    parser.add_argument("-t",
                        "--title",
                        dest="title",
                        default="CPU Usage By Container",
                        help="Specifies title of graph.")
    parser.add_argument("-s",
                        "--source",
                        dest="source",
                        default=f"{path}/Kubernetes_Container_-_CPU_usage_time_[RATE].csv",
                        help="Specifies source of data.")
    parser.add_argument("-o",
                        "--output",
                        dest="output",
                        default="cpu_plot.png",
                        help="Specifies where graph is saved.")
    arguments = parser.parse_args()
    main(arguments)


# Ejercicio 14
# Muchas personas piensan en su altura en pies y pulgadas, incluso en algunos países que utilizan principalmente el sistema métrico.
# Escriba un programa que lea un número de pies del usuario, seguido de un número de pulgadas.
# Una vez que se leen estos valores, su programa, debe calcular y mostrar el número equivalente de centímetros.
# Sugerencia: un pie mide 12 pulgadas. Una pulgada es 2.54 centímetros.

pies = int(input('Introduce el numero en pies: '))
pulgadas = int(input('Introduce el numero en pulgadas: '))

pulgadas += pies * 12

centimetros = pulgadas * 2.54

print('La altura es: ',(centimetros), 'cm')


import calendar
import datetime

from condensedinlinepanel.edit_handlers import CondensedInlinePanel
from django.core.validators import MinValueValidator, MaxValueValidator
from django.db import models
from django.template.defaultfilters import truncatechars
from django.utils.dates import MONTHS
from django.utils.functional import cached_property
from modelcluster.fields import ParentalKey
from modelcluster.models import ClusterableModel
from wagtail.admin.edit_handlers import (FieldPanel, MultiFieldPanel, PageChooserPanel)
from wagtail.core.fields import RichTextField
from wagtail.core.models import Orderable, Page
from wagtail.documents.edit_handlers import DocumentChooserPanel
from wagtail.documents.models import Document
from wagtail.images.edit_handlers import ImageChooserPanel
from wagtail.snippets.edit_handlers import SnippetChooserPanel
from wagtailmetadata.models import MetadataPageMixin

from cms_pages.webicons.edit_handlers import WebIconChooserPanel
from core.models import ProductCategory, ProductType, ProductGroup, GeographicalRegion
from core.utils import get_years, paginate, query_param_to_list, get_first_non_empty_p_string
from icpacweb.settings.base import SUMMARY_RICHTEXT_FEATURES
from .utils import getEndDate

class TendayForecastIndexPage(MetadataPageMixin, Page):
    template = 'tendayforecast_index.html'
    parent_page_types = ['home.HomePage']
    subpage_types = ['tendayforecast.TendayForecastDetailPage']
    max_count = 1
    show_in_menus_default = True

    product = models.OneToOneField(ProductCategory, on_delete=models.PROTECT)

    introduction_title = models.CharField(max_length=100, help_text="Introduction section title")
    introduction_text = RichTextField(
        help_text="A description of what how ICPAC produces the 10 day forecast",
        features=SUMMARY_RICHTEXT_FEATURES)
    introduction_image = models.ForeignKey(
        'webicons.WebIcon',
        verbose_name="Illustration Image",
        help_text="A high quality illustration related to 10 day forecast",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+',
    )
    introduction_button_text = models.TextField(max_length=20, blank=True, null=True)
    introduction_button_link = models.ForeignKey(
        'wagtailcore.Page',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+',
    )
    items_per_page = models.PositiveIntegerField(default=6, validators=[
        MinValueValidator(6),
        MaxValueValidator(20),
    ], help_text="How many items should be visible on the landing page filter section ?")

    content_panels = Page.content_panels + [
        SnippetChooserPanel('product'),
        MultiFieldPanel(
            [
                FieldPanel('introduction_title'),
                WebIconChooserPanel('introduction_image'),
                FieldPanel('introduction_text'),
                FieldPanel('introduction_button_text'),
                PageChooserPanel('introduction_button_link'),
            ],
            heading="Introduction Section",
        ),
        MultiFieldPanel(
            [
                FieldPanel('items_per_page'),
            ],
            heading="Other Settings",
        ), ]

    @cached_property
    def forecast_product_groups(self):
        groups = ProductGroup.objects.all()
        return groups

    @cached_property
    def latest_dekad_forecast(self):
        return TendayForecastDetailPage.objects.live().order_by('-end_date').first()

    def get_latest_forecast_products(self, request):
        regions = query_param_to_list(request.GET.get('region'), as_int=True)

        latest_dekad_forecast = self.latest_dekad_forecast

        if regions:
            # get the first region
            products = latest_dekad_forecast.products.filter(region__in=[regions[0]])
        else:
            products = latest_dekad_forecast.products.filter(region__is_default=True)

        return products

    @cached_property
    def filters(self):
        resource_types = ProductType.objects.all()
        regions = GeographicalRegion.objects.all().order_by('-is_default')
        return {'year': get_years(2010), 'month': MONTHS, 'resource_type': resource_types, 'regions': regions}

    @cached_property
    def all_dekad_products(self):
        # all products with the detail page live
        return TenDayForecastProduct.objects.filter(page__live=True).order_by('-page__end_date')

    def filter_dekad_forecasts(self, request):
        queryset = self.all_dekad_products

        years = query_param_to_list(request.GET.get("year"))
        months = query_param_to_list(request.GET.get("month"))
        resource_types = query_param_to_list(request.GET.get("resource_type"))
        dekads = query_param_to_list(request.GET.get("dekad"), as_int=True)
        regions = query_param_to_list(request.GET.get('region'), as_int=True)

        filters = models.Q()

        if years:
            filters &= models.Q(page__begin_date__year__in=years)
        if months:
            filters &= models.Q(page__begin_date__month__in=months)
        if resource_types:
            filters &= models.Q(resource_type__in=resource_types)
        if dekads:
            filters &= models.Q(page__dekad__in=dekads)
        if regions:
            filters &= models.Q(region__in=regions)
        return queryset.filter(filters)

    def filter_and_paginate_products(self, request):
        page = request.GET.get('page')

        filtered_products = self.filter_dekad_forecasts(request)

        paginated_products = paginate(filtered_products, page, self.items_per_page)

        return paginated_products

    def get_context(self, request, *args, **kwargs):
        context = super(TendayForecastIndexPage,
                        self).get_context(request, *args, **kwargs)

        context['latest_dekad_products'] = self.get_latest_forecast_products(request)

        context['products'] = self.filter_and_paginate_products(request)

        return context

    def save(self, *args, **kwargs):
        if not self.search_description and self.introduction_text:
            p = get_first_non_empty_p_string(self.introduction_text)
            if p:
                # Limit the search meta desc to google's 160 recommended chars
                self.search_description = truncatechars(p, 160)
        return super().save(*args, **kwargs)

class TendayForecastDetailPage(MetadataPageMixin, Page):
    DEKAD_CHOICES = (
        (1, "First"),
        (2, "Second"),
        (3, "Third"),
    )

    template = 'tendayforecast_page.html'
    parent_page_types = ['tendayforecast.TendayForecastIndexPage']

    begin_date = models.DateField(default=datetime.date.today)
    end_date = models.DateField(default=getEndDate)

    dekad = models.PositiveIntegerField(blank=True, null=True, choices=DEKAD_CHOICES)

    @cached_property
    def products(self):
        return self.ten_day_forecast_products.all()

    @cached_property
    def filters(self):
        regions = GeographicalRegion.objects.all().order_by('-is_default')
        return {'regions': regions}

    content_panels = Page.content_panels + [
        FieldPanel('begin_date'),
        FieldPanel('end_date'),
        FieldPanel('dekad'),
        CondensedInlinePanel('ten_day_forecast_products',
                             label="Add Product",
                             heading="Ten Day Forecast Products", ),
    ]

    @cached_property
    def period(self):
        return "{} - {} {} {}".format(self.begin_date.day, self.end_date.day, calendar.month_name[self.end_date.month],
                                      self.end_date.year)

    class Meta:
        verbose_name = "Ten day forecast product"

    def get_context(self, request, *args, **kwargs):
        context = super(TendayForecastDetailPage, self).get_context(request, *args, **kwargs)

        resource_type = query_param_to_list(request.GET.get('resource_type'), as_int=True)
        regions = query_param_to_list(request.GET.get('region'), as_int=True)

        if regions:
            # get the first region
            products = self.products.filter(region__in=[regions[0]])
        else:
            products = self.products.filter(region__is_default=True)
        context['products'] = products

        if resource_type:
            active_product = self.products.filter(resource_type=resource_type[0])
            if active_product.exists():
                context['active_product'] = active_product[0]
        return context

    def save(self, *args, **kwargs):
        parent = self.get_parent().specific

        # Get meta details from parent
        if not self.search_image and parent.search_image:
            self.search_image = parent.search_image
        if not self.search_description:
            self.search_description = self.title
        return super().save(*args, **kwargs)

class TenDayForecastProduct(Orderable):
    page = ParentalKey(TendayForecastDetailPage,
                       related_name='ten_day_forecast_products')

    resource_type = models.ForeignKey(ProductType, on_delete=models.PROTECT, related_name='dekadal_forecast_products')

    map_image = models.ForeignKey(
        'wagtailimages.Image',
        verbose_name="Image",
        help_text="A high quality map image",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+',
    )

    document = models.ForeignKey(
        'core.CustomDocumentModel',
        null=True,
        blank=True,
        help_text="PDF document if the product is a bulletin or similar",
        on_delete=models.SET_NULL,
        related_name='+'
    )
    analysis_heading = models.CharField(max_length=255, blank=True, null=True)
    analysis_summary = RichTextField(features=SUMMARY_RICHTEXT_FEATURES)
    region = models.ForeignKey(GeographicalRegion, on_delete=models.CASCADE, related_name='dekad_products', blank=True,
                               null=True)

    def __str__(self):
        return "{} - {}".format(self.resource_type.name, self.page.title)

    panels = [
        SnippetChooserPanel('resource_type'),
        SnippetChooserPanel('region'),
        ImageChooserPanel('map_image'),
        MultiFieldPanel([
            FieldPanel('analysis_heading'),
            FieldPanel('analysis_summary'),
        ],
            heading="Analysis"
        ),

        DocumentChooserPanel('document'),
    ]

    @cached_property
    def product_url(self):
        return "{}?resource_type={}".format(self.page.url, self.resource_type.pk)

    @cached_property
    def product_category(self):
        parent = self.page.get_parent().specific
        return parent.product.name

    @property
    def period(self):
        return self.page.period

    @cached_property
    def card_props(self):
        return {
            "card_image": self.map_image,
            "card_title": self.resource_type,
            "card_text": self.analysis_summary,
            "card_meta": self.page.period,
            "card_more_link": self.product_url,
            "card_tag": self.product_category,
        }

    def save(self, *args, **kwargs):
        # if region not set
        if not self.region:
            # get default region
            default_region = GeographicalRegion.objects.filter(is_default=True).first()

            if default_region:
                self.region = default_region

        super(TenDayForecastProduct, self).save(*args, **kwargs)


#
# ovirt-engine-setup -- ovirt engine setup
# Copyright (C) 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""database plugin."""

import urlparse
import gettext
_ = lambda m: gettext.dgettext(message=m, domain='ovirt-engine-setup')

from otopi import constants as otopicons
from otopi import util
from otopi import plugin
from otopi import transaction

from ovirt_engine_setup import constants as osetupcons
from ovirt_engine_setup.engine_common \
    import constants as oengcommcons
from ovirt_engine_setup.engine import constants as oenginecons
from ovirt_engine_setup.engine_common import database
from ovirt_engine_setup.engine import vdcoption
from ovirt_engine_setup import domains

@util.export
class Plugin(plugin.PluginBase):
    """database plugin."""

    class DBTransaction(transaction.TransactionElement):
        """yum transaction element."""

        def __init__(self, parent):
            self._parent = parent

        def __str__(self):
            return _("Database Transaction")

        def prepare(self):
            pass

        def abort(self):
            connection = self._parent.environment[
                oenginecons.EngineDBEnv.CONNECTION
            ]
            if connection is not None:
                connection.rollback()
                self._parent.environment[
                    oenginecons.EngineDBEnv.CONNECTION
                ] = None

        def commit(self):
            connection = self._parent.environment[
                oenginecons.EngineDBEnv.CONNECTION
            ]
            if connection is not None:
                connection.commit()

    def __init__(self, context):
        super(Plugin, self).__init__(context=context)

    @plugin.event(
        stage=plugin.Stages.STAGE_INIT,
    )
    def _init(self):
        self.environment[otopicons.CoreEnv.MAIN_TRANSACTION].append(
            self.DBTransaction(self)
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_MISC,
        name=oengcommcons.Stages.DB_CONNECTION_AVAILABLE,
    )
    def _connection(self):
        self.environment[
            oenginecons.EngineDBEnv.STATEMENT
        ] = database.Statement(
            dbenvkeys=oenginecons.Const.ENGINE_DB_ENV_KEYS,
            environment=self.environment,
        )
        # must be here as we do not have database at validation
        self.environment[
            oenginecons.EngineDBEnv.CONNECTION
        ] = self.environment[oenginecons.EngineDBEnv.STATEMENT].connect()

    @plugin.event(
        stage=plugin.Stages.STAGE_VALIDATION,
    )
    def _validation(self):
        dbovirtutils = database.OvirtUtils(
            plugin=self,
            dbenvkeys=oenginecons.Const.ENGINE_DB_ENV_KEYS,
        )
        dbovirtutils.tryDatabaseConnect()
        dbstatement = database.Statement(
            dbenvkeys=oenginecons.Const.ENGINE_DB_ENV_KEYS,
            environment=self.environment,
        )
        my_domains = []
        rows = dbstatement.execute(
            statement="""
                select
                    storage_name,
                    connection
                from
                    storage_domain_static s,
                    storage_server_connections c
                where
                    s.storage = c.id and
                    s.storage_type=%(storage_type)s and
                    s.storage_domain_type=%(storage_domain_type)s
            """,
            args=dict(
                storage_type=domains.StorageType.NFS,
                storage_domain_type=domains.StorageDomainType.ISO,
            ),
            ownConnection=True,
        )
        for row in rows:
            host, path = row['connection'].split(':', 1)
            if host == self.environment[osetupcons.ConfigEnv.FQDN]:
                my_domains.append(row['storage_name'])
        if my_domains:
            self.logger.warning(_('Engine host hosting Storage Domains'))
            self.dialog.note(
                text=_(
                    'The following Storage Domains use the engine host\n'
                    'as an NFS server:\n'
                    '\n'
                    '{domains}\n'
                    '\n'
                    'Cannot rename the engine host. Please backup relevant\n'
                    'data if needed, remove all of these domains, and then\n'
                    'run this utility again.\n'
                ).format(
                    domains='\n'.join(sorted(my_domains))
                ),
            )
            raise RuntimeError(_('Cannot rename host hosting Storage Domains'))

    @plugin.event(
        stage=plugin.Stages.STAGE_MISC,
        after=(
            oengcommcons.Stages.DB_CONNECTION_AVAILABLE,
        ),
    )
    def _misc(self):
        option = 'RedirectServletReportsPage'
        vdc = vdcoption.VdcOption(
            statement=self.environment[
                oenginecons.EngineDBEnv.STATEMENT
            ]
        )
        value = vdc.getVdcOption(name=option)

        if value:
            newfqdn = self.environment[osetupcons.RenameEnv.FQDN]
            u = urlparse.urlparse(value)
            ulist = list(u)
            ulist[1] = newfqdn + ":" + str(u.port)
            vdc.updateVdcOptions(
                options=(
                    {
                        'name': option,
                        'value': urlparse.urlunparse(ulist),
                    },
                )
            )

# vim: expandtab tabstop=4 shiftwidth=4


# -*- coding: utf-8 -*-
"""
Created on Tue Dec 24 11:34:51 2019

@author: thinkpad
修改内容：用LTP对单个词进行词性标注
再用LTP对整句话先分词，再进行词性标注，最后再去除停用词
"""
import json
import re
import thulac
import os
from entity_verb.nlp import NLP
from pyltp import Segmentor,SentenceSplitter,Postagger,NamedEntityRecognizer,Parser

class entity_verb_new:
    def not_empty(self,s):
        return s and "".join(s.split())

    def splitSentence(self,text):
        pattern = r'。|！|？|；|='
        result_list = re.split(pattern, text)
        result_list = list(filter(self.not_empty, result_list))
        #    print(result_list)
        return result_list

    def stopwordsList(self):
        """
        return:停用词列表
        """
        stop_list_Path = 'D:\python-file\\北京市旅游知识图谱\\verb-entity\\中文停用词.txt'
        f = open(stop_list_Path, 'r', encoding='utf-8')
        stopwords = [line.strip() for line in f.readlines()]
        #    print(stopwords)
        return stopwords

    def splitWord(self,sentence_list, thu1):
        result_list = []
        stopwords = self.stopwordsList()

        for sentence in sentence_list:
            sentence = sentence.strip()
            result_words = []
            words = thu1.cut(sentence)  # 进行一句话分词
            for word in words:
                if word[0] not in stopwords and len(word[0].strip()) != 0:
                    result_words.append(word)
            result_list.append(result_words)
        #    print(result_list)
        return result_list

    def splitWordOneSentence(self,sentence, thu1):
        stopwords = self.stopwordsList()

        sentence = sentence.strip()
        result_words = []
        words = thu1.cut(sentence)  # 进行一句话分词
        for word in words:
            if word[0] not in stopwords and len(word[0].strip()) != 0:
                result_words.append(word)
        return result_words

    def getKeySentence(sentence_list, keyWords):
        result_list = []
        for sentence in sentence_list:
            for keyWord in keyWords:
                if keyWord in sentence:
                    result_list.append(sentence)
                    break
        return result_list

    def mapVerbEntity(split_result, entity_list):
        result_list = []
        for sentence in split_result:
            index = -1
            dict_list = []
            for word in sentence:
                dic = dict()
                index = index + 1
                if word[1] == 'v' and len(word[0]) > 1:
                    dic[word[0] + "_" + word[1]] = index
                    dict_list.append(dic)
                else:
                    for entity in entity_list:
                        if word[0] == entity[0]:
                            dic[word[0] + "_" + word[1]] = index
                            dict_list.append(dic)
                            break
            result_list.append(dict_list)
        return result_list

    def mapEntity(self,sentence_list, all_entity,thu1,nlp,segmentor,postagger):
        stopwords = self.stopwordsList()
        entity_in_all_sentence =[]
        for sentence in sentence_list:
            # entity_in_each_sentence =[]
            flag = False
            entity_verb_dict = {}
            for entity in all_entity:
                # if entity in sentence:
                if sentence.find(entity)!=-1:
                    flag = True
                    location = 0
                    index = 0
                    while location<len(sentence):
                        location = sentence.find(entity,location)
                        if location ==-1:
                            break
                        else:
                            index +=1
                            entity_verb_dict[str(index)+'#'+entity + '_' + nlp.get_postag(entity)] = location
                            location+=len(entity)
                    # entity_in_each_sentence.append(entity+'_'+str(sentence.find(entity)))

            if flag:
                splitWordList = segmentor.segment(sentence)
                result_words = []
                for word in splitWordList:
                    result_words.append(word)
                print(result_words)
                wordPosList = postagger.postag(result_words)
                result_pos = []
                for pos in wordPosList:
                    result_pos.append(pos)
                print(result_pos)
                # splitWordList = self.splitWordOneSentence(sentence,thu1)
                count = 0
                for index in range(len(result_words)):

                    word = result_words[index]
                    pos = result_pos[index]
                    if word not in stopwords and pos== 'v':  # 如果该单词不在停用词表中，且为动词
                        entity_verb_dict[word+'_v_'+str(count)] = count  # 由于一句话中可能会出现多次，所以我们使用count来计数，该动词所在的位置。
                        # 而且一个动词也可能在一句话中出现多次，所以我们给动词+count，使动词唯一。
                    count += len(word)
            entity_verb_dict = sorted(entity_verb_dict.items(), key=lambda item: item[1])
            print(entity_verb_dict)
            entity_in_all_sentence.append(entity_verb_dict)
        return entity_in_all_sentence

    def readAllEntity(self,file = 'entity_verb_result\\' + "all_entity.json"):
        f = open(file
                 , 'r', encoding='utf-8')
        file = f.read()
        all_entity = json.loads(file)['all_entity']
        return all_entity

if __name__ == "__main__":
    # 读取文件
    entity_verb_new = entity_verb_new()
    """
    加载LTP的分词器和词性标注器
    """
    default_model_dir = 'D:\python-file\knowledge_extraction-master-tyz\\ltp_data_v3.4.0\\'  # LTP模型文件目录
    segmentor = Segmentor()
    user_dict = "source\\user.txt"
    segmentor_flag = segmentor.load_with_lexicon(os.path.join(default_model_dir, 'cws.model'), user_dict)

    postagger = Postagger()
    postag_flag = postagger.load(os.path.join(default_model_dir, 'pos.model'))

    nlp = NLP()
    thu1 = thulac.thulac()  # 默认模式
    path = r"D:\python-file\北京市旅游知识图谱\\verb-entity\\bj_travel"
    file_list = os.listdir(path)
    f =open('entity_verb_result\\' + "all_entity.json"
                 , 'r', encoding='utf-8')
    file = f.read()
    all_entity = json.loads(file)['all_entity']

    print(all_entity)
    f.close()
    for file_name in file_list:
        print(file_name)
        f = open('D:\python-file\北京市旅游知识图谱\\verb-entity\\bj_travel\\' + file_name
                 , 'r', encoding='utf-8')
        file = f.read()
        #    print(file)
        json_file = json.loads(file)  # 转化为json格式
        text = json_file.get("text")  # 读取text

        #        print(text)
        sentence_list = entity_verb_new.splitSentence(text)  # 将text分为句子列表
        """不需要考虑句子必须含有大景点"""
        # location = file_name.find("_")
        # jingdian_name = file_name[location + 1:]
        # keyWords = []
        # splitKeyWords = thu1.cut(jingdian_name)
        # for splitKeyWord in splitKeyWords:
        #     keyWords.append(splitKeyWord[0])
        # keySentence = getKeySentence(sentence_list, keyWords)
        # split_result = splitWord(keySentence, thu1)  # 使用thulac进行分词
        Entity_in_sentence = entity_verb_new.mapEntity(sentence_list, all_entity,thu1,nlp,segmentor,postagger)
        print(Entity_in_sentence)
        with open('entity_verb_result\\实体+额外名词\\' + file_name[:-3]+"json", 'w',
                  encoding='utf-8') as json_file:
            i = -1
            for line in Entity_in_sentence:
                i +=1
                line_dict = dict()
                if len(line) != 0:
                    line_dict[i] = line
                    json_file.write(json.dumps(line_dict,ensure_ascii=False) + '\n')

        # with open('entity_verb_result\\' + file_name, 'w',
        #           encoding='utf-8') as json_file:
        #     for line in Entity_in_sentence:
        #         if len(line) != 0:
        #             json_file.write(str(line) + '\n')

        f.close()


#!/usr/bin/python3.5

#To-Do List:
# - Change _c_evalution cancel by inputting -1
# - Fix integer number extraction of 0 results queries

from .bing_search_api import BingSearchAPI
from .pos_tagger import POS_tag_cleaner
from collections import Counter # To count the number of occurrances of each word in the n-gram
import math
import re
import sys
import os

"""
Procedure:
----------
- Given an input text file, punctuation and non-ascii characters are removed from the text
- All the unique words from the input file are extracted and total search results are extracted using Bing search API
- The total number of search results for the alphabet 'a' are obtained. Since 'a' is both an alphabet and a word in English, it gives us a
	universe of all English pages from the web which serves us as a denominator when trying to find the probability value
- Given a phrase 'x y z', probability are calculated as follows:
	probability(x y z) = search_result_total(x y z) / search_result_total(a)
	probability(x) = probability of the word 'x' = search_result_total(x) / search_result_total(a)
	probability(y) = probability of the word 'y' = search_result_total(y) / search_result_total(a)
	probability(z) = probability of the word 'z' = search_result_total(z) / search_result_total(a)
- Technique-1:
	If probability(x y z) > (probability(x) * probability(y) * probability (z)) / search_result_total(a),
	we conclude that the phrase 'x y z' is a Collocation
- Technique-2:
	Takes into consideration, the uniqueness of the words in the n-gram into consideration. Details in Thesis documentation OR read the code below

"""

def POS_Check(_n_gram):
	valid_POS_tags = ['NN', 'VB', 'RB', 'JJ']
	for _pos_tag in valid_POS_tags:
		if ('_' + _pos_tag) in _n_gram:
			return True
	return False

def Collocations_Method_5(_bing_api_key, _n_grams_from_input_text_file, _input_file_path, _apply_POS_restrictions, _input_queries_hit_results, _individual_word_hit_results, _corpus, original_input_queries_file, individual_words, _verbose, _p_value = 1.0, _c_value = 1.0, _version = 1):
	if _verbose:
		# A file to save the verbose output of the program
		_output_file_verbose = str(_input_file_path).replace(_input_file_path.split('/')[-1], 'verbose.txt')
		_output_file_verbose = open(_output_file_verbose, 'a')
		print("\n----------------------------------------------------------------------------------------", file = _output_file_verbose)
		print("\tMethod-5: Probability technique - %s - Extracting collocations:" %(_corpus), file = _output_file_verbose)
		print("----------------------------------------------------------------------------------------\n\n", file = _output_file_verbose)
		print("\tMethod-5: Probability technique - %s - Extracting collocations ..." %(_corpus))

	Universe_of_the_webpages = 0
	_n_value = 0
	if _corpus == 'Bing':
		_bing_search = BingSearchAPI(_bing_api_key)
		Universe_of_the_webpages, _bing_api_key = _bing_search.search_total(False, 'a') # A total search result count from Bing.com of the word 'a'
	elif _corpus == 'Wiki_07':
		Universe_of_the_webpages = 1.7 * (10 ** 6)
	elif _corpus == 'Wiki_06':
		Universe_of_the_webpages = 1.5 * (10 ** 6)
	elif _corpus == 'Voice':
		Universe_of_the_webpages = 1695
	elif _corpus == 'Reuters':
		Universe_of_the_webpages = 806791
	elif _corpus == 'DUC':
		Universe_of_the_webpages = 47940
	elif _corpus == 'BNC':
		Universe_of_the_webpages = 4234
	elif _corpus == 'ANC':
		Universe_of_the_webpages = 1695
	elif _corpus == '20NewsGroups':
		Universe_of_the_webpages = 18828

	assert Universe_of_the_webpages != 0, _corpus
	# Python dictionary to save the hit-results of the input search queries
	_hitResults_input_queries = {}
	# A Python dictionary to save the hot-results of the individual words in the input search queries
	_hitResults_individual_words = {}

	if _apply_POS_restrictions:
		_n_grams_from_input_text_file = list(filter(POS_Check, _n_grams_from_input_text_file))

	# If _corpus used is 'Bing', search result totals of the phrases are passes in a dictionary
	if _corpus == 'Bing':
		_hitResults_input_queries = _input_queries_hit_results
	else:
		# Read input from the files, _input_queries_hit_results and _individual_word_hit_results
		#       And append the hit-results to the dictionaries created in the step above
		_input_queries_hitResults = open(_input_queries_hit_results).readlines()
		# Read original search queries
		_original_input_queries = open(original_input_queries_file).readlines()
		if len(_input_queries_hitResults) == len(_original_input_queries):
			for _x in xrange(0, len(_input_queries_hitResults)):
				_each_line = _input_queries_hitResults[_x].strip('\n').replace("Total # Hits for: ", '').split(" = ")
				# Last second element will be the query, enclosed in quotes followed by '~1' as long as the program retuns an exact match.
				#       Some times when an exact match is not found, an approximate match, i.e. hit-results for individual
				#       words are returned. This will be reflected in the search query returned by the program. hence, we
				#       look for an exact match. In these scenarios, query will not be enclosed in quoted
				_hitResults_input_queries[_original_input_queries[_x].strip('\n').replace('"', '').replace('~1', '')] = int(_each_line[-1])
		else:
			print("ERROR: Number of input queries does not match with the number of lucene hit-results.\nIn Method-5.")
			sys.exit(0)

		if _verbose:
			print("Hit results of individual n-grams:\n", _hitResults_input_queries, file = _output_file_verbose)

	# If _corpus used is 'Bing', search result totals of the words are passed in a dictionary
	if _corpus == 'Bing':
		_hitResults_individual_words = _individual_word_hit_results
	else:
		_individual_word_hitResults = open(_individual_word_hit_results).readlines()
		_individual_words = open(individual_words).readlines()

		if len(_individual_word_hitResults) == len(_individual_words):
			for _x in xrange(0, len(_individual_word_hitResults)):
				_each_line = _individual_word_hitResults[_x].strip('\n').replace("Total # Hits for: ", '').split(" = ")
				# Last second element will be the query, enclosed in quotes followed by '~1' as long as the program retuns an exact match.
				#       Some times when an exact match is not found, an approximate match, i.e. hit-results for individual
				#       words are returned. This will be reflected in the search query returned by the program. hence, we
				#       look for an exact match. In these scenarios, query will not be enclosed in quoted
				_hitResults_individual_words[_individual_words[_x].strip('\n').replace('"', '').replace('~1', '')] = int(_each_line[-1])
		else:
			print("ERROR: Number of individual words does not match with the number of lucene hit-results.\nIn Method-5.\n", _corpus)
			sys.exit(0)

		if _verbose:
			print("Hit results of individual words:\n", _hitResults_individual_words, file = _output_file_verbose)

	# Step 1: Calculating individual word probabilities

	word_probabilities = {} # A dictionary to store the probabilities of individual words
	if _verbose:
		print("\nWord \t\t Total search results \t\t Universe page count \t\t Probability", file = _output_file_verbose)
	for _word in _hitResults_individual_words:
		_probability = float(_hitResults_individual_words[_word]) / Universe_of_the_webpages
		word_probabilities[_word] = _probability
		if _verbose:
			print("%s \t\t %s \t\t %d \t\t %f" %(_word, _hitResults_individual_words[_word], Universe_of_the_webpages, _probability), file = _output_file_verbose)

	# Step 2: Calculating n-gram phrase probabilities
	phrase_probabilities = {} # A dictionary to store the probabilities of the n-gram phrases
	if _verbose:
		print("\nN-gram \t\t Total search results \t\t Universe page count \t\t Probability", file = _output_file_verbose)
	for n_gram in _n_grams_from_input_text_file:
		# Remove the POS tags attached
		_n_gram = POS_tag_cleaner(n_gram)
		if _n_value <= 0:
			_n_value = len(n_gram.split())
		#_n_gram = n_gram + ' '
		#_n_gram = re.sub(r'_.*? ', ' ', _n_gram.strip('\n').lstrip(' ')).rstrip(' ')
		try:
			n_gram_search_total = _hitResults_input_queries[_n_gram]
		except Exception as e:
			print(str(e))

		n_gram_probability = float(n_gram_search_total) / Universe_of_the_webpages
		if _verbose:
			print("%s \t\t %s \t\t %d \t\t %f" %(_n_gram, n_gram_search_total, Universe_of_the_webpages, n_gram_probability), file = _output_file_verbose)
		phrase_probabilities[_n_gram] = n_gram_probability

	Collocations_without_uniqueness = [] # A python list to store n-grams that are collocations, without uniqueness
	Just_ngrams_without_uniqueness = [] # A python script to store n-grams that are not collocations, without uniqueness

	_evaluate_value = pow(float(_c_value), pow(_n_value, _p_value) - 1)
	if _verbose:
		print(("\nTechnique-1: Uniqueness of words in the n-gram is not taken into consideration.\n\n"
			"N-gram \t\tProbability \t\t Product of individual word probabilities"), file = _output_file_verbose)
	for n_gram in phrase_probabilities:

		if _verbose:
			print("N value is %d" % (_n_value))
			print("Evaluate value is: " + str(_evaluate_value))

		_words_in_ngram = n_gram.lstrip(' ').rstrip(' ').strip('\n').split()
		product_of_individual_word_probabilities = 1
		for _word in _words_in_ngram:
			_word = _word.lstrip(' ').rstrip(' ').strip('\n')
			if _word in word_probabilities:
				product_of_individual_word_probabilities *= word_probabilities[_word]
			else: # If the word is not present in the word dictionary already, then probability is 0
				product_of_individual_word_probabilities *= float(0) / Universe_of_the_webpages
		if _verbose:
			print(n_gram, ' \t\t ', phrase_probabilities[n_gram], ' \t\t ', product_of_individual_word_probabilities, file = _output_file_verbose)
		if phrase_probabilities[n_gram] > _evaluate_value * product_of_individual_word_probabilities:
			Collocations_without_uniqueness.append(n_gram)
		else:
			Just_ngrams_without_uniqueness.append(n_gram)

	# Technique-2: Taking uniqueness of words into consideration
	Collocations_with_uniqueness = [] # A python list to store n-grams that are collocations
	Just_ngrams_with_uniqueness = [] # A python script to store n-grams that are not collocations

	if _verbose:
		print("\nTechnique-2: Uniqueness of words in n_gram is taken into consideration.\n", file = _output_file_verbose)
	for n_gram in phrase_probabilities:
		_words_in_ngram = n_gram.split() # Splits the n-gram at each occurrance of a space

		unique_word_count = len(set(re.findall('\w+', n_gram.lower()))) # counts the total number of unique words in a given collocation
		n_factorial = math.factorial(len(_words_in_ngram))

		if unique_word_count == len(_words_in_ngram):
			value_to_multiply_product_of_individual_probabilities = float(1) / n_factorial
			if _verbose:
				print("N-gram '%s' has unique words. n! = %d" %(n_gram, n_factorial), file = _output_file_verbose)
		else:
			factorial_of_unique_words = 1 # Store the product, 1! * 2! * 3! ... k! for 'k' unique words in the n-gram
			word_count = Counter(_words_in_ngram) # counts the number of times each word appears in the n-gram
			for _word in word_count:
				factorial_of_unique_words *= math.factorial(word_count[_word])
				if _verbose:
					print(_word, word_count[_word], factorial_of_unique_words, file = _output_file_verbose)
			value_to_multiply_product_of_individual_probabilities = float(factorial_of_unique_words) / n_factorial
			if _verbose:
				print("N-gram '%s' has repeated words! n! = %d, product of individual factorials = %d " %(n_gram, n_factorial, factorial_of_unique_words), file = _output_file_verbose)

		product_of_individual_word_probabilities = 1
		for _word in _words_in_ngram:
			_word = _word.lstrip(' ').rstrip(' ').strip('\n')
			try:
				product_of_individual_word_probabilities *= word_probabilities[_word]
			except Exception as e:
				print("ERROR: Method-5 \n\t%s" %(str(e)))
				product_of_individual_word_probabilities *= 1
		if _verbose:
			print(n_gram, phrase_probabilities[n_gram], value_to_multiply_product_of_individual_probabilities * product_of_individual_word_probabilities, file = _output_file_verbose)

		if phrase_probabilities[n_gram] > _evaluate_value * value_to_multiply_product_of_individual_probabilities * product_of_individual_word_probabilities:
			Collocations_with_uniqueness.append(n_gram)
		else:
			Just_ngrams_with_uniqueness.append(n_gram)

	if _version == 1:
		return Collocations_without_uniqueness, _bing_api_key
	else:
		return Collocations_with_uniqueness, _bing_api_key


from numpy import array, argpartition, savetxt
from pandas import DataFrame
import csv

def create_header(self, out):

    '''Creates the Header column
    On the first round creates the header columns
    for the experiment output log.
    '''

    _rr_out = {}

    _rr_out['round_epochs'] = []
    for key in out.history.keys():
        _rr_out[key] = []
    for key in self.params.keys():
        _rr_out[key] = []

    for key in out.history.keys():
        self.peak_epochs[key] = []

    return _rr_out

def run_round_results(self, out):

    '''THE MAIN FUNCTION FOR CREATING RESULTS FOR EACH ROUNDself.
    Takes in the history object from model.fit() and handles it.

    NOTE: The epoch level data will be dropped here each round.

    '''

    _rr_out = self.result

    self._round_epochs = len(list(out.history.values())[0])

    # otherwise proceed to create the value row
    _rr_out['round_epochs'].append(self._round_epochs)

    # iterates through the keys and records last or peak for metrics
    for key in out.history.keys():
        t_t = array(out.history[key])

        # this handles metrics (NOTE: 'acc' have to be in metric name)
        if 'acc' in key:
            best_epoch = argpartition(t_t, len(t_t) - 1)[-1]

        # this handles losses (takes minimum value epoch)
        else:
            best_epoch = argpartition(t_t, 0)[0]

        if self.last_epoch_value:
            value_to_report = out.history[key][-1]
        else:
            value_to_report = array(out.history[key])[best_epoch]

        _rr_out[key].append(value_to_report)

        # this takes care of the separate entity with just peak epoch data
        self.peak_epochs[key].append(best_epoch)

    for key in self.round_params.keys():
        _rr_out[key].append(self.round_params[key])

def save_result(self):
    '''SAVES THE RESULTS/PARAMETERS TO A CSV SPECIFIC TO THE EXPERIMENT'''

    with open(self.experiment_name + '.csv', 'w') as f:
        w = csv.DictWriter(f, self.result.keys())
        w.writeheader()
        w.writerow(self.result)

def result_todf(self):
    '''ADDS A DATAFRAME VERSION OF THE RESULTS TO THE CLASS OBJECT'''

    self.result = DataFrame(self.result)

    return self

def peak_epochs_todf(self):

    return DataFrame(self.peak_epochs)


"""
Written by Shoyo Inokuchi (June 2019)

Scripts for the acoustic keylogger surrounding cluster labeling.
Repository is located at: https://github.com/shoyo-inokuchi/acoustic-keylogger
"""
from .audio_processing import *
from .unsupervised import extract_features

import numpy as np
from sklearn.manifold import TSNE
from sklearn.preprocessing import MinMaxScaler

import time

def signal_to_digit_sequence(signal, sr=44100, output=True):
    """Converts an audio signal to a sequence of digits corresponding to the
    clusters that each keystroke was categorized into.

    Example:
        Original Signal
            => 'this is an example.'
        Returned Sequence
            => [0, 1, 2, 3, 4, 2, 3, 4, 5, 6, 4, 7, 8, 5, 9, 10, 11, 12, 13]

    Explanation:
        The original signal contains 14 unique keystrokes ('t', 'h', 'i', etc),
        each of which are presumably grouped into separate clusters. Each
        cluster is given a numeric label, and the returned sequence is a
        one-to-one mapping of the original keystrokes to their respective
        cluster labels.

        Note that the returned sequence above is meant to merely illustrate the
        pattern in which the original signal is mapped. In practice, the first
        unique letter does not necessarily map to '0', the second unique letter
        to '1' etc.
    """
    # Keystroke Detection
    t0 = time.time()
    if output: print('> Detecting keystrokes', end='')
    input_vec = detect_keystrokes(signal, sample_rate=sr)
    if output: print(f' => took {time.time() - t0}s')

    # Feature Extraction
    t0 = time.time()
    if output: print('> Extracting features', end='')
    feature_vec = np.array([extract_features(key) for key in input_vec])
    if output: print(f' => took {time.time() - t0}s')

    # Scaling Features
    t0 = time.time()
    if output: print('> Scaling features', end='')
    scaled_vec = MinMaxScaler().fit_transform(feature_vec)
    if output: print(f' => took {time.time() - t0}s')

    # Dimensionality Reduction with t-SNE
    t0 = time.time()
    if output: print(f'> Performing dimensionality reduction', end='')
    embedded_vec = TSNE().fit_transform(scaled_vec)
    if output: print(f' => took {time.time() - t0}s')

    # Clustering t-SNE output

    # Add label to return vector

    ret = np.empty_like(keys_raw)


from tinyms import layers
from tinyms.primitives import OnesLike

class CombinedGenerator(layers.Layer):
    """
    Generator of CycleGAN, return fake_A, fake_B, rec_A, rec_B, identity_A and identity_B.

    Args:
        G_A (layers.Layer): The generator network of domain A to domain B.
        G_B (layers.Layer): The generator network of domain B to domain A.
        use_identity (bool): Use identity loss or not. Default: True.

    Returns:
        Tensors, fake_A, fake_B, rec_A, rec_B, identity_A and identity_B.
    """

    def __init__(self, G_A, G_B, use_identity=True):
        super().__init__()
        self.G_A = G_A
        self.G_B = G_B
        self.ones = OnesLike()
        self.use_identity = use_identity

    def construct(self, img_A, img_B):
        """If use_identity, identity loss will be used."""
        fake_A = self.G_B(img_B)
        fake_B = self.G_A(img_A)
        rec_A = self.G_B(fake_B)
        rec_B = self.G_A(fake_A)
        if self.use_identity:
            identity_A = self.G_B(img_A)
            identity_B = self.G_A(img_B)
        else:
            identity_A = self.ones(img_A)
            identity_B = self.ones(img_B)
        return fake_A, fake_B, rec_A, rec_B, identity_A, identity_B


# -*- coding: utf-8 -*-
"""
Created on Sat Jun  6 16:42:15 2020

@author: dell
"""

from textblob import TextBlob
import numpy as np
import re
import pandas as pd
data=pd.read_csv("tweet.tsv",sep="\t",quoting=3)
def clean(text):
    text=re.sub(r'@[A-Za-z0-9]+','',text)#remove @
    text=re.sub(r'#','',text)#remove #

    text=re.sub(r'RT[\s]+','',text)#remove RT(Re Tweet)
    text=re.sub(r'https?:\/\/\S+','',text)#remove links
    text=re.sub(r'[^A-Za-z]',' ',text)
    return text
def polarity(text):
    return TextBlob(text).sentiment.polarity
def rate(score):
    if score>=0:
        return 1 #non toxic
    else:
        return 0 #toxic
data["text"]=data["text"].apply(clean)
data['text'].replace('', np.nan, inplace=True)
data.dropna(inplace=True)
data["polarity"]=data["text"].apply(polarity)
data["review"]=data["polarity"].apply(rate)
x=data["text"].values
y=data["review"].values
import nltk
nltk.download("stopwords")
from nltk.corpus import stopwords

nltk.download('wordnet')
from nltk.stem import WordNetLemmatizer #containing only root words
corpus=[]
for i in range(0,1326):
    text=x[i]
    text=text.lower()
    text=text.split()
    wl=WordNetLemmatizer()
    all_stopwords=stopwords.words("english")
    all_stopwords.remove("not")
    all_stopwords.remove("no")
    text=[wl.lemmatize(word) for word in text if not word in set(all_stopwords)] #steeming nd removing stop words
    text=' '.join(text)
    corpus.append(text)
#creating bag of words
from sklearn.feature_extraction.text import CountVectorizer
cv=CountVectorizer(max_features=3500)
X=cv.fit_transform(corpus).toarray()
from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test=train_test_split(X,y,random_state=0,stratify=y,test_size=0.35)

from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix

from sklearn.ensemble import RandomForestClassifier
rf=RandomForestClassifier()
rf.fit(x_train,y_train)
ypred=rf.predict(x_test)
accuracy=accuracy_score(ypred,y_test)
print("accuracy score=",accuracy*100)
print("accuracy=",confusion_matrix(y_test,ypred))


import os
import time

Import("env_base")
env = env_base.Copy()
#================== Perform local customization here ==================

# Local Variables
src_top=env.AbsDirPathInSou('#')

if env['MACHINE'] == 'LX86':
   MscArch='linux32'
elif env['MACHINE'] == 'LX8664':
   MscArch='linux64'
elif env['MACHINE'] == 'WINNT':
   MscArch='win32'
elif env['MACHINE'] == 'WIN8664':
   MscArch='win64'
else:
   MscArch=env['MACHINE']

libdir=src_top + os.sep + 'lib' + os.sep + MscArch + os.sep

# scons build environment
env.Append(CPPDEFINES=['DRESP3_SERVER', '_MSCUTIL', '_OEM_NASTRAN'])
env.Append(MSCFPP_DEFINES = ['_IMPLICITNONE', 'DRESP3_SERVER'])
env.Append(CPPPATH=[src_top + os.sep + 'include'])

# Compile Flags
if env['MACHINE'].startswith('LX86'):
  env.Append(CCFLAGS='-fPIC')
  env.Append(FORTRANFLAGS='-fPIC')
elif env_base['MACHINE'] == 'linuxipf':
  env.Append(CCFLAGS='-fPIC')
  env.Append(FORTRANFLAGS='-fPIC')
elif env_base['MACHINE'] == 'hpux':
  env.Append(CCFLAGS='+Z')
  env.Append(FORTRANFLAGS='+pic=long')

# Link Libraries
env.AddLinkSource(libdir+'cnxx'+env['OBJSUFFIX'])
env.AddLinkSource(libdir+'initgmsrvcmns'+env['OBJSUFFIX'])
env.AddLinkSource(libdir+'main'+env['OBJSUFFIX'])
env.AddLinkSource(libdir+'semd'+env['OBJSUFFIX'])

env.AddLinkLibrary(libdir + 'dr3srv')

if env['MACHINE'] == 'hpux':
  env.Append(LIBS=['nsl', 'pthread','c','dld'])

elif env['MACHINE'] == 'hpuxipf':
  env.Append(LIBS=['nsl'])

elif env['MACHINE'] == 'irix':
  env.Append(LIBS=['fortran'])

elif env['MACHINE'] == 'solaris':
  env.Append(LIBS=['aio','rt','sunmath','socket','nsl','intl'])
  env.Append(LIBS=['Cstd','Crun','malloc','dl','mvec'])

elif env['MACHINE'].startswith('WIN'):
  env.AddLinkLibrary(src_top + '/../../api/' + MscArch + '/xdr')
  env.Append(LIBS=['netapi32', 'Advapi32', 'ws2_32'])
  env.AddLinkSource(libdir+'getlserm'+env['OBJSUFFIX'])

if env['MACHINE'] == 'solaris8664':
  if env['MODE_I8']:
    env.Append(LIBS=['sunquad'])

env.BuildProgram('pcbuck',  aliases=None, tree='apps', fortranmain=False, fortranlibs=True)

#================== Perform local customization here ==================
retval = env.ProcessDir(env_base)
Return('retval')


# Want to ask someone's birthday and validate the data

import webapp2
import cgi

form="""
<form method="post">
    What is your birthday?

     Month <input type="text" name="month" value="%(month)s">
     Day <input type="text" name="day" value="%(day)s">
     Year <input type="text" name="year" value="%(year)s">

%(error)s

    <input type="submit">
</form>
"""

valid_msg = """
Thanks! That's a valid date.
"""

months = ['January',
          'February',
          'March',
          'June',
          'July',
          'August',
          'September',
          'October'
          'November',
          'December']

class MainPage(webapp2.RequestHandler):
    def write_form(self, error="", month="", day="", year=""):
        self.response.out.write(form % {"error": error,
                                        "month": self.escape_html(month),
                                        "day": self.escape_html(day),
                                        "year": self.escape_html(year)})

    def get(self):
        self.write_form()

    def post(self):
        user_month = self.request.get('month')
        user_day = self.request.get('day')
        user_year = self.request.get('year')

        month = self.valid_month(user_month)
        day = self.valid_day(user_day)
        year = self.valid_year(user_year)

        if not (month and day and year):
            self.write_form("Please, enter a valid date.", user_month, user_day, user_year)
        else:
            self.response.out.write(valid_msg)

    def valid_month(self, month):
        if month:
            cap_month = month.capitalize()
            if cap_month in months:
                return cap_month

    def valid_day(self, day):
        if day and day.isdigit():
            day = int(day)
            if day > 0 and day < 32:
                return day

    def valid_year(self, year):
        if year and year.isdigit():
            year=int(year)
            if year > 1900 and year < 2020:
                return year

    def escape_html(self, s):
        return cgi.escape(s, quote = True)

app = webapp2.WSGIApplication([('/', MainPage)], debug=True)


#!/usr/bin/env python
# -*- coding:utf-8 -*-

import sys

__author__ = "Onceabu"
__version__ = "v2.0"

"""
    Time
    describe
    copyright (c) 2019 by Abu
"""

reload(sys)
sys.setdefaultencoding('utf-8')

"""
# Definition for a Node.
class Node(object):
    def __init__(self, val=None, children=None):
        self.val = val
        self.children = children
"""

class Solution(object):

    # 方法一 递归遍历所有的子节点 最后再添加当前根节点的值
    # 当然如果是前序遍历就先添加当前根节点的值再遍历子节点
    def postorder(self, root):
        """
        :type root: Node
        :rtype: List[int]
        """
        res = []
        if root is None:
            return res

        def recursive(root, res):
            for c in root.children:
                recursive(c, res)
            res.append(root.val)  # 如果是前序遍历就放到for循环上面

        recursive(root, res)
        return res

    # 方法二 利用栈，以后序遍历的顺序将各个节点入栈，然后逐个出栈记录val，最后倒序 时间复杂度应该是O(1)
    def _postorder(self, root):
        res = []
        if root is None:
            return res
        stack = [root]
        while stack:
            cur = stack.pop()
            res.append(cur.val)
            stack.extend(cur.children)
        return res[::-1]