#!/usr/bin/env python
from __future__ import absolute_import, print_function

import sys
import argparse

import numpy as np
import re
from numpy.linalg import norm

try:
    import matplotlib.pyplot as plt
except ImportError:
    print("Matplotlib not installed; do not use plotting functions.")

# PN modules from dislopy package
from dislopy.pn import pn_1D as pn1
from dislopy.pn import pn_2D as pn2
from dislopy.pn import fit_gsf as fg
from dislopy.pn import peierls_barrier as pb
from dislopy.pn import energy_coeff as coeff
from dislopy.pn import visualise_pn as vp
from dislopy.atomic import aniso
from dislopy.utilities.control_functions import control_file, from_mapping, change_type,  \
                                            to_bool, change_or_map, print_control

atomic_to_GPa =  160.2176487

def to_vector(in_str, datatype=float):
    '''Converts <in_str>, which is a list of numbers, in an array with elements
    of type float.
    '''

    new_vec = []
    for x in re.finditer('-?\d+\.?\d*', in_str):
        new_vec.append(datatype(x.group()))

    return np.array(new_vec, dtype=datatype)

def to_int_vector(in_str):
    '''Calls <to_vector> with datatype=int. Used for parsing the PN input file.
    '''

    arr = to_vector(in_str, datatype=int)
    return arr

def handle_pn_control(param_dict):
    '''handle each possible card in turn. If default value is <None>, the card is
    considered "mission critical" and the program will abort if a value is not
    provided.

    TO DO: List defining control parameters
    '''

    # cards for the <&control> namelist. If <run_sim> is False, we won't actually
    # run the PN simulation. Do this only if you want to visualise the results
    # without re-running the simulation
    control_cards = (('run_sim', {'default':True, 'type':to_bool}),
                     ('gsf_file', {'default':None, 'type':str}),
                     ('output', {'default':'dislopyPN.out', 'type':str}),
                     ('title_line', {'default':'Peierls-Nabarro model', 'type':str}),
                     ('n_iter', {'default':1, 'type':int}),
                     ('dimensions', {'default':2, 'type':int}),
                     ('max_x', {'default':100, 'type':int}),
                     ('n_funcs', {'default':np.nan, 'type':int}),
                     ('disl_type', {'default':'edge', 'type':str}),
                     ('use_sym', {'default':True, 'type':to_bool}),
                     ('plot', {'default':False, 'type':to_bool}),
                     ('plot_name', {'default':'pn_plot.tif', 'type':str}),
                     ('plot_both', {'default':False, 'type':to_bool}),
                     ('visualize', {'default':False, 'type':to_bool}),
                     ('record', {'default':True, 'type':to_bool}),
                     ('nplanes', {'default':20, 'type':int}),
                     ('noisy', {'default': False, 'type': to_bool}),
                   )

    # cards for the <&struc> namelist. Need to find some way to incorporate
    # anisotropic elasticity
    struc_cards = (('a', {'default':None, 'type':float}),
                   ('b', {'default':'map struc > a: x -> x', 'type':float}),
                   ('burgers', {'default':'map struc > a: x -> x', 'type':float}),
                   ('spacing', {'default':'map struc > a: x -> x', 'type':float}),
                   ('parameters', {'default':np.array([]), 'type':to_vector})
                  )

    # cards for the &elast namelist. <k_para> and <k_norm> allow the user to
    # supply predetermined elastic coefficients for the systems parallel and
    # perpendicular to the Burgers vector
    elast_cards = (('bulk', {'default':None, 'type':float}),
                   ('shear', {'default':None, 'type':float}),
                   ('poisson', {'default':None, 'type':float}),
                   ('k_parallel', {'default':None, 'type':float}),
                   ('k_normal', {'default':np.nan, 'type':float}),
                   ('cij', {'default':None, 'type':aniso.readCij}),
                   ('b_edge', {'default':np.array([1., 0., 0.]), 'type':to_vector}),
                   ('b_screw', {'default':np.array([0., 0., 1.]), 'type':to_vector}),
                   ('normal', {'default':np.array([0., 1., 0.]), 'type':to_vector}),
                   ('in_gpa', {'default':True, 'type':to_bool})
                  )

    # cards for the <&surface> namelist. Where dimensions == 1 but the gsf
    # file is two-dimensional (ie. a gamma surface), <map_ux> corresponds to the
    # gamma line direction. If <dimensions> == 2, <map_ux> (<max_uy>) is the
    # direction of the edge (screw) component of the displacement.
    surf_cards = (('x_length', {'default':'map struc > a: x -> x', 'type':float}),
                  ('y_length', {'default':'map struc > b: x -> x', 'type':float}),
                  ('angle', {'default':np.pi/2, 'type':float}),
                  ('map_ux', {'default':'remap: (ux, uy) -> ux', 'type':str}),
                  ('map_uy', {'default':'remap: (ux, uy) -> uy', 'type':str}),
                  ('gamma_shift', {'default':0., 'type':float}),
                  ('has_vacuum', {'default':True, 'type':to_bool}),
                  ('do_fourier', {'default':False, 'type':to_bool}),
                  ('fourier_N', {'default':3, 'type': int}),
                  ('fourier_M', {'default':-1, 'type':int})
                 )

    # cards for the <&properties> namelist
    prop_cards = (('max_rho', {'default':True, 'type':to_bool}),
                  ('width', {'default':True, 'type':to_bool}),
                  ('center', {'default':True, 'type':to_bool})
                 )

    # cards for the <&stress> namelist
    stress_cards = (('calculate_stress', {'default':True, 'type':to_bool}),
                    ('calculate_barrier', {'default':False, 'type':to_bool}),
                    ('dtau', {'default':0.0001, 'type':float}),
                    ('use_GPa', {'default':True, 'type':to_bool}),
                    ('threshold', {'default':0.5, 'type':float})
                   )

    # cards for the <&visualize> namelist
    vis_cards = (('xyz_name', {'default':'pn.xyz', 'type':str}),
                 ('unitcell', {'default':'', 'type':str}),
                 ('path_to_cell', {'default':'./', 'type':str}),
                 ('threshold', {'default':1., 'type':float}),
                 ('sym_thresh', {'default':0.3, 'type':float}),
                 ('radius', {'default':10., 'type':float}),
                 ('program', {'default':'', 'type':str}),
                 ('shift', {'default':np.zeros(3), 'type':to_vector}),
                 ('permutation', {'default':np.array([0, 1, 2], dtype=int), 'type':to_int_vector}),
                 ('thickness', {'default': 1, 'type':int})
                )

    # list of valid namelists
    namelists = ['control', 'struc', 'elast', 'surface', 'properties', 'stress',
                                                        'visualize']
    # check that all namelists were in the control file and add them as keys
    # (with empty dicts as values) in the control dictionary
    for name in namelists:
        if name not in param_dict.keys():
            param_dict[name] = dict()

    # handle the input namelists, except for the elasticity namelist, which
    # requires special handling.
    for i, cards in enumerate([control_cards, struc_cards, elast_cards, surf_cards,
                                               prop_cards, stress_cards, vis_cards]):
        if namelists[i] == 'elast':
            # handle the elastic parameters later
            continue
        # else
        for var in cards:
            try:
                change_or_map(param_dict, namelists[i], var[0], var[1]['type'])
            except ValueError:
                default_val = var[1]['default']
                # test to see if variable is "mission-critical" using None != None
                if default_val is None:
                    raise ValueError("No value supplied for mission-critical" +
                                                 " variable {}.".format(var[0]))
                else:
                    param_dict[namelists[i]][var[0]] = default_val
                    # if <default_val> is a mapping, needs to be evaluated
                    if type(default_val) == str and 'map' in default_val:
                        from_mapping(param_dict, namelists[i], var[0])

    # extract values from the &elast namelist
    for var in elast_cards:
        try:
            change_or_map(param_dict, 'elast', var[0], var[1]['type'])
        except ValueError:
            param_dict['elast'][var[0]] = var[1]['default']

    # test to make sure that all of the parameters required for one of isotropic
    # or anisotropic elasticity have been supplied. Should we also test to see
    # which function to use when calculating the energy coefficients?
    if not (param_dict['elast']['k_parallel'] is None):
        param_dict['elast']['coefficients'] = 'specific'
    elif not (param_dict['elast']['cij'] is None):
        # if an elastic constants tensor is given, use anisotropic elasticity
        param_dict['elast']['coefficients'] = 'aniso'
    elif not (param_dict['elast']['shear'] is None):
        # test to see if another isotropic elastic property has been provided.
        # Preference poisson's ratio over bulk modulus, if both have been provided
        if not (param_dict['elast']['poisson']  is None):
            param_dict['elast']['coefficients'] = 'iso_nu'
        elif not (param_dict['elast']['bulk'] is None):
            param_dict['elast']['coefficients'] = 'iso_bulk'
        else:
            raise AttributeError("No elastic properties have been provided.")
    else:
        raise AttributeError("No elastic properties have been provided.")

    return

class PNSim(object):
    # handles the control file for a PN simulation in <dislopy>

    def __init__(self, filename):
        self.sim = control_file(filename)
        handle_pn_control(self.sim)

        # functions to access values in namelists easily
        self.control = lambda card: self.sim['control'][card]
        self.struc = lambda card: self.sim['struc'][card]
        self.elast = lambda card: self.sim['elast'][card]
        self.surf = lambda card: self.sim['surface'][card]
        self.prop = lambda card: self.sim['properties'][card]
        self.stress = lambda card: self.sim['stress'][card]
        self.vis = lambda card: self.sim['visualize'][card]

        if self.control('run_sim'):
            # calculate the energy coefficients
            if self.elast('coefficients') == 'specific':
                self.K = coeff.predefined(self.elast('k_parallel'), self.elast('k_normal'),
                                                    using_atomic=not(self.elast('in_gpa')))
            elif self.elast('coefficients') == 'aniso':
                self.K = coeff.anisotropic_K(self.elast('cij'),
                                             self.elast('b_edge'),
                                             self.elast('b_screw'),
                                             self.elast('normal'),
                                             using_atomic=not(self.elast('in_gpa'))
                                            )
            elif self.elast('coefficients') == 'iso_nu':
                self.K = coeff.isotropic_nu(self.elast('poisson'), self.elast('shear'),
                                                 using_atomic=not(self.elast('in_gpa')))
            elif self.elast('coefficients') == 'iso_bulk':
                self.K = coeff.isotropic_K(self.elast('bulk'), self.elast('shear'),
                                               using_atomic=not(self.elast('in_gpa')))

            # if restricting calculation to one component of displacement, extract
            # the appropriate energy coefficient.
            if self.control('dimensions') == 1:
                # store a copy of the edge and screw energy coefficients
                self.K_store = self.K
                if self.control('disl_type').lower() == 'edge':
                    self.K = self.K[0]
                else:
                    # for the moment, default to shear
                    self.K = self.K[1]

            # construct the gamma surface/gamma line
            if self.control('noisy'):
                print("Constructing gamma surface/line")
            self.construct_gsf()

            # calculate fit parameters and energy of the dislocation
            if self.control('noisy'):
                print("Relaxing dislocation core structure.")

            self.run_sim()

            if self.control('noisy'):
                print("Core relaxation complete.")

            # calculate Peierls stress and Peierls barrier, if requested
            if self.control('noisy'):
                print('Calculating Peierls stress.')

            self.peierls()

            if self.control('noisy'):
                print('Peierls stress calculated.')

            # do post-processing and/or plotting, if requested by the user
            if self.control('noisy'):
                print('Doing post-processing.')
            self.post_processing()

            # write results to ouput
            self.write_output()

        if self.control('visualize'):
            if self.control('noisy'):
                print('Visualising core structure.')
            self.visualise()

    def run_sim(self):
        '''Calculates an optimum(ish) dislocation structure.
        '''

        # check to see if the user has provided parameters specifying a trial
        # disregistry profile (in the order A x0 c).
        if len(self.struc('parameters')) != 0:
            # use supplied parameters and perform only a single energy minimization
            inpar = self.struc('parameters')
            niter = 1
        else:
            # no parameters -> tell program that it should generate a trial
            # configuration
            inpar = None
            niter = self.control('n_iter')

        if self.control('dimensions') == 1:
            # calculate dislocation energy and disregistry profile
            self.E, self.par = pn1.run_monte1d(
                                               niter,
                                               self.control('n_funcs'),
                                               self.K,
                                               max_x=self.control('max_x'),
                                               energy_function=self.gsf,
                                               use_sym=self.control('use_sym'),
                                               b=self.struc('burgers'),
                                               spacing=self.struc('spacing'),
                                               noisy=self.control('noisy'),
                                               params=inpar
                                              ) # Done Monte Carlo 1D

        elif self.control('dimensions') == 2:
            self.E, self.par = pn2.run_monte2d(
                                               niter,
                                               self.control('n_funcs'),
                                               self.control('disl_type'),
                                               self.K,
                                               max_x=self.control('max_x'),
                                               energy_function=self.gsf,
                                               use_sym=self.control('use_sym'),
                                               b=self.struc('burgers'),
                                               spacing=self.struc('spacing'),
                                               noisy=self.control('noisy'),
                                               params=inpar
                                              ) # Done Monte Carlo 2D

        return

    def construct_gsf(self):
        '''Makes a function mapping disregistry values to misfit energies.
        '''

        gsf_grid, self.units = fg.read_numerical_gsf(self.control('gsf_file'))

        # determine which fitting function to use from the shape of <gsf_grid>
        # original test was self.control("dimensions")
        n_columns = np.shape(gsf_grid)[-1]
        if n_columns == 2:
            self.gsf = fg.spline_fit1d(gsf_grid, self.surf('x_length'),
                                                 self.surf('y_length'),
                                                 angle=self.surf('angle'),
                                                 units=self.units,
                                                 hasvac=self.surf('has_vacuum'),
                                                 do_fourier_fit=self.surf('do_fourier'),
                                                 n_order=self.surf('fourier_N'))

        elif n_columns == 3: # 2-dimensional misfit function
            # fit the gamma surface
            if self.control('dimensions') == 2:
                if self.surf('fourier_M') < 0:
                    m_order = self.surf('fourier_N')
                else:
                    m_order = self.surf('fourier_M')

                base_func = fg.spline_fit2d(gsf_grid, self.surf('x_length'),
                                                      self.surf('y_length'),
                                                   angle=self.surf('angle'),
                                                           units=self.units,
                                             hasvac=self.surf('has_vacuum'),
                                     do_fourier_fit=self.surf('do_fourier'),
                                             n_order=self.surf('fourier_N'),
                                             m_order=m_order)

                self.gsf = fg.new_gsf(base_func, self.surf('map_ux'), self.surf('map_uy'))

            elif self.control('dimensions') == 1:
                # begin by performing a spline fit to the surface
                base_func = fg.spline_fit2d(gsf_grid, self.surf('x_length'),
                                                      self.surf('y_length'),
                                                   angle=self.surf('angle'),
                                                           units=self.units,
                                             hasvac=self.surf('has_vacuum'),
                                                       do_fourier_fit=False)

                # remap input so that x and y correspond to ux and uy
                temp_gsf = fg.new_gsf(base_func, self.surf('map_ux'), self.surf('map_uy'))

                # project out the dislocation parallel component
                # determine which axis to use (0/x if edge, 1/y if screw)
                if self.control('disl_type') == 'edge':
                    use_axis = 0
                elif self.control('disl_type') == 'screw':
                    use_axis = 1

                temp_1d = fg.projection(temp_gsf, self.surf('gamma_shift'), use_axis)

                # set the correct periodicity
                temp_1d2 = lambda x: temp_1d(x % self.struc('burgers'))

                if self.surf('do_fourier'):
                    # fit a fourier series to <temp_1d2>.
                    self.gsf = fg.gline_fourier(temp_1d, self.surf('fourier_N'),
                                                           self.struc('burgers'))
                else:
                    self.gsf = temp_1d2

        else:
            raise ValueError("GSF grid has invalid number of dimensions")

        return

    def post_processing(self):
        '''Calculate the displacement field and dislocation density, record these
        values in appropriate output files, plot them (if requested to by the
        user), and then calculate the dislocation height and width.
        '''

        if (self.control('plot') or self.prop('max_rho') or self.prop('width') or
                                                        self.control('record')):

            if self.control('record'):
                # open output files for the disregistry and dislocation density
                outstreamu = open('disreg.{}'.format(self.control('output')), 'w')
                outstreamrho = open('rho.{}'.format(self.control('output')), 'w')

            # lattice points
            r = self.struc('spacing')*np.arange(-self.control('max_x'), self.control('max_x'))

            # calculate misfit profile and dislocation density and write to output
            # files
            if self.control('dimensions') == 1:
                self.ux = pn1.get_u1d(self.par, self.struc('burgers'),
                                   self.struc('spacing'), self.control('max_x'))

                # record disregistry and dislocation density
                if self.control('record'):
                    # header for output files
                    outstreamu.write('# r (\AA) u (\AA)\n')
                    outstreamrho.write('# r (\AA) rho\n')

                    # record values of rho and u at each lattice point
                    rho = pn1.rho(self.ux, r)
                    for i in range(len(r)):
                        outstreamu.write('{:.3f} {:.6f}\n'.format(r[i], self.ux[i]))
                        if i != 0:
                            # rho not defined for first lattice plane
                            # Record at intervals between lattice planes
                            ri = 0.5*(r[i]+r[i-1])
                            outstreamrho.write('{:.3f} {:.6f}\n'.format(ri, rho[i-1]))

                    outstreamu.close()
                    outstreamrho.close()

            elif self.control('dimensions') == 2:
                self.ux, self.uy = pn2.get_u2d(self.par, self.struc('burgers'),
                                    self.struc('spacing'), self.control('max_x'),
                                                      self.control('disl_type'))

                # record both components of disregistry and dislocation density
                if self.control('record'):
                    # header lines
                    outstreamu.write('# r (\AA) ux (\AA) uy (\AA)\n')
                    outstreamrho.write('# r (\AA) rhox  rhoy\n')

                    # record values of u and rho at each lattice point
                    rhox = pn1.rho(self.ux, r)
                    rhoy = pn1.rho(self.uy, r)
                    for i in range(len(r)):
                        outstreamu.write('{:.3f} {:.6f} {:.6f}\n'.format(r[i], self.ux[i],
                                                                              self.uy[i]))
                        if i != 0:
                            # as above, rho not defined for first lattice plane.
                            # Record at intervals between lattice planes
                            ri = 0.5*(r[i]+r[i-1])
                            outstreamrho.write('{:.3f} {:.6f} {:.6f}\n'.format(ri,
                                                                rhox[i-1], rhoy[i-1]))

                    outstreamu.close()
                    outstreamrho.close()

            # create plot of dislocation density and misfit profile
            if self.control('plot'):
                if self.control('dimensions') == 1:
                    self.fig, self.ax = pn1.plot_both(self.ux, r, self.struc('burgers'),
                                                        self.struc('spacing'),
                                                        nplanes=self.control('nplanes'))
                    plt.savefig(self.control('plot_name'))
                    plt.close()
                elif self.control('plot_both'): # dimension == 2
                    self.fig, self.ax = pn1.plot_both(self.ux, r, self.struc('burgers'),
                                                               self.struc('spacing'),
                                                        nplanes=self.control('nplanes'))
                    plt.savefig('edge.' + self.control('plot_name'))
                    plt.close()
                    self.fig, self.ax = pn1.plot_both(self.uy, r, self.struc('burgers'),
                                                               self.struc('spacing'),
                                                        nplanes=self.control('nplanes'))
                    plt.savefig('screw.' + self.control('plot_name'))
                    plt.close()
                else: # dimensions == 2
                    if self.control('disl_type') in 'edge':
                        self.fig, self.ax = pn1.plot_both(self.ux, r, self.struc('burgers'),
                                                               self.struc('spacing'),
                                                        nplanes=self.control('nplanes'))
                    else: # screw
                        self.fig, self.ax = pn1.plot_both(self.uy, r, self.struc('burgers'),
                                                               self.struc('spacing'),
                                                        nplanes=self.control('nplanes'))
                    plt.savefig(self.control('plot_name'))
                    plt.close()

            # calculate dislocation width and height
            if (self.control('disl_type') in 'edge'
                or self.control('dimensions') == 1):
                self.rhos = pn1.rho(self.ux, r)
            else: # screw
                self.rhos = pn1.rho(self.uy, r)

            if self.control('disl_type') in 'edge' or self.control('dimensions') == 1:
                self.dis_width = pn1.dislocation_width(self.ux, r, self.struc('burgers'))
            else: # screw dislocation with edge component of displacement
                self.dis_width = pn1.dislocation_width(self.uy, r, self.struc('burgers'))

            self.max_density = pn1.max_rho(self.rhos, self.struc('spacing'))

            # calculate the centre of mass
            if self.control('dimensions') == 1:
                self.com = pn1.com_from_pars(self.par, self.struc('burgers'),
                               self.struc('spacing'), self.control('max_x'))
            else:
                self.com = pn2.com_from_pars2d(self.par, self.struc('burgers'),
                                  self.struc('spacing'), self.control('max_x'),
                                                      self.control('disl_type'))

        return

    def peierls(self):
        '''Calculate the Peierls stress for the lowest-energy dislocation.
        '''

        if self.stress('calculate_barrier'):
            # calculate change in dislocation energy with position
            e_shifted, pars = pb.shift_energies(self.par,
                                          self.control('max_x'),
                                          self.gsf,
                                          self.K,
                                          self.struc('burgers'),
                                          self.struc('spacing'),
                                          dims=self.control('dimensions'),
                                          disl_type=self.control('disl_type'))

            # Check to see if any of the shifted energies is lower than the
            # original energy calculated for the dislocation. If so, use the
            # parameters for the shifted dislocation
            En = e_shifted[np.argmin(e_shifted[:, 1]), 1]
            if En < self.E:
                self.E = En
                self.par = pars[np.argmin(e_shifted[:, 1])]

            self.wp_shift = e_shifted[:, 1].max()-e_shifted[:, 1].min()

            # calculate the Peierls stress
            self.taup_shift = pb.sigmap_from_wp(self.struc('spacing'), e_shifted[:, 1],
                                                        self.struc('burgers'))
            if self.stress('use_GPa'):
                self.taup_shift *= atomic_to_GPa

            if self.control('record'):
                # retain a copy of the shifted dislocation energies
                outstream_e = open('e_peierls.{}'.format(self.control('output')), 'w')

                for x, E in e_shifted:
                    outstream_e.write('{:.3f} {:.6f}\n'.format(x, E))

                outstream_e.close()

        if self.stress('calculate_stress'):
            self.taup, self.taup_av = pb.taup(
                                              self.par,
                                              self.control('max_x'),
                                              self.gsf,
                                              self.K,
                                              self.struc('burgers'),
                                              self.struc('spacing'),
                                              self.control('dimensions'),
                                              self.control('disl_type'),
                                              dtau=self.stress('dtau'),
                                              in_GPa=self.stress('use_GPa'),
                                              thr=self.stress('threshold')
                                             )

            # calculate average and direction-dependent Peierls barriers
            self.wp_av = pb.peierls_barrier(self.taup_av, self.struc('burgers'),
                                                  in_GPa=self.stress('use_GPa'))

        return

    def write_output(self):
        '''Write the results of the PN calculation to the specified output file
        (<control('output')>)
        '''

        outstream = open(self.control('output'), 'w')

        # write front matter
        boiler_plate = 'Peierls-Nabarro calculation with dislopy - a Python' + \
                       ' interface for atomistic modelling of dislocations\n'
        outstream.write(boiler_plate)
        outstream.write('{}\n\n'.format(self.control('title_line')))

        # write parameters used in the simulation
        outstream.write('SIMULATION PARAMETERS:\n')
        outstream.write('Dimensions: {}\n'.format(self.control('dimensions')))
        outstream.write('Dislocation type: {}\n'.format(self.control('disl_type')))
        outstream.write('Burgers vector: {:.3f} ang.\n'.format(self.struc('burgers')))
        outstream.write('Interlayer spacing: {:.3f} ang.\n'.format(self.struc('spacing')))
        outstream.write('Number of iterations: {}\n'.format(self.control('n_iter')))
        outstream.write('Number of atomic planes used: {}\n'.format(2*self.control('max_x')+1))

        # write stacking fault energy at midpoint of gamma line
        if self.control('dimensions') == 1:
            sfe = self.gsf(self.struc('burgers')/2.)
        else:
            if self.control('disl_type') == 'edge':
                sfe = self.gsf(self.struc('burgers')/2., 0.)
            else:
                sfe = self.gsf(0., self.struc('burgers')/2.)

        if not self.surf('do_fourier'):
            sfe = float(sfe)

        outstream.write('Stacking fault energy: {:.4f} eV/ang.^2\n'.format(sfe))

        # energy coefficients -> only write relevant coefficient if 1D
        if self.control('dimensions') == 2:
            outstream.write('Edge energy coefficient: {:.4f} eV/ang.^3\n'.format(self.K[0]))
            outstream.write('Screw energy coefficient: {:.4f} eV/ang.^3\n'.format(self.K[1]))
        else: # dimensions == 1
            outstream.write('Edge energy coefficient: {:.4f} eV/ang.^3\n'.format(self.K_store[0]))
            outstream.write('Screw energy coefficient: {:.4f} eV/ang.^3\n'.format(self.K_store[1]))

        outstream.write('\n\n')

        # write fit parameters
        outstream.write('Fit parameters: \n')

        # needed in case user set input <parameters> manually
        N = len(self.par)/3

        if self.control('dimensions') == 2:
            # write parameters for the edge component of displacement
            outstream.write('----X1----\n')

            outstream.write('A\n')
            for A in self.par[:N/2]:
                outstream.write('{:.3f} '.format(A))
            outstream.write('\n')

            outstream.write('x0\n')
            for x0 in self.par[N:N+N/2]:
                outstream.write('{:.3f} '.format(x0))
            outstream.write('\n')

            outstream.write('c\n')
            for c in self.par[2*N:2*N+N/2]:
                outstream.write('{:.3f} '.format(c))
            outstream.write('\n')

            # write parameters for screw component of displacement
            outstream.write('----X2----\n')

            outstream.write('A\n')
            for A in self.par[N/2:N]:
                outstream.write('{:.3f} '.format(A))
            outstream.write('\n')

            outstream.write('x0\n')
            for x0 in self.par[N+N/2:2*N]:
                outstream.write('{:.3f} '.format(x0))
            outstream.write('\n')

            outstream.write('c\n')
            for c in self.par[2*N+N/2:]:
                outstream.write('{:.3f} '.format(c))
            outstream.write('\n')

        elif self.control('dimensions') == 1:
            outstream.write('----X----\n')

            outstream.write('A\n')
            for A in self.par[:N]:
                outstream.write('{:.3f} '.format(A))
            outstream.write('\n')

            outstream.write('x0\n')
            for x0 in self.par[N:2*N]:
                outstream.write('{:.3f} '.format(x0))
            outstream.write('\n')

            outstream.write('c\n')
            for c in self.par[2*N:]:
                outstream.write('{:.3f} '.format(c))
            outstream.write('\n')

        outstream.write('\n\n')

        # write results
        outstream.write('Results:\n')
        outstream.write('E = {:.3f} eV\n'.format(self.E))

        # write Peierls stress related stuff
        if self.stress('use_GPa'):
            units = 'GPa'
        else:
            units = 'eV/ang.^3'

        if self.stress('calculate_stress'):
            outstream.write('Using applied stress method:\n')

            outstream.write('Left Peierls stress: {:.3f} {}\n'.format(
                                                   self.taup[0], units))
            outstream.write('Right Peierls stress: {:.3f} {}\n'.format(
                                                    self.taup[1], units))

            # average Peierls stress with APPROXIMATE uncertainty
            e_taup = abs(self.taup[0] - self.taup_av)
            outstream.write('Average Peierls stress: {:.3f} +/- {:.3f}{}\n'.format(
                                                     self.taup_av, e_taup, units))

            outstream.write('Peierls barrier: {:.3f} eV/ang\n\n'.format(self.wp_av))

        if self.stress('calculate_barrier'):
            outstream.write('Using dislocation translation method:\n')
            outstream.write('Peierls stress: {:.3f} {}\n'.format(self.taup_shift, units))
            outstream.write('Peierls barrier: {:.3f} eV/ang.\n\n'.format(self.wp_shift))

        if self.prop('max_rho'):
            outstream.write('Maximum density: {:.3f}\n'.format(self.max_density))
        if self.prop('width'):
            outstream.write('Dislocation width: {:.3f} ang.\n'.format(abs(self.dis_width)))
        if self.prop('center'):
            outstream.write('Dislocation centre (/spacing): {:.3f}\n'.format(
                                            self.com/self.struc('spacing')))
            outstream.write('Dislocation centre (ang.): {:.3f}\n'.format(self.com))

        outstream.write('\n\n**Finished**')
        outstream.close()
        return

    def visualise(self):
        '''Constructs an atomistic representation of the dislocation core.
        '''

        if not self.control('run_sim'):
            # read in dislocation parameters
            dims, self.par = vp.import_pn_pars(self.control('output'))

        if self.vis('program') == '':
            raise AttributeError("Atomistic simulation code not specified")
        else:
            basestruc = vp.read_unit_cell(self.vis('unitcell'),
                                          self.vis('program'),
                                          self.vis('shift'),
                                          permutation=self.vis('permutation'),
                                          path=self.vis('path_to_cell'))

        # construct elastic displacement field function
        field = aniso.makeAnisoField(self.elast('cij'), n=self.elast('b_edge'),
                                        m=self.elast('normal'))

        vp.make_xyz(basestruc, self.par, self.control('dimensions'), self.struc('burgers'),
                                   self.struc('spacing'), self.control('disl_type'), field,
                       self.vis('radius'), self.vis('xyz_name'), thr=self.vis('threshold'),
                      sym_thr=self.vis('sym_thresh'), description=self.control('title_line'),
                                                        thickness=self.vis('thickness'))

def main():
    '''Runs an Peierls-Nabarro simulation.
    '''

    parser = argparse.ArgumentParser()
    parser.add_argument('-i', type=str, nargs='?', dest='filename', default='0')

    args = parser.parse_args()
    if args.filename != 0:
        new_sim = PNSim(args.filename)
    else:
        # read in filename from the command line
        if sys.version_info.major == 2:
            filename = raw_input('Enter name of input file: ')
        elif sys.version_info.major == 3:
            filename = input('Enter name of input file: ')

        new_simulation = PNSim(filename)

if __name__ == "__main__":
    main()


# Helper Functions
import pytest
from tests.common.helpers.assertions import pytest_assert

pytestmark = [
    pytest.mark.topology('any')
]

def get_status_redisout(status_out):
    """Extract status value for feature in redis
    """
    status_list = status_out[1:]
    status = ""
    for s in status_list:
        status = s.encode('UTF-8')
        return status

# Test Functions
def test_show_features(duthost):
    """Verify show features command output against CONFIG_DB
    """
    features_dict, succeeded = duthost.get_feature_status()
    pytest_assert(succeeded, "failed to obtain feature status")
    for cmd_key, cmd_value in features_dict.items():
        feature = str(cmd_key)
        status_out = duthost.shell('/usr/bin/redis-cli -n 4 hgetall "FEATURE|{}"'.format(feature), module_ignore_errors=False)['stdout_lines']
        redis_value = get_status_redisout(status_out)
        status_value_expected = str(cmd_value)
        assert str(redis_value) == status_value_expected, "'{}' is '{}' which does not match with config_db".format(cmd_key, cmd_value)


import PyPDF2

# Open file you want to encrypt and read it
pdfFile = open('meetingminutes.pdf', 'rb')
pdfReader = PyPDF2.PdfFileReader(pdfFile)
pdfWriter = PyPDF2.PdfFileWriter()
3
# for each page copy to pdfFileWriter object
for pageNum in range(pdfReader.numPages):
    pdfWriter.addPage(pdfReader.getPage(pageNum))

# encrypt this object with password
pdfWriter.encrypt('swordfish')

# create an ecncrypted PDF suing this object
resultPdf = open('encryptedminutes.pdf', 'wb')
pdfWriter.write(resultPdf)

# close the resulting file
resultPdf.close()


from django.shortcuts import render, redirect
from django.contrib import messages
from django.http import HttpResponse
from django.utils.html import escape

from products.models import Product, Orders, OrderDetails

# Create your views here.

# Admin's product functions
def product_change(request, pid):
    if request.user.is_superuser:
        url = "/admin/products/accept_change/" + pid
        product = Product.objects.get(product_id=pid)
        return render(request, 'product_change.html', {'product': product, 'url': url})
    return HttpResponse("You don't have permission to view this page")

def product_add(request):
    if request.user.is_superuser:
        url = "/admin/products/accept_add/"
        return render(request, 'product_add.html', {'url': url})
    return HttpResponse("You don't have permission to view this page")

def product_accept_change(request, pid):
    if request.user.is_superuser:
        product = Product.objects.get(product_id=pid)
        if request.method == 'POST':
            product_id = request.POST['product_id']
            product_name = request.POST['product_name']
            product_image = request.POST['product_image']
            company = request.POST['company']
            product_description = request.POST['product_description']
            quantity = request.POST['quantity']
            price = request.POST['price']

            # Update any fields that's different
            if product_id != product.product_id:
                product.product_id = product_id
                product.save(update_fields=["product_id"])

            if product_name != product.product_name:
                product.product_name = product_name
                product.save(update_fields=["product_name"])

            if product_image != product.product_image:
                product.product_image = product_image
                product.save(update_fields=["product_image"])

            if company != product.company:
                product.company = company
                product.save(update_fields=["company"])

            if product_description != product.product_description:
                product.product_description = product_description
                product.save(update_fields=["product_description"])

            if quantity != product.quantity_in_stock:
                product.quantity_in_stock = quantity
                product.save(update_fields=["quantity_in_stock"])

            if price != product.sell_price:
                product.sell_price = price
                product.save(update_fields=["sell_price"])

        return redirect('/products')
    return HttpResponse("You don't have permission to view this page")

def product_accept_add(request):
    if request.user.is_superuser:
        if request.method == 'POST':
            product_id = request.POST['product_id']
            product_name = request.POST['product_name']
            product_image = request.POST['product_image']
            company = request.POST['company']
            product_description = request.POST['product_description']
            quantity = request.POST['quantity']
            price = request.POST['price']

            product = Product(product_id=product_id, product_name=product_name, product_image=product_image, company=company, product_description=product_description, quantity_in_stock=quantity, sell_price=price)
            product.save()

            return redirect('/products')
        messages.add_message(request, messages.INFO, 'Error! Please refill the form to add a product')
        return redirect('/admin/products/add')
    return HttpResponse("You don't have permission to view this page")

def product_delete(request, pid):
    if request.user.is_superuser:
        Product.objects.get(product_id=pid).delete()
        return redirect('/products')
    return HttpResponse("You don't have permission to view this page")

# Admin's order functions
def order_status_change(request, oid):
    if request.user.is_superuser:
        order = Orders.objects.get(order_id=oid)

        if order.status == 'Pending':
            order.status = 'Confirmed'

        order.save(update_fields=["status"])
        return redirect('/products/orders')
    return HttpResponse("You don't have permission to view this page")

def order_filter(request):
    if request.user.is_superuser:
        order = Orders.objects.all()
    elif request.user.is_authenticated:
        order = Orders.objects.filter(user=request.user.username)

    if 'status' in request.GET:
        status = escape(request.GET["status"])
        if status == 'confirmed':
            order = order.filter(status="Confirmed")
        elif status == 'pending':
            order = order.filter(status="Pending")
        elif status == 'cancelled':
            order = order.filter(status="Canceled")

    return render(request, 'order.html', {'orders': order})

def order_details(request, oid):
    order = Orders.objects.get(order_id=oid)
    details = OrderDetails.objects.filter(order_id=oid).select_related('product_id')

    return render(request, 'order_details.html', {'id': oid, 'status': order.status, 'details': details, 'total': order.total_price})


import sys
from PyQt5 import QtWidgets
import design
import random

class MatrixApp(QtWidgets.QMainWindow, design.Ui_Matrix):

    def __init__(self):
        super().__init__()
        self.setupUi(self)
        self.randomiseMatrix.clicked.connect(self.newMatrix)
        self.pushButton_2.clicked.connect(self.showMatrix)
        self.showMinimum.clicked.connect(self.printMin)
        self.sumRow.clicked.connect(self.printSumC)
        self.M = 0
        self.N = 0
        self.matr = [[1 for y in range(self.M)] for x in range(self.N)]
    def newMatrix(self):
        self.listWidget.clear()
        self.matr = [[random.randrange(0, 10) for y in range(self.M)] for x in range(self.N)]
        for i in self.matr:
            self.listWidget.addItem(str(i))
    def showMatrix(self):
        self.listWidget.clear()
        self.M = self.rows.value()
        self.N = self.columns.value()
        self.matr = [[1 for y in range(self.M)] for x in range(self.N)]
        for i in self.matr:
            self.listWidget.addItem(str(i))
    def printMin(self):
        i = self.columnNumber.value()
        min = 10
        for j in self.matr:
            if j[i] < min:
                min = j[i]
        self.listWidget.addItem("minimum of {} colomn - ".format(i) + str(min))
    def printSumC(self):
        i = self.rowNumber.value()
        s = sum(self.matr[i])
        self.listWidget.addItem("sum of {} row - ".format(i) + str(s))
    def printSumAll(self):
        s = 0
        for i in self.matr:
            s += sum(i)
        self.listWidget.addItem("sum of all elements - ".format(i) + str(s))
def main():
    app = QtWidgets.QApplication(sys.argv)
    window = MatrixApp()
    window.show()
    app.exec()

if __name__ == '__main__':
    main()


# coding: utf-8

# In[ ]:

import torch
from torchtext import data

SEED = 1234

torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
torch.backends.cudnn.deterministic = True

TEXT = data.Field(tokenize='spacy')
LABEL = data.LabelField(dtype=torch.float)

# In[2]:

#https://towardsdatascience.com/use-torchtext-to-load-nlp-datasets-part-i-5da6f1c89d84
from torchtext import datasets

import logging

import pandas as pd
import numpy as np

VAL_RATIO = 0.2
def prepare_csv(trainCsvPath, testCsvPath, seed=999):
    df_train = pd.read_csv(trainCsvPath)
    df_train["comment_text"] =         df_train.comment_text.str.replace("\n", " ")
    idx = np.arange(df_train.shape[0])
    np.random.seed(seed)
    np.random.shuffle(idx)
    val_size = int(len(idx) * VAL_RATIO)
    df_train.iloc[idx[val_size:], :].to_csv(
        "cache/dataset_train.csv", index=False)
    df_train.iloc[idx[:val_size], :].to_csv(
        "cache/dataset_val.csv", index=False)

    df_test = pd.read_csv(testCsvPath)
    df_test["comment_text"] =         df_test.comment_text.str.replace("\n", " ")
    df_test.to_csv("cache/dataset_test.csv", index=False)

import re
import spacy
NLP = spacy.load('en')
MAX_CHARS = 20000
def tokenizer(comment):
    comment = re.sub(
        r"[\*\"“”\n\\…\+\-\/\=\(\)‘•:\[\]\|’\!;]", " ",
        str(comment))
    comment = re.sub(r"[ ]+", " ", comment)
    comment = re.sub(r"\!+", "!", comment)
    comment = re.sub(r"\,+", ",", comment)
    comment = re.sub(r"\?+", "?", comment)
    if (len(comment) > MAX_CHARS):
        comment = comment[:MAX_CHARS]
    return [
        x.text for x in NLP.tokenizer(comment) if x.text != " "]

import torch
from torchtext import data
LOGGER = logging.getLogger("toxic_dataset")
def get_dataset(trainCsvPath, testCsvPath, fix_length=100, lower=False, vectors=None):
    if vectors is not None:
        # pretrain vectors only supports all lower cases
        lower = True
    LOGGER.debug("Preparing CSV files...")
    prepare_csv(trainCsvPath, testCsvPath)
    comment = data.Field(
        sequential=True,
        fix_length=fix_length,
        tokenize=tokenizer,
        pad_first=True,
        #tensor_type=torch.cuda.LongTensor,
        lower=lower
    )
    LOGGER.debug("Reading train csv file...")
    train, val = data.TabularDataset.splits(
        path='cache/', format='csv', skip_header=True,
        train='dataset_train.csv', validation='dataset_val.csv',
        fields=[
            ('id', None),
            ('comment_text', comment),
            ('toxic', data.Field(
                use_vocab=False, sequential=False)), #tensor_type=torch.cuda.ByteTensor
            ('severe_toxic', data.Field(
                use_vocab=False, sequential=False)), #tensor_type=torch.cuda.ByteTensor
            ('obscene', data.Field(
                use_vocab=False, sequential=False)), #tensor_type=torch.cuda.ByteTensor
            ('threat', data.Field(
                use_vocab=False, sequential=False)), #tensor_type=torch.cuda.ByteTensor
            ('insult', data.Field(
                use_vocab=False, sequential=False)), #tensor_type=torch.cuda.ByteTensor
            ('identity_hate', data.Field(
                use_vocab=False, sequential=False
                )),   #tensor_type=torch.cuda.ByteTensor
        ])
    LOGGER.debug("Reading test csv file...")
    test = data.TabularDataset(
        path='cache/dataset_test.csv', format='csv',
        skip_header=True,
        fields=[
            ('id', None),
            ('comment_text', comment)
        ])
    LOGGER.debug("Building vocabulary...")
    comment.build_vocab(
        train, val, test,
        max_size=20000,
        min_freq=50,
        vectors=vectors
    )
    LOGGER.debug("Done preparing the datasets")
    return train, val, test

def get_iterator(dataset, batch_size, train=True,
    shuffle=True, repeat=False):
    dataset_iter = data.Iterator(
        dataset, batch_size=batch_size, device=0,
        train=train, shuffle=shuffle, repeat=repeat,
        sort=False
    )
    return dataset_iter

dataSet = get_dataset("../../../data/nlp/jigsawtoxic/train.csv", "../../../data/nlp/jigsawtoxic/test.csv")

# In[3]:

#print(dataSet.shape)

# In[ ]:

import torchtext
from torchtext import data

# tokenizer function using spacy
nlp = spacy.load('en',disable=['parser', 'tagger', 'ner'])
def tokenizer(s):
    return [w.text.lower() for w in nlp(tweet_clean(s))]

def tweet_clean(text):
    text = re.sub(r'[^A-Za-z0-9]+', ' ', text) # remove non alphanumeric character
    text = re.sub(r'https?:/\/\S+', ' ', text) # remove links
    return text.strip()

# define the columns that we want to process and how to process
txt_field = data.Field(sequential=True,
                       tokenize=tokenizer,
                       include_lengths=True,
                       use_vocab=True)
label_field = data.Field(sequential=False,
                         use_vocab=False,
                         pad_token=None,
                         unk_token=None)

train_val_fields = [
    ('ItemID', None), # we dont need this, so no processing
    ('Sentiment', label_field), # process it as label
    ('SentimentSource', None), # we dont need this, so no processing
    ('SentimentText', txt_field) # process it as text
]

trainData = data.TabularDataset.splits(path='../../../data/nlp/',
                                            format='csv',
                                            train='Sentiment.csv',
                                            fields=train_val_fields,
                                            skip_header=True)

print(len(trainData))
#print(len(validationData))

from torchtext import vocab
# specify the path to the localy saved vectors
vec = vocab.Vectors('glove.6B.50d.txt', '../../../data/nlp/')
# build the vocabulary using train and validation dataset and assign the vectors
txt_field.build_vocab(trainData, max_size=100000, vectors=vec)
# build vocab for labels
label_field.build_vocab(trainData)

print(txt_field.vocab.vectors.shape)
# torch.Size([100002, 100])

txt_field.vocab.vectors[txt_field.vocab.stoi['the']]


"""
Base45 Data Encoding as described in RFC 9285
"""

from typing import Union

BASE45_CHARSET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ $%*+-./:"
BASE45_DICT = {v: i for i, v in enumerate(BASE45_CHARSET)}

def b45encode(buf: bytes) -> bytes:
    """Convert bytes to base45-encoded string"""
    res = ""
    buflen = len(buf)
    for i in range(0, buflen & ~1, 2):
        x = (buf[i] << 8) + buf[i + 1]
        e, x = divmod(x, 45 * 45)
        d, c = divmod(x, 45)
        res += BASE45_CHARSET[c] + BASE45_CHARSET[d] + BASE45_CHARSET[e]
    if buflen & 1:
        d, c = divmod(buf[-1], 45)
        res += BASE45_CHARSET[c] + BASE45_CHARSET[d]
    return res.encode()

def b45decode(s: Union[bytes, str]) -> bytes:
    """Decode base45-encoded string to bytes"""
    try:
        if isinstance(s, str):
            buf = [BASE45_DICT[c] for c in s.rstrip("\n")]
        elif isinstance(s, bytes):
            buf = [BASE45_DICT[c] for c in s.decode()]
        else:
            raise TypeError("Type must be 'str' or 'bytes'")

        buflen = len(buf)
        if buflen % 3 == 1:
            raise ValueError("Invalid base45 string")

        res = []
        for i in range(0, buflen, 3):
            if buflen - i >= 3:
                x = buf[i] + buf[i + 1] * 45 + buf[i + 2] * 45 * 45
                if x > 0xFFFF:
                    raise ValueError
                res.extend(divmod(x, 256))
            else:
                x = buf[i] + buf[i + 1] * 45
                if x > 0xFF:
                    raise ValueError
                res.append(x)
        return bytes(res)
    except (ValueError, KeyError, AttributeError):
        raise ValueError("Invalid base45 string")


from sys import stdin
from collections import defaultdict
from collections import deque

n = int(stdin.readline())
graph = defaultdict(list)

for _ in range(n):
    a, b = map(int, stdin.readline().split())
    graph[a].append(b)
    graph[b].append(a)

src = None
dest = None
for node in graph:
    if not src and len(graph[node]) == 1:
        src = node

    elif src and len(graph[node]) == 1:
        dest = node
        break

path = {src: None}
queue = deque([src])

while queue:
    node = queue.popleft()

    if node == dest:
        goal = node
        break

    for neigh in graph[node]:
        if neigh not in path:
            path[neigh] = node
            queue.append(neigh)

final_path = []
while goal:
    final_path.append(goal)
    goal = path[goal]

print(*final_path)


from selenium import webdriver
from selenium.webdriver.chrome.options import Options

import time

#url = 'https://www.youtube.com/watch?v=k2eEyOZmi2c&ab_channel=CanalemDia'
options = Options()
options.add_experimental_option=True

webBrowser = webdriver.Chrome(options=options)
url = 'https://www.youtube.com/watch?v=k2eEyOZmi2c&ab_channel=CanalemDia'

def open_new_tab(url, tab_number):
    webBrowser.execute_script("window.open('about:blank', "+ str(tab_number) +");")
    webBrowser.switch_to.window(str(tab_number) )
    webBrowser.get(url)

while True:
    for i in range(10):
        open_new_tab(url, i)

    time.sleep(90)


__all__ = [
	'get_field',
	'list_to_single_value',
	'normalize_value',
]

import os
import re
from collections.abc import Mapping

import audio_metadata
from audio_metadata import AudioMetadataException
from multidict import MultiDict

from .constants import FIELD_MAP

def get_field(tags, field, default='', *, field_map=FIELD_MAP):
	value = default
	try:
		value = tags[field]
	except KeyError:
		if isinstance(field_map, MultiDict):
			for alias in field_map.getall(field, []):
				if tags.get(alias):  # pragma: no branch
					value = tags[alias]
					break
		elif isinstance(field_map, Mapping):  # pragma: no branch
			alias = field_map.get(field)

			if alias in tags:  # pragma: no branch
				value = tags[alias]

	return value

def get_item_tags(item):
	if isinstance(item, (str, os.PathLike)):
		try:
			tags = audio_metadata.load(item).tags
		except AudioMetadataException:
			tags = None
	elif isinstance(item, audio_metadata.Format):
		tags = item.tags
	else:
		tags = item

	return tags

def list_to_single_value(value):
	if not isinstance(value, (list, tuple)):
		return value

	return value[0]

def normalize_value(value):
	"""Normalize metadata value to improve match accuracy."""

	value = str(value)
	value = value.casefold()

	value = re.sub(r'\/\s*\d+', '', value)  # Remove "/<totaltracks>" from track number.
	value = re.sub(
		r'^0+([0-9]+)', r'\1', value
	)  # Remove leading zero(s) from track number.
	value = re.sub(r'^(\d+)\.+', r'\1', value)  # Remove dots from track number.
	value = re.sub(r'[^\w\s]', '', value)  # Remove leading non-word characters.
	value = re.sub(r'^the\s+', '', value)  # Remove leading "the".
	value = re.sub(r'^\s+', '', value)  # Remove leading spaces.
	value = re.sub(r'\s+$', '', value)  # Remove trailing spaces.
	value = re.sub(r'\s+', ' ', value)  # Reduce multiple spaces to a single space.

	return value


import os
import glob
import numpy as np
import sys
import re
from io import StringIO
import matplotlib.pyplot as plt
import math
import pickle

from outliers import smirnov_grubbs as grubbs

def Zscan_data_extract_singlechip(runID,result_path,save_path):
    chipdir=os.path.join(result_path,runID)
    print(chipdir)
    nameFolds=glob.glob(os.path.join(chipdir,'X*'))
    tileNum=len(nameFolds)
    print(len(nameFolds))
    if tileNum==0:
        print('No data in %s'%runID)
        sys.exit()
    if not os.path.exists(os.path.join(save_path,runID)):
        os.makedirs(os.path.join(save_path,runID))
    save_path1=os.path.join(save_path,runID)

    shift_info_wav=np.zeros((tileNum,3))
    shift_info_sml=np.zeros((tileNum,3))
    X=np.zeros((tileNum,1))
    Y=np.zeros((tileNum,1))
    Z=np.zeros((tileNum,1))
    Z_WDI=np.zeros((tileNum,1))
    Fv_max=np.zeros((tileNum,1))
    plt.figure(figsize=(35,20))
    for n in range(tileNum):
        fileName=nameFolds[n].split('\\')[-1]
        A=re.split(r'[XY,_]',fileName)
        X[n]=A[1]
        Y[n]=A[3]
        infoFile=os.path.join(nameFolds[n],'%s.txt'%fileName)
        with open(infoFile) as file:
            content=file.readlines()
            content=content[4:len(content)-2]
            C= np.loadtxt(content)
        if n<10:
            '''draw plot'''
            plt.subplot(5,4,2*(n+1)-1)
            plt.plot(C[:,0],C[:,1])
            plt.plot(C[:,0],C[:,3])
            max_value=np.max(C[:,3])
            max_pos=np.argmax(C[:,3],axis=0)
            plt.plot([C[:,0][max_pos],C[:,0][max_pos]],[0,max_value])
            Z[n]=C[:,0][max_pos]
            Fv_max[n]=max_value
            fv_shift1,fv_shift2,WDI_shift=F_getFvShift(C[:,1],C[:,3])
            plt.title('Fv_WAV_tile%d,shift:%.2f,%.2f,WDI_shift:%.2f'%(n,fv_shift1,fv_shift2,WDI_shift))
            shift_info_wav[n,:]=[fv_shift1,fv_shift2,WDI_shift]

            plt.subplot(5,4,2*(n+1))
            plt.plot(C[:,0],C[:,2])
            plt.plot(C[:,0],C[:,4])
            max_value=np.max(C[:,4])
            max_pos=np.argmax(C[:,4],axis=0)
            plt.plot([C[:,0][max_pos],C[:,0][max_pos]],[0,max_value])
            fv_shift1,fv_shift2,WDI_shift=F_getFvShift(C[:,2],C[:,4])
            plt.title('Fv_SML_tile%d,shift:%.2f,%.2f,WDI_shift:%.2f'%(n,fv_shift1,fv_shift2,WDI_shift))
            shift_info_sml[n,:]=[fv_shift1,fv_shift2,WDI_shift]
        max_value=np.max(C[:,3])
        max_pos=np.argmax(C[:,4],axis=0)
        Z[n]=C[:,0][max_pos]
        Z_WDI[n]=C[:,0][math.floor((len(C[:,0])+1)/2)]
        Fv_max[n]=max_value
        fv_shift1,fv_shift2,WDI_shift=F_getFvShift(C[:,1],C[:,3])
        shift_info_wav[n,:]=[fv_shift1,fv_shift2,WDI_shift]
        fv_shift1,fv_shift2,WDI_shift=F_getFvShift(C[:,2],C[:,4])
        shift_info_sml[n,:]=[fv_shift1,fv_shift2,WDI_shift]
    plt.tight_layout()
    plt.subplots_adjust(left=None, bottom=None, right=None, top=None,wspace=None, hspace=0.15)
    plt.savefig(os.path.join(save_path1,'fv_curve_'+runID+'.png'),dpi=500,bbox_inches = 'tight')
    plt.close()
    #plt.show()

    '''shift info figure'''
    range_shift1 = np.ptp(shift_info_wav[:, 0])
    range_shift2 = np.ptp(shift_info_wav[:, 1])
    range_wdishift = np.ptp(shift_info_wav[:, 2])

    shift1 = shift_info_wav[:, 0]
    shift2 = shift_info_wav[:, 1]
    wdi_shift = shift_info_wav[:, 2]
    shift1 = grubbs.test(shift1, alpha=.05)
    shift2 = grubbs.test(shift2, alpha=.05)
    wdi_shift = grubbs.test(wdi_shift, alpha=.05)

    std_shift1 = np.std(shift1, ddof=1)
    std_shift2 = np.std(shift2, ddof=1)
    std_WDI = np.std(wdi_shift, ddof=1)
    mean_shift1 = np.mean(shift1)
    mean_shift2 = np.mean(shift2)
    mean_WDI = np.mean(wdi_shift)
    plt.figure(figsize=(20,10))
    plt.subplot(3,2,1)
    line1,=plt.plot(shift_info_wav[:,0],marker='o',markerfacecolor='none',markersize = '5',linestyle='-',linewidth='1')
    plt.legend(handles=[line1], labels=['shift1,range %s std %s mean %s'%(range_shift1,std_shift1,mean_shift1)], loc='best',fontsize=8)
    plt.title('WAV offset summary')

    plt.subplot(3,2,3)
    line2,=plt.plot(shift_info_wav[:,1],marker='*',markerfacecolor='none',markersize = '5',linestyle='-',linewidth='1')
    plt.legend(handles=[line2], labels=['shift2,range %s std %s mean %s'%(range_shift2,std_shift2,mean_shift2)], loc='best',fontsize=8)

    plt.subplot(3,2,5)
    line3,=plt.plot(shift_info_wav[:,2],marker='^',markerfacecolor='none',markersize = '5',linestyle='-',linewidth='1')
    plt.plot([0,tileNum],[mean_WDI,mean_WDI],'r',linewidth='0.5')
    plt.legend(handles=[line3], labels=['WDI_s,range %s std %s mean %s'%(range_wdishift,std_WDI,mean_WDI)], loc='best',fontsize=8)

    range_shift1 = np.ptp(shift_info_sml[:, 0])
    range_shift2 = np.ptp(shift_info_sml[:, 1])
    range_wdishift = np.ptp(shift_info_sml[:, 2])
    shift1 = shift_info_sml[:, 0]
    shift2 = shift_info_sml[:, 1]
    wdi_shift = shift_info_sml[:, 2]
    shift1 = grubbs.test(shift1, alpha=.05)
    shift2 = grubbs.test(shift2, alpha=.05)
    wdi_shift = grubbs.test(wdi_shift, alpha=.05)

    std_shift1 = np.std(shift1, ddof=1)
    std_shift2 = np.std(shift2, ddof=1)
    std_WDI = np.std(wdi_shift, ddof=1)
    mean_shift1 = np.mean(shift1)
    mean_shift2 = np.mean(shift2)
    mean_WDI = np.mean(wdi_shift)
    plt.subplot(3,2,2)
    line4,=plt.plot(shift_info_sml[:,0],marker='o',markerfacecolor='none',markersize = '5',linestyle='-',linewidth='1')
    plt.legend(handles=[line4], labels=['shift1,range %s std %s mean %s'%(range_shift1,std_shift1,mean_shift1)], loc='best',fontsize=8)
    plt.title('SML offset summary')

    plt.subplot(3,2,4)
    line5,=plt.plot(shift_info_sml[:,1],marker='*',markerfacecolor='none',markersize = '5',linestyle='-',linewidth='1')
    plt.legend(handles=[line5], labels=['shift2,range %s std %s mean %s'%(range_shift2,std_shift2,mean_shift2)], loc='best',fontsize=8)

    plt.subplot(3,2,6)
    line6,=plt.plot(shift_info_sml[:,2],marker='^',markerfacecolor='none',markersize = '5',linestyle='-',linewidth='1')
    plt.plot([0,tileNum],[mean_WDI,mean_WDI],'r',linewidth='0.5')
    plt.legend(handles=[line6], labels=['WDI_s,range %s std %s mean %s'%(range_wdishift,std_WDI,mean_WDI)], loc='best',fontsize=8)

    plt.tight_layout()
    plt.subplots_adjust(left=None, bottom=None, right=None, top=None,wspace=None, hspace=0.15)
    plt.savefig(os.path.join(save_path1,'offset_summary_'+runID+'.png'),dpi=500,bbox_inches = 'tight')
    plt.close()
    #plt.show()

    '''shift spatial figure'''
    plt.figure(figsize=(40,10))
    ax1 = plt.subplot(1,5,1)
    '''关闭裁剪功能'''
    plt.scatter(X,Y,s=300,c=Z,clip_on = False)
    plt.gca().set_aspect(1)
    ax1.spines['top'].set_visible(False)
    ax1.spines['right'].set_visible(False)
    ax1.spines['bottom'].set_visible(False)
    ax1.spines['left'].set_visible(False)
    plt.colorbar()
    plt.title('focus_Z',fontsize=20)

    ax2 = plt.subplot(1,5,2)
    plt.scatter(X,Y,s=300,c=Z_WDI,clip_on = False)
    #XY轴单位长度相同
    plt.gca().set_aspect(1)
    ax2.spines['top'].set_visible(False)
    ax2.spines['right'].set_visible(False)
    ax2.spines['bottom'].set_visible(False)
    ax2.spines['left'].set_visible(False)
    plt.colorbar()
    plt.title('WDI_Z',fontsize=20)

    ax3 = plt.subplot(1,5,3)
    c=np.squeeze(shift_info_sml[:,2])
    #C的值不能是列向量并且和XY保持一致
    plt.scatter(X,Y,s=300,c=shift_info_sml[:,2].reshape(len(X),1),clip_on = False)
    #XY轴单位长度相同
    plt.gca().set_aspect(1)
    ax3.spines['top'].set_visible(False)
    ax3.spines['right'].set_visible(False)
    ax3.spines['bottom'].set_visible(False)
    ax3.spines['left'].set_visible(False)
    plt.colorbar()
    plt.title('WDI_offset',fontsize=20)

    ax4 = plt.subplot(1,5,4)
    plt.scatter(X,Y,s=300,c=shift_info_sml[:,0].reshape(len(X),1),clip_on = False)
    #XY轴单位长度相同
    plt.gca().set_aspect(1)
    ax4.spines['top'].set_visible(False)
    ax4.spines['right'].set_visible(False)
    ax4.spines['bottom'].set_visible(False)
    ax4.spines['left'].set_visible(False)
    plt.colorbar()
    plt.title('shift1',fontsize=20)

    ax5 = plt.subplot(1,5,5)
    plt.scatter(X,Y,s=300,c=Fv_max,clip_on = False,vmin=0.7,vmax=1.2)
    #XY轴单位长度相同
    plt.gca().set_aspect(1)
    ax5.spines['top'].set_visible(False)
    ax5.spines['right'].set_visible(False)
    ax5.spines['bottom'].set_visible(False)
    ax5.spines['left'].set_visible(False)
    plt.colorbar()
    plt.title('Fv_max',fontsize=20)

    plt.suptitle(runID.replace("_","-"))
    plt.tight_layout()
    plt.subplots_adjust(top=0.85)
    plt.savefig(os.path.join(save_path1,'offset_spatialinfo_'+runID+'.png'),dpi=500,bbox_inches = 'tight')
    plt.close()
    #plt.show()

    '''保存变量'''
    with open(os.path.join(save_path1,'Fv_max.txt'),'wb') as A1:
        pickle.dump(Fv_max,A1)
    with open(os.path.join(save_path1,'C.txt'),'wb') as A1:
        pickle.dump(C,A1)
    with open(os.path.join(save_path1,'shift_info_sml.txt'),'wb') as A1:
        pickle.dump(shift_info_sml,A1)
    with open(os.path.join(save_path1,'shift_info_wav.txt'),'wb') as A1:
        pickle.dump(shift_info_wav,A1)

def F_getFvShift(C_BF,C_FF):
    step=0.2
    indexLength=len(C_BF)
    FF_max_value=np.max(C_FF)
    FF_max_pos=np.argmax(C_FF)
    if FF_max_pos>0:
        BF_1=C_BF[0:FF_max_pos]
    else:
        BF_1 = C_BF[FF_max_pos]
    BF_2=C_BF[FF_max_pos:indexLength]
    BF_max_value1=np.max(BF_1)
    BF_max_pos1=np.argmax(BF_1)
    BF_max_value2=np.max(BF_2)
    BF_max_pos2=np.argmax(BF_2)
    '''python不用加1'''
    BF_max_pos2=BF_max_pos2+FF_max_pos
    fv_shift1=(FF_max_pos-BF_max_pos1)*step
    fv_shift2=(BF_max_pos2-FF_max_pos)*step
    '''这里需要加1'''
    WDI_shift=((indexLength+1)/2-(FF_max_pos+1))*step
    return fv_shift1,fv_shift2,WDI_shift

# runID = '123'
# result_path =r'D:\bost_ui\output\z_get_imge'
# save_path = r'D:\bost_ui\output\z_get_imge'
#
# Zscan_data_extract_singlechip(runID, result_path, save_path)


import sys
import threading
import time
from pynput import keyboard
from tkinter import *
ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"

class Display:
    def __init__(self, settings, enigma, pairs, reset, console):
        self.rT1, self.rN1 = settings[0], settings[3]
        self.rT2, self.rN2 = settings[1], settings[4]
        self.rT3, self.rN3 = settings[2], settings[5]
        self.refT = settings[6]
        self.enigma = enigma
        self.pairs = pairs
        self.reset = reset
        self.console = console
        self.line = ""
        self.exit_code = 0

    def editEnigma(self):
        pass

    def interactiveConsole(self, key):
        try:
            if key.char in ALPHABET:
                self.line += self.enigma.process(key.char)
                print(f" |{self.enigma.rotors[0].rotations+1:02}-{self.enigma.rotors[0].type[3]}|{self.enigma.rotors[1].rotations+1:02}-{self.enigma.rotors[1].type[3]}|{self.enigma.rotors[2].rotations+1:02}-{self.enigma.rotors[2].type[3]}|{self.enigma.reflector.typeStr}| enigma £> {key.char} | {self.line}      ", end='\r')
        except:
            #print(f"Not a valid key.")
            if (key == keyboard.Key.alt_l) or (key == keyboard.Key.alt_r):
                # editing mode
                self.exit_code = 2
                sys.exit(1)
            elif key == keyboard.Key.space:
                self.line += " "
                print(f" |{self.enigma.rotors[0].rotations+1:02}-{self.enigma.rotors[0].type[3]}|{self.enigma.rotors[1].rotations+1:02}-{self.enigma.rotors[1].type[3]}|{self.enigma.rotors[2].rotations+1:02}-{self.enigma.rotors[2].type[3]}|{self.enigma.reflector.typeStr}| enigma £> _ | {self.line}      ", end='\r')

            elif key == keyboard.Key.delete:
                self.exit_code = 3
                sys.exit(1)
            elif key == keyboard.Key.enter:
                self.enigma.reset()
                cipher = self.enigma.process(self.line)
                print(f"\n{cipher}      ")

                if self.reset: self.enigma.reset()
                self.line = ""
                print(f" |{self.enigma.rotors[0].rotations+1:02}-{self.enigma.rotors[0].type[3]}|{self.enigma.rotors[1].rotations+1:02}-{self.enigma.rotors[1].type[3]}|{self.enigma.rotors[2].rotations+1:02}-{self.enigma.rotors[2].type[3]}|{self.enigma.reflector.typeStr}| enigma £>                       ", end='\r')
            elif key == keyboard.Key.backspace:
                if (self.line != ""):
                    if (self.line[-1] != "_") or (self.line[-1] != " "):
                        turn_first = self.enigma.rotors[0].rotate(force=True, offset=-1)
                        if turn_first: turn_next = True
                        else: turn_next = False
                        for rotor in self.enigma.rotors[1:]:
                            turn_next = rotor.rotate(force=(turn_next and turn_first), offset=-1)
                    self.line = self.line[:-1]

                print(f" |{self.enigma.rotors[0].rotations+1:02}-{self.enigma.rotors[0].type[3]}|{self.enigma.rotors[1].rotations+1:02}-{self.enigma.rotors[1].type[3]}|{self.enigma.rotors[2].rotations+1:02}-{self.enigma.rotors[2].type[3]}|{self.enigma.reflector.typeStr}| enigma £> - | {self.line}      ", end='\r')
            elif key == keyboard.Key.esc:
                self.exit_code = 0
                sys.exit(1)
            else:
                pass

    def userInterface(self, key):
        if key == keyboard.Key.esc:
            sys.exit(1)

    def start_thread(self):
        self.root = Tk()
        self.root.geometry("800x1000")

        editing = Tk.frame(self.root)

        #Create a canvas object
        c= Canvas(self.root,width=400, height=400)
        c.pack()
        c.create_oval(60,60,210,210)
        self.root.mainloop()

    def on_press(self, key):
        if self.console:
            self.interactiveConsole(key)
        else:
            self.userInterface(key)

    def headless(self):
        # aka interactive console
        print("Starting interactive console.    ", end='\r')
        time.sleep(.5)
        print("Starting interactive console..   ", end='\r')
        time.sleep(.5)
        print("Starting interactive console...  ", end='\r')
        time.sleep(.5)
        print("                                                                                ", end='\r')
        print(f" |{self.enigma.rotors[0].rotations+1:02}-{self.enigma.rotors[0].type[3]}|{self.enigma.rotors[1].rotations+1:02}-{self.enigma.rotors[1].type[3]}|{self.enigma.rotors[2].rotations+1:02}-{self.enigma.rotors[2].type[3]}|{self.enigma.reflector.typeStr}| enigma £>                 ", end='\r')
        listener = keyboard.Listener(on_press=self.on_press, suppress=True)
        listener.start()
        listener.join()
        if self.exit_code == 0 or self.exit_code == 1:
            string = 'Returning to non-interactive console'
        elif self.exit_code == 2:
            string = 'Entering Edit Machine Mode'
        elif self.exit_code == 3:
            string = 'Exiting Application'

        print(f"{string}                        ", end='\r')
        time.sleep(.5)
        print(f"{string}.                       ", end='\r')
        time.sleep(.5)
        print(f"{string}..                      ", end='\r')
        time.sleep(.5)
        print(f"{string}...                     ", end='\r')
        time.sleep(.5)
        if (self.exit_code == 0) or (self.exit_code == 1): print("                                                                  ", end='\r')
        if (self.exit_code == 0) or (self.exit_code == 1): print("General Console Controls: Type input and press enter to see output, type quit() to quit, reset() to reset machine, edit() to edit machine, and inter() to start the interactive mode.")

        return self.exit_code

    def run(self):
        thread = threading.Thread(target=self.start_thread)
        thread.start()
        listener = keyboard.Listener(on_press=self.on_press, suppress=True)
        listener.start()
        listener.join()
        thread.join()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Purpose.

This program analyses the news database for most popular
articles, most popular authors, and the days in which
the percentage of requests that return 404 error is more than one.

"""

import psycopg2

DBNAME = "news"

title_one = "\nThe three most popular articles of all time"
title_two = "\nThe three most popular authors of all time"
title_three = "\nThe days with more than one percent of" \
              " requests that leads to an error"

query_one = "SELECT title, COUNT(*) as views FROM articles a," \
            "log l WHERE a.slug=substring(l.path, 10) GROUP BY " \
            "title ORDER BY views desc limit 3"
query_two = "SELECT name, SUM(views) FROM title_views_author" \
            " GROUP BY name ORDER BY sum desc limit 3"
query_three = "SELECT * FROM error_view WHERE percentage_error > 1"

def get_query_result(query):
    """Get query results from database."""
    db = psycopg2.connect(database=DBNAME)
    c = db.cursor()
    c.execute(query)
    results = c.fetchall()
    db.close()
    return results

def print_query_result(query_result):
    """Print query result from first and second queries."""
    print(query_result[1])
    for results in (query_result[0]):
        print(str(results[0]) + ' - ' + str(results[1]) + ' views')

def print_error_result(error_result):
    """Print query result for third query."""
    print(error_result[1])
    for results in error_result[0]:
        print(str(results[0]) + ' - ' + str(results[1]) + '%')

if __name__ == '__main__':
    most_popular_articles = get_query_result(query_one), title_one
    most_popular_authors = get_query_result(query_two), title_two
    highest_error_days = get_query_result(query_three), title_three

    print_query_result(most_popular_articles)
    print_query_result(most_popular_authors)
    print_error_result(highest_error_days)


from notebook.utils import url_path_join
from notebook.base.handlers import IPythonHandler
import tornado.escape
import os
import time
import jextension.log as log
import jextension.utils as utils
from jextension.utils import formatSize

logger = log.getLogger(app_name='jextension', filename='/var/log/BDViewer/jextension.log', level=log.DEBUG)

class FileSizeHandler(IPythonHandler):
    def get(self, _filepath):
        file_path = _filepath
        logger.debug('In FileSizeHandler, filepath: %s' % _filepath)
        if not os.path.exists(file_path):
            self.write("File does not exist")
            return
        size = os.path.getsize(file_path)
        format_size = formatSize(size)
        self.write(format_size)

class FileDateHandler(IPythonHandler):
    def get(self, _filepath):
        file_path = _filepath
        logger.debug('In FileDateHandler, filepath: %s' % _filepath)
        if not os.path.exists(file_path):
            self.write("File does not exist")
            return
        statinfo = os.stat(file_path)
        st_mtime = statinfo.st_mtime
        format_date = time.strftime("%Y-%m-%d %H:%M %p", time.localtime(st_mtime))
        self.write(format_date)

class ViewTableHandler(IPythonHandler):
    def get(self, _filepath):
        file_path = _filepath
        logger.debug('In ViewTableHandler, filepath: %s' % _filepath)
        if not os.path.exists(file_path):
            self.write("File %s does not exist" % file_path)
            return
        self.write(self.render_template('table.html'))

class FileContentHandler(IPythonHandler):
    def get(self, _filepath):
        arr = _filepath.split('/')
        beg, end = int(arr[-2]), int(arr[-1])
        _filepath = '/'.join(arr[:-2])
        logger.debug('In FileContentHandler, filepath: %s' % _filepath)
        beg = int(beg)
        end = int(end)
        if not os.path.exists(_filepath):
            self.write("File does not exist")
            return
        lines = utils.get_lines_skip_rows(_filepath, beg, end)
        self.write(lines)

class FileLineNumberHandler(IPythonHandler):
    def get(self, _filepath):
        _filepath = str(_filepath)
        logger.debug('In FileLineNumberHandler, filepath: %s' % _filepath)
        if not os.path.exists(_filepath):
            self.write("File does not exist")
            return
        line_num = str(utils.get_line_num(_filepath))
        logger.info("file %s with %s lines" % (_filepath, line_num))
        self.write(line_num)

class SortContentHandler(IPythonHandler):
    def get(self, _filepath):
        arr = _filepath.split('/')
        col = int(arr[-1])
        _filepath = '/'.join(arr[:-1])
        logger.debug('In SortContentHandler, filepath: %s' % _filepath)
        if not os.path.exists(_filepath):
            self.write("File does not exist")
            return

        lines = utils.get_lines_after_sort(_filepath, col)
        self.write(lines)

class DataFeatureHandler(IPythonHandler):
    def get(self, _filepath):
        logger.debug('In DataFeatureHandler, filepath: %s' % _filepath)
        arr = _filepath.split('/')
        feature, dim, index = int(arr[-3]), int(arr[-2]), int(arr[-1])
        _filepath = '/'.join(arr[:-3])
        if not os.path.exists(_filepath):
            self.write("File does not exist")
            return

        ret = str(utils.get_data_feature(_filepath, feature, dim, index))
        print(ret)
        self.write(ret)

def _jupyter_server_extension_paths():
    return [{
        "module": "jextension.jextension"
    }]

def load_jupyter_server_extension(nb_server_app):
    """
    Called when the extension is loaded.

    Args:
        nb_server_app (NotebookWebApplication): handle to the Notebook webserver instance.
    """
    web_app = nb_server_app.web_app
    host_pattern = '.*$'

    #route patterns
    file_size_pattern = url_path_join(web_app.settings['base_url'], '/filesize/(.+$)')
    file_date_pattern = url_path_join(web_app.settings['base_url'], '/filedate/(.+$)')
    view_table_pattern = url_path_join(web_app.settings['base_url'], '/table_view/(.+$)')
    file_content_pattern = url_path_join(web_app.settings['base_url'], '/file_content/(.+$)')
    line_number_pattern = url_path_join(web_app.settings['base_url'], '/line_num/(.+$)')
    sort_content_pattern = url_path_join(web_app.settings['base_url'], '/sort_content/(.+$)')
    data_feature_pattern = url_path_join(web_app.settings['base_url'], '/data_feature/(.+$)')

    web_app.add_handlers(host_pattern, [
                (file_size_pattern, FileSizeHandler),
                (file_date_pattern, FileDateHandler),
                (view_table_pattern, ViewTableHandler),
                (file_content_pattern, FileContentHandler),
                (line_number_pattern, FileLineNumberHandler),
                (sort_content_pattern, SortContentHandler),
                (data_feature_pattern, DataFeatureHandler)
    ])


#!/usr/bin/env python3
"""move (rename) files

"""
from sys import exit
from pathlib import PurePosixPath
from argparse import ArgumentParser

from client_backend.fs_db_io import FSDatabase
from client_backend.fs_db_file import File, Directory
from client_backend.fs_db_file import IncorrectFileTypeError, MissingFileError

def parse_args():
    global ARGV, SHELL
    parser = ArgumentParser(description='move (rename) files')
    parser.add_argument('source',
                        help='file(s) that should be renamed/moved',
                        nargs='+')
    parser.add_argument('destination',
                        help='New filename or directory to move the file to')

    return parser.parse_args(ARGV)

# See FSDatabase._resolve_relative_path
def to_abs_path(path):
    global SHELL
    ctx_path = PurePosixPath("/")
    path = PurePosixPath(path)
    if path.parts and path.parts[0] == "~":
        ctx_path = PurePosixPath(SHELL.HOME)
    else:
        ctx_path = PurePosixPath(SHELL.PWD)

    assert ctx_path.anchor
    # if path starts with /, will ignore ctx_path automatically
    return ctx_path.joinpath(path)

def move_to(src_path, dst_path):
    global FS
    try:
        File.resolve_to(FS, dst_path, Directory)
    except (MissingFileError, IncorrectFileTypeError):
        print(f"mv: target '{str(dst_path)}' is not a directory")
        return 1

    return not rename_to(src_path, dst_path)

def rename_to(src_path, dst_path):
    global FS

    try:
        # Either exists or does not. If does not exist, generic error is fine
        # If wrong type then should've been deleted
        src_file = File(FS, src_path)
        src_path = PurePosixPath(src_file.full_name)
        dst_path = to_abs_path(dst_path)
        try:
            resolved_dir = File.resolve_to(FS, dst_path, Directory)
            dst_path = PurePosixPath(resolved_dir.full_name).joinpath(src_path.name)
        except IncorrectFileTypeError:
            File(FS, dst_path).remove()
        except MissingFileError:
            pass
        dst_dir = Directory(FS, dst_path.parent)
        if src_path.parent != dst_path.parent:
            src_file.move(dst_dir)
        src_file.name = dst_path.name

    except (MissingFileError, IncorrectFileTypeError):
        print(f"cannot move '{str(src_path)}' to '{str(dst_path)}': No such file or directory")
        return 1

def main(args):
    global FS
    if len(args.source) > 1:
        for source in args.source:
            if not move_to(source, args.destination):
                return 1
    else:
        return rename_to(args.source[0], args.destination)

if __name__ == "__main__":
    import sys
    from client_backend import shell_context
    FS = FSDatabase('.fs_db_rdbsh')
    ARGV = sys.argv[1:]
    SHELL = shell_context
    exit(main(parse_args()))

if __name__ == "__rdbsh__":
    exit(main(parse_args()))


import cycle
import rx

def log(msg):
  print(msg)

def main(responses):
  responses["dumbDriver"].subscribe(on_next=log)
  obs = rx.Observable.interval(1000)

  return {
    "dumbDriver": obs
  }

def dumbDriver():

  def driver(responses):
    responses.subscribe(on_next=log)
    obs = rx.Observable.interval(200).map(lambda x: x*x)
    return obs

  return driver

drivers = {
  "dumbDriver": dumbDriver()
}

cycle.run(main, drivers)


import numpy as np
import pylab as pyl
import matplotlib.pyplot as plt
import bcolz
from random import sample
from sklearn.cluster import MiniBatchKMeans
from sklearn.metrics.pairwise import pairwise_distances_argmin
from scipy.spatial.distance import cdist,pdist

def save_array(fname, arr):
    carr=bcolz.carray(arr, rootdir=fname, mode='w'); carr.flush()

def load_array(fname):
    return bcolz.open(fname)[:]

def plot_image(imname):
    pyl.imshow(imname)
    pyl.show()

def plot_cluster(image_list, labels, c_id, n_images=20, cols=4):
    """
    helper to plot shoes/images within a cluster

    Params:
    --------
    image_list: list with all the images
    labels    : cluster labels
    c_id      : id of the cluster to plot
    n_images  : number of images in the plot
    cols      : columns of the plot
    """

    idx = np.where(labels == c_id)[0]
    idx = sample(idx, n_images)
    imgs = [image_list[i] for i in idx]

    nrow = len(idx) / cols
    ncol = cols

    if ((ncol * nrow) != len(idx)): nrow = nrow + 1

    plt.figure()
    for i,img in enumerate(imgs):
        plt.subplot(nrow,ncol,i+1)
        plt.imshow(img)
        plt.axis('off')
    plt.show()

def avg_within_ss(X, k):
    """
    Compute the average within-cluster sum of squares. The code here can be
    found "almost" anywhere online

    Params:
    --------
    X: numpy array with observations and features to be clustered
    k: number of clusters

    Returns:
    --------
    avgwithinss: average within-cluster sum of squares
    """

    model = MiniBatchKMeans(init='k-means++', n_clusters=k, batch_size=50,
                          n_init=3, max_no_improvement=10, verbose=0)
    model.fit(X)

    centroids = model.cluster_centers_
    dist_c = cdist(X, centroids, 'euclidean')
    dist   = np.min(dist_c, axis=1)
    avgwithinss = sum(dist**2)/X.shape[0]

    return avgwithinss

def perc_var_explained(X,k):
    """
    Compute the percentage of variance explained defined as between sum of squares
    divided but the total sum of squares.
    WARNING: It will take a while.
    The code here can be found "almost" anywhere online.

    Params:
    --------
    X: numpy array with observations and features to be clustered
    k: number of clusters

    Returns:
    --------
    pve: percentage of variance explained
    """

    model = MiniBatchKMeans(init='k-means++', n_clusters=k, batch_size=50,
                          n_init=3, max_no_improvement=10, verbose=0)
    model.fit(X)

    centroids = model.cluster_centers_
    dist_c = cdist(X, centroids, 'euclidean')
    dist   = np.min(dist_c, axis=1)
    tot_withinss = sum(dist**2)
    totss = sum(pdist(X)**2)/X.shape[0]
    betweenss = totss - tot_withinss
    pve = (betweenss/totss  *100)

    return pve

def bic(X, k):
    """
    Compute the BIC score.
    Implementarion from here:
    http://www.aladdin.cs.cmu.edu/papers/pdfs/y2000/xmeans.pdf
    with corrections from here:
    https://stats.stackexchange.com/questions/90769/using-bic-to-estimate-the-number-of-k-in-kmeans

    Params:
    --------
    X: numpy array with observations and features to be clustered
    k: number of clusters

    Returns:
    --------
    BIC: bic score
    """

    model = MiniBatchKMeans(init='k-means++', n_clusters=k, batch_size=50,
                          n_init=3, max_no_improvement=10, verbose=0)
    model.fit(X)

    centers = model.cluster_centers_
    centers = np.expand_dims(centers, axis=1)
    labels  = model.labels_
    N_C = np.bincount(labels)
    R, M = X.shape

    wcss = sum([sum(cdist(X[np.where(labels == c)], centers[c], 'euclidean')**2) for c in range(k)])
    var = (1.0/(R-k)/M) * wcss
    const_term = 0.5 * k * np.log(R) * (M+1)

    BIC = np.sum([ ( Rn * np.log(Rn) ) -
                   ( Rn * np.log(R) ) -
                   ( ((Rn * M) / 2) * np.log(2*np.pi*var) )  -
                   ( (Rn - 1) * M/ 2 )
                   for Rn in N_C]) - const_term

    return BIC


import imp
from fastapi import FastAPI,Header,Depends
from fastapi.security import oauth2
from fastapi.middleware.cors import CORSMiddleware
from fastapi_pagination import add_pagination
from sqladmin import Admin
from main.db import engine,asyncengine
from main.utils import OAuth2AppBearer
from user.approutes import auth
from store.approutes import store_app
from user.admin import UserAdmin,RoleAdmin,ContentTypesAdmin
from user.models import User
def create_dev_app():

    app=FastAPI(title="Development User Admin App",debug=False)
    origins = [ "*"]
    app.add_middleware(
            CORSMiddleware,
            allow_origins=origins,
            allow_credentials=True,
            allow_methods=["*"],
            allow_headers=["*"],
            )
    tok_bear=OAuth2AppBearer(tokenUrl="/auth/login")
    app.include_router(auth,prefix="/auth")
    app.include_router(store_app,prefix="/store")
    admin=Admin(app,asyncengine,title="Store Admin Dashboard")
    @app.get("/")
    def index(current_user : str=Depends(tok_bear)):
        return {"Message":current_user}
    add_pagination(app)
    admin.add_view(UserAdmin)
    admin.add_view(RoleAdmin)
    admin.add_view(ContentTypesAdmin)

    return app

def create_testing_app():
    app=FastAPI()
    origins = [ "*"]
    app.add_middleware(
            CORSMiddleware,
            allow_origins=origins,
            allow_credentials=True,
            allow_methods=["*"],
            allow_headers=["*"],
            )

    @app.get("/")
    def index():
        return {"Message":"You should make your own index page"}
    return app


import os
from datetime import datetime

APP_NAME = "Kotar"
APP_DATA = 'APPDATA'
# TODO: set the telegram bot token and the chat id
# TELEGRAM Connection based on
# https://pythoncircle.com/post/265/how-to-create-completely-automated-telegram-channel-with-python/
TELEGRAM_TOKEN = None
TELEGRAM_CHAT_ID = None

NEW_DRIVER_DATE = datetime.strptime('2019-12-09', '%Y-%m-%d').date()

def create_temp_folder(notify: bool):
    app_data = os.getenv(APP_DATA)
    if app_data is not None:
        app_data = os.path.join(app_data, APP_NAME)
        os.makedirs(app_data, exist_ok=True)

    if notify:
        print("App temp directory: {}".format(app_data))
    return app_data

def rreplace(s: str, old: str, new: str, occurences):
    li = s.rsplit(old, occurences)
    return new.join(li)

def change_file_name(filepath, to):
    path, ext = os.path.splitext(filepath)
    filename = os.path.split(path)[-1]
    new_path = rreplace(filepath, filename, to, -1)
    return new_path

def str_to_bool(value: str) -> bool:
    return value.lower() in ["true", "t", "1", "yes""y"]

def try_parse_int(value):
    """
    Attempts to parse the given value to int and return its value, if fails returns original value
    :param value: the value to parse to int
    :return: int representation for value or value
    """
    try:
        return int(value)
    except ValueError:
        return value

class Config(object):
    """
    A class that should handle all app configurations
    """
    ENV = "environment"
    DB_URL = "DB_URL"

    def __init__(self, **kwargs):
        import configparser
        self.read_ok = False
        self.config = configparser.ConfigParser()
        config_path = kwargs.get("config_path", None)
        if config_path is not None:
            self.read(config_path)

    def get(self, section, option, default):
        if not self.read_ok:
            print("No config file found, working with defaults")
            self.read_ok = True
        return try_parse_int(self.config.get(section, option)) if self.config.has_option(section, option) else default

    def read(self, config_file):
        self.read_ok = len(self.config.read(config_file)) > 0
        print("Reading config file: '{}'".format(config_file))

    def expose_env(self):
        if self.config.has_section(Config.ENV):
            exposed_db_url = False
            for var_name in self.config.options(Config.ENV):
                var_name = var_name.upper()
                var_value = self.config.get(Config.ENV, var_name).replace('"', "").replace("'", '')

                if var_name == Config.DB_URL:
                    if not os.path.isabs(var_value):
                        var_value = os.path.abspath(var_value)
                    exposed_db_url = True

                print("Exposing {} - {}: {}".format(Config.ENV, var_name, var_value))

                os.environ[var_name] = var_value

            if not exposed_db_url:
                print("WARNING! DB_URL is not defined")
        else:
            os.environ["DB_URL"] = os.path.abspath("database.db")
            print("Exposing {} - {}: {}".format(Config.ENV, "DB_URL", os.path.abspath("database.db")))

class Singleton(type):
    _instances = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
        return cls._instances[cls]

class AppDir(object, metaclass=Singleton):
    def __init__(self):
        self.app_directory = self.create_app_dir(True)

    def create_app_dir(self, notify: bool):
        app_dir = os.getenv(APP_DATA)
        if app_dir is not None:
            app_dir = os.path.join(app_dir, APP_NAME)
            try:
                os.makedirs(app_dir)
            except:
                pass

        if notify:
            print("App temp directory: {}".format(app_dir))
        return app_dir


import re
from operator import add

from mrjob.job import MRJob

class MRSparkWordcount(MRJob):

    def spark(self, input_path, output_path):
        # Spark may not be available where script is launched
        from pyspark import SparkContext

        sc = SparkContext(appName='mrjob Spark image ount script')

        lines = sc.textFile(input_path)

        counts = (
            lines.flatMap(self.get_words)
            .map(lambda word: (word, 1))
            .reduceByKey(add))

        counts.saveAsTextFile(output_path)

        sc.stop()

    def get_words(self, line):
        words = re.split(",", line)
        return words[0].lower().strip()

if __name__ == '__main__':
    MRSparkWordcount.run()


import numpy as np
import cv2

def image_t(im, scale=1.0, rot=45, trans=(50,-50)):
    # transform the rotation angle to radian
    rot_rad = np.pi * rot / 180

    # sample three points and compute their point coordinates in the target image
    m = np.float32([[scale * np.cos(rot_rad), scale * np.sin(rot_rad), trans[0]],
                    [-scale * np.sin(rot_rad), scale * np.cos(rot_rad), trans[1]],
                    [0, 0, 1]])
    src_pt1 = np.float32([0, 0, 1])
    src_pt2 = np.float32([0, 1, 1])
    src_pt3 = np.float32([1, 0, 1])
    dst_pt1 = np.dot(m, src_pt1)
    dst_pt2 = np.dot(m, src_pt2)
    dst_pt3 = np.dot(m, src_pt3)
    h, w, _ = im.shape
    center = np.float32([h/2, w/2])
    src_tri = np.float32([center + src_pt1[0:2], center + src_pt2[0:2], center + src_pt3[0:2]])
    dst_tri = np.float32([center + dst_pt1[0:2], center + dst_pt2[0:2], center + dst_pt3[0:2]])

    # calculate transformation matrix
    trans_mat = cv2.getAffineTransform(src_tri, dst_tri)

    # obtain the target image by using transform matrix
    result = cv2.warpAffine(im, trans_mat, (h, w))

    # return transformed result image
    return result

if __name__ == '__main__':
    im = cv2.imread('./misc/pearl.jpeg')

    scale  = 0.5
    rot    = 45
    trans  = (50, -50)
    result = image_t(im, scale, rot, trans)
    cv2.imwrite('./results/affine_result.png', result)

Assignment 1 - Image Processing Basic/code/runTransform.py


"""Module for visualizing the first convolution layer filter images"""
import torch
import network
import torchvision.transforms as transforms
from PIL import Image
import torchvision.models as models
import matplotlib.pyplot as plt
import numpy as np

OUT_PATH = "../filter_images/"

def generate_filters(model_path, out_file_name):
	"""Generate the first layer filters"""
	net = network.SimClRNet(num_classes=12)
	net.load_state_dict(torch.load(model_path))
	model_children = list(net.backbone.children())
	print(model_children[0], type(model_children[0]), model_children[0].weight.shape)
	convlayer1 = model_children[0]

	# visualize the first conv layer filters
	plt.figure(figsize=(20, 17), tight_layout=True)
	for i, fltr in enumerate(model_children[0].weight):
		plt.subplot(8, 8, i+1)  # (8, 8) because in conv0 we have 7x7 filters and total of 64 (see printed shapes)
		fltr = fltr.detach().numpy()
		fltr = fltr - fltr.min()
		fltr = fltr / fltr.max()
		fltr = np.transpose(fltr, (1, 2, 0))
		plt.imshow(fltr)
		plt.axis('off')
		plt.savefig(OUT_PATH + out_file_name)
	# plt.show()

generate_filters("../simclr_models/simclr_toybox_self_final_1_supervised.pt", "self_1.png")
generate_filters("../simclr_models/simclr_toybox_self_final_2_supervised.pt", "self_2.png")
generate_filters("../simclr_models/simclr_toybox_transform_final_1_supervised.pt", "transform_1.png")
generate_filters("../simclr_models/simclr_toybox_transform_final_2_supervised.pt", "transform_2.png")
generate_filters("../simclr_models/simclr_toybox_object_final_1_supervised.pt", "object_1.png")
generate_filters("../simclr_models/simclr_toybox_object_final_2_supervised.pt", "object_2.png")
generate_filters("../simclr_models/toybox_class_final_supervised.pt", "class_1.png")
generate_filters("../simclr_models/toybox_class_final_2_supervised.pt", "class_2.png")


import scrapy
import pickle
import os
import ast
from urllib import parse
from scrapy.selector import Selector

class AnhuiSpider(scrapy.Spider):
    name = "Anhui"
    if not os.path.exists('../../data/HTML_pk/%s' % name):
        os.makedirs('../../data/HTML_pk/%s' % name)
    if not os.path.exists('../../data/text/%s' % name):
        os.makedirs('../../data/text/%s' % name)
    def start_requests(self):
        total_page = 448
        # total_page = 3
        url_base = 'http://www.ah.gov.cn/site/label/8888?IsAjax=1&dataType=html&_=0.42568734060518776&siteId=6781961&pageSize=16&pageIndex={0}&action=list&isDate=true&dateFormat=yyyy-MM-dd&length=46&organId=1681&type=4&catId=&cId=&result=%E6%9A%82%E6%97%A0%E7%9B%B8%E5%85%B3%E4%BF%A1%E6%81%AF&searchType=&keyWords=&specialCatIds=6708451%2C6708461%2C6708471&catIdExplainType=6711101tp_explain%2C6711111xwfbh_explain%2C6711121sp_explain%2C6711131ft_explain&labelName=publicInfoList&file=%2Fahxxgk%2FpublicInfoList-an-new'
        for i in range(total_page):
            yield scrapy.Request(url=url_base.format(i+1), callback=self.parse)

    def parse(self,response):
        detail_page_links = []
        for div in response.css('div.xxgk_navli'):
            url = response.urljoin(div.css('a::attr(href)').get())
            UID = url.split('/')[-1][:-5]
            detail_page_links.append(url)
            yield {
                'UID': UID,
                'title': div.css('a::attr(title)').get(),
                'date': div.css('span.date::text').get(),
                'FileNumber':None,
                'url': url,
                'crawl state':'half'
            }
        yield from response.follow_all(detail_page_links, callback = self.parse_content)

    def parse_content(self, response):
        UID = response.url.split('/')[-1][:-5]
        with open('../../data/HTML_pk/%s/%s.pkl' % (self.name,UID), 'wb') as f:
            pickle.dump(response.text,f)
        doc_info_dict = {}
        td_list = response.css('tbody')[0].css('td')
        th_list = response.css('tbody')[0].css('th')
        for i in range(len(th_list)):
            key = ''.join(th_list[i].css('::text').getall())
            value = ''.join(td_list[i].css('::text').getall())
            doc_info_dict[key] = value
        File_num = None
        if '文号：' in doc_info_dict.keys():
            File_num = doc_info_dict['文号：']
        paragraph_list =  response.css('div.wzcon p *::text').getall()
        with open('../../data/text/%s/%s.txt' % (self.name,UID), 'w') as f:
            f.write('\n'.join(paragraph_list))
        return {
            'UID': UID,
            'FileNumber':File_num,
            'mainText': paragraph_list,
            'doc_info_dict':doc_info_dict,
            'crawl state':'full',
        }