# Copyright 2016, RadiantBlue Technologies, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use
# this file except in compliance with the License. You may obtain a copy of the
# License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

import logging
import math
import re
from datetime import datetime
from typing import Optional

import dateutil.parser
import requests

from bfapi import db
from bfapi.config import CATALOG, DOMAIN

PATTERN_SCENE_ID = re.compile(r'^(planetscope|rapideye):[\w_-]+$')
PLATFORM_PLANETSCOPE = 'planetscope'
PLATFORM_RAPIDEYE = 'rapideye'
STATUS_ACTIVE = 'active'
STATUS_ACTIVATING = 'activating'
STATUS_INACTIVE = 'inactive'

#
# Types
#

class Scene:
    def __init__(
            self,
            *,
            capture_date: datetime,
            cloud_cover: float,
            geotiff_coastal: str = None,
            geotiff_multispectral: str = None,
            geotiff_swir1: str = None,
            geometry: dict,
            platform: str,
            resolution: int,
            scene_id: str,
            sensor_name: str,
            status: str,
            tide: float,
            tide_min: float,
            tide_max: float,
            uri: str):
        self.capture_date = capture_date
        self.cloud_cover = cloud_cover
        self.id = scene_id
        self.geometry = geometry
        self.geotiff_coastal = geotiff_coastal
        self.geotiff_multispectral = geotiff_multispectral
        self.geotiff_swir1 = geotiff_swir1
        self.platform = platform
        self.resolution = resolution
        self.sensor_name = sensor_name
        self.status = status
        self.tide = tide
        self.tide_min = tide_min
        self.tide_max = tide_max
        self.uri = uri

#
# Actions
#

def activate(scene: Scene, planet_api_key: str, user_id: str) -> Optional[str]:
    log = logging.getLogger(__name__)

    if scene.status == STATUS_ACTIVE:
        return scene.geotiff_multispectral
    elif scene.status == STATUS_ACTIVATING:
        return None

    # Request activation
    platform, external_id = _parse_scene_id(scene.id)

    activation_url = 'https://{}/planet/activate/{}/{}'.format(CATALOG, platform, external_id)
    log.info('Activating `%s`', scene.id, actor=user_id, action='activate scene', actee=scene.id)
    try:
        log.debug('Requesting activation; url=`%s`', activation_url)
        response = requests.get(
            activation_url,
            params={
                'PL_API_KEY': planet_api_key,
            }
        )
        response.raise_for_status()
    except requests.ConnectionError:
        raise CatalogError()
    except requests.HTTPError as err:
        status_code = err.response.status_code
        if status_code == 404:
            raise NotFound(scene.id)
        raise CatalogError()

def create_download_url(scene_id: str, planet_api_key: str = '') -> str:
    # HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK
    # FIXME -- hopefully this endpoint can move into the IA Broker eventually
    return 'https://bf-api.{}/v0/scene/{}.TIF?planet_api_key={}'.format(
        DOMAIN,
        scene_id,
        planet_api_key,
    )
    # HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK

def get(scene_id: str, planet_api_key: str, *, with_tides: bool = True) -> Scene:
    log = logging.getLogger(__name__)

    platform, external_id = _parse_scene_id(scene_id)

    uri = 'https://{}/planet/{}/{}'.format(CATALOG, platform, external_id)
    log.info('Fetching `%s`', uri, action='fetch scene metadata', actee=scene_id)
    try:
        response = requests.get(
            uri,
            params={
                'PL_API_KEY': planet_api_key,
                'tides': with_tides,
            }
        )
        response.raise_for_status()
    except requests.ConnectionError:
        raise CatalogError()
    except requests.HTTPError as err:
        status_code = err.response.status_code
        if status_code == 404:
            raise NotFound(scene_id)
        raise CatalogError()

    feature = response.json()

    status = _extract_status(scene_id, feature)

    geotiff_multispectral = feature['properties'].get('location')
    if status == STATUS_ACTIVE and not geotiff_multispectral:
        raise ValidationError(scene_id, 'Scene is activated but missing GeoTIFF URL')

    scene = Scene(
        scene_id=scene_id,
        uri=uri,
        capture_date=_extract_capture_date(scene_id, feature),
        cloud_cover=_extract_cloud_cover(scene_id, feature),
        geometry=_extract_geometry(scene_id, feature),
        geotiff_multispectral=geotiff_multispectral,
        platform=platform,
        resolution=_extract_resolution(scene_id, feature),
        sensor_name=_extract_sensor_name(scene_id, feature),
        status=status,
        tide=_extract_tide(scene_id, feature),
        tide_min=_extract_tide_min(scene_id, feature),
        tide_max=_extract_tide_max(scene_id, feature),
    )

    _save_to_database(scene)
    return scene

#
# Helpers
#

def _extract_capture_date(scene_id: str, feature: dict) -> datetime:
    value = feature['properties'].get('acquiredDate')
    if value is None:
        raise ValidationError(scene_id, 'missing `acquiredDate`')
    try:
        value = dateutil.parser.parse(value)
    except Exception as err:
        raise ValidationError(scene_id, 'could not parse `acquiredDate`: ({})'.format(err))
    return value

def _extract_cloud_cover(scene_id: str, feature: dict) -> float:
    value = feature['properties'].get('cloudCover')
    if value is None:
        raise ValidationError(scene_id, 'missing `cloudCover`')
    try:
        value = round(value, 2)
    except:
        raise ValidationError(scene_id, '`cloudCover` is not a float')
    return value

def _extract_geometry(scene_id: str, feature: dict) -> dict:
    value = feature.get('geometry')
    if value is None:
        raise ValidationError(scene_id, 'missing `geometry`')
    if not isinstance(value, dict):
        raise ValidationError(scene_id, '`geometry` is not a dictionary')
    return value

def _extract_resolution(scene_id: str, feature: dict) -> int:
    value = feature['properties'].get('resolution')
    if value is None:
        raise ValidationError(scene_id, 'missing `resolution`')
    try:
        value = int(math.ceil(value))
    except:
        raise ValidationError(scene_id, '`resolution` is not an int')
    return value

def _extract_sensor_name(scene_id: str, feature: dict) -> str:
    value = feature['properties'].get('sensorName')
    if value is None:
        raise ValidationError(scene_id, 'missing `sensorName`')
    return value.strip()

def _extract_status(scene_id: str, feature) -> str:
    value = feature['properties'].get('status')
    if value is None:
        raise ValidationError(scene_id, 'missing `status`')
    if value not in (STATUS_ACTIVE, STATUS_ACTIVATING, STATUS_INACTIVE):
        raise ValidationError(scene_id, 'value of `status` is ambiguous')
    return value

def _extract_tide(scene_id: str, feature: dict) -> float:
    value = feature['properties'].get('CurrentTide')
    if value is not None:
        try:
            value = round(value, 5)
        except:
            raise ValidationError(scene_id, '`CurrentTide` is not a float')
    return value

def _extract_tide_min(scene_id: str, feature: dict) -> float:
    value = feature['properties'].get('24hrMinTide')
    if value is not None:
        try:
            value = round(value, 5)
        except:
            raise ValidationError(scene_id, '`24hrMinTide` is not a float')
    return value

def _extract_tide_max(scene_id: str, feature: dict) -> float:
    value = feature['properties'].get('24hrMaxTide')
    if value is not None:
        try:
            value = round(value, 5)
        except:
            raise ValidationError(scene_id, '`24hrMaxTide` is not a float')
    return value

def _parse_scene_id(scene_id) -> (str, str):
    if not PATTERN_SCENE_ID.match(scene_id):
        raise MalformedSceneID(scene_id)
    return scene_id.split(':', 1)

def _save_to_database(scene: Scene):
    log = logging.getLogger(__name__)
    conn = db.get_connection()
    try:
        db.scenes.insert(
            conn,
            scene_id=scene.id,
            captured_on=scene.capture_date,
            catalog_uri=scene.uri,
            cloud_cover=scene.cloud_cover,
            geometry=scene.geometry,
            resolution=scene.resolution,
            sensor_name=scene.sensor_name,
        )
    except db.DatabaseError as err:
        log.error('Could not save scene `%s` to database', scene.id)
        db.print_diagnostics(err)
        raise
    finally:
        conn.close()

#
# Errors
#

class CatalogError(Exception):
    def __init__(self):
        super().__init__('error communicating with image catalog')

class MalformedSceneID(Exception):
    def __init__(self, scene_id: str):
        super().__init__('malformed scene id `{}`'.format(scene_id))
        self.scene_id = scene_id

class NotFound(Exception):
    def __init__(self, scene_id: str):
        super().__init__('scene `{}` not found in catalog'.format(scene_id))
        self.scene_id = scene_id

class ValidationError(Exception):
    def __init__(self, scene_id: str, message: str):
        super().__init__('scene `{}` has invalid metadata: {}'.format(scene_id, message))
        self.scene_id = scene_id


import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
mpl.rcParams['xtick.labelsize']=16
mpl.rcParams['ytick.labelsize']=16
datafile = 'spec_on_foreground_reflected_power_21cmfast_14.6m_150.0_MHz_subband_v2.npz'

npzdata = np.load(datafile)
tau = npzdata['tau']
achrm = npzdata['achrmbeam']

apdata = np.load('refl_cstr.npz')
aptau = apdata['tau_cstr']
vk10 = apdata['refl_cstr_dspec_k0.10']
vk15 = apdata['refl_cstr_dspec_k0.15']
vk20 = apdata['refl_cstr_dspec_k0.20']
ok10 = apdata['refl_cstr_oqe_k0.10']
ok15 = apdata['refl_cstr_oqe_k0.15']
ok20 = apdata['refl_cstr_oqe_k0.20']
aptm = apdata['tau_meas']
aprm = apdata['refl_meas']

bottom = np.zeros(len(tau))
plt.plot(tau,achrm[0],lw=2,label=r'$k_{||}>0.10$ h/Mpc')
plt.fill_between(tau,bottom,achrm[0],color='b',alpha='0.25')
plt.plot(tau,achrm[1],lw=2,label=r'$k_{||}>0.15$ h/Mpc')
plt.fill_between(tau,bottom,achrm[1],color='g',alpha='0.25')
plt.plot(tau,achrm[2],lw=2,label=r'$k_{||}>0.20$ h/Mpc')
plt.fill_between(tau,bottom,achrm[2],color='k',alpha='0.25')
t60 = [60,60,500]
a60 = [0,60,60]
a0 = [0.0,0.0,0.0]
plt.fill_between(t60,a0,a60,color='k',alpha='0.05')
plt.plot(t60,a60)
plt.axis([0,500,67,0])
plt.xlabel(r'$\tau$ [ns]',fontsize=18)
plt.ylabel('Attenuation [dB]',fontsize=18)
plt.legend(fontsize=18)

delay = np.load('delay_spectrum.npz')
#plt.plot(delay['tau'],delay['dpaper'])
plt.plot(delay['tau_hera'],delay['dhera'],'k',lw=2)
#plt.figure('aaa')
#plt.plot(delay['tau'],delay['dhera'])

plt.figure('ap')
plt.plot(aptau,vk10,'b')
plt.plot(aptau,vk15,'b')
plt.plot(aptau,vk20,'b')
plt.plot(aptau,ok10,'g')
plt.plot(aptau,ok15,'g')
plt.plot(aptau,ok20,'g')
plt.plot(aptm,aprm,'k')
plt.axis([0,500,-70,0])

plt.figure(1)
dash = []
dash.append(np.array([[89.4,22.3],
         [188.5,24.0]]))
plt.plot(dash[0][:,0],dash[0][:,1],'b',lw=0.5)
plt.plot(dash[0][:,0],dash[0][:,1],'b--',lw=2)

dash.append(np.array([dash[0][1],
         [286.3,25.3]]))
plt.plot(dash[1][:,0],dash[1][:,1],'g',lw=0.5)
plt.plot(dash[1][:,0],dash[1][:,1],'g--',lw=2)

dash.append(np.array([dash[1][1],
         [500,28.6]]))
plt.plot(dash[2][:,0],dash[2][:,1],'r',lw=0.5)
plt.plot(dash[2][:,0],dash[2][:,1],'r--',lw=2)

plt.savefig('delayspecplot.pdf',dpi=200)


#!/usr/bin/env python
import argparse
import os
import numpy as np
import h5py

def cntSteps(fname):
    with h5py.File(fname,'r') as hf:
        grps = hf.values()
        grpNames = [str(grp.name) for grp in grps]
        #Steps = [stp if "/Step#" in stp for stp in grpNames]
        Steps = [stp for stp in grpNames if "/Step#" in stp]
        nSteps = len(Steps)
        return nSteps
def cntParticles(fname):
    with h5py.File(fname,'r') as hf:
        id0 = np.array(hf.get("/Step#0/id"))
    return id0.min(),id0.max()
def getPVars(fname):
    #Get the particle valued functions of this h5p file
    #Use step0

    with h5py.File(fname,'r') as hf:
        stp0 = hf["Step#0"]
        Np = stp0["id"].shape[0] #Number of particles
        pVars = []
        for k in stp0.keys():
            L = stp0[k].shape[0]
            #Only take particle-valued data
            if (L == Np):
                pVars.append(str(k))
    return pVars

def copyRootAttr(fIn,fOut):
    with h5py.File(fIn,'r') as hfr, h5py.File(fOut,'a') as hfw:
        for k in hfr.attrs.keys():
            aStr = str(k)
            hfw.attrs.create(k,hfr.attrs[aStr])
def sumToAttr(hId,redStr,pStr):
    V = hId[pStr].value.sum()
    hId.attrs.create(redStr,V)
    return V

if __name__ == "__main__":

	#Set defaults
	hfdType = np.float32
	parser = argparse.ArgumentParser(description="Concatenates numerous h5part files generated by LFMTP into a single h5p file.")
	#Get required info
	parser.add_argument('-eq',help="Only include equatorial data",action="store_true")
	parser.add_argument('-min3d',help="Only include minimal 3D data",action="store_true")
	parser.add_argument('tpdat',nargs='+',metavar='output.h5part',help="LFMTP data files to combine.")

	args = parser.parse_args()

	fIns = args.tpdat
	doEq = False
	doMin3D = False
	if (args.eq):
		doEq = True
	if (args.min3d):
		doMin3D = True

	#Create output file name
	pre,ext = os.path.splitext(fIns[0])
	stubs = pre.split(".")

	#Parse data from input files
	Nf = len(fIns)
	nSteps = np.zeros(Nf,dtype=np.int)
	nP = np.zeros((Nf,2),dtype=np.int)

	#Loop over input files and pull info
	for n in range(Nf):
		h5p = fIns[n]
		nSteps[n] = cntSteps(h5p)
		nP[n,0],nP[n,1] = cntParticles(h5p)

	print("Max/Min Steps = %d / %d"%(nSteps.max(),nSteps.min()))

	#Do any checks necessary
	Ns = nSteps.max()
	NumP = nP.max() #Assume particles are contiguous for now
	ids = np.arange(1,NumP+1) #[) interval

	#Get variables, assume matched across all inputs
	#Either use all or only equatorial
	if (doEq):
		pVars = ["x","y","z","id","in","Teq","xeq","yeq","mueq","alpheq","keveq"]
		oStub = "eqAll"
	elif (doMin3D):
		pVars = ["x","y","z","id","in","alpha","kev","xeq","yeq"]
		oStub = "Min3D"
	else:
		pVars = getPVars(fIns[0])
		oStub = "All"
	NumV = len(pVars)
	fOut = stubs[0] + "." + oStub + ".h5part"

	print("Reading %d variables:  %s"%(NumV,str(pVars)))
	print("Merging %d files into %s ..."%(Nf,fOut))

	#Create big data holder, size NumP x NumV
	pDat = np.zeros((NumP,NumV),dtype=hfdType)

	#Do big output write
	with h5py.File(fOut,'w') as hfw:
		#Loop over steps, variables, and input files
		for n in range(Ns):
			gId = "Step#%d"%(n)
			hfw.create_group(gId)
			grp = hfw.get(gId)
			print("\tMerging %s"%(gId))
			pDat[:,:] = 0.0

			#Open a file, scoop out all relevant data for this timestep
			for i in range(Nf):
				#Get Step#n unless the file ended early
				#Then grab last slice
				stpId = "Step#%d"%(min(n,nSteps[i]-1))
				sStr = stpId + "/"
				pId0 = nP[i,0]
				pId1 = nP[i,1]
				with h5py.File(fIns[i],'r') as hfr:
					if (Ns == nSteps[i]):
						time = hfr[sStr].attrs["time"]
						timestep = hfr[sStr].attrs["timestep"]
					for k in range(NumV):
						vStr = sStr + pVars[k] #Variable string
						pDat[pId0-1:pId1,k] = np.array(hfr[vStr])
			#Write full timestep data for each variable
			for k in range(NumV):
				grp.create_dataset(pVars[k],data=pDat[:,k].squeeze())

			#Handle attributes for this step
			hfw[sStr].attrs.create("time",time)
			hfw[sStr].attrs.create("TimeValue",time)
			hfw[sStr].attrs.create("timestep",timestep)

	print("Copying root attributes from first input file and finalizing ...")
	#Copy root attributes from first input file
	copyRootAttr(fIns[0],fOut)


import open3d as o3d
import numpy as np
import time
import copy
from sklearn.neighbors import KDTree
import matplotlib.pyplot as plt

def draw_registration_result(source, target, transformation):
    source_temp = copy.deepcopy(source)
    target_temp = copy.deepcopy(target)
    source_temp.estimate_normals(
        o3d.geometry.KDTreeSearchParamHybrid(radius=0.15, max_nn=30))
    target_temp.estimate_normals(
        o3d.geometry.KDTreeSearchParamHybrid(radius=0.15, max_nn=30))
    source_temp.paint_uniform_color([1, 0.706, 0])
    target_temp.paint_uniform_color([0, 0.651, 0.929])
    source_temp.transform(transformation)
    o3d.visualization.draw_geometries([source_temp, target_temp])

def preprocess_point_cloud(pcd, voxel_size):
    print(":: Downsample with a voxel size %.3f." % voxel_size)
    pcd_down = pcd.voxel_down_sample(voxel_size)

    radius_normal = voxel_size * 2
    # print(":: Estimate normal with search radius %.3f." % radius_normal)
    pcd_down.estimate_normals(
        o3d.geometry.KDTreeSearchParamHybrid(radius=radius_normal, max_nn=30))

    radius_feature = voxel_size * 5
    # print(":: Compute FPFH feature with search radius %.3f." % radius_feature)
    pcd_fpfh = o3d.pipelines.registration.compute_fpfh_feature(
        pcd_down,
        o3d.geometry.KDTreeSearchParamHybrid(radius=radius_feature, max_nn=100))
    return pcd_down, pcd_fpfh

def prepare_dataset(source, target, voxel_size):
    source_down, source_fpfh = preprocess_point_cloud(source, voxel_size)
    target_down, target_fpfh = preprocess_point_cloud(target, voxel_size)
    # print('source_fpfh', source_fpfh.num, target_fpfh.num)
    #     print('source_fpfh',source_fpfh,np.asarray(source_fpfh.data))
    return source, target, source_down, target_down, source_fpfh, target_fpfh

def execute_global_registration(source_down, target_down, source_fpfh,
                                target_fpfh, voxel_size):
    distance_threshold = voxel_size * 1.5
    # print(":: RANSAC registration on downsampled point clouds.")
    # print("   Since the downsampling voxel size is %.3f," % voxel_size)
    # print("   we use a liberal distance threshold %.3f." % distance_threshold)
    result = o3d.pipelines.registration.registration_ransac_based_on_feature_matching(
        source_down, target_down, source_fpfh, target_fpfh, distance_threshold,
        o3d.pipelines.registration.TransformationEstimationPointToPoint(False),
        4, [
            o3d.pipelines.registration.CorrespondenceCheckerBasedOnEdgeLength(
                0.9),
            o3d.pipelines.registration.CorrespondenceCheckerBasedOnDistance(
                distance_threshold)
        ], o3d.pipelines.registration.RANSACConvergenceCriteria(2000000, 500))
    return result

def file2matrix(filename):
    fr = open(filename)
    numberOfLines = len(fr.readlines())  # get the number of lines in the file
    trans = np.eye(4)  # prepare matrix to return
    truth = []  # prepare labels return
    fr = open(filename)
    index = 0
    for line in fr.readlines():
        line = line.strip()
        # listFromLine = line.split('\t')
        listFromLine = line.split()
        listFromLine = [float(x) for x in listFromLine]

        if (index % 5 == 0):
            index = 0
        elif (index % 5 == 1):
            trans[0, :] = np.array(listFromLine)
        elif (index % 5 == 2):
            trans[1, :] = np.array(listFromLine)
        elif (index % 5 == 3):
            trans[2, :] = np.array(listFromLine)
        elif (index % 5 == 4):
            trans[3, :] = np.array(listFromLine)
            truth.append(trans.copy())  # 这里不用copy的话，，，每个元素都是一样的
        index += 1
    return truth

if __name__ == '__main__':
    root_path = '/Bill/DataSet/RedWood/'
    dataset_names = ['loft', 'lobby', 'apartment', 'bedroom', 'boardroom']
    root_save_path = '/ransac_2M/src2ref'
    dataset_numbers = [252, 199, 319, 219, 243]
    for i in range(len(dataset_names)):
        # for i in  range(1):
        file_path = root_path + dataset_names[i]
        end = dataset_numbers[i]
        save_path = dataset_names[i] + root_save_path

        print(file_path)
        groud_truth = file2matrix(file_path + '/reg_output.log')

        voxel_size = 0.05  # means 5cm for this dataset
        err_R = []
        err_T = []
        trans_all = []
        fail_list = []
        start = 0
        # end = 251
        for j in range(start, end):
            print(
                'j', j
            )
            # index_src = j + 1
            # index_ref = j

            index_src = j
            index_ref = j + 1

            source_show = o3d.io.read_point_cloud(file_path + "/mesh_%s.ply" % (index_src))
            target_show = o3d.io.read_point_cloud(file_path + "/mesh_%s.ply" % (index_ref))
            source, target, source_down, target_down, source_fpfh, target_fpfh = prepare_dataset(source_show,
                                                                                target_show,
                                                                                voxel_size)
            result_ransac = execute_global_registration(source_down, target_down,
                                                        source_fpfh, target_fpfh,
                                                        voxel_size)
            print(result_ransac.transformation)
            total_trans = result_ransac.transformation

            R = total_trans[:3, :3].reshape(3, 3)
            t = total_trans[:3, 3].reshape(-1, 1)
            if index_src > index_ref:
                err_R.append(np.arccos((np.trace(R.T @ groud_truth[j][:3, :3]) - 1) / 2) * 180 / np.pi)
                err_T.append(np.linalg.norm(t - groud_truth[j][:3, 3].reshape(-1, 1), ord=2, axis=0))
                trans_all.append((total_trans))
            else:
                err_R.append(np.arccos((np.trace(R @ groud_truth[j][:3, :3]) - 1) / 2) * 180 / np.pi)
                err_T.append(np.linalg.norm(-R.T @ t - groud_truth[j][:3, 3].reshape(-1, 1), ord=2, axis=0))
                trans_all.append((total_trans))

            # print(total_trans[:3,:3] @ groud_truth[j][:3,:3], np.trace(total_trans[:3,:3] @ groud_truth[j][:3,:3] - np.eye(3)))
            # print(total_trans, groud_truth[j])
            print('err_R err_T', err_R[j - start], err_T[j - start], total_trans)
        if index_src > index_ref:
            #
            #     location = str(start) + '_' + str(end)

            err_all = [err_R, err_T]
            plt.figure("ERR_R ref2src")  # 图像窗口名称
            plt.plot(err_R)
            plt.savefig(save_path + '/%s_%s_err_All_ref2src.jpg' % (start, end))
            # plt.show()
            plt.close()
            plt.figure("ERR_T ref2src")  # 图像窗口名称
            plt.plot(err_T)
            plt.savefig(save_path + '/%s_%s_trans_all_ref2src.jpg' % (start, end))
            # plt.show()
            plt.close()
            np.savetxt(save_path + '/%s_%s_fail_list_ref2src.txt' % (start, end), fail_list)
            np.save(save_path + '/%s_%s_err_All_ref2src.npy' % (start, end), err_all)
            np.savetxt(save_path + '/%s_%s_err_All_ref2src.txt' % (start, end), err_all)
            np.save(save_path + '/%s_%s_trans_all_ref2src.npy' % (start, end), trans_all)
            np.savetxt(save_path + '/%s_%s_trans_all_ref2src.txt' % (start, end), np.array(trans_all).reshape(-1, 4),
                       fmt='%0.8f')

        else:
            err_all = [err_R, err_T]
            plt.figure("ERR_R src2ref")  # 图像窗口名称
            plt.plot(err_R)
            plt.savefig(save_path + '/%s_%serr_All_src2ref.jpg' % (start, end))
            # plt.show()
            plt.close()
            plt.figure("ERR_T src2ref")  # 图像窗口名称
            plt.plot(err_T)
            plt.savefig(save_path + '/%s_%strans_all_src2ref.jpg' % (start, end))
            # plt.show()
            plt.close()
            np.savetxt(save_path + '/%s_%s_fail_list_src2ref.txt' % (start, end), fail_list)
            np.savetxt(save_path + '/%s_%serr_All_src2ref.txt' % (start, end), err_all)
            np.save(save_path + '/%s_%serr_All_src2ref.npy' % (start, end), err_all)
            np.save(save_path + '/%s_%strans_all_src2ref.npy' % (start, end), trans_all)
            np.savetxt(save_path + '/%s_%strans_all_src2ref.txt' % (start, end), np.array(trans_all).reshape(-1, 4),
                       fmt='%0.8f')


#!/usr/bin/python3

# AIO product is here!
# Version: 0.0.1

from engine import *
from cli import *

def main():
    # Defaults...
    Env.g_project_location = "/home/sagid/proj1/"
    # Env.g_binary = "tests/binaries/file.exe"
    Env.g_binary = "/home/sagid/Files/_/plain_payload"

    cli = Cli()
    cli.run()

    return 0

if __name__ == "__main__":
    exit(main())


'''
Created on 24 Mar 2021
테이블에 사원 목록 출력하기
@author: shane
'''
from flask import Flask
from flask.templating import render_template

from day12.mydao import MyEmpDao

app = Flask(__name__)

@app.route('/emp01')
def emp():
    empList = MyEmpDao().getEmps();
    return render_template("emp01.html", empList=empList)

if __name__ == "__main__":
    app.run(host='0.0.0.0', port=80)


import logging
import os
from pyrogram import (
    Client,
    Filters
)
from m3u8 import (
    BLACKLIST_USERS,
    DOWNLOAD_LOCATION
)
# the Strings used for this "thing"
from translation import Translation

logging.basicConfig(
    level=logging.DEBUG,
    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
LOGGER = logging.getLogger(__name__)
logging.getLogger("pyrogram").setLevel(logging.WARNING)

@Client.on_message(Filters.photo)
async def save_photo(bot, update):
    if update.from_user.id in BLACKLIST_USERS:
        await bot.delete_messages(
            chat_id=update.chat.id,
            message_ids=update.message_id,
            revoke=True
        )
        return
    # received single photo
    download_location = os.path.join(
        DOWNLOAD_LOCATION + "/" +
        str(update.from_user.id) + ".jpg"
    )
    await bot.download_media(
        message=update,
        file_name=download_location
    )
    await bot.send_message(
        chat_id=update.chat.id,
        text=Translation.SAVED_CUSTOM_THUMB_NAIL,
        reply_to_message_id=update.message_id
    )

@Client.on_message(Filters.command(["deletethumbnail"]))
async def delete_thumbnail(bot, update):
    if update.from_user.id in BLACKLIST_USERS:
        await bot.delete_messages(
            chat_id=update.chat.id,
            message_ids=update.message_id,
            revoke=True
        )
        return
    download_location = os.path.join(
        DOWNLOAD_LOCATION,
        str(update.from_user.id)
    )
    try:
        os.remove(download_location + ".jpg")
        # os.remove(download_location + ".json")
    except:
        pass
    await bot.send_message(
        chat_id=update.chat.id,
        text=Translation.DEL_ETED_CUSTOM_THUMB_NAIL,
        reply_to_message_id=update.message_id
    )


import gym
import matplotlib.pyplot as plt

observations = []
rewards = []

def runRandom(model, maxSteps=1000):
    env = gym.make(model)
    env.reset()
    for step in range(maxSteps):
        env.render()
        observation, reward, done, info = env.step(env.action_space.sample())
        observations.append(observation)
        rewards.append(reward)
        # if done:
            # print("Finished after", step+1, "time steps.")
            # break

def rewardHistory():
    plt.plot(rewards)
    plt.show()


import numpy as np
import matplotlib

matplotlib.use('Qt5Agg')
import matplotlib.pyplot as plt
from scipy.optimize import minimize
from plotData import plotData
from sigmoid import sigmoid
from costFunction import costFunction
from costFunction import objective_cost
from costFunction import objective_grad
from costFunction import gradientDescent
from plotDecisionBoundary import plotDecisionBoundary
from predict import predict

def main():
    # Machine Learning Online Class - Exercise 2: Logistic Regression
    #
    #  Instructions
    #  ------------
    #
    #  This file contains code that helps you get started on the logistic
    #  regression exercise. You will need to complete the following functions
    #  in this exericse:
    #
    #     sigmoid.m
    #     costFunction.m
    #     predict.m
    #     costFunctionReg.m
    #
    #  For this exercise, you will not need to change any code in this file,
    #  or any other files other than those mentioned above.
    #

    # Load Data
    #  The first two columns contains the exam scores and the third column
    #  contains the label.
    data = np.loadtxt('ex2data1.txt', delimiter=',')
    X = data[:, 0:2]
    y = data[:, 2:]

    # ==================== Part 1: Plotting ====================
    #  We start the exercise by first plotting the data to understand the
    #  the problem we are working with.

    print('Plotting data with + indicating (y = 1) examples and o indicating (y = 0) examples.\n')
    fig1, ax1 = plt.subplots(figsize=(7.2, 5.4))
    # print(plt.rcParams.get('figure.figsize'))
    plt.title('Logistic Regression')
    plotData(ax1, X, y, 'Admitted', 'Not admitted')
    plt.xlabel('Exam 1 score')
    plt.ylabel('Exam 2 score')
    plt.xlim(30, 100)
    plt.ylim(30, 100)

    # ============ Part 2: Compute Cost and Gradient ============
    #  In this part of the exercise, you will implement the cost and gradient
    #  for logistic regression. You need to complete the code in
    #  costFunction.m

    #  Setup the data matrix appropriately, and add ones for the intercept term
    m, n = X.shape

    # Add intercept term to x and X_test
    X = np.hstack((np.ones((m, 1)), X))

    # Initialize fitting parameters
    initial_theta = np.zeros((n + 1, 1))

    # Compute and display initial cost and gradient
    cost, grad = costFunction(initial_theta, X, y)

    print('Cost at initial theta (zeros):', cost)
    print('Gradient at initial theta (zeros):\n', grad, '\n', sep='')

    # TODO: Using gradient descent algorithm seems not working ...
    # Some gradient descent settings
    iterations = 100
    alpha = 3e-5

    # run gradient descent
    theta, J_history = gradientDescent(initial_theta, X, y, alpha, iterations)

    # print theta to screen
    print('// TODO: Using gradient descent algorithm seems not working ...')
    print('Theta found by gradient descent:\n', theta, '\n', sep='')

    # Plot the convergence graph
    fig2, ax2 = plt.subplots(figsize=(7.2, 5.4))
    plt.title('Convergence Graph')
    plt.plot(np.arange(1, len(J_history) + 1), J_history, label='Logistic regression')
    plt.xlabel('Number of iterations')
    plt.ylabel('Cost J')
    plt.ion()
    plt.show()

    # ============= Part 3: Optimizing using fminunc  =============
    #  In this exercise, you will use a built-in function (fminunc) to find the
    #  optimal parameters theta.
    #  Run fminunc to obtain the optimal theta
    #  This function will return theta and the cost
    # Python implementation: use scipy.optimize.minimize
    # only methods Nelder-Mead and TNC will work
    # x0.shape can be (3,) or (3, 1)
    # x0 = initial_theta.flatten()

    # Initialize fitting parameters
    initial_theta = np.zeros((n + 1, 1))

    res = minimize(objective_cost, initial_theta, args=(X, y), method='TNC', jac=objective_grad)
    cost = res.fun
    theta = res.x.reshape(-1, 1)

    # Print theta to screen
    print('Cost at theta found by scipy.optimize.minimize:', cost)
    print('theta:', theta, sep='\n')
    input('Program paused. Press enter to continue.\n')

    # Plot Boundary
    plotDecisionBoundary(ax1, theta, X, y)

    # ============== Part 4: Predict and Accuracies ==============
    #  After learning the parameters, you'll like to use it to predict the outcomes
    #  on unseen data. In this part, you will use the logistic regression model
    #  to predict the probability that a student with score 45 on exam 1 and
    #  score 85 on exam 2 will be admitted.
    #
    #  Furthermore, you will compute the training and test set accuracies of
    #  our model.
    #
    #  Your task is to complete the code in predict.m
    #
    #  Predict probability for a student with score 45 on exam 1
    #  and score 85 on exam 2

    prob = sigmoid(np.dot(np.array([1, 45, 85]), theta))
    print('For a student with scores 45 and 85, we predict an admission probability of', prob)

    # Compute accuracy on our training set
    p = predict(theta, X)
    match = p == y
    print('Train Accuracy: ', np.mean(match.astype(float)) * 100, '\n', sep='')
    input('Program paused. Press enter to continue.')

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Tue Feb 26 07:19:49 2019

@author: luisa
"""
import sys
import numpy as np
import pickle
from skimage import color
from skimage.transform import resize

#Classification
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.ensemble import RandomForestClassifier

#Add filter library path
sys.path.append('python')

#Download and open the database
from cifar10 import load_cifar10
Train = load_cifar10(meta='cifar-10-batches-py', mode=5)
Test = load_cifar10(meta='cifar-10-batches-py',mode='test')

TrainImages=Train["data"]
TrainLabels=Train["labels"]
TestImages=Test["data"]
TestLabels=Test["labels"]

#Transform the images to grayscale
Train = []
for i in range(0,len(TrainImages)):
    img=color.rgb2gray(resize(np.asanyarray(TrainImages[i]), (32, 32)))
    Train.append(img)

Test = []
for i in range(0,len(TestImages)):
    img=color.rgb2gray(resize(TestImages[i], (32, 32)))
    Test.append(img)
print('Data loaded')

#Create the filters
from fbCreate import fbCreate
fb = fbCreate(support=2, startSigma=0.6)

#Open Dictionary
infile=open('Diccion300','rb')
Dict=pickle.load(infile)
infile.close()

def histc(X, bins):
    import numpy as np
    map_to_bins = np.digitize(X,bins)
    r = np.zeros(bins.shape)
    for i in map_to_bins:
        r[i-1] += 1
    return np.array(r)

# AssignTextons
from assignTextons import assignTextons
from fbRun import fbRun

mapTrain=[]
#Calculate texton representation with current texton dictionary
for img in Train:
    mymap=assignTextons(fbRun(fb,img),Dict.transpose())
    mapTrain.append(mymap)
print('Train Images textones assign')

histTrain=[]
for map in mapTrain:
    histTrain.append(histc(map.flatten(), np.arange(300)))
print('Train Images histograms created')

mapTest=[]
for img in Test:
    mymap=assignTextons(fbRun(fb,img),Dict.transpose())
    mapTest.append(mymap)
print('Test Images textones assign')

histTest=[]
for map in mapTest:
    histTest.append(histc(map.flatten(), np.arange(300)))
print('Test Images histograms created')

# RandomForest
RF = RandomForestClassifier(n_estimators=600)
RF.fit(histTrain,TrainLabels)
testresultRf=RF.predict(histTest)
AcaTestRF= accuracy_score(TestLabels,testresultRf)
CTestRF=confusion_matrix(TestLabels,testresultRf)/ confusion_matrix(TestLabels,testresultRf).sum(axis=0)

print('Test Results with Random Forest')
print(AcaTestRF)
print(CTestRF)


#!/usr/bin/env python
# coding=utf-8
# __project__ = eams
# __author__ = shiyongfei@donews.com
# __date__ = 2018-12-11
# __time__ = 17:53
import random
import re
import string

from sanic import response
from common import tpl_env
from config import ResCode
from lib.pub_cipher import md5_digest

RE_PHONE = re.compile("^(\+86)?1\d{10}$")
RE_TELEPHONE = re.compile("^\d{3,4}?-\d{7,8}$")
RE_EMAIL = re.compile("^[A-Za-z0-9]+([-_.][A-Za-z0-9]+)*@([A-Za-z0-9]+[-.])+[A-Za-z0-9]{2,4}$")
RE_CONTAINS_CHINESE = re.compile("[\u4e00-\u9fa5]")
RE_DOMAIN = re.compile(r"[0-9a-z]-?[0-9a-z]+(.[0-9a-z]-?[0-9a-z]+)?.[a-z]{2,26}", re.IGNORECASE)
RE_ICP_APPROVE = re.compile(r"^[京津冀晋蒙辽吉黑沪苏浙皖闽赣鲁豫鄂湘粤桂琼渝川黔滇藏陕甘青宁新]ICP[备证]\d+号[-\d+]?$", re.IGNORECASE)
RE_POLICE_APPROVE = re.compile(r"^[京津冀晋蒙辽吉黑沪苏浙皖闽赣鲁豫鄂湘粤桂琼渝川黔滇藏陕甘青宁新]公网安备\d{14,}号$")

def resp_json(status: int = ResCode.OK, count: int = 0, msg: str = None, data=None, headers: dict = None):
    """
    响应json
    :param status: 返回的状态码
    :param count: 总条数
    :param msg: 状态解释
    :param data: 数据
    :param headers:响应头
    :return:
    """
    res = {"code": status, "count": count, "msg": ResCode.desc.get(status)}
    if msg is not None:
        res["msg"] = msg
    if data is not None:
        res["data"] = data
    if count is not None:
        res["count"] = count

    return response.json(res, headers=headers)

def resp_table_json(header, data):
    """
    响应表格类型的数据
    :param header: 表格头
    :param data: 表格数据
    :return:
    """
    res = {
        "resultCode": ResCode.OK,
        "msg": ResCode.desc.get(ResCode.OK),
        "data": {
            "header": header,
            "data": data
        }
    }
    return response.json(res)

def html(request, template_name: str, status=200, **vars):
    """
    渲染并响应网页
    :param request:
    :param template_name:
    :param status:
    :param vars:
    :return:
    """
    vars.update({
        "account_name": request["account_name"],
        "left_menu": request.get("session", {}).get("menu", [])
    })
    body = tpl_env.get_template(template_name).render(vars)
    return response.html(body=body, status=status)

def gen_password(text: str) -> str:
    """
    生成密码
    :param text:
    :return:
    """
    tmp = f"1qaz{text}@WSX"
    result = md5_digest(tmp)
    return result

def contains_chinese(text: str) -> bool:
    """
    文本中是否包含中文
    :param text:
    :return:
    """
    contains = re.match(RE_CONTAINS_CHINESE, text)
    return bool(contains)

def is_valid_email(text: str) -> bool:
    """
    文本是否是有效的邮箱地址
    :param text:
    :return:
    """
    is_valid = re.match(RE_EMAIL, text)
    return bool(is_valid)

def is_valid_phone(text: str) -> bool:
    """
    文本是否是有效的手机号
    :param text:
    :return:
    """
    is_valid = re.match(RE_PHONE, text)
    return bool(is_valid)

def is_valid_telephone(text: str) -> bool:
    """
    文本是否是有效的座机号码
    :param text:
    :return:
    """
    is_valid = re.match(RE_TELEPHONE, text)
    return bool(is_valid)

def is_valid_domain(text: str) -> bool:
    """
    文本是否是有效的域名
    :param text:
    :return:
    """
    is_valid = re.match(RE_DOMAIN, text)
    return bool(is_valid)

def is_valid_icp_approve(text: str) -> bool:
    """
    文本是否是有效的ICP备案号
    :param text:
    :return:
    """
    is_valid = re.match(RE_ICP_APPROVE, text)
    return bool(is_valid)

def is_valid_police_approve(text: str) -> bool:
    """
    文本是否是有效的公安网备案号
    :param text:
    :return:
    """
    is_valid = re.match(RE_POLICE_APPROVE, text)
    return bool(is_valid)

def _is_in_ipv4_range(text, ip_range: tuple) -> bool:
    """
    检测文本是否在指定的IP段内
    :param ip_range: IP段
    :return:
    """
    try:
        start_ip, end_ip = ip_range
        assert is_valid_ipv4(text), "invalid ipv4"
        assert is_valid_ipv4(start_ip), "invalid start ip"
        assert is_valid_ipv4(end_ip), "invalid end ip"

        ip_slice = [int(x) for x in text.split(".")]
        start_ip_slice = [int(x) for x in start_ip.split(".")]
        end_ip_slice = [int(x) for x in end_ip.split(".")]

        for ip_fragment, start_fragment, end_fragment in zip(ip_slice, start_ip_slice, end_ip_slice):
            assert start_fragment <= ip_fragment <= end_fragment, f"{text} out of range {start_ip}-{end_ip}"

        return True
    except Exception as e:
        print(e)
        return False

def is_valid_ipv4(text: str) -> bool:
    """
    检查文本是否是有效的IPv4地址
    :param text:
    :return:
    """
    tmp = text.split(".")
    try:
        assert len(tmp) == 4, "bad length"
        assert all(map(lambda x: 0 <= int(x) <= 255, tmp)), "bad range"
        return True
    except:
        return False

def is_valid_private_ipv4(text: str) -> bool:
    """
    检测文本是否是有效的内网IP
    :param text:
    :return:
    """
    private_ipv4_ranges = [("10.0.0.0", "10.255.255.255"),
                           ("172.16.0.0", "172.31.255.255"),
                           ("192.168.0.0", "192.168.255.255")]
    is_valid = any(map(lambda x: _is_in_ipv4_range(text, x), private_ipv4_ranges))
    return is_valid

def is_valid_public_ipv4(text: str) -> bool:
    """
    检测文本是否是有效的公网IP
    :param text:
    :return:
    """
    return not is_valid_private_ipv4(text)

def random_string(length: int = 10) -> str:
    """
    生成随机字符串
    :param length: 字符串长度
    :return:
    """
    str_list = [random.choice(string.digits + string.ascii_letters) for i in range(length)]
    random_str = ''.join(str_list)

    return random_str

if __name__ == '__main__':
    # print(is_valid_domain("178.com"))
    # print(is_valid_icp_approve("京ICP备16021487号-10"))
    print(is_valid_private_ipv4("192.169.1.1"))


#!/usr/bin/env python

from __future__ import division

from math import ceil
from socket import AF_INET, SOCK_DGRAM, socket
from struct import pack

from cv2 import IMWRITE_JPEG_QUALITY, VideoCapture, destroyAllWindows, imencode

class FrameSegment(object):
    """
    Object to break down image frame segment
    if the size of image exceed maximum datagram size
    """
    MAX_DGRAM = 2 ** 16

    # extract 64 bytes in case UDP frame overflown
    MAX_IMAGE_DGRAM = MAX_DGRAM - 64

    def __init__(self, port=12345, remote='127.0.0.1', quality=100):
        self.sock = socket(AF_INET, SOCK_DGRAM)
        self.quality = [
            int(IMWRITE_JPEG_QUALITY), quality
        ]
        self.port = port
        self.addr = remote

    def udp_frame(self, img):
        """
        Compress image and Break down
        into data segments
        """

        compress_img = imencode(
            '.jpg', img, self.quality
        )[1]
        dat = compress_img.tobytes()
        size = len(dat)
        count = ceil(size / self.MAX_IMAGE_DGRAM)
        start = 0

        while count:
            end = min(
                size, start + self.MAX_IMAGE_DGRAM
            )
            self.sock.sendto(
                pack("B", count) + dat[start:end],
                (
                    self.addr,
                    self.port
                )
            )
            start = end
            count -= 1

    def quit(self, *largs):
        self.sock.close()

def main():
    """ Top level main function """

    remote = '127.0.0.1'
    port = 12345
    quality = 50

    fs = FrameSegment(port, remote, quality)
    cap = VideoCapture(2)

    while (cap.isOpened()):
        ret, frame = cap.read()

        # if frame is read correctly
        if ret:
            fs.udp_frame(frame)

    cap.release()
    destroyAllWindows()
    fs.quit()

if __name__ == "__main__":
    main()


from __future__ import absolute_import

from rest_framework import serializers

from .list import ListField

class NoteSerializer(serializers.Serializer):
    text = serializers.CharField()
    mentions = ListField(required=False)

    def validate_mentions(self, attrs, source):
        if source in attrs and 'group' in self.context:
            mentions = attrs[source]
            project = self.context['group'].project
            member_ids = set(
                project.member_set.filter(user_id__in=mentions).values_list('user_id', flat=True)
            )
            invalid_user_ids = [m for m in mentions if int(m) not in member_ids]
            if invalid_user_ids:
                raise serializers.ValidationError('Cannot mention a non-team member')

        return attrs


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Aug 21 12:04:07 2017

@author: hsadeghi
"""

from __future__ import division, print_function, absolute_import
import tensorflow as tf
import numpy as np
import scipy.io as sio

from spectrogram_data_loader import data_loader, data_parser
from dcgan import regressor, discriminator

#%% Main parameters

input_dim = 2 ** 9
both_adam = True
apply_clip_weights =  False
clip_limit = 0.9
cost_lambda = 0.

gpu_frac = 1.0
disc_update_cycles = 1

n_batch = 1
training_epochs = 1
num_training_files = 10

n_batch = 100
training_epochs = 3
num_training_files = 10

learning_rate_init = 0.0001
lr_dec_fac = 1.
noise_std_init = [0.1]
noise_dec_fac = 1.

#%% Other Parameters
batch_size = 128
display_step = 10

print('noise_std_init', noise_std_init)
print('learning_rate_init', learning_rate_init)
print('input_dim', input_dim)

#%% ##############################################################################
L = tf.placeholder("float", [batch_size, None, None], name='low_input')
H = tf.placeholder("float", [batch_size, None, None], name='high_input')
#drop_out_p=tf.placeholder("float", [1])
training = tf.placeholder("float",None, name = 'training_indicator')
learning_rate = tf.placeholder("float", None, name = 'learning_rate')
noise_std = tf.placeholder("float", 1, name = 'noise_std')
maxi = tf.placeholder("float", None, name = 'maximum_spec_value')

#%%##############################################################################
# Building the model
Sxx_l = L
Sxx_h = H

Sxx_real = tf.concat([Sxx_l, Sxx_h], axis = 1, name='Sxx_real')

#noise = tf.random_normal(shape = Sxx_l.get_shape(), stddev = noise_std)
#x_in_reg = tf.concat([Sxx_l, noise], axis=1)
x_in_reg = Sxx_l

with tf.variable_scope('reg'):
    reg_output_h = regressor(x_in_reg, training = training)

reg_output = tf.concat([Sxx_l, reg_output_h], axis = 1, name = 'Sxx_reconstructed')

with tf.variable_scope('disc'):
    d_real = discriminator( Sxx_real, training = training)
#    d_real = discriminator( Sxx_h, training = training)
with tf.variable_scope('disc', reuse = True):
    d_fake = discriminator(reg_output, training = training)
#    d_fake = discriminator(reg_output_h, training = training)

#%% Cost definitions
# mean squared
disc_cost_real = tf.reduce_mean( tf.squared_difference(d_real, 1.))
disc_cost_fake = tf.reduce_mean( tf.squared_difference(d_fake, 0.))
# log norm
#disc_cost_real =  - tf.reduce_mean( tf.log(d_real))
#disc_cost_fake = tf.reduce_mean( tf.log( 1. - d_fake))
disc_cost = disc_cost_real + disc_cost_fake

#reg_adv_cost =  - tf.reduce_mean(tf.log(d_fake))
reg_adv_cost = tf.reduce_mean(tf.squared_difference(d_fake, 1.))
reg_l2_cost = tf.reduce_mean(tf.squared_difference(Sxx_h, reg_output_h), name = 'reg_cost')
#reg_cost = reg_adv_cost
reg_cost = reg_adv_cost + cost_lambda * reg_l2_cost

#%% Variables
t_vars = tf.trainable_variables()
disc_vars = []
reg_vars = []
for var in t_vars:
    if var.name.startswith('disc'):
        disc_vars.append(var)
    if var.name.startswith('reg'):
        reg_vars.append(var)
for x in disc_vars:
    print('disc_variable', x.name)
    assert x not in reg_vars
for x in reg_vars:
    print('reg_variable', x.name)
    assert x not in disc_vars
for x in t_vars:
    print('all_variables', x.name)
    assert x in reg_vars or x in disc_vars, x.name
if apply_clip_weights:
    print('Clipping D weights')
    clip_disc_vars = [v.assign(tf.clip_by_value(v, -1 * clip_limit, clip_limit)) for v in disc_vars]
else:
    print('Not clipping D weights')

#%% Optimizers
if both_adam:
    disc_opt = tf.train.AdamOptimizer(learning_rate).minimize(disc_cost, var_list=disc_vars)
    reg_opt = tf.train.AdamOptimizer(learning_rate).minimize(reg_cost, var_list=reg_vars)
else:
    disc_opt = tf.train.RMSPropOptimizer(learning_rate).minimize(disc_cost, var_list=disc_vars)
    reg_opt = tf.train.AdamOptimizer(learning_rate).minimize(reg_cost, var_list=reg_vars)

#%%##############################################################################
# Training
init = tf.global_variables_initializer()
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction = gpu_frac)
sess = tf.Session(config=tf.ConfigProto(log_device_placement = False, gpu_options=gpu_options))
sess.run(init)

#%% Tensorboard setup
tf.summary.scalar('reg_l2_cost', reg_l2_cost)
tf.summary.scalar('reg_adv_cost', reg_adv_cost)
tf.summary.scalar('disc_cost', disc_cost)
merged_summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter('/home/hsadeghi/Dropbox/august/spectrogram/tensorboard/dc_BWE/1/')
summary_writer.add_graph(sess.graph)

#%% Training cycle
for epoch in range(training_epochs):
    training_data = [[]]* num_training_files
    reading_phase = True
    for file_idx in range(num_training_files):
        if reading_phase:
            training_data [file_idx] = data_loader(file_idx, input_dim)
        for  i in range(n_batch):
            #pre_emph is zero since we do it on X if we want separatrely
            batch_xs = data_parser(training_data[file_idx], input_dim, batch_size)

            ################################################################
            # Update disc
            with tf.device('/gpu:0'):
                for i in range(disc_update_cycles):
                    _, disc_cost_ = sess.run([disc_opt, disc_cost],
                                             feed_dict={L: batch_xs[0], H : batch_xs[1], maxi : batch_xs[2],
                                                        learning_rate: learning_rate_init,
                                                        noise_std : noise_std_init,
                                                        training : 1.})
#                     Display logs per epoch step
#                    if i % display_step == 0:
#                        print("Discriminator_cost =", "{:.9f}".format( (10**4) * disc_cost_))

                # clip disc weights
                if apply_clip_weights:
                    sess.run(clip_disc_vars)

            ################################################################
            # Update regressor
            _, reg_adv_cost_, reg_l2_cost_, ss= sess.run([reg_opt, reg_adv_cost, reg_l2_cost, merged_summary], feed_dict={L: batch_xs[0],
                                              H : batch_xs[1], maxi : batch_xs[2],
                                                    learning_rate: learning_rate_init,
                                                    noise_std : noise_std_init,
                                                    training : 1.})
#           Display logs per epoch step
            if i % display_step == 0:
                summary_writer.add_summary(ss, i)  #tensorboard
                print("epoch:", '%02d' % (epoch + 1),
                        "File:", '%02d' % (file_idx),
                      "iteration:", '%04d' % (i + 1),
                      "disc_cost =", "{:.9f}".format( (10**4) * disc_cost_),
                      "reg_adv_cost =", "{:.9f}".format( (10**4) * reg_adv_cost_),
                      "reg_l2_cost =", "{:.9f}".format( (10**4) * reg_l2_cost_))
            ################################################################3

        learning_rate_init = max( 0.001, learning_rate_init /lr_dec_fac)
        noise_std_init[0] =   max(0.01, noise_std_init[0]/ noise_dec_fac  )
    reading_phase = False
print("Optimization Finished!")

#%%##########################################################################
# Training error calculation
training_data= training_data[9] #data_loader(9, input_dim)
training_error = 0
avg_num = 50
for i in range(avg_num):
    sampled_x = data_parser(training_data, input_dim, batch_size)
    training_error += sess.run(reg_cost, feed_dict={L: sampled_x[0],
                                              H : sampled_x[1], maxi : batch_xs[2], noise_std : [0.],
                                                    training : 0. })
training_error = (training_error/ avg_num) ** 0.5

#%% Test error calculation
test_data = data_loader(10, input_dim)
test_error = 0
avg_num = 50

nfft = 128
n_time_bins = input_dim//128
y_pred_test = np.zeros([ avg_num*(batch_size), nfft+1       , n_time_bins])
y_true_test = np.zeros([ avg_num*(batch_size), nfft+1       , n_time_bins])
y_h_true = np.zeros([avg_num*(batch_size),     nfft //2     , n_time_bins])
y_h_pred = np.zeros([avg_num*(batch_size),     nfft //2     , n_time_bins])
maximum_spec = np.zeros(avg_num)

for i in range(avg_num):
    sampled_x = data_parser(test_data, input_dim, batch_size)
    Sxx_h_ , y_reg_h_,\
    y_true_test_, y_pred_test_,\
    test_error_, maxi_ = sess.run([Sxx_h , reg_output_h, Sxx_real, reg_output, reg_cost, maxi],\
                                        feed_dict={L: sampled_x[0],
                                              H : sampled_x[1], maxi : sampled_x[2],
                                              noise_std : [0.],
                                              training : 0.})
    test_error += test_error_
    y_pred_test [ i * batch_size : (i + 1) * batch_size  ,:,:] = y_pred_test_
    y_true_test [ i * batch_size : (i + 1) * batch_size ,:,:] = y_true_test_
    y_h_true [ i * batch_size : (i + 1) * batch_size ,:,:] = Sxx_h_
    y_h_pred [ i * batch_size : (i + 1) * batch_size ,:,:] = y_reg_h_
    maximum_spec[i] = maxi_

test_error = (test_error/ avg_num) ** 0.5

#%% PRINTING COSTS
print( 'training_error', "{:.9f}".format(training_error))
print( 'test_error', "{:.9f}".format(test_error))

#%%##########################################################################
# Savings network
AE_output={};
AE_output['y_true_test'] = y_true_test
AE_output['y_pred_test'] = y_pred_test
AE_output['input_dim'] = input_dim
AE_output['y_h_true'] = y_h_true
AE_output['y_h_pred'] = y_h_pred
AE_output['maximum_spec'] = maximum_spec

save_path = "/home/hsadeghi/Dropbox/august/spectrogram/gan_dc_BWE_output.mat"
sio.savemat(save_path, AE_output);
sess.close()


#!/usr/bin/env python3

import os
import sys
import shlex
import logging
import argparse
import importlib
import commands as _commands

from commands import CommandBase, CommandType, CommandInfo
from subprocess import check_output
from telegram.ext import Updater
from telegram.ext import CommandHandler, MessageHandler, Filters
from ruamel.yaml import YAML

yaml = YAML(typ="safe", pure=True)
logging.basicConfig(
    format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    level = logging.INFO
)

baseconf = dict()
commands = []
regdhelp = dict()

def version(bot, update):
    res = check_output(["git", "rev-list", "--count", "HEAD"])
    res = res.decode("utf-8").strip()
    bot.send_message(chat_id = update.message.chat_id,
                     text = "kadebot rev. {}".format(res),
                     disable_notification = True)

def kill(bot, update):
    if update.message.from_user.id in baseconf["admins"]:
        logging.info("Admin killed the bot, shutting down.")
        for cmd in commands:
            cmd.on_exit()
        logging.info("Cleanup done, exiting.")
        sys.stdout.flush()
        os._exit(0)

def reload(bot, update):
    if update.message.from_user.id in baseconf["admins"]:
        logging.info("Reloading chat bot now...")
        for cmd in commands:
            cmd.on_exit()
        python = sys.executable
        os.execv(python, ['python3'] + sys.argv)

def update(bot, update):
    if update.message.from_user.id in baseconf["admins"]:
        logging.info("Updating bot...")
        os.system("git fetch origin master")
        os.system("git reset --hard origin/master")
        reload(bot, update)

def cmdlist(bot, update):
    out = []
    for cmd in commands:
        for ci in cmd.to_register:
            if not ci.name in baseconf['disabled'] and ci.type == CommandType.Default:
                out.append('/' + ci.name)
    bot.send_message(chat_id = update.message.chat_id,
                     text = 'Command list:\n' + ', '.join(sorted(out)),
                     disable_notification = True)

def help(bot, update):
    args = shlex.split(update.message.text)
    if len(args) > 1:
        cmd = args[1]
        if cmd in regdhelp.keys():
            dat = regdhelp[cmd].get_help_msg(cmd)
            out = "Help for /{}:\n\n{}".format(cmd, dat)
            bot.send_message(chat_id = update.message.chat_id,
                             text = out,
                             disable_notification = True)
    else:
        out = ""
        for cmd in commands:
            for ci in cmd.to_register:
                if not ci.name in baseconf["disabled"] and ci.type == CommandType.Default:
                    if ci.alias != None:
                        out += "/{}, /{} - {}\n".format(ci.name, ci.alias, ci.helpmsg)
                    else:
                        out += "/{} - {}\n".format(ci.name, ci.helpmsg)
        bot.send_message(chat_id = update.message.chat_id,
                         text = out,
                         disable_notification = True)

def error_handler(bot, update, error):
    try:
        raise error
    except Exception as e:
        logging.error(e)

def main():
    global regdhelp
    global commands
    updater = Updater(token = baseconf["api_key"])
    dispatcher = updater.dispatcher
    dispatcher.add_handler(CommandHandler("help", help))
    dispatcher.add_handler(CommandHandler("list", cmdlist))
    dispatcher.add_handler(CommandHandler("reload", reload))
    dispatcher.add_handler(CommandHandler("update", update))
    dispatcher.add_handler(CommandHandler("kill", kill))
    dispatcher.add_handler(CommandHandler("version", version))
    to_schedule = []
    for cmd in commands:
        for ci in cmd.to_register:
            if ci.name in baseconf["disabled"]:
                logging.info("Disabled command /{}".format(ci.name))
                continue
            if ci.name in baseconf["disabled_monitors"]:
                logging.info("Disabled monitor {}".format(ci.name))
                continue
            if ci.name in baseconf["disabled_schedules"]:
                logging.info("Disabled scheduled task {}".format(ci.name))
                continue
            if ci.type == CommandType.Default:
                dispatcher.add_handler(CommandHandler(ci.name, ci.func))
                regdhelp[ci.name] = cmd
                if ci.alias != None:
                    dispatcher.add_handler(CommandHandler(ci.alias, ci.func))
                    regdhelp[ci.alias] = cmd
                    logging.info("Registered command /{}, /{}".format(ci.name, ci.alias))
                else:
                    logging.info("Registered command /{}".format(ci.name))
            elif ci.type == CommandType.Schedule:
                to_schedule.append(ci)
            else:
                dispatcher.add_handler(MessageHandler(ci.filter, ci.func))
                logging.info("Registered monitor {}".format(ci.name))
    dispatcher.add_error_handler(error_handler)
    updater.start_polling()
    logging.info("Started polling for commands.")
    if len(to_schedule) > 0:
        logging.info("Registering scheduled tasks..")
    for ci in to_schedule:
        ci.func(updater)
        logging.info("Registered scheduled task {}".format(ci.name))

def load_config(filename):
    global baseconf
    global commands
    conf = dict()
    with open(filename, 'r') as f:
        conf = yaml.load(f)
    baseconf = conf["base"]
    logging.info("Loaded base configuration.")
    for command in _commands.__all__:
        if 'Command' not in command and command not in baseconf['disabled_modules']:
            mod = importlib.import_module('commands')
            commands.append(getattr(mod, command)(logging))
        elif command in baseconf['disabled_modules']:
            logging.info("Disabled module {}.".format(command))
    for cmd in commands:
        section = "command.{}".format(cmd.safename)
        if section in conf.keys():
            cmd.load_config(conf[section])
    logging.info("Loaded module configurations.")

if __name__ == "__main__":
    parser  = argparse.ArgumentParser()
    parser.add_argument(
        "--config",
        default = os.path.expanduser("~/.config/kadebot/kadebot.yaml"),
        help = "configuration file to use",
        type = str
    )
    args = parser.parse_args()
    if os.path.exists(args.config):
        load_config(args.config)
        main()
    else:
        logging.error("No config file found.")


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from consumption.management.commands.load import Command as ImportCommand
from consumption.models import User, Consumption, get_total_and_average_consumptions
from django.test import TestCase

class ImportingTestCase(TestCase):
    def test_load(self):
        # run importing data task
        cmd = ImportCommand()
        cmd.handle()

        # make sure new data records were added
        self.assertGreater(User.objects.count(), 0)
        self.assertNotEqual(Consumption.objects.count(), 0)

class UserTestCase(TestCase):
    def test_average(self):
        new_user = User(id=100, area='area1', tariff='t1')
        new_user.save()
        Consumption(user_id=100, consumption=30.0, datetime='2016-07-15 09:30:00').save()
        Consumption(user_id=100, consumption=20.0, datetime='2016-07-19 12:30:00').save()

        # average consumptions during July must be (20 + 30) / 2 = 50
        self.assertEqual(new_user.calculate_monthly_consumptions()[1][6], 50.00)

class TotalConsumptionsTestCase(TestCase):
    def setup(self):
        new_user = User(id=100, area='area1', tariff='t1')
        new_user.save()
        Consumption(user_id=100, consumption=30.0, datetime='2016-07-15 09:30:00').save()
        Consumption(user_id=100, consumption=20.0, datetime='2016-07-19 12:30:00').save()
        Consumption(user_id=100, consumption=10.0, datetime='2016-07-21 11:30:00').save()

        new_user = User(id=101, area='area1', tariff='t1')
        new_user.save()
        Consumption(user_id=101, consumption=45.0, datetime='2016-07-15 09:30:00').save()
        Consumption(user_id=101, consumption=27.0, datetime='2016-07-21 10:30:00').save()
        Consumption(user_id=101, consumption=21.0, datetime='2016-07-24 11:30:00').save()
        Consumption(user_id=101, consumption=43.0, datetime='2016-09-02 06:30:00').save()

    def test_average(self):
        self.setup()
        self.assertEqual(get_total_and_average_consumptions().filter(user_id=100)[0]['average'], 20.00)
        self.assertEqual(get_total_and_average_consumptions().filter(user_id=101)[0]['average'], 34.00)

    def test_total(self):
        self.setup()
        self.assertEqual(get_total_and_average_consumptions().filter(user_id=100)[0]['total'], 60.00)
        self.assertEqual(get_total_and_average_consumptions().filter(user_id=101)[0]['total'], 136.00)


from __future__ import print_function
import os, sys
from include.fmt import psql, pfmtd
import subprocess, logging

e=sys.exit
log=logging.getLogger('loader')

class SimpleNamespace (object):
	def __init__ (self, **kwargs):
		self.__dict__.update(kwargs)
	def __repr__ (self):
		keys = sorted(self.__dict__)
		items = ("{}={!r}".format(k, self.__dict__[k]) for k in keys)
		return "{}({})".format(type(self).__name__, ", ".join(items))
	def __eq__ (self, other):
		return self.__dict__ == other.__dict__

def get_cli(a, ):
	cfg 	= 'config/db_config.%s.json' % a.runtime_env
	cloc	= 'config/proc/position/%s' % wflow
	assert os.path.isfile (cfg), 'Db config file does not exists.'
	assert os.path.isdir (cloc), 'Pileline dir does not exists.'
	return 'time %s cli.py -nopp {PARAM_CNT} {DUMP} -dcf %s -pcf %s/{TABLE}.json {LAME_DUCK} --proc_params' % (PY, cfg, cloc)
def load_day(a, mon, day=None):
	cli= get_cli(a)
	pars = params[a.table][0].format( **dict( CENTER = a.center, CLIENT = a.client, YEAR = a.year, MONTH = mon, DAY = day, EOM = a.day_to,  BUNIT= a.bunit ))

	pycli  = cli.format(**dict( TABLE = a.table, PARAM_CNT = len([p for p in pars.strip().split(' ') if p]), DUMP = '--dump' if a.dump else '--no-dump', LAME_DUCK='-ld %d ' % a.lame_duck))
	cmd= '%s %s' % (pycli, pars)

	if not a.dry:
		pfmtd([dict(Command=os.linesep.join(cmd.split()))], a.table)
		pipe=subprocess.Popen([cmd], stdout=subprocess.PIPE,stderr=subprocess.PIPE, shell=True)

		line= pipe.stdout.readline()

		while line:
			print ('OUTPUT:', line.strip())
			line= pipe.stdout.readline()

		line= pipe.stderr.readline()

		while line:
			print ('ERROR:', line)
			line= pipe.stderr.readline()

		while pipe.poll() is None:
			print ('Waiting...')
			time.sleep(1)

		if pipe.returncode != 0:
			print('returncode = %d' % pipe.returncode)
			e(pipe.returncode)
	else:
		print (cmd)


from django.urls import path, include
from django.contrib.auth import views as auth_views
from . import views

app_name = 'clickpostapp'

urlpatterns = [
    path('', views.DashboardView.as_view(), name='dashboard'),
    path('uploadawbs/', views.UploadAwbs.as_view(), name='uploadawbs'),
    path('cleardatabase/', views.ClearDatabase.as_view(), name='cleardatabase'),
    path('api/<slug:awbnumber>/', views.Api.as_view(), name='api'),
    path('calculate/', views.Calculate.as_view(), name='calculate'),
]


"""
    CTC-like decoder utilitis.
"""

from itertools import groupby
import numpy as np
import copy
import kenlm

def ctc_best_path_decode(probs_seq, vocabulary):
    """
    Best path decoding, also called argmax decoding or greedy decoding.
    Path consisting of the most probable tokens are further post-processed to
    remove consecutive repetitions and all blanks.

    :param probs_seq: 2-D list of probabilities over the vocabulary for each
                      character. Each element is a list of float probabilities
                      for one character.
    :type probs_seq: list
    :param vocabulary: Vocabulary list.
    :type vocabulary: list
    :return: Decoding result string.
    :rtype: baseline
    """
    # dimension verification
    for probs in probs_seq:
        if not len(probs) == len(vocabulary) + 1:
            raise ValueError("probs_seq dimension mismatchedd with vocabulary")
    # argmax to get the best index for each time step
    max_index_list = list(np.array(probs_seq).argmax(axis=1))
    # remove consecutive duplicate indexes
    index_list = [index_group[0] for index_group in groupby(max_index_list)]
    # remove blank indexes
    blank_index = len(vocabulary)
    index_list = [index for index in index_list if index != blank_index]
    # convert index list to string
    return ''.join([vocabulary[index] for index in index_list])

class Scorer(object):
    """
    External defined scorer to evaluate a sentence in beam search
               decoding, con

    :param alpha: Parameter associated with language model,
    :type alpha: float
    :param beta: Parameter associated with word count.
    :type beta: float
    :model_path: Path to load language model.
    :type model_path: basestring
    """
    def __init__(self,
                 alpha,
                 beta,
                 model_path):

        self._alpha = alpha
        self._beta = beta
        self._language_model = kenlm.LanguageModel(model_path)

    def language_model_score(self, sentence, bos=True, eos=False):
        log_prob = self._language_model.score(sentence, bos, eos)
        return log_prob

    def word_count(self, sentence):
        words = sentence.strip().split(' ')
        return len(words)

    def evaluate(self, sentence, bos=True, eos=False):
        lm = self._alpha*self.language_model_score(sentence, bos, eos)
        word_cnt = self._beta*self.word_count(sentence)
        return (lm, word_cnt)

def ctc_beam_search_decoder(probs_seq,
                            beam_size,
                            vocabulary,
                            ext_scoring_func=None,
                            blank_id=0):
    '''
    Beam search decoder for CTC-trained network, using beam search with width
    beam_size to find many paths to one label, return  beam_size labels in
    the order of probabilities. The implementation is based on Prefix Beam
    Search(https://arxiv.org/abs/1408.2873), and the unclear part is
    redesigned, need to be verified.

    :param probs_seq: 2-D list with length max_time_steps, each element
                      is a list of normalized probabilities over vocabulary
                      and blank for one time step.
    :type probs_seq: 2-D list
    :param beam_size: Width for beam search.
    :type beam_size: int
    :param vocabulary: Vocabulary list.
    :type vocabulary: list
    :param ext_scoring_func: External defined scoring function for
                            partially decoded sentence, e.g. word count
                            and language model.
    :type external_scoring_function: function
    :param blank_id: id of blank, default 0.
    :type blank_id: int
    :return: Decoding log probability and result string.
    :rtype: list

    '''
    # dimension check
    for prob_list in probs_seq:
        if not len(prob_list) == len(vocabulary) + 1:
            raise ValueError("probs dimension mismatchedd with vocabulary")
    max_time_steps = len(probs_seq)

    # blank_id check
    probs_dim = len(probs_seq[0])
    if not blank_id < probs_dim:
        raise ValueError("blank_id shouldn't be greater than probs dimension")

    # assign space_id
    if ' ' not in vocabulary:
        raise ValueError("space doesn't exist in vocabulary")
    space_id = vocabulary.index(' ')

    # function to convert ids in string to list
    def ids_str2list(ids_str):
        ids_str = ids_str.split(' ')
        ids_list = [int(elem) for elem in ids_str]
        return ids_list
    # function to convert ids list to sentence
    def ids2sentence(ids_list, vocab):
        return  ''.join([vocab[ids] for ids in ids_list])

    ## initialize
    # the set containing selected prefixes
    prefix_set_prev = {'-1': 1.0}
    probs_b, probs_nb = {'-1': 1.0}, {'-1': 0.0}

    ## extend prefix in loop
    for time_step in range(max_time_steps):
        # the set containing candidate prefixes
        prefix_set_next = {}
        probs_b_cur, probs_nb_cur = {}, {}
        for l in prefix_set_prev:
            prob = probs_seq[time_step]

            # convert ids in string to list
            ids_list = ids_str2list(l)
            end_id = ids_list[-1]
            if not prefix_set_next.has_key(l):
                probs_b_cur[l], probs_nb_cur[l] = 0.0, 0.0

            # extend prefix by travering vocabulary
            for c in range(0, probs_dim):
                if c == blank_id:
                    probs_b_cur[l] += prob[c] * (probs_b[l] + probs_nb[l])
                else:
                    l_plus = l + ' ' + str(c)
                    if not prefix_set_next.has_key(l_plus):
                        probs_b_cur[l_plus], probs_nb_cur[l_plus] = 0.0, 0.0

                    if c == end_id:
                        probs_nb_cur[l_plus] += prob[c] * probs_b[l]
                        probs_nb_cur[l] += prob[c] * probs_nb[l]
                    else:
                        probs_nb_cur[l_plus] += prob[c] * (
                            probs_b[l] + probs_nb[l])
                    # add l_plus into prefix_set_next
                    prefix_set_next[l_plus] = probs_nb_cur[
                        l_plus] + probs_b_cur[l_plus]
            # add l into prefix_set_next
            prefix_set_next[l] = probs_b_cur[l] + probs_nb_cur[l]
        # update probs
        probs_b, probs_nb = copy.deepcopy(probs_b_cur), copy.deepcopy(
            probs_nb_cur)

        ## store top beam_size prefixes
        prefix_set_prev = sorted(
            prefix_set_next.iteritems(), key=lambda asd: asd[1], reverse=True)
        if beam_size < len(prefix_set_prev):
            prefix_set_prev = prefix_set_prev[:beam_size]
        prefix_set_prev = dict(prefix_set_prev)

    beam_result = []
    for (seq, prob) in prefix_set_prev.items():
        if prob > 0.0:
            ids_list = ids_str2list(seq)[1: ]
            result = ids2sentence(ids_list, vocabulary)
            lm_score, word_count = ext_scoring_func(result, True, True)
            log_prob = np.log(prob)
            total_score = log_prob+lm_score+word_count
            beam_result.append([total_score, log_prob, lm_score, word_count, result])

    ## output top beam_size decoding results
    beam_result = sorted(beam_result, key=lambda asd: asd[0], reverse=True)
    return beam_result


# -*- coding: utf-8 -*-
import datetime

from bson.objectid import ObjectId

from ig_api import db
from ig_api.helpers import send_trans_email

## Helpers

class FormException(Exception):
    def __init__(self, message):
        self.message = message

class InstanceException(Exception):
    def __init__(self, message):
        self.message = message

class FeedbackException(Exception):
    def __init__(self, message):
        self.message = message

## Models

class FormFieldSubModel(db.EmbeddedDocument):
        FIELD_TYPES = (
            ('YN', 'YES/NO FIELD'),
            ('ST', 'STAR RATING FIELD'),
            ('MT', 'MULTIPLE FIELD'),
            ('TT', 'TEXT FIELD')
        )

        field_type = db.StringField(max_length=2, choices=FIELD_TYPES, required=True)
        text = db.StringField(required=True)
        choices = db.ListField(db.StringField()) # only used in case of MULTIPLE FIELD
        required = db.BooleanField(required=True, default=False);
        id = db.ObjectIdField(required=True, default=ObjectId)

        def validate(self, *args, **kwargs):
            # check there are the correct amount of choices in case of multiple field
            if self.field_type == 'MT': # in case of MULTIPLE FIELD
                if len(self.choices) < 2 or len(self.choices) > 7:
                    raise db.ValidationError(message='Number of choices provided for multiple field should be between 2 and 5.')

            # run the original validation method
            super(FormFieldSubModel, self).validate(*args, **kwargs)

        def validate_customer_response(self, response):
            """Validates the response given by a customer for a particular field.
            Raises a `ValueError` if the response given is wrong. In case of no error
            it just return `None`
            """
            # YES/NO FIELD
            if self.field_type == 'YN':
                if not response in ['YES', 'NO']:
                    raise ValueError('The response to a Yes/No Field can only be "YES" or "NO". Case-Sensitive.')

            # STAR RATING FIELD
            if self.field_type == 'ST':
                if not int(response) in [1, 2, 3, 4, 5]:
                    raise ValueError('The response to a Star Rating Field can only be among [1,2,3,4,5]. Integer.')

            # MULTIPLE FIELD
            if self.field_type == 'MT':
                if not response in self.choices:
                    raise ValueError('The response to a Mutliple Field should be among one of the choices of the field.')

            # TEXT FIELD
            if self.field_type == 'TT':
                if not type(response) is str and not type(response) is unicode:
                    raise ValueError('The response to a Text Field can only be a string of unicode or str type.')

        def __repr__(self):
            return '<FormFieldSubModel: {0} ({1})>'.format([i[1] for i in self.FIELD_TYPES if i[0] == self.field_type][0], self.text)

class FormIncrementCounter(db.Document):

    counter = db.IntField(required=True, default=0) # the first value will be 1

    # form associated with the counter
    form = db.ReferenceField('FormModel', required=True)

    @staticmethod
    def create(form):
        """Creates a new FormIncrementCounter."""
        # save the counter
        counter = FormIncrementCounter(form=form)
        counter.save()

        return counter

    def get_next_counter(self):
        """Updates the DB with the incremented (+1) value of the counter
        and returns the new counter value"""
        # atomic update of the counter
        self.update(inc__counter=1)
        self.reload()

        return self.counter

    def __repr__(self):
        return '<FormIncrementCounter: {0} ({1})>'.format(self.counter, self.form.name)

class FormModel(db.Document):
    # basic info about the form
    name = db.StringField(required=True)
    description = db.StringField(required=True)

    # flag if the form needs to produce a incremetal id for every feedback
    incremental_id = db.BooleanField(required=True, default=False)

    # fields comprising of the form
    fields = db.ListField(db.EmbeddedDocumentField(FormFieldSubModel))

    # heading text for 'Your Details', 'NPS Score', 'Tell Us More' sections
    customer_details_heading = db.StringField(required=True)
    feedback_heading = db.StringField(required=True)
    nps_score_heading = db.StringField(required=True)

    # CSS class name to be added for this form
    css_class_name = db.StringField(required=True)

    # HTML for the display card
    display_card_html = db.StringField(required=True)

    # which merchant is the form associated with?
    merchant = db.ReferenceField('MerchantModel', required=True)

    meta = {'collection': 'forms'}

    @property
    def counter(self):
        try:
            counter = FormIncrementCounter.objects.get(form=self)
        except db.DoesNotExist:
            counter = None

        return counter

    def get_analytics(self, instance_ids, start_date=None, end_date=None):
        """Returns the analytics for the form fields.

        Keyword Arguments:
        instance_ids -- ids of the instance for which the analytics are needed.
        start_date -- date (inclusive) from which the analytics are needed.
        end_date -- date (inclusive) till which the analytics are needed.

        Note: 1. No analytics are generated for the text field.
              2. If the analytics need to be filtered on basis of time both
                 `start_date` and `end_date` need to be specified.
              3. The value of `start_date` and `end_date` should be in UTC time.
        """
        # all the feedbacks attached with the given instance ids
        if start_date and end_date: # add filter if `start_date` and `end_date` are provided
            feedbacks = FeedbackModel.objects.filter(received_at__gte=start_date, received_at__lte=end_date,
                    form_instance__in=instance_ids)
        else:
            feedbacks = FeedbackModel.objects.filter(form_instance__in=instance_ids)
        # this dict will contain all the analytics and the corresponding fields
        responses = {}
        # adding the `FormFieldSubModel` instance in the `responses` dict
        for field in self.fields:
            # the analytics will not be provided for TEXT FIELD
            if field.field_type != 'TT':
                responses[str(field.id)] = {
                    'responses': [],
                    'field': field
                }
        # adding the actual list of responses to the `responses` dict
        for feedback in feedbacks:
            for f_id, response in feedback.field_responses.items():
                if str(f_id) in responses:
                    responses[str(f_id)]['responses'].append(response)
        # counting the number of responses for providing as analytics
        for f_id, response in responses.items():
            #TODO: this is just a temproary work around
            colors = ['#4D5361', '#FFB553', '#3EBFBE', '#949FB2', '#FA4444', '#736d64']
            if response['field'].field_type == 'YN':
                responses[f_id]['numbers'] = {
                    'YES': {
                        'number': response['responses'].count('YES'),
                        'color': colors.pop(),
                        'text': 'YES'
                    },
                    'NO': {
                        'number': response['responses'].count('NO'),
                        'color': colors.pop(),
                        'text': 'NO'
                    }
                }
                responses[f_id]['total_responses'] = sum([i['number'] for i in responses[f_id]['numbers'].values()])
            if response['field'].field_type == 'ST':
                responses[f_id]['numbers'] = {
                    '1': {
                        'number': response['responses'].count('1'),
                        'color': colors.pop(),
                        'text': '1 stars'
                    },
                    '2': {
                        'number': response['responses'].count('2'),
                        'color': colors.pop(),
                        'text': '2 stars'
                    },
                    '3': {
                        'number': response['responses'].count('3'),
                        'color': colors.pop(),
                        'text': '3 stars'
                    },
                    '4': {
                        'number': response['responses'].count('4'),
                        'color': colors.pop(),
                        'text': '4 stars'
                    },
                    '5': {
                        'number': response['responses'].count('5'),
                        'color': colors.pop(),
                        'text': '5 stars'
                    },
                }
                responses[f_id]['total_responses'] = sum([i['number'] for i in responses[f_id]['numbers'].values()])
            if response['field'].field_type == 'MT':
                responses[f_id]['numbers'] = {}
                for choice in response['field'].choices:
                    responses[f_id]['numbers'][choice] = {
                            'number': response['responses'].count(choice),
                            'color': colors.pop(),
                            'text': choice
                    }
                    responses[f_id]['total_responses'] = sum([i['number'] for i in responses[f_id]['numbers'].values()])

        return responses

    @staticmethod
    def create(name, description, customer_details_heading, feedback_heading, nps_score_heading, fields,
            merchant, incremental_id, css_class_name, display_card_html):
        """Creates a new form instance.

        The `fields` keyword argument will take up a list with `FormFieldSubModel` instances. The order of these
        instances in the `fields` list will decide the order for the customer facing form.
        """

        # saving the form
        form = FormModel(name=name, description=description, merchant=merchant, customer_details_heading=customer_details_heading,
                feedback_heading=feedback_heading, nps_score_heading=nps_score_heading, fields=fields,
                incremental_id=incremental_id, css_class_name=css_class_name, display_card_html=display_card_html)
        try:
            form.save()
        except db.ValidationError:
            raise FormException('Form data provided was wrong')

        # create a form counter object if `incremental_id` is True
        if incremental_id:
            FormIncrementCounter.create(form)

        return form

    def create_instance(self, name, description, location):
        """name, description & location of the instance are to be provided."""
        instance = InstanceModel(name=name, description=description, location=location, form=self)
        try:
            instance.save()
        except db.ValidationError:
            raise InstanceException('Instance data provided was wrong.')

        return instance

    def __repr__(self):
        return '<FormModel: {0} ({1})>'.format(self.name, self.merchant.name)

class InstanceModel(db.Document):
    name = db.StringField(required=True)
    description = db.StringField(required=True)
    location = db.StringField(required=True)

    # which form is the instance associated with?
    form = db.ReferenceField('FormModel', required=True)

    meta = {'collection': 'intances'}

    def __repr__(self):
        return '<InstanceModel: {0}>'.format(self.name)

class FeedbackModel(db.Document):
    # nps score and feedback text
    nps_score = db.IntField(required=True, choices=range(1,11))
    feedback_text = db.StringField(required=False)

    # customer response to the different fields of the form
    field_responses = db.DictField(required=False) # will not be required for feedbacks without any custom cards

    # time the feedback was received at (utc time)
    received_at = db.DateTimeField(required=True, default=datetime.datetime.utcnow)

    # which customer is feedback associated with?
    customer = db.ReferenceField('CustomerModel')

    # which instance is feedback associated with?
    form_instance = db.ReferenceField('InstanceModel', required=True)

    # which merchant is feedback associated with?
    merchant = db.ReferenceField('MerchantModel', required=True)

    # unique counter of the feedback (only exists if `incremental_id` flag is true on form)
    has_counter = db.BooleanField(default=False)
    counter = db.IntField()

    meta = {'collection': 'feedbacks'}

    @staticmethod
    def get_timeline(merchant, nps_score_start=None, nps_score_end=None, start_date=None, end_date=None, instances=None):
        """Returns the MongoEngine queryset for the timeline filtered on basis of
        various filters ordered in a reverse chronogical order.

        Note: This method does not take care of any sort of filters. They can be implemented using the
              queryset returned by this method.

        Keyword Arguments:

        merchant: (required) Whose timeline is to be returned?

        nps_score_start: (optional) Starting (inclusive) range of NPS score. (between 0 to 10)
        nps_score_end (optional) Ending (inclusive) range of NPS score. (between 0 to 10)

        start_date: (optional) Results returned would be from dates equal to or greater than this date. (datetime.datetime)
        end_date: (optional) Results returned would be from dates equal to or less than this date. (datetime.datetime)

        instances: (optional) List of instances of whose results need to be returned.

        Return Value: Returns a tuple of (feedbacks, all_start_date, all_end_date) where start_date and end_date are
                      beginning and end dates of all feedbacks of the given merchant.
        """

        # get all feedbacks (without a filter) for the merchant in reverse chronological order
        feedbacks = FeedbackModel.objects.filter(merchant=merchant).order_by('-received_at')

        # start & end date (this is start and end date of all feedbacks of this merchant)
        all_start_date = feedbacks.skip(len(feedbacks)-1).limit(1)[0].received_at
        all_end_date = feedbacks[0].received_at

        # filter on basis of nps score if provided
        if nps_score_start and nps_score_end:
            feedbacks = FeedbackModel.objects.filter(merchant=merchant, nps_score__gte=nps_score_start,
                    nps_score__lte=nps_score_end)

        # filter on basis of start and end date
        if start_date and end_date:
            feedbacks = feedbacks.filter(merchant=merchant, received_at__gte=start_date,
                    received_at__lte=end_date)

        # filter on basis of instances
        if instances:
            feedbacks = feedbacks.filter(form_instance__in=instances)

        return feedbacks, all_start_date, all_end_date

    @property
    def responses(self):
        """This method returns a list of responses given to the
        feedback with the details of fields the responses are
        associated with.
        """

        form = self.form_instance.form
        # list of dictionaries of responses
        responses = []

        # iterating over all fields of the form
        for field in form.fields:
            f_id = str(field.id)
            responses.append({
                'text': field.text,
                'type': field.field_type,
                'id': f_id,
                'response': self.field_responses.get(f_id)
            })

        return responses

    @staticmethod
    def send_notification(feedback):
        """Checks if a notification needs to be sent for the feedback provided
        and also send the required notificaton (currently e-Mail) to the e-Mail
        IDs of the merchant.

        Keyword Arguments:
        feedback -- feedback object for which notification needs to be sent
        """

        # abort if NPS notifications for the merchant are not enabled
        if not feedback.merchant.nps_notifs:
            return False

        # abort if feedback's NPS score is above the threshold
        if feedback.nps_score > feedback.merchant.nps_threshold:
            return False

        # send an e-mail to the provided e-mail ids
        emails = feedback.merchant.notif_emails
        send_trans_email('feedback_nps_notification', emails, {'feedback': feedback})

    @staticmethod
    def create(nps_score,field_responses, form_instance, feedback_text=None, customer_details=None):
        # validate the responses of the fields of the form
        form = form_instance.form
        for field in form.fields:
            # validate the response of the field
            response = field_responses.get(str(field.id))
            if not response and field.required:
                raise FeedbackException('Response to all the fields of the form has not been provided.')
            # check if correct response is given to all of the fields
            try:
                if response: field.validate_customer_response(response)
            except ValueError:
                raise FeedbackException('Correct response to one or more fields has not been given.')

        # feedback form object
        feedback = FeedbackModel(nps_score=nps_score, feedback_text=feedback_text, field_responses=field_responses,
                form_instance=form_instance, merchant=form_instance.form.merchant)

        # add counter info to feedback (if exists on form)
        if form.incremental_id:
            feedback.has_counter = True
            feedback.counter = form.counter.get_next_counter()

        # create customer object only if customer exists
        customer = None
        if customer_details:
            name = customer_details.get('name')
            mobile = customer_details.get('mobile')
            email = customer_details.get('email')
            customer = CustomerModel(name=name, mobile=mobile, email=email)

        # validate customer & feedback objects
        try:
            feedback.validate()
            if customer:
                customer.validate()
        except db.ValidationError:
            raise FeedbackException('Feedback data provided was wrong.')

        # add customer id to feedback object
        if customer:
            customer.save()
            feedback.customer = customer

        # add feedback in db
        feedback.save()

        # send notification (if required)
        feedback.send_notification(feedback)

        return feedback

    def __repr__(self):
        return '<FeedbackModel: {0}>'.format(self.received_at)

class CustomerModel(db.Document):
    name = db.StringField(required=False)
    mobile = db.StringField(required=False)
    email = db.StringField(required=False)

    meta = {'collection': 'customers'}

    def __repr__(self):
        return '<CustomerModel: {0} ({1})>'.format(self.name, self.email)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# BSD 3-Clause License
#
# Copyright (c) 2017 xxxx
# All rights reserved.
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# ============================================================================
#
"""
pytorch-dl
Created by raj at 11:05
Date: January 18, 2020
"""

from torch import nn
from models.bert import Bert
import torch.npu
import os
NPU_CALCULATE_DEVICE = 0
if os.getenv('NPU_CALCULATE_DEVICE') and str.isdigit(os.getenv('NPU_CALCULATE_DEVICE')):
    NPU_CALCULATE_DEVICE = int(os.getenv('NPU_CALCULATE_DEVICE'))
if torch.npu.current_device() != NPU_CALCULATE_DEVICE:
    torch.npu.set_device(f'npu:{NPU_CALCULATE_DEVICE}')

class BertLanguageModel(nn.Module):
    """
    """
    def __init__(self, bert: Bert, vocab_size):
        super().__init__()
        self.bert = bert
        self.masked_lm = MaskedLanguageModel(bert.h, vocab_size)
        self.next_sentence = NextSentencePrediction(bert.h)

    def forward(self, x, segment=None):
        x = self.bert(x, segment)
        if segment:
            return self.masked_lm(x), self.next_sentence(x)
        else:
            return self.masked_lm(x)

class MaskedLanguageModel(nn.Module):
    """
    """
    def __init__(self, h, vocab_size):
        super().__init__()
        self.ff = nn.Linear(h, vocab_size)
        self.softmax = nn.LogSoftmax(dim=-1)

    def forward(self, x):
        return self.softmax(self.ff(x))

class NextSentencePrediction(nn.Module):
    """
    """
    def __init__(self, h, c=2):
        super().__init__()
        self.ff = nn.Linear(h, c)
        self.softmax = nn.LogSoftmax(dim=-1)

    def forward(self, x):
        return self.softmax(self.ff(x[:, 0]))