#!/usr/bin/env python
# coding: utf-8

# In[22]:

import numpy as np
import os
import sys
import xarray as xr
import scipy.io as sio
import matplotlib.pyplot as plt
from scipy.interpolate import griddata
import h5py
import pandas as pd

proj_dir = os.path.join(os.pardir)
data_dir = os.path.join(proj_dir,'data','analysis')
int_dir = os.path.join(data_dir,'interim')
ext_dir = os.path.join(data_dir,'external')
src_dir = os.path.join(proj_dir,'src')
fig_dir = os.path.join(proj_dir,'reports','shallow_melting','figures')

sys.path.append(src_dir)
from tools.log_progress import log_progress

# # Rignot data from Susheel

# ## interpolation onto ROMS grid

# In[8]:

grd2_path = os.path.join(proj_dir,'data','preprocessing','processed','waom2_grd.nc')
grd2 = xr.open_dataset(grd2_path)

# In[9]:

rignot_path = os.path.join(data_dir,'external','rignot','RignotEtAl2013-melt-rates.h5')
#rig = xr.open_dataset(rignot_path)
rig = {}
f = h5py.File(rignot_path)
for k, v in f.items():
    rig[k] = np.flipud(np.rot90(np.array(v)))

# In[10]:

plt.close()
for k,v in rig.items():
    plt.pcolormesh(v[::20,::20])
    plt.colorbar()
    plt.title(k)
    plt.show()

# In[15]:

mr=2
x,y = np.meshgrid(np.arange(-3000,3300+mr/2,mr/2),np.arange(-2700,2600+mr/2,mr/2))
x_rho = x[1::2,1::2]
y_rho = y[1::2,1::2]

x_rig,y_rig = np.meshgrid(np.arange(-2800,2801),np.arange(-2800,2801))

wb_rho = xr.DataArray(griddata((x_rig[::2,::2].flatten(),y_rig[::2,::2].flatten()),rig['wb'][::2,::2].flatten(),
                             (x_rho[::1,::1],y_rho[::1,::1])),dims=('eta_rho','xi_rho'))

wb_rho.values[(x_rho < -2800) | (x_rho > 2800)]=0

# In[16]:

wb_rho.to_netcdf(os.path.join(int_dir,'wb_rho.nc'))

# ## melt rate map and depth distribiution

# In[2]:

grd2_path = os.path.join(proj_dir,'data','preprocessing','processed','waom2_grd.nc')
grd2 = xr.open_dataset(grd2_path)

wb_rho_path = os.path.join(int_dir,'wb_rho.nc')
wb_rho = xr.open_dataarray(wb_rho_path)

# In[139]:

from visualization.shiftedColorMap import shiftedColorMap
import matplotlib
def plot_ismr_map(m,mask,title,m_min=-1,m_max=6,out_path=False):

    m_cmap = shiftedColorMap(plt.cm.seismic,midpoint= (1 - m_max/(m_max + abs(m_min))))

    mask_land = (grd2.zice < 0.0)&(grd2.mask_rho==1)
    land_zice = (grd2.mask_rho==0) | (grd2.zice < 0.0)
    matplotlib.rcParams.update({'font.size': 18})

    plt.close()
    fig,ax = plt.subplots(figsize=(15,8))
    ax.contourf(grd2.mask_rho.where(land_zice).values,colors=(('0.6','0.6','0.6')),alpha=0.5)
    (m.where(mask)).plot(vmin=m_min,vmax=m_max,ax=ax,cmap=m_cmap,cbar_kwargs={'label':''})
    ax.contour(-grd2.zice.where(grd2.mask_rho).values, levels=['0.01'], colors=('black'),linewidths=0.5,alpha=0.5)
    ax.contour(grd2.mask_rho.values, levels=['0.01'], colors=('black'),linewidths=0.5,alpha=0.5)

    plt.title(title, fontsize=20)
    ax.set_aspect('equal')
    ax.axis('off')
    plt.tight_layout()
    if out_path:
        plt.savefig(out_path,format='png',dpi=375)
    plt.show()

# In[206]:

plot_ismr_map(wb_rho,wb_rho!=0,'Rignot 2013 melt rates on ATOM grid',m_min=-6,m_max=6,
              out_path=os.path.join(fig_dir,'Rignot_wb_on_roms_grid.png'))

# In[58]:

def integrate_melt(m,dA,mask):
    rhoi = 916

    dA = dA.where(mask)
    weights = dA/dA.sum()

    ismr = (m*weights).sum()
    bmb = (m*dA*rhoi*10**-12).sum()

    print('Area of all ice shelves in 10^3 km^2: ',dA.sum().values*10**-9)
    print('Area average melt rate in m/yr: ',ismr.values)
    print('Basal mass loss in Gt/a: ',bmb.values)

def plot_depth_melting(m,dA,mask,bins,save=False):
    rhoi = 916

    dA = dA.where(mask)

    depths = (grd2.zice*-1)
    ismr2bmb = dA*rhoi*(10**-12)

    ismr = (m.where(mask).groupby_bins(depths,bins).mean()).to_series()
    A = dA.groupby_bins(depths,bins).sum().to_series()/10**9
    bmb =  ((m.where(mask)*ismr2bmb).groupby_bins(depths,bins).sum()).to_series()

    print('400 m fraction: ',bmb.loc[slice(0,400)].sum()/bmb.sum(),bmb.loc[slice(0,400)].sum())
    print('200 m fraction: ',bmb.loc[slice(0,200)].sum()/bmb.sum(),bmb.loc[slice(0,200)].sum())
    print('200-400 m fraction: ',bmb.loc[slice(201,400)].sum()/bmb.sum(),bmb.loc[slice(201,400)].sum())

    plt.rcParams.update({'font.size': 18})
    plt.close()
    fig,axes = plt.subplots(3,figsize=(12,12))
    ax1,ax2,ax3 = axes.flatten()
    ismr.plot.bar(ax=ax1,color='k')
    bmb.plot.bar(ax=ax2,color='k')
    A.plot.bar(ax=ax3,color='k')
    ax1.set_ylabel('Melt rate (m/yr)')
    ax2.set_ylabel('Basal mass loss (Gt/yr)')
    ax3.set_ylabel('Area (10$^3$ km$^2$)')
    for ax in [ax1,ax2]:
        #ax.set_xticks([])
        ax.set_xticklabels('')
        ax.set_xlabel('')
        ax.xaxis.grid(True)
    ax3.xaxis.grid(True)
    depth_str = list()
    for i in bins:
        depth_str.append(f'{i}-{i+100}')
    ax3.set_xticklabels(depth_str)
    ax3.set_xlabel('Depth ranges (m)')
    ax1.set_ylim(0,5.2)
    ax2.set_ylim(0,300)
    ax3.set_ylim(0,440)
    plt.savefig(os.path.join(fig_dir,'ATOM_depth_bins_data.png'),transparent=True,bbox_inches = "tight")
    plt.show()

    return ismr,A,bmb

# In[6]:

dA = (grd2.pm*grd2.pn)**-1
integrate_melt(wb_rho,dA,wb_rho==0)

# In[59]:

ismr_R,A_R,bmb_R = plot_depth_melting(wb_rho,dA,wb_rho!=0,np.arange(0,2800,100),save=False)

# ### comparison against ATOM

# In[12]:

m_path = os.path.join(int_dir,'m2.nc')
#m_path = os.path.join(data_dir,'raw','waom2','ocean_avg_0538-0610_m_avg.nc')
ismr = xr.open_dataarray(m_path).squeeze()

mask_vostock = (grd2.lat_rho<-75) & (grd2.lat_rho>-80) & (grd2.lon_rho>95) & (grd2.lon_rho<115)
mask_ice = (grd2.mask_rho ==1) & (grd2.zice < 0) & (mask_vostock == 0)

s2a=365.25*24*3600

# In[13]:

integrate_melt(ismr*s2a,dA,mask_ice)

# In[61]:

ismr_A,A_A,bmb_A = plot_depth_melting(ismr*s2a,dA,mask_ice,np.arange(0,2800,100),save=True)

# In[57]:

get_ipython().run_line_magic('matplotlib', 'inline')
#sns.set(style='darkgrid',font_scale=2)
plt.rcParams.update({'font.size': 18})
plt.close()
fig,axes = plt.subplots(3,figsize=(12,12))
ax1,ax2,ax3 = axes.flatten()
pd.concat({'Rignot':ismr_R,'ATOM':ismr_A},axis=1).plot.bar(ax=ax1,legend=False)
pd.concat({'Rignot':bmb_R,'ATOM':bmb_A},axis=1).plot.bar(ax=ax2,legend=False)
pd.concat({'Rignot':A_R,'ATOM':A_A},axis=1).plot.bar(ax=ax3,legend=False)
ax1.set_ylabel('Melt rate (m/yr)')
ax2.set_ylabel('Basal mass loss (Gt/yr)')
ax3.set_ylabel('Area (10$^3$ km$^2$)')
for ax in [ax1,ax2]:
    #ax.set_xticks([])
    ax.set_xticklabels('')
    ax.set_xlabel('')
    ax.xaxis.grid(True)
ax3.xaxis.grid(True)
depth_str = list()
for i in np.arange(0,2800,100):
    depth_str.append(f'{i}-{i+100}')
ax3.set_xticklabels(depth_str)
ax3.set_xlabel('Depth ranges (m)')
ax1.legend(('This study','Rignot et al., 2013 (Volume div.)'))
#ax1.set_ylim(0,5.2)
#ax2.set_ylim(0,300)
#ax3.set_ylim(0,440)
plt.savefig(os.path.join(fig_dir,'Rignot_depth_bins_comparison.png'),transparent=True,bbox_inches = "tight")
plt.show()

# In[ ]:

# In[107]:

shelves_masks_path = os.path.join(int_dir,'shelves2.npy')
shelves2 = np.load(shelves_masks_path,allow_pickle=True).item()
mask_shelves = np.zeros_like(grd2.mask_rho)
for key,values in log_progress(shelves2.items(),every=2):
        mask_shelves[values['mask']==1]=1

# In[241]:

sector_mask = os.path.join(int_dir,'sector2+.npy')
sector2 = np.load(sector_mask,allow_pickle=True).item()

# In[243]:

ismr.values[(sector2['Amundsen Sea'] == 1)|(sector2['Bellingshausen Sea'] == 1)] *=2

# In[312]:

from scipy.ndimage.morphology import binary_dilation as bd

mask_oo = (grd2.mask_rho == 1) & (grd2.zice==0)
mask_fz = bd(mask_oo,iterations=1)

mask_comp = (mask_ice!=0) & (wb_rho!=0)

# ## ice shelf average quantities compared to ATOM

# In[19]:

dA = (grd2.pm*grd2.pn)**-1

for k,v in log_progress(shelves2.items(),every = 2):
    rhoi = 916
    mask =( v['mask'] == 1) & (wb_rho != 0)
    mask_shallow = (mask == 1) & (grd2.zice > -400)
    mask_very_shallow = (mask_shallow == 1) & (grd2.zice > -200)

    weights = dA.where(mask)/dA.where(mask).sum()

    shelves2[k]['A_r'] = dA.where(mask).sum()/10**9
    shelves2[k]['ismr_r'] = (wb_rho*weights).sum()
    shelves2[k]['bmb_r'] = (wb_rho.where(mask)*dA*rhoi*10**-12).sum()

    shelves2[k]['A_shallow_r'] = dA.where(mask_shallow).sum()/10**9
    shelves2[k]['bmb_shallow_r'] = (wb_rho.where(mask_shallow)*dA*rhoi*10**-12).sum()
    shelves2[k]['frac_bmb_shallow_r'] = (wb_rho.where(mask_shallow)*dA).sum()/(wb_rho.where(mask)*dA).sum()

# In[20]:

m_path = os.path.join(data_dir,'raw','waom2','ocean_avg_0538-0610_m_avg.nc')
ismr = xr.open_dataarray(m_path)

s2a = 3600*24*365.25
for k,v in log_progress(shelves2.items(),every = 2):
    rhoi = 916
    mask = (v['mask'] == 1) & (wb_rho != 0)
    mask_shallow = (mask == 1) & (grd2.zice > -400)
    weights = dA.where(mask)/dA.where(mask).sum()

    shelves2[k]['rm'] = mask
    shelves2[k]['rm_shallow'] = mask

    shelves2[k]['A_rm'] = dA.where(mask).sum()/10**9
    shelves2[k]['ismr_rm'] = (ismr*weights*s2a).sum()
    shelves2[k]['bmb_rm'] = (ismr.where(mask)*s2a*dA*rhoi*10**-12).sum()

    shelves2[k]['A_shallow_rm'] = dA.where(mask_shallow).sum()/10**9
    shelves2[k]['bmb_shallow_rm'] = (ismr.where(mask_shallow)*s2a*dA*rhoi*10**-12).sum()
    shelves2[k]['frac_bmb_shallow_rm'] = (ismr.where(mask_shallow)*dA).sum()/(ismr.where(mask)*dA).sum()

# In[35]:

from scipy.ndimage.morphology import binary_dilation as bd

mask_oo = (grd2.mask_rho == 1) & (grd2.zice==0)
mask_bd = bd(mask_oo,iterations=1)

for k,v in log_progress(shelves2.items(),every = 2):
    rhoi = 916
    mask = (v['mask'] == 1) & (mask_bd == 0)
    mask_shallow = (mask == 1) & (grd2.zice > -400)
    weights = dA.where(mask)/dA.where(mask).sum()

    shelves2[k]['fm'] = mask
    shelves2[k]['fm_shallow'] = mask

    shelves2[k]['A_fm'] = dA.where(mask).sum()/10**9
    shelves2[k]['ismr_fm'] = (ismr*weights*s2a).sum()
    shelves2[k]['bmb_fm'] = (ismr.where(mask)*s2a*dA*rhoi*10**-12).sum()

    shelves2[k]['A_shallow_fm'] = dA.where(mask_shallow).sum()/10**9
    shelves2[k]['bmb_shallow_fm'] = (ismr.where(mask_shallow)*s2a*dA*rhoi*10**-12).sum()
    shelves2[k]['frac_bmb_shallow_fm'] = (ismr.where(mask_shallow)*dA).sum()/(ismr.where(mask)*dA).sum()

# In[66]:

import pandas as pd
import seaborn as sns

avg = pd.DataFrame.from_dict(shelves2).T[['A_r','ismr_r','bmb_r','frac_bmb_shallow_r','A_shallow_r',
                                          'A_rm','ismr_rm','bmb_rm','frac_bmb_shallow_rm','A_shallow_rm',
                                         'A_fm','ismr_fm','bmb_fm','frac_bmb_shallow_fm','A_shallow_fm']].dropna(axis=0,how='any').astype(float)

#avg.to_feather(os.path.join(int_dir,'df_rignot'))

# In[146]:

for key in ['A','bmb']:
    avg[avg['A_r']>5][[key+'_rm',key+'_r']].plot(kind='barh',width=0.7)
plt.tight_layout()
#plt.xlim([-0.5,2])

# In[304]:

avg['bmb_r'][['Fimbul','Jelbart']].sum()

# In[309]:

avg['bmb_r']['George VI']

# In[73]:

import seaborn as sns
sns.set();
for key in ['A','frac_bmb']:
    avg[avg['A_r']>5][[key+'_shallow_rm',key+'_shallow_r']].plot(kind='barh',width=0.7)
plt.tight_layout()
plt.xlim([-0.5,2])

# In[47]:

(ismr*s2a-wb_rho).where(wb_rho!=0).plot(size=7,vmax=3,vmin=-3,cmap='seismic')

# In[ ]:

avg[avg['A_r']>10][['A_shallow_r','A_shallow_rm']].plot(kind='barh')

# In[ ]:

mask = shelves2['Ross West']['mask']
plt.close()
(ismr.where(mask)*s2a-wb_rho).plot()
plt.show()

# In[ ]:

plt.close()
fig,ax1 = plt.subplots(figsize=(15,15))
mlc.iloc[3:39:3].plot(kind='barh',y='This study',ax=ax1,color='silver',width=0.9,fontsize=24)
data.plot(kind='barh',xerr=errors,ax=ax1,width=0.7,fontsize=24,alpha=0.7)
ax1.legend(['This study','Liu 2015','Depoorter 2013','Rignot 2013'],markerscale=1,fontsize=24,loc=4)
#ax1.set_title('Ice shelf basal mass loss compared to observations',fontsize=24)
ax1.set_xlabel('mass loss in Gt/yr',fontsize=24)
ax1.grid(axis='x')
#ax1.set_xlim(0,350)

plt.savefig(out_path,format='png',bbox_to_anchor=True,bbox_inches='tight')
plt.show()

# # Digitize Rignot
# ## Georef

# In[64]:

import cv2
get_ipython().run_line_magic('matplotlib', 'notebook')
plt.close()
plt.figure(figsize=(9,7))
impath = os.path.join(data_dir,'external','rignot','266.png')
nemo_img = cv2.imread(impath)[::,::,:]
plt.imshow(nemo_img)
plt.show()

# In[24]:

points_R = np.array([[280,1074],[2173,862],[1097,120],[1377,1827]])

# In[41]:

grd_path = os.path.join(int_dir,'grd2_dV.nc')
grd2 = xr.open_dataset(grd_path)

# In[18]:

grd.mask_rho.where(grd.zice<0)[::2,::2].plot(size=7)

# In[25]:

points_A = np.array([[272,624],[1361,743],[743,1171],[904,188]])

# In[26]:

points_R[:,1]=nemo_img.shape[0]-points_R[:,1]
points_A *=2

# In[33]:

x_fac = (points_A[1,0] - points_A[0,0])/(points_R[1,0] - points_R[0,0])
y_fac = (points_A[2,1] - points_A[3,1])/(points_R[2,1] - points_R[3,1])
fac = np.mean((x_fac,y_fac))
x_shift = np.mean(points_A[:,0]-points_R[:,0]*fac)
y_shift = np.mean(points_A[:,1]-points_R[:,1]*fac)

# In[42]:

eta_nemo_img = np.arange(nemo_img.shape[0])*fac+y_shift
xi_nemo_img = np.arange(nemo_img.shape[1])*fac+x_shift
xi_nemo_img_2d,eta_nemo_img_2d, = np.meshgrid(xi_nemo_img,eta_nemo_img)
points = (eta_nemo_img_2d.flatten(),xi_nemo_img_2d.flatten())
data = np.flipud(nemo_img)
data = np.reshape(data,(data.shape[0]*data.shape[1],3))
xi_2d = np.broadcast_to(grd2.xi_rho.values,grd2.mask_rho.shape)
eta_2d = np.repeat(grd2.eta_rho.values[:,np.newaxis], grd2.xi_rho.size, axis=1)
tar = (eta_2d,xi_2d)
nemo_img_rho = griddata(points,data,tar)
mask_ice = (grd2.mask_rho == 1) & (grd2.zice < 0)
nemo_da = xr.DataArray(nemo_img_rho,dims=('eta_rho','xi_rho','rgb')).where(mask_ice)

# In[48]:

nemo_da[:,:,0].plot(size=7)

# ### Replicate colorbar

# In[113]:

plt.close()
plt.figure(figsize=(6,2))
impath = os.path.join(data_dir,'external','rignot','g3841.png')
c_img = cv2.imread(impath)[::,::,:]
plt.imshow(c_img)
#plt.grid()
plt.xticks(np.arange(100))
plt.show()

# In[110]:

ct = {'red':[],'green':[],'blue':[]}
for i,step in enumerate(np.linspace(0,1,100)):
    c = c_img[5,i,:]/255
    for j,rgb in enumerate(['red','green','blue']):
        ct[rgb].append([step,c[j],c[j]])

# In[112]:

cmn = LinearSegmentedColormap('colormap_nemo', segmentdata=ct, N=256)
a=np.outer(np.arange(0,1,0.01),np.ones(10)).T
plt.figure(figsize=(6,2))
plt.imshow(a,aspect='auto',cmap=cmn,origin="lower")

# In[51]:

def unmap_nearest(img, rgb):
    """ img is an image of shape [n, m, 3], and rgb is a colormap of shape [k, 3]. """
    d = np.sum(np.abs(img[np.newaxis, ...] - rgb[:, np.newaxis, np.newaxis, :]), axis=-1)
    i = np.argmin(d, axis=0)
    return i / (rgb.shape[0] - 1)

# In[114]:

cmap = cmn
rgb = cmap(np.linspace(0, 1, cmap.N))[:, :3]
recon = unmap_nearest(nemo_da.values[::2,::2]/255,rgb)
recon_da = (xr.DataArray(recon, dims=('eta_rho','xi_rho')).where(mask_ice[::2,::2])-0.5)*10

# ## masking and depth melting

# In[53]:

shelves_masks_path = os.path.join(int_dir,'shelves2.npy')
shelves2 = np.load(shelves_masks_path,allow_pickle=True).item()

shelf_mask = np.zeros_like(grd2.mask_rho)[::2,::2]
for key,values in log_progress(shelves2.items(),every=2):
        shelf_mask[values['mask'][::2,::2]==1]=1
#front_map = xr.DataArray(front_map,dims=('eta_rho','xi_rho')).where(front_map==1)

# In[115]:

plt.close()
recon_da.where(shelf_mask).plot(cmap = cmn,size=7)
plt.savefig(os.path.join(fig_dir,'Rignot_digi2.png'),transparent=True,dpi=300,bbox_inches = "tight")
plt.show()

# In[142]:

recon_ad = recon_da.copy()
#recon_ad.values[recon_da==-5]=-100

rhoi = 916

dA = (((grd2.pm*grd2.pn)[::2,::2]**-1)*4).where(shelf_mask)
weights = dA/dA.sum()

ismr = (recon_ad*weights).sum()
bmb = (recon_ad*dA*rhoi*10**-12).sum()

print('Area of all ice shelves in 10^3 km^2: ',dA.sum().values*10**-9)
print('Area average melt rate in m/yr: ',-ismr.values)
print('Basal mass loss in Gt/a: ',-bmb.values)

# In[143]:

bins = np.arange(0,2800,100)
depths = grd2.zice[::2,::2]*-1
ismr2bmb = dA*rhoi*(10**-12)

ismr = (-recon_ad.where(shelf_mask).groupby_bins(depths,bins).mean()).to_series()
A = dA.groupby_bins(depths,bins).sum().to_series()/10**9
bmb =  ((-recon_ad.where(shelf_mask)*ismr2bmb).groupby_bins(depths,bins).sum()).to_series()

# In[144]:

plt.rcParams.update({'font.size': 18})
plt.close()
fig,axes = plt.subplots(3,figsize=(12,12))
ax1,ax2,ax3 = axes.flatten()
ismr.plot.bar(ax=ax1,color='k')
bmb.plot.bar(ax=ax2,color='k')
A.plot.bar(ax=ax3,color='k')
ax1.set_ylabel('ismr in m/yr')
ax2.set_ylabel('bmb in Gt/yr')
ax3.set_ylabel('A in 10^3 km^2')
for ax in [ax1,ax2]:
    #ax.set_xticks([])
    ax.set_xticklabels('')
    ax.set_xlabel('')
    ax.xaxis.grid(True)
ax3.xaxis.grid(True)
ax3.set_xlabel('depth bins in m')
ax1.set_ylim(0,5.2)
ax2.set_ylim(0,300)
ax3.set_ylim(0,440)
plt.savefig(os.path.join(fig_dir,'Rignot_depth_bins2.png'),transparent=True,bbox_inches = "tight")
plt.show()

# In[145]:

bmb.loc[slice(0,200)].sum()/bmb.sum()

# In[38]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[2]:

import cv2
get_ipython().run_line_magic('matplotlib', 'notebook')
plt.close()
impath = os.path.join(data_dir,'external','rignot','meltRates.jpg')
nemo_img = cv2.imread(impath)[::,::,:]
plt.imshow(nemo_img)
plt.show()

# In[4]:

from scipy.interpolate import griddata
lat_lon = np.array([[-70,60],[-70,140],[-70,80],[-65,-80],[-75,-140]])
Rxy = np.array([[1083,320],[972,952],[1143,483],[34,460],[412,855]])
Rxy[:,1]=nemo_img.shape[0]-Rxy[:,1]

grd_path = os.path.join(proj_dir,'data','preprocessing','processed','waom2_grd.nc')
grd2= xr.open_dataset(grd_path)

lat_rho = grd2.lat_rho.values
lon_rho = grd2.lon_rho.values
xi_2d = np.broadcast_to(grd2.xi_rho.values,grd2.mask_rho.shape)
eta_2d = np.repeat(grd2.eta_rho.values[:,np.newaxis], grd2.xi_rho.size, axis=1)

tar = lat_lon
xi = griddata((lat_rho.flatten()[::10],lon_rho.flatten()[::10]),xi_2d.flatten()[::10],tar)
eta = griddata((lat_rho.flatten()[::10],lon_rho.flatten()[::10]),eta_2d.flatten()[::10],tar)

# In[5]:

x_ratio = 500/227+0.02
y_ratio = x_ratio
print(xi-Rxy[:,0]*x_ratio)
print(eta - Rxy[:,1]*y_ratio)
x_shift = np.mean(xi-Rxy[:,0]*x_ratio)
y_shift = np.mean(eta - Rxy[:,1]*y_ratio)
eta_nemo_img = np.arange(nemo_img.shape[0])*y_ratio+y_shift
xi_nemo_img = np.arange(nemo_img.shape[1])*x_ratio+x_shift
xi_nemo_img_2d,eta_nemo_img_2d, = np.meshgrid(xi_nemo_img,eta_nemo_img)
points = (eta_nemo_img_2d.flatten(),xi_nemo_img_2d.flatten())
data = np.flipud(nemo_img)
data = np.reshape(data,(data.shape[0]*data.shape[1],3))
tar = (eta_2d,xi_2d)
nemo_img_rho = griddata(points,data,tar)
mask_ice = (grd2.mask_rho == 1) & (grd2.zice < 0)
nemo_da = xr.DataArray(nemo_img_rho,dims=('eta_rho','xi_rho','rgb')).where(mask_ice)

# In[95]:

plt.close()
nemo_da[:,:,0].plot(size=7)
plt.show()

# In[129]:

shelves_masks_path = os.path.join(int_dir,'shelves2.npy')
shelves2 = np.load(shelves_masks_path,allow_pickle=True).item()

shelf_mask = np.zeros_like(grd2.mask_rho)[::2,::2]
for key,values in log_progress(shelves2.items(),every=2):
        shelf_mask[values['mask'][::2,::2]==1]=1
#front_map = xr.DataArray(front_map,dims=('eta_rho','xi_rho')).where(front_map==1)

# ## Digitize colormap
# ### get rid of gray and black and white

# In[6]:

from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from matplotlib import colors

nemo = nemo_da.values[::10,::10].astype('uint8')
r, g, b = cv2.split(nemo)
fig = plt.figure()
axis = fig.add_subplot(1, 1, 1, projection="3d")

pixel_colors = nemo.reshape((np.shape(nemo)[0]*np.shape(nemo)[1], 3))
norm = colors.Normalize(vmin=-1.,vmax=1.)
norm.autoscale(pixel_colors)
pixel_colors = norm(pixel_colors).tolist()

axis.scatter(r.flatten(), g.flatten(), b.flatten(), facecolors=pixel_colors, marker=".")
axis.set_xlabel("Red")
axis.set_ylabel("Green")
axis.set_zlabel("Blue")
plt.show()

# In[7]:

get_ipython().run_line_magic('matplotlib', 'notebook')
hsv_nemo = cv2.cvtColor(nemo, cv2.COLOR_RGB2HSV)
h, s, v = cv2.split(hsv_nemo)
plt.close()
fig = plt.figure(figsize=(10,8))
axis = fig.add_subplot(1, 1, 1, projection="3d")

axis.scatter(h.flatten(), s.flatten(), v.flatten(), facecolors=pixel_colors, marker=".")
axis.set_xlabel("Hue")
axis.set_ylabel("Saturation")
axis.set_zlabel("Value")
plt.show()

# In[8]:

get_ipython().run_line_magic('matplotlib', 'inline')
plt.close()
fig = plt.figure(figsize=(12,9))
axis = fig.add_subplot(1, 2, 1)
axis.scatter(h.flatten(), v.flatten(), facecolors=pixel_colors, marker=".")
axis.set_xlabel("Hue")
axis.set_ylabel("Value")
axis = fig.add_subplot(1, 2, 2)
axis.scatter(h.flatten(), s.flatten(), facecolors=pixel_colors, marker=".")
axis.set_xlabel("Hue")
axis.set_ylabel("Saturation")
#axis.set_xlim(0,10)
plt.show()

# In[9]:

from matplotlib.colors import hsv_to_rgb,rgb_to_hsv
get_ipython().run_line_magic('matplotlib', 'inline')
light_gray = (0,0,0)
dark_gray = (1,1,255)

hsv_nemo = cv2.cvtColor(nemo, cv2.COLOR_RGB2HSV)
mask = ~cv2.inRange(hsv_nemo, light_gray, dark_gray)
result = cv2.bitwise_and(nemo, nemo, mask=mask)
plt.close()
fig,axes = plt.subplots(ncols=2,figsize=(9,6))
ax1,ax2 = axes.flatten()
ax1.imshow(mask, cmap="gray")
ax2.imshow(result)
plt.show()

# ### interpolate the gray lines

# In[49]:

gray_mask = ((recon_da==-5) | (recon_da==5))
plt.close()
recon_da.where(gray_mask).plot(cmap=cmn,size=7)
plt.show()

# In[50]:

from tools.NDinterp import NDinterp
test = recon_da.copy()
test.values[gray_mask]=np.nan
test.values = NDinterp(test.values)

# In[53]:

plt.close()
test.where(shelf_mask).plot(cmap=cmn,size=7)
plt.show()

# In[ ]:

# In[ ]:

# # Ben's estimates

# In[188]:

from scipy.ndimage import gaussian_filter
ben_path = os.path.join(data_dir,'external','ben','Richter_20190801.mat')
ben_raw = sio.loadmat(ben_path)
ben_raw = dict((k, np.flipud(np.rot90(np.array(ben_raw[k])))) for k in ['m','draft','uv_la','uv_lo'])
ben_raw['m'] = gaussian_filter(ben_raw['m'].copy(),sigma=3)*-1

# In[160]:

waom_grd_path = os.path.join(proj_dir,'data','preprocessing','processed','waom10_grd.nc')
waom_grd = xr.open_dataset(waom_grd_path)

# In[162]:

dA = (waom_grd.pn*waom_grd.pm)**-1

wla = waom_grd.lat_rho.values
wlo = waom_grd.lon_rho.values

bla = ben_raw['uv_la']
blo = ben_raw['uv_lo']

dA_on_ben=griddata((wla.flatten(),wlo.flatten()),dA.values.flatten(),(bla,blo),method='nearest')

# In[163]:

draft = waom_grd.zice.where((waom_grd.mask_rho ==1) & (waom_grd.zice<0))

draft_on_ben = griddata((wla.flatten(),wlo.flatten()),draft.values.flatten(),(bla,blo),method='nearest')

# In[164]:

shift = 0.85-ben.m.mean()

# In[190]:

ii = ~np.isnan(ben_raw['m'])
ben = xr.Dataset({'m':(('y','x'),ben_raw['m'][ii]+shift),
            'draft':(('y','x'),ben_raw['draft']),
            'draft_bm':(('y','x'),draft_on_ben),
            'mask':(('y','x'),~np.isnan(ben_raw['m'])),
            'lat':(('y','x'),ben_raw['uv_la']),
            'lon':(('y','x'),ben_raw['uv_lo']),
             'dA':(('y','x'),dA_on_ben*0.01)})

# In[158]:

get_ipython().run_line_magic('matplotlib', 'notebook')
ben.m.where(ben.mask)[::10,::10].plot()

from tools.cartesian_grid_2d import haversine
from tools.rotate_vector_roms import rotate_vector_roms

dx = haversine(ben['lon'][:,:-1].values,ben['lat'][:,:-1].values,ben['lon'][:,1:].values,ben['lat'][:,1:].values)
dy = haversine(ben['lon'][:-1:,:].values,ben['lat'][:-1,:].values,ben['lon'][1:,:].values,ben['lat'][1:,:].values)
dx,dy = rotate_vector_roms(dx,dy,np.zeros_like(ben['m']))
ben['dx']=(('y','x'),dx)
ben['dy']=(('y','x'),dy)
ben['dA']=(('y','x'),np.ones_like(dx)*1000000)
# In[115]:

dA = ben.dA.where(ben.mask)
weights = dA/dA.sum()

rhoi = 917
ismr = (ben.m_shift*weights).sum()
bmb = (ben.m_shift*dA*rhoi*10**-12).sum()

print('Area of all ice shelves in 10^3 km^2: ',dA.sum().values*10**-9)
print('Area average melt rate in m/yr: ',ismr.values)
print('Basal mass loss in Gt/a: ',bmb.values)

# In[47]:

from dask.distributed import Client
C = Client()
C

# In[116]:

ben['m_shift'] = ben.m_shift.chunk({'x':1000,'y':1000})

# In[145]:

mask = ben.mask
bins = np.arange(0,2800,100)
depths = ben.draft
ismr2bmb = dA*rhoi*(10**-12)

# In[146]:

plt.close()
depths.where(ben.mask)[::10,::10].plot()
plt.show()

# In[130]:

ismr = (ben.m_shift*weights).groupby_bins(depths,bins).sum().to_series()
A = dA.groupby_bins(depths,bins).sum().to_series()*10**-9
bmb =  ((ben.m_shift*ismr2bmb).groupby_bins(depths,bins).sum()).to_series()

# In[123]:

plt.rcParams.update({'font.size': 18})
plt.close()
fig,axes = plt.subplots(3)
ax1,ax2,ax3 = axes.flatten()
ismr.plot.bar(ax=ax1,color='k')
bmb.plot.bar(ax=ax2,color='k')
A.plot.bar(ax=ax3,color='k')
ax1.set_ylabel('ismr in m/yr')
ax2.set_ylabel('bmb in Gt/yr')
ax3.set_ylabel('A in 10^3 km^2')
for ax in [ax1,ax2]:
    #ax.set_xticks([])
    ax.set_xticklabels('')
    ax.set_xlabel('')
    ax.xaxis.grid(True)
ax3.xaxis.grid(True)
ax3.set_xlabel('depth bins in m')
#ax1.set_ylim(0,5.2)
ax2.set_ylim(0,300)
ax3.set_ylim(0,440)
plt.savefig(os.path.join(fig_dir,'ben_depth_bins.png'),transparent=True,bbox_inches = "tight")
plt.show()

# In[ ]:

# In[ ]:

mask_sector = np.load(os.path.join(int_dir,'sector2.npy'))

sec_depth = {}

for sec_key,sec_mask in log_progress(mask_sector.items(),name='sector'):
    sec_depth[sec_key]={}

    for depth_key,depth_mask in mask_depth.items():

        mask = sec_mask & depth_mask

        dA = (1/(grd.pm*grd.pn)).where(mask)
        weights = dA/dA.sum()

        ismr2bmb = dA*rhoi*(10**-12)

        sec_depth[sec_key]["A "+depth_key] = dA.sum().values*10**-9
        sec_depth[sec_key]["ismr "+depth_key] = (m.where(mask)*weights).sum().values*s2a
        sec_depth[sec_key]["bmb "+depth_key] = (m.where(mask)*ismr2bmb).sum().values*s2a

df_melt = pd.DataFrame.from_dict(sec_depth, orient='index').T

# In[290]:

def make_mask_front(grd,nb_cells):

    mask_rho = grd.mask_rho.values
    mask_land = np.zeros_like(mask_rho)
    mask_land[mask_rho == 0] = 1
    mask_zice = np.zeros_like(mask_land)
    mask_zice[grd.zice.values*mask_rho != 0] = 1

    mask_front = np.zeros_like(grd.mask_rho.values)

    for j in grd.eta_rho.values:
        for i in grd. xi_rho.values:
            if mask_zice[j,i] == 1:
                j_min = max(j-nb_cells,0)
                j_max = min(j+nb_cells, np.size(mask_rho,0))
                i_min = max(i-nb_cells,0)
                i_max = min(i+nb_cells+1, np.size(mask_rho,1))

                if np.any(mask_zice[j_min:j_max,i_min:i_max] + mask_land[j_min:j_max,i_min:i_max]== 0):
                        mask_front[j,i] = 1

    grd['mask_front'] = (('eta_rho','xi_rho'),mask_front)

    return grd

# In[291]:

waom_grd = make_mask_front(waom_grd,2)

# In[292]:

import pandas as pd
def make_mass_loss_table(m,grd,
                         google_sheet_url='https://docs.google.com/spreadsheets/d/1BpI6iRB569kF7TxdiFT3ColNVH9zwHgNwq49Dat7Ok4/edit?usp=sharing',
                        idx_start=0,idx_end=69,include_no_front=True):

    csv_export_url = google_sheet_url.replace('edit?', 'export?gid=0&format=csv&')
    IS = pd.read_csv(csv_export_url)

    rhoi = 916

    mask_ice = (grd.mask_rho==1) & (grd.zice<0)

    for idx,row in log_progress(IS.iloc[idx_start:idx_end].iterrows(),name='Ice shelf',every=1):

        if row.lon_min<row.lon_max:
            mask_coord = (grd.lon_rho>row.lon_min) & (grd.lon_rho<=row.lon_max) & (grd.lat_rho>row.lat_min) & (grd.lat_rho<=row.lat_max)
        else:
            mask_coord = ((grd.lon_rho>row.lon_min) | (grd.lon_rho<=row.lon_max)) & (grd.lat_rho>row.lat_min) & (grd.lat_rho<=row.lat_max)

        mask_shelf = mask_ice & mask_coord

        dA = (1/(grd.pm*grd.pn)).where(mask_shelf)
        weights = dA/dA.sum()

        dA_l = dA.where(m > 0.0)
        weights_l = dA_l/dA_l.sum()

        dA_g = dA.where(m < 0.0)
        weights_g = dA_g/dA_g.sum()

        ismr2bmb = dA*rhoi*(10**-12)

        IS.at[idx,"A"] = float(dA.sum().values)*10**-9
        IS.at[idx,"ismr"] = float((m.where(mask_shelf)*weights).sum().values)
        IS.at[idx,"ismr_l"] = float((m.where(mask_shelf & (m > 0.0))*weights_l).sum().values)
        IS.at[idx,"ismr_g"] = float((m.where(mask_shelf & (m < 0.0))*weights_g).sum().values)
        IS.at[idx,"bmb"] = float((m.where(mask_shelf)*ismr2bmb).sum().values)
        IS.at[idx,"bml"] = float((m.where(mask_shelf & (m > 0.0))*ismr2bmb).sum().values)
        IS.at[idx,"bmg"] = float((m.where(mask_shelf & (m < 0.0))*ismr2bmb).sum().values)

        if include_no_front:

            dA = dA.where(grd.mask_front == 0)
            weights = dA/dA.sum()

            dA_l = dA.where(m > 0.0)
            weights_l = dA_l/dA_l.sum()

            dA_g = dA.where(m < 0.0)
            weights_g = dA_g/dA_g.sum()

            ismr2bmb = dA*rhoi*(10**-12)

            IS.at[idx,"A_nf"] = float(dA.sum().values)*10**-9
            IS.at[idx,"ismr_nf"] = float((m.where(mask_shelf & (grd.mask_front == 0))*weights).sum().values)
            IS.at[idx,"ismr_nf_l"] = float((m.where(mask_shelf & (grd.mask_front == 0) & (m > 0.0))*weights_l).sum().values)
            IS.at[idx,"ismr_nf_g"] = float((m.where(mask_shelf & (grd.mask_front == 0) & (m < 0.0))*weights_g).sum().values)
            IS.at[idx,"bmb_nf"] = float((m.where(mask_shelf & (grd.mask_front == 0))*ismr2bmb).sum().values)
            IS.at[idx,"bml_nf"] = float((m.where(mask_shelf & (grd.mask_front == 0) & (m > 0.0))*ismr2bmb).sum().values)
            IS.at[idx,"bmg_nf"] = float((m.where(mask_shelf & (grd.mask_front == 0) & (m < 0.0))*ismr2bmb).sum().values)

    return IS

# In[293]:

IS = make_mass_loss_table(wb,waom_grd,include_no_front=True)
IS

# In[294]:

def merge_patches(df,name1,name2,name):
    p1 = df.loc[df['Names'] == name1]
    p2 = df.loc[df['Names'] == name2]

    m = {'Names':name,'A':np.nan,'bmb':np.nan,'bml':np.nan,'bmg':np.nan,'ismr':np.nan,'ismr_l':np.nan,'ismr_g':np.nan,
         'A_nf':np.nan,'bmb_nf':np.nan,'bml_nf':np.nan,'bmg_nf':np.nan,'ismr_nf':np.nan,'ismr_nf_l':np.nan,'ismr_nf_g':np.nan,}
    for key in ['A','bmb','bml','bmg','A_nf','bmb_nf','bml_nf','bmg_nf']:
        m[key] = p1[key].values + p2[key].values
    for key in ['ismr','ismr_l','ismr_g','ismr_nf','ismr_nf_l','ismr_nf_g']:
        m[key] = ((p1[key]*p1.A_nf).values + (p2[key]*p2.A_nf).values) / m['A_nf']

    df = pd.concat([df,pd.DataFrame(m)],ignore_index=True)

    return df

# In[295]:

IS_merged = merge_patches(IS,'George VI 1','George VI 2','George VI')
IS_merged = merge_patches(IS_merged,'Ronne','Filchner','Ronne-Filchner')
IS_merged = merge_patches(IS_merged,'Moscow University','Totten','Moscow Uni + Totten')
IS_merged = merge_patches(IS_merged,'Brunt/Stancomb','Riiser-Larsen','Brunt + Riiser-Larsen')
IS_merged = merge_patches(IS_merged,'Fimbul','Jelbart','Fimbul + Jelbart')
IS_merged = merge_patches(IS_merged,'Ross West','Ross East','Ross')

# In[296]:

IS_out = IS_merged[['Names','A','bmb','ismr',
                    'A_nf','bmb_nf','ismr_nf',
                    'bml','bmg','ismr_l','ismr_g',
                    'bml_nf','bmg_nf','ismr_nf_l','ismr_nf_g',
                    'lon_min','lat_min','lon_max','lat_max']]

pd.options.display.float_format = '{:,.2f}'.format

IS_out

# In[297]:

IS_out.to_csv(mass_loss_table_out_path)

# In[ ]:


#!/usr/bin/env python3

import json
import os
import subprocess

"""
The output format will be:
    GitHub_Entity_Filename,Oasis_Entity_ID,Oasis_Account,ETH_Address,Stake_Amount_Day1,Stake_Amount_Day2,...Stake_Amount_Day30
    If the paratime node is dead, your stake amount will be set to zero.
"""

RUNTIME_ID = 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/wQ='

class Entity():
    def __init__(self):
        self.entity_filename = ""
        self.oasis_id = ""
        self.oasis_account = ""
        self.eth_address = ""
        self.stake_amount = {}
    def __str__(self):
        s = [self.entity_filename, self.oasis_id, self.oasis_account, self.eth_address]
        for k in self.stake_amount:
            #print('Stake ... ', k, self.stake_amount[k])
            s.append(self.stake_amount[k])
        return ','.join(s)
    def __repr__(self):
        s = [self.entity_filename, self.oasis_id, self.oasis_account, self.eth_address]
        for k in self.stake_amount:
            #print('Stake ... ', k, self.stake_amount[k])
            s.append(self.stake_amount[k])
        return ','.join(s)

    def load_entity(self, line):
        p = line.split(',')
        self.entity_filename = p[0]
        self.oasis_id = p[1]
        self.oasis_account = p[2]
        self.eth_address = p[3]
    def load_stake_amount(self, date, nodes):
        p = os.path.join(date, 'accounts_info', 'account_info.'+self.oasis_account+'.'+date+'.txt')
        try:
            with open(p) as f:
                print("Handle ... ", p)
                lines = f.readlines()
                targetI = 0;
                for i in range(len(lines)):
                    if lines[i].startswith("Active Delegations to this Account:"):
                        targetI = i+1
                        break
                print(lines[targetI].split()[1].strip())
                self.stake_amount[date] = lines[targetI].split()[1].strip()
                if self.oasis_id not in nodes:
                    print("NOT in the nodes")
                    self.stake_amount[date] = '-1'
        except:
            print("ERROR")
            self.stake_amount[date] = '-2'

def load_entities():
    entities = []
    with open('./whitelisted_entities.csv') as f:
        for _,line in enumerate(f):
            e = Entity()
            e.load_entity(line)
            entities.append(e)
    return entities

def get_dates():
    return [p for p in os.listdir('.') if os.path.isdir(os.path.join('.',p))]

def main():
    dates = sorted(get_dates())
    entities = load_entities()
    print(entities)
    for date in dates:
        nodes = {}
        p = os.path.join('.', date, 'node_list.'+date+'.json')
        with open(p) as f:
            for _,line in enumerate(f):
                node = json.loads(line)
                if node['roles'] == 1:
                    print(date, "GET COMPUT NODE", node['entity_id'])
                    # Compute node
                    if node['runtimes'][0]['id'] == RUNTIME_ID:
                        nodes[node['entity_id']] = 1
                elif node['roles'] == 35:
                    print(date, "GET GENESIS NODE", node['entity_id'])
                    # Storage + Compute node
                    if node['runtimes'][0]['id'] == RUNTIME_ID:
                        nodes[node['entity_id']] = 35
                else:
                    continue
        for e in entities:
            e.load_stake_amount(date, nodes)
    print(entities)
    with open('reward_base.csv', 'w') as f:
        b = ['GitHub_Entity_Filename,Oasis_Entity_ID,Oasis_Account,ETH_Address']
        b.extend(dates)
        f.write(','.join(b)+'\n')
        for e in entities:
            f.write(str(e)+'\n')

if __name__ == '__main__':
    main()


import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
import cv2
import time
import os
preprocess_paraments={}
example_name = {}

##########################要改的东西#######################################
#tfrecords文件的路径(在本代码测试可填写相对路径，但给项目用喂要写完整路径，否则总是报错找不到文件)
train_tfrecord = ['/home/mo/work/caps_face/make_italy_tfrecord/italy_tf/italy_train_00000-of-00003.tfrecord',
                  '/home/mo/work/caps_face/make_italy_tfrecord/italy_tf/italy_train_00001-of-00003.tfrecord',
                  '/home/mo/work/caps_face/make_italy_tfrecord/italy_tf/italy_train_00002-of-00003.tfrecord',
                  ]
test_tfrecord = ['/home/mo/work/caps_face/make_italy_tfrecord/italy_tf/italy_validation_00000-of-00003.tfrecord',
                 '/home/mo/work/caps_face/make_italy_tfrecord/italy_tf/italy_validation_00001-of-00003.tfrecord',
                 '/home/mo/work/caps_face/make_italy_tfrecord/italy_tf/italy_validation_00002-of-00003.tfrecord']

labels_txt = ['/home/mo/work/caps_face/make_italy_tfrecord/italy_tf/labels.txt']

# 解码部分：填入解码键值和原图大小以便恢复
example_name['image'] = 'image/encoded'  #主要是这个(原图)p
example_name['label'] = 'image/class/label' #主要是这个(标签)
origenal_size =[32,32,1] #要还原原先图片尺寸

#预处理方式
to_random_brightness = True
to_random_contrast = True
to_resize_images = False
resize_size =[20,20]
to_random_crop = True
crop_size= [28, 28, 1]

#多队列、多线程、batch读图部分
num_threads = 8
batch_size = 32
shuffle_batch =True
#训练多少轮，string_input_producer的num_epochs就写多少，
# 否则会爆出OutOfRangeError的错误（意思是消费量高于产出量）
num_epochs = 150

#显示方式
cv2_show = False  # 用opencv显示或plt显示
#######################  end  ############################################

def ReadTFRecord(tfrecords,example_name):
    if len(tfrecords) == 1:
        record_queue = tf.train.string_input_producer(tfrecords,num_epochs=num_epochs+1)#只有一个文件，谈不上打乱顺序
    else:
        # shuffle=False，num_epochs为3，即每个文件复制成3份，再打乱顺序，否则按原顺序
        record_queue = tf.train.string_input_producer(tfrecords,shuffle=True, num_epochs=num_epochs+1)

    reader = tf.TFRecordReader()
    key, value = reader.read(record_queue)
    features = tf.parse_single_example(value,
            features={
                # 取出key为img_raw和label的数据,尤其是int位数一定不能错!!!
                example_name['image']: tf.FixedLenFeature([],tf.string),
                example_name['label']: tf.FixedLenFeature([], tf.int64)
            })

    img = tf.decode_raw(features[example_name['image']], tf.uint8)
    # 注意定义的为int多少位就转换成多少位,否则容易出错!!

    if len(origenal_size) == 2:
        w, h = origenal_size[0],origenal_size[1]
    else:
        w, h, c = origenal_size[0],origenal_size[1],origenal_size[2]
    img = tf.reshape(img, [w, h, c])

    # 不清楚为何加了这个tf.cast会让cv2.imshow显示不正常，图片变成黑白二值图，模糊不清
    img = tf.cast(img, tf.float32)

    label = tf.cast(features[example_name['label']], tf.int64)
    label = tf.cast(label, tf.int32)

    return img, label

def preprocess_data(is_train,image, label):
    if is_train:

        if to_random_brightness:
            image = tf.image.random_brightness(image, max_delta=32. / 255.)
        if to_random_contrast:
            image = tf.image.random_contrast(image, lower=0.5, upper=1.5)
        if to_resize_images:
            # 只有method = 1没有被破坏最严重
            image = tf.image.resize_images(image, resize_size,method=1)
        if to_random_crop:
            image = tf.random_crop(image, crop_size)

    else:
        if to_resize_images:
            image = tf.image.resize_images(image, [28, 28])
        if to_random_crop:
            image = tf.random_crop(image, crop_size)

    return image, label

def feed_data_method(image,label):
    if shuffle_batch:
        images, labels = tf.train.shuffle_batch(
            [image, label],
            batch_size=batch_size,
            num_threads=num_threads,
            capacity=batch_size*64,
            min_after_dequeue=batch_size*32,
            allow_smaller_final_batch=False)
    else:
        images, labels = tf.train.batch(
            [image, label],
            batch_size=batch_size,
            num_threads=num_threads,
            capacity=batch_size*64,
            allow_smaller_final_batch=False)
    return images, labels

def plt_imshow_data(data):
    #调成标准格式和标准维度，免得爆BUG
    data = np.asarray(data)

    if data.ndim == 3:
        if data.shape[2] == 1:
            data = data[:, :, 0]
    plt.imshow(data)
    plt.show()
    time.sleep(2)
def read_label_txt_to_dict(labels_txt =None):
    if os.path.exists(labels_txt):
        labels_maps = {}
        with open(labels_txt) as f:
            while True:
                line = f.readline()
                if not line:
                    break
                line = line[:-1]  # 去掉换行符
                line_split = line.split(':')
                labels_maps[line_split[0]] = line_split[1]
        return labels_maps
    return None

def create_inputs_italy(is_train):
    if is_train:
        data_tfrecord = train_tfrecord
    else:
        data_tfrecord = test_tfrecord
    image, label = ReadTFRecord(data_tfrecord,example_name)          #恢复原始数据
    image, label = preprocess_data(is_train,image, label)            #预处理方式
    images,labels = feed_data_method(image, label)                   #喂图方式

    return images,labels

if  __name__== '__main__':
    images, labels = create_inputs_italy(is_train = False)
    labels_maps = read_label_txt_to_dict(labels_txt=labels_txt[0])   #标签映射
    #观察自己设置的参数是否符合心意，合适的话在别的项目中直接调用 create_inputs_xxx() 函数即可喂数据
    with tf.Session() as sess:
        tf.global_variables_initializer().run()
        tf.local_variables_initializer().run()
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess=sess, coord=coord)

        # 输出100个batch观察
        batch_size_n = 5  #观察batch_size的第n张图片
        if cv2_show:
            for i in range(100):
                x, y = sess.run([images, labels])
                label = y[batch_size_n]
                label = str(label)
                label_name = labels_maps[label]
                title = 'label:{}'.format(label_name)
                print('image_shape:',x.shape, 'label_shape:', y.shape,title)
                cv2.namedWindow(title, 0)
                cv2.startWindowThread()
                cv2.imshow(title, x[batch_size_n])
                cv2.waitKey(2000)
                cv2.destroyAllWindows()
        else:
            for i in range(100):
                x, y = sess.run([images, labels])
                label = y[batch_size_n]
                label = str(label)
                label_name =labels_maps[label]
                title = 'label:{}'.format(label_name)
                print('image_shape:',x.shape, 'label_shape:', y.shape,title)

                plt_imshow_data(x[batch_size_n])

        coord.request_stop()
        coord.join(threads)


# -*- coding: utf-8 -*-

import sys

from compmake.exceptions import UserError
from compmake.jobs.storage import all_jobs, get_job_cache
from compmake.storage.filesystem import StorageFilesystem
from compmake.structures import Cache
from contracts import check_isinstance
from . import logger

def comptest_to_junit_main():
    args = sys.argv[1:]
    if not args:
        msg = 'Require the path to a Compmake DB.'
        raise UserError(msg)

    dirname = args[0]
    # try compressed
    try:
        db = StorageFilesystem(dirname, compress=True)
    except Exception:
        db = StorageFilesystem(dirname, compress=False)

    jobs = list(all_jobs(db))

    if not jobs:
        msg = 'Could not find any job, compressed or not.'
        logger.error(msg)
        sys.exit(1)

    s = junit_xml(db)
    check_isinstance(s, six.text_type)
    s = s.encode('utf8')
    sys.stdout.buffer.write(s)

def junit_xml(compmake_db):
    from junit_xml import TestSuite

    jobs = list(all_jobs(compmake_db))
    logger.info('Loaded %d jobs' % len(jobs))
    N = 10
    if len(jobs) < N:
        logger.error('too few jobs (I expect at least %s)' % N)
        sys.exit(128)

    test_cases = []
    for job_id in jobs:
        tc = junit_test_case_from_compmake(compmake_db, job_id)
        test_cases.append(tc)

    ts = TestSuite("comptests_test_suite", test_cases)

    res = TestSuite.to_xml_string([ts])
    check_isinstance(res, six.text_type)
    return res

import six

# def flatten_ascii(s):
#     if s is None:
#         return None
#     # if six.PY2:
#     #     # noinspection PyCompatibility
#     #     s = unicode(s, encoding='utf8', errors='replace')
#     #     s = s.encode('ascii', errors='ignore')
#     return s

def junit_test_case_from_compmake(db, job_id):
    from junit_xml import TestCase
    cache = get_job_cache(job_id, db=db)
    if cache.state == Cache.DONE:  # and cache.done_iterations > 1:
        # elapsed_sec = cache.walltime_used
        elapsed_sec = cache.cputime_used
    else:
        elapsed_sec = None

    check_isinstance(cache.captured_stderr, (type(None), six.text_type))
    check_isinstance(cache.captured_stdout, (type(None), six.text_type))
    check_isinstance(cache.exception, (type(None), six.text_type))
    stderr = remove_escapes(cache.captured_stderr)
    stdout = remove_escapes(cache.captured_stdout)

    tc = TestCase(name=job_id, classname=None, elapsed_sec=elapsed_sec,
                  stdout=stdout, stderr=stderr)

    if cache.state == Cache.FAILED:
        message = cache.exception
        output = cache.exception + "\n" + cache.backtrace
        tc.add_failure_info(message, output)

    return tc

def remove_escapes(s):
    if s is None:
        return None
    import re
    escape = re.compile('\x1b\[..?m')
    return escape.sub("", s)

if __name__ == '__main__':
    comptest_to_junit_main()


class Person:
    """
    Базовый класс Person
    """
    print('-' * 80)
    # интерпретатор выполняет инструкции тела класса во время загрузки модуля
    print('[~] Вызвано в теле класса Person')

    # приватные переменные объекта класса (class object) Person. Два способа
    # реализации сокрытия атрибутов класса
    _PERSON_COUNTER = 0
    __CLASS_AUTHOR = 'Andrii'

    # публичная переменная объекта класса (свойство объекта класса)
    CLASS_VERSION = 'v1.0'

    def __init__(self, surname, first_name, age=None):
        """
        Конструктор класса Person (интерпретатор передает в качестве аргумента
        вызова конструктора объект экземпляра класса, по соглашению называется
        self)
        :param surname: фамилия
        :param first_name: имя
        :param age: возраст
        """
        print('[~] вызов конструктора класса Person')
        self._surname = surname
        self._first_name = first_name
        if age is not None:
            # вызов метода экземпялра класса
            self._age = self._check_age(age)
        else:
            self._age = 1

        # вызов метода объекта класса (метод может быть вызван в контесте
        # объекта экземпляра класса (instance object))
        self._person_create()

    # определение свойства при помощи декоратора property
    @property
    def surname(self):
        return self._surname

    @surname.setter
    def surname(self, surname):
        self._surname = surname

    @property
    def first_name(self):
        return self._first_name

    @first_name.setter
    def first_name(self, first_name):
        self._first_name = first_name

    @property
    def age(self):
        return self._age

    @age.setter
    def age(self, age):
        if age < 0:
            raise ValueError('[#] возраст не может быть отрицательным числом')
        self._age = age

    # приватные методы класса (принимают в качестве аргумента объект класса)
    @classmethod
    def _person_create(cls):
        cls._PERSON_COUNTER += 1

    @classmethod
    def _check_age(cls, age):
        try:
            age = int(age)
            if age <= 0:
                raise ValueError(
                    '[#] возраст не должен быть отрицательным числом'
                )
            return age
        except ValueError:
            raise ValueError('[#] возраст должен быть целым числом')

    # публичные методы класса
    @classmethod
    def show_person_count(cls):
        print(str.format(
            '[~] количество объектов класса {}: {}',
            cls.__name__, cls._PERSON_COUNTER
        ))

    @classmethod
    def show_person_author(cls):
        print(str.format(
            '[~] автор класса {}: {}', cls.__name__, cls.__CLASS_AUTHOR
        ))

    # переопределяем методы родительского класса builtins.object
    def __str__(self):
        return str.format(
            '{}\nИмя: {}\nФамилия: {}\nВозраст: {}\n{}',
            '-' * 40, self.surname, self.first_name, self.age, '-' * 40
        )

    # методы экземпляра класса (методы объекта класса Person)
    def show_person(self):
        # вызов метод self.__str__()
        print(self)

class User(Person):
    """
    Класс User наследует класс Person
    """
    pass

def main():
    print('-' * 80)
    # название класса
    print(str.format('[*] название экземпляра класса: {}', Person.__name__))

    # объект родительского класса (кастомные классы наследуются от объекта
    # класса object по умолчанию)
    print(str.format(
        '[*] базовый класс экземпляра класса Person: {}', Person.__base__
    ))

    # корневой объект класса для всех кастомных классов так же имеет имя
    print(str.format(
        '[*] название базового класса: {}',  Person.__base__.__name__
    ))

    # объект класса, как любой объект в python имеет пространство имен,
    # которое хранится в словаре self.__dict__ (такое поведение может
    # изменится в будущем)
    print(str.format(
        '[*] пространство имен класса Person: {}', Person.__dict__
    ))

    # наличие атрибута необходимо проверять при помощи встроенной ф-и hasatrr()
    print(str.format(
        '[*] класс Person имеет атрибут CLASS_VERSION: {}',
        hasattr(Person, 'CLASS_VERSION')
    ))

    # встроенная ф-я dir() возвращает список имен объекта: собственные и
    # системные аттрибуты
    print(str.format('[*] dir(Person): {}', dir(Person)))
    print(str.format(
        '[*] len(dir(Person) == len(Person.__dict__): {}',
        len(dir(Person)) == len(Person.__dict__)
    ))
    print(str.format('[*] len(dir(Person): {}', len(dir(Person))))
    print(str.format('[*] len(Person.__dict__): {}', len(Person.__dict__)))

    # вызов метода класса можно осуществить не создавая объект экземпляра
    # класса
    Person.show_person_count()

    # создание объектов класса Person
    p1 = Person('Sapel', 'Andrii', 30)
    p2 = Person('Sapel', 'Kri', 30)

    # доступ к методам класса можно осуществить через экзепляр этого класса
    p1.show_person_count()

    # вызов метода self.__str__()
    print(p1)

    print(str.format(
        '[*] объект p2 является экземпляром класса Person: {}',
        isinstance(p2, Person)
    ))
    print(str.format('[*] пространство имен объекта p1: {}', p1.__dict__))
    print('[*] атрибуты объекта p1:')
    [print(attribute_name, end=', ') for attribute_name in dir(p1)]
    print()

    print('-' * 80)
    # наследование
    print(str.format(
        '[*] класс Person родительский класс для User: {}',
        Person is User.__base__
    ))

    print(str.format('[*] кортеж базовый классов: {}', User.__bases__))

    print(str.format('[*] MRO класса User: {}', User.__mro__))

    print('-' * 80)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
import socket
import numpy as np

class Get_face:
    def __init__(self, host, port):

        self.num_face = 5
        self.num_time = 5
        self.milli = 2

        serversock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        serversock.bind((host,port)) #IPとPORTを指定してバインドします
        serversock.listen(10) #接続の待ち受けをします（キューの最大数を指定）
        print('connecting')

        self.clientsock, self.client_address = serversock.accept() #接続されればデータを格納
        print('connected')

    def check_face(self,face_tmp_list):
        ret = 0
        for t in range(self.num_face+self.num_time):
            if face_tmp_list[t]:
                if face_tmp_list[t].isdigit() == False:
                    ret = 0
                    return ret
                else:
                    ret = 1
        if ret == 1:
            face_int = map(int,face_tmp_list[0:self.num_face+self.num_time])
            for i in range(self.num_face+self.num_time):
                if face_int[i]<100 and face_int[i]>=0:
                    ret = 1
                else:
                    ret = 0
                    break
            return ret

    def check_time(self,face_tmp_list):
        ret = 0
        time_int = map(int,face_tmp_list[self.num_face:self.num_face+self.num_time])
        for i in range(self.num_time):
            if time_int[0]==0 and time_int[4]==0:
                ret = 0
                break
            else:
                ret = 1
        return ret

    def read_face(self):
        get_face = np.zeros(self.num_face)
        tmp_time = np.zeros(self.num_time)
        get_time = np.zeros(self.num_time-self.milli+1)
        rcvmsg = self.clientsock.recv(1024)
        face = rcvmsg.split(",")
        if self.check_face(face) == 1 and self.check_time(face) == 1:
            int_face = map(int,face[0:self.num_face+self.num_time])
            get_face = np.array(int_face[0:self.num_face])
            milli_num = int_face[-2]*10 + int_face[-1]
            get_time = np.hstack((np.array(int_face[self.num_face:self.num_face+self.num_time-self.milli]),np.array([milli_num])))

        return np.array(np.hstack((get_face,get_time)))

    def close():
        clientsock.close()

if __name__ == "__main__" :

    host = "192.168.146.128" #お使いのサーバーのホスト名を入れます
    port = 50000 #クライアントと同じPORTをしてあげます
    type_face = 5
    type_ir = 5

    get = Get_face(host,port)
    while True:
        print(get.read_face())


import json
import sys
import os

from FileParser import FileParser
from FolderParser import FolderParser

class LogParser:
    def __init__(self, propertyFile):
        self.propertyFile = propertyFile
        self.parseSpecificDate = False

    def GetSource(self):
        try:
            property_file = open(self.propertyFile, 'r')
            for line in property_file:
                if "SourcePath" in line:
                    source_pathway = line.split("=")[-1].strip()
                    if not os.path.exists(source_pathway):
                        raise Exception("File does not exist")
                    else:
                        return source_pathway
                else:
                    break
        except:
            raise Exception("Something went wrong with your file, or destination path")

    def GetDestination(self):
        try:
            property_file = open(self.propertyFile, 'r')
            for line in property_file:
                if "DestinationPath" in line:
                    return line.split("=")[-1].strip()
        except:
            raise Exception("Something went wrong with your file, or destination path")

    def CreateJsonFile(self, listOfWarnings):
        destination = self.GetDestination()

        if listOfWarnings:
            with open(destination + "AllResults" + '.json', 'w') as log_file:
                log_file.write('\n'.join(json.dumps(i) for i in listOfWarnings) + '\n')
                #json.dump(listOfWarnings, log_file, indent=4)

    def RunParser(self):
        command_line_arguments = sys.argv[1:]
        source_path = self.GetSource()
        destination_path = self.GetDestination()

        if (source_path or destination_path) is None:
            raise Exception("Something went wrong with your file, source path, or destination path")

        elif not command_line_arguments:
            parser = FolderParser(source_path, destination_path, 0, 0, "")
            list_of_warnings = parser.ParseFolder()
            self.CreateJsonFile(list_of_warnings)

        elif command_line_arguments[0] == "help":
            print("You must put an input parameter along with the necessary arguments. For example: LogParser.py 5 sql")
            print("InputParam=0, Input is given by user in terminal to indicate that a folder needs to be parsed")
            print("InputParam=1, Input is given by user in terminal to indicate that a specific log file in the "
                  "folder needs to be parsed")
            print("InputParam=2, Input is given by user in terminal to indicate that two specific log files in the "
                  "folder needs to be parsed")
            print("InputParam=3, Input is given by user in terminal to indicate that three specific log files in the "
                  "folder needs to be parsed")
            print("InputParam=4, Input is given by user in terminal to indicate a specific start date & end date")
            print("InputParam=5, Input is given by user in terminal to indicate a specific pattern")

        else:
            command_input = int(command_line_arguments[0])

            if len(command_line_arguments) < 1:
                raise Exception("Please enter an argument")

            elif command_input < 0 or command_input > 5:
                raise Exception("Please enter a valid input paramater")

            elif command_input == 0:
                parser = FolderParser(source_path, destination_path, 0, 0, "")
                list_of_warnings = parser.ParseFolder()
                self.CreateJsonFile(list_of_warnings)

            elif command_input == 1:
                # create command for multiple files....
                file_name = source_path + "/" + command_line_arguments[1].strip()
                parser = FileParser(file_name, destination_path, 0, 0, "")
                list_of_all_warnings = parser.ParseFile()
                self.CreateJsonFile(list_of_all_warnings)

            elif command_input == 2:
                first_file_name = source_path + "/" + command_line_arguments[1].strip()
                parser_one = FileParser(first_file_name, destination_path, 0, 0, "")
                list_one = parser_one.ParseFile()

                second_file_name = source_path + "/" + command_line_arguments[2].strip()
                parser_two = FileParser(second_file_name, destination_path, 0, 0, "")
                list_two = parser_two.ParseFile()

                self.CreateJsonFile(list_one + list_two)

            elif command_input == 3:
                first_file_name = source_path + "/" + command_line_arguments[1].strip()
                parser_one = FileParser(first_file_name, destination_path, 0, 0, "")
                list_one = parser_one.ParseFile()

                second_file_name = source_path + "/" + command_line_arguments[2].strip()
                parser_two = FileParser(second_file_name, destination_path, 0, 0, "")
                list_two = parser_two.ParseFile()

                third_file_name = source_path + "/" + command_line_arguments[3].strip()
                parser_three = FileParser(third_file_name, destination_path, 0, 0, "")
                list_three = parser_three.ParseFile()

                self.CreateJsonFile(list_one + list_two + list_three)

            elif command_input == 4:
                start_date = command_line_arguments[1]
                end_date = command_line_arguments[2]
                parser = FolderParser(source_path, destination_path, start_date, end_date, "")
                list_one = parser.ParseFolder()
                self.CreateJsonFile(list_one)

            elif command_input == 5:
                pattern = command_line_arguments[1]
                parser = FolderParser(source_path, destination_path, 0, 0, pattern)
                list_one = parser.ParseFolder()
                self.CreateJsonFile(list_one)

if __name__ == '__main__':
    LogParser = LogParser("/Users/shrirangbagdi/PycharmProjects/LogParser/LogParser.properties")
    LogParser.RunParser()


#units are in inches, 3 decimals are kept (as a default)

#File name of your airfoil .dat as resulting from optiFOIL
wing = 'manini v4.dat'

#All Dimensioning Values - REQUIRES A COMMENT
#When adding variables, be sure to update the lower segment as well. *division exists for design_set_parameters.py module
##Start
wing_chord              =5.5#Inches
wing_ar                 =8.0#Dimensionless
wing_alpha              =3.5#Degrees
wing_taper              =1.0#Ratio of Chord at tip to Chord at base
wing_taper_start        =0.6#Location where tapering begins as a ratio of the length of the outer wing panel
wing_dihedral           =0.0#Degrees *unimplemented
wing_interference       =1.0#amount of interference between wing panels in inches *unimplemented
wing_panel_ratio        =0.6#Ratio between outer panel and center panel

wing_rib_thickness      =0.125#Thickness of internal wing ribs
wing_rib_count          =12.0#Not implemented in solidworks model, implemented in mass analysis
wing_spar_thickness     =0.125#thickness of spar in inches
wing_spar_length        =20.0#spar length in inches

fuse_base_length        =8.75#overall length of fuselage in inches
fuse_base_width         =1.8#overall width of fuselage in inches
fuse_base_thickness     =0.25#thickness of the bottom panels in inches
fuse_height             =3.0#height of fuselage in inches
fuse_wall_thickness     =0.125#thickness of the fuselage wall in inches
fuse_front_xchamf       =1.0#X length of the taper in the front in inches
fuse_front_ychamf       =1.0#Y length of the taper in the front in inches
fuse_back_xchamf        =1.0#X length of the taper in the back in inches
fuse_back_ychamf        =1.0#Y length of the taper in the back in inches
fuse_wing_pos           =0.35#location of the wing mount as a ratio of the overall length from the front
fuse_wingmount_length   =1.75#length of the wing mount in inches *unimplemented
fuse_wingmount_height   =0.68#height of wing mount in inches *unimplemented

conn_length             =12.0#Length of the connector rod in inches
conn_side               =0.3937#Side length of the connector rod
conn_id                 =0.315#internal diameter of the connector rod

empe_hori_alpha		=0.0#Degrees *below* horizontal, !negative angles are not supported!
empe_hori_tc            =10.0#Last 2 digits of a NACA4 symmetric airfoil designation for the horizontal empennage
empe_hori_chord 	=3.0#Chord of the horizontal empennage in inches
empe_hori_taper		=1.0#Ratio of Chord at tip to Chord at base for the horizontal emp.
empe_hori_ar		=5.0#Dimensionless AR of horizontal empennage
empe_hori_taper_start	=0.6#Location where tapering begins as a ratio of the length

empe_vert_tc            =10.0#Last 2 digits of a NACA4 symmetric airfoil designation for the vertical empennage
empe_vert_chord 	=2.8#Chord of the vertical empennage in inches
empe_vert_taper		=1.0#Ratio of Chord at tip to Chord at base for the vertical emp.
empe_vert_ar		=2.143#Dimensionless AR of vertical empennage
empe_vert_taper_start	=0.5#Location where tapering begins as a ratio of the length
empe_vert_skew		=0.0#offset angle to account for propeller yaw moment (Not implemented)

case_length		=20.5#Length of the aircraft container in inches
case_thickness		=0.125#Thickness of the aircraft container in inches

elec_motor_diameter	=1.46#Approximate diameter of drive motor in inches
elec_motor_length	=1.46#Approximate length of drive motor in inches
elec_motor_axel_diameter=0.3125#drive axel diameter
elec_battery_length     =2.0#Approximate length of battery
elec_battery_width      =1.125#Approximate width of battery
elec_battery_height     =0.6#Approximate height of battery
elec_esc_length         =2.5#Approximate length of esc
elec_esc_width          =1.0#Approximate width of esc
elec_esc_height         =0.25#Approximate height of esc

payl_length		=5.0#Length of the payload in inches
payl_width		=1.5#Width of the payload in inches
payl_height		=1.5#Height of the payload in inches

ctrl_aileron_offset	=2.0#Distance from tip to start of aileron in inches
ctrl_aileron_length	=6.6#Spanwise length of aileron in inches
ctrl_aileron_width	=1.8#Chordwise width of aileron in inches

ctrl_elevator_length	=18.0#Spanwise length of elevator in inches
ctrl_elevator_width	=1.0#Chordwise width of elevator in inches

ctrl_rudder_offset	=0.1#Distance from horizontal tail to start of rudder in inches
ctrl_rudder_length	=5.8#Spanwise length of rudder in inches (not implemented)
ctrl_rudder_width	=1.0#Chordwise width of rudder in inches
##End

with open('vars.txt','w') as var:

    #Wing Vars
    var.write('"wing_chord" = %5.3f\n'               %wing_chord)
    var.write('"wing_ar" = %5.3f\n'                  %wing_ar  )
    var.write('"wing_alpha" = %5.3f\n'               %wing_alpha  )
    var.write('"wing_taper_ratio" = %5.3f\n'         %wing_taper)
    var.write('"wing_taper_start" = %5.3f\n'         %wing_taper_start  )
    var.write('"wing_dihedral" = %5.3f\n'            %wing_dihedral  )
    var.write('"wing_interference" = %5.3f\n'        %wing_interference  )
    var.write('"wing_panel_ratio" = %5.3f\n'         %wing_panel_ratio )
    var.write('"wing_rib_thickness" = %5.3f\n'       %wing_rib_thickness )
    var.write('"wing_rib_count" = %5.3f\n'           %wing_rib_count )
    var.write('"wing_spar_thickness" = %5.3f\n'      %wing_spar_thickness )
    var.write('"wing_spar_length" = %5.3f\n'         %wing_spar_length )

    #Body
    var.write('"fuse_base_length" = %5.3f\n'         %fuse_base_length )
    var.write('"fuse_base_width" = %5.3f\n'          %fuse_base_width  )
    var.write('"fuse_base_thickness" = %5.5f\n'      %fuse_base_thickness )
    var.write('"fuse_height" = %5.3f\n'              %fuse_height  )
    var.write('"fuse_wall_thickness" = %5.3f\n'      %fuse_wall_thickness )
    var.write('"fuse_front_xchamf" = %5.3f\n'        %fuse_front_xchamf  )
    var.write('"fuse_front_ychamf" = %5.3f\n'        %fuse_front_ychamf  )
    var.write('"fuse_back_xchamf" = %5.3f\n'         %fuse_back_xchamf )
    var.write('"fuse_back_ychamf" = %5.3f\n'         %fuse_back_ychamf  )
    var.write('"fuse_wing_pos" = %5.3f\n'            %fuse_wing_pos )
    var.write('"fuse_wingmount_length" = %5.3f\n'    %fuse_wingmount_length )
    var.write('"fuse_wingmount_height" = %5.3f\n'    %fuse_wingmount_height)

    #Connector
    var.write('"conn_length" = %5.3f\n'              %conn_length )
    var.write('"conn_side" = %5.3f\n'                %conn_side)
    var.write('"conn_id" = %5.4f\n'                  %conn_id)

    #Empenage
    var.write('"empe_hori_tc" = %5.3f\n'             %empe_hori_tc)
    var.write('"empe_hori_chord" = %5.3f\n'          %empe_hori_chord)
    var.write('"empe_hori_ar" = %5.3f\n'             %empe_hori_ar  )
    var.write('"empe_hori_alpha" = %5.3f\n'          %empe_hori_alpha )
    var.write('"empe_hori_taper_ratio" = %5.3f\n'    %empe_hori_taper)
    var.write('"empe_hori_taper_start" = %5.3f\n'    %empe_hori_taper_start  )

    var.write('"empe_vert_tc" = %5.3f\n'             %empe_vert_tc)
    var.write('"empe_vert_chord" = %5.3f\n'          %empe_vert_chord)
    var.write('"empe_vert_ar" = %5.3f\n'             %empe_vert_ar )
    var.write('"empe_vert_taper_ratio" = %5.3f\n'    %empe_vert_taper)
    var.write('"empe_vert_taper_start" = %5.3f\n'    %empe_vert_taper_start )
    var.write('"empe_vert_skew" = %5.3f\n'           %empe_vert_skew  )

    #Container
    var.write('"case_length" = %5.3f\n'              %case_length )
    var.write('"case_thickness" = %5.3f\n'           %case_thickness)

    #Electronics
    var.write('"elec_motor_diameter" = %5.3f\n'      %elec_motor_diameter )
    var.write('"elec_motor_length" = %5.3f\n'        %elec_motor_length )
    var.write('"elec_motor_axel_diameter" = %5.3f\n' %elec_motor_axel_diameter )
    var.write('"elec_battery_length" = %5.3f\n'      %elec_battery_length  )
    var.write('"elec_battery_width" = %5.3f\n'       %elec_battery_width  )
    var.write('"elec_battery_height" = %5.3f\n'      %elec_battery_height  )
    var.write('"elec_esc_length" = %5.3f\n'          %elec_esc_length  )
    var.write('"elec_esc_width" = %5.3f\n'           %elec_esc_width  )
    var.write('"elec_esc_height" = %5.3f\n'          %elec_esc_height  )

    #Payload
    var.write('"payl_length" = %5.3f\n'              %payl_length  )
    var.write('"payl_width" = %5.3f\n'               %payl_width  )
    var.write('"payl_height" = %5.3f\n'              %payl_height  )

    #Control Surface
    var.write('"ctrl_aileron_offset" = %5.3f\n'      %ctrl_aileron_offset  )
    var.write('"ctrl_aileron_length" = %5.3f\n'      %ctrl_aileron_length  )
    var.write('"ctrl_aileron_width" = %5.3f\n'       %ctrl_aileron_width  )
    var.write('"ctrl_elevator_length" = %5.3f\n'     %ctrl_elevator_length  )
    var.write('"ctrl_elevator_width" = %5.3f\n'      %ctrl_elevator_width  )
    var.write('"ctrl_rudder_offset" = %5.3f\n'       %ctrl_rudder_offset  )
    var.write('"ctrl_rudder_length" = %5.3f\n'       %ctrl_rudder_length  )
    var.write('"ctrl_rudder_width" = %5.3f\n'        %ctrl_rudder_width  )

    #ASSEMBLY VARIABLES

with open('wing.txt','w') as file_:
    with open(wing,'r') as raw:
        for ind,line in enumerate(raw):
            if ind > 0:
                templine = line.split('    ')
                templine = [str(float(x)*wing_chord) for x in templine]
                templine.append('0.0')
                file_.writelines(' '.join(templine)+'\n')
with open('wing_taper.txt','w') as file_:
    with open(wing,'r') as raw:
        for ind,line in enumerate(raw):
            if ind > 0:
                templine = line.split('    ')
                templine = [float(x)*wing_chord*wing_taper for x in templine]
                templine[0] += wing_chord*(1-wing_taper)
                templine = [str(x) for x in templine]
                templine.append('0.0')
                file_.writelines(' '.join(templine)+'\n')


import ws_dataset
import datasets
import imgaug
import tensorflow as tf
import random
import detection
from tensorflow.compat.v1.keras.backend import set_session
import sklearn.model_selection
import numpy as np

"""
Set session to avoid bug
"""
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True  # dynamically grow the memory used on the GPU
config.log_device_placement = True  # to log device placement (on which device the operation ran)
sess = tf.compat.v1.Session(config=config)
set_session(sess)

"""
datasets
"""

detector = detection.Detector()

# augmenter = imgaug.augmenters.Sequential([
#     imgaug.augmenters.Affine(
#         scale=(1.0, 1.2),
#         rotate=(-5, 5)
#     ),
#     imgaug.augmenters.GaussianBlur(sigma=(0, 0.5)),
#     imgaug.augmenters.Multiply((0.8, 1.2), per_channel=0.2)
# ])

"""
training
"""
# image, character_bboxes, new_words, confidence_mask, confidences = data

def single_image_loss(pre_loss, loss_label):
    batch_size = pre_loss.shape[0]
    sum_loss = tf.math.reduce_mean(pre_loss.reshape(-1))*0
    pre_loss = pre_loss.reshape(batch_size, -1)
    loss_label = loss_label.reshape(batch_size, -1)

    for i in range(batch_size):
        average_number = 0
        positive_pixel = len(pre_loss[i][(loss_label[i] >= 0.1)])
        average_number += positive_pixel
        if positive_pixel != 0:
            posi_loss = tf.math.reduce_mean(pre_loss[i][(loss_label[i] >= 0.1)])
            sum_loss += posi_loss

            if len(pre_loss[i][(loss_label[i] < 0.1)]) < 3*positive_pixel:
                nega_loss = tf.math.reduce_mean(pre_loss[i][(loss_label[i] < 0.1)])
                average_number += len(pre_loss[i][(loss_label[i] < 0.1)])
            else:
                nega_loss = tf.math.reduce_mean(tf.math.top_k(pre_loss[i][(loss_label[i] < 0.1)], 3*positive_pixel)[0])
                average_number += 3*positive_pixel
            sum_loss += nega_loss
        else:
            nega_loss = tf.math.reduce_mean(tf.math.top_k(pre_loss[i], 500)[0])
            average_number += 500
            sum_loss += nega_loss

    return sum_loss

def maploss(gh_label, gah_label, p_gh, p_gah, mask):
    loss1 = tf.losses.mse(gh_label, p_gh)
    loss2 = tf.losses.mse(gah_label, p_gah)
    loss_g = tf.multiply(loss1, mask)
    loss_a = tf.multiply(loss2, mask)

    char_loss = single_image_loss(loss_g, gh_label)
    affi_loss = single_image_loss(loss_a, gah_label)
    return char_loss/loss_g.shape[0] + affi_loss/loss_a.shape[0]

if __name__ == '__main__':
    import math
    import os

    # synth_dataset = datasets.prepare_dataset('../icdar2013/images')
    # synth_train, synth_validation = sklearn.model_selection.train_test_split(
    #     synth_dataset, train_size=0.8, random_state=42
    # )
    # synth_train_length = len(synth_train)
    # generator_kwargs = {'width': 640, 'height': 640}
    #
    # training_synth_generator = datasets.get_detector_image_generator(
    #     labels=synth_train,
    #     augmenter=None,
    #     **generator_kwargs
    # )
    # validation_synth_generator = datasets.get_detector_image_generator(
    #     labels=synth_validation,
    #     **generator_kwargs
    # )

    real_dataset = ws_dataset.prepare_dataset('../ICDAR2017/training_images/')
    real_train, real_validation = sklearn.model_selection.train_test_split(
        real_dataset, train_size=0.8, random_state=42
    )
    augmenter = imgaug.augmenters.Sequential([
        imgaug.augmenters.Affine(
            scale=(1.0, 1.2),
            rotate=(-5, 5)
        ),
        imgaug.augmenters.GaussianBlur(sigma=(0, 0.5)),
        imgaug.augmenters.Multiply((0.8, 1.2), per_channel=0.2)
    ])

    training_image_generator = ws_dataset.real_data_generator(real_train,
                                                             width=640,
                                                             height=640,
                                                             augmenter=augmenter,
                                                             )
    validation_image_generator = ws_dataset.real_data_generator(real_validation,
                                                               width=640,
                                                               height=640,
                                                               augmenter=None)

    # import tools
    # import cv2
    # canvas = tools.drawBoxes(image=image, boxes=lines, boxes_format='lines')
    # cv2.imwrite('test.png', canvas)

    batch_size = 1
    training_generator, validation_generator = [
        detector.get_batch_generator(
            image_generator=image_generator, batch_size=batch_size
        ) for image_generator in
        [training_image_generator, validation_image_generator]
    ]
    detector.model.fit_generator(
        generator=training_generator,
        steps_per_epoch=math.ceil(len(real_train) / batch_size),
        epochs=10,
        workers=0,
        callbacks=[
            tf.keras.callbacks.EarlyStopping(restore_best_weights=True, patience=5),
            tf.keras.callbacks.CSVLogger(os.path.join('./log', 'detector_icdar2013.csv')),
            tf.keras.callbacks.ModelCheckpoint(filepath=os.path.join('./model', 'detector_icdar2013.h5'))
        ],
        validation_data=validation_generator,
        validation_steps=math.ceil(len(real_validation) / batch_size)
    )


from pprint import pprint
from uuid import uuid4

from app.domain.context import Context
from app.domain.entity import Entity
from app.domain.event import Event
from app.domain.events import Events
from app.domain.payload.tracker_payload import TrackerPayload
from app.domain.record.flow_record import FlowRecord
from app.domain.named_entity import NamedEntity
from app.domain.profile import Profile
from app.domain.rule import Rule
from app.domain.session import Session
from app.domain.type import Type
from app.process_engine.rules_engine import RulesEngine
import asyncio

if __name__ == "__main__":
    async def main():
        source = NamedEntity(id="mobile-app", name="mobile-app")

        # Create profile, rule needs it
        p = Profile.new()
        await p.storage().save()
        profile = await p.storage().load()
        assert p.id == profile.id
        assert isinstance(profile, Profile)

        flow_record = FlowRecord(id="1", name="flow-1")
        await flow_record.storage().save()

        rule = Rule(
            id="string",
            source=source,
            name="my-rule",
            event=Type(type="xxx1"),
            flow=flow_record
        )
        await rule.storage().save()

        saved_rule = await rule.storage().load()
        assert saved_rule.id == rule.id
        x = await RulesEngine.load_rules("xxx2")
        pprint(x)

        payload = {
            "id": str(uuid4()),
            "event_server": Context(),
            "source": source.dict(),
            "profile": p.dict(),
            "context": {},
            "session": {"id": "0e6121a5-3ea0-45ed-a9ad-552e1765167f",
                        "event_server": {
                            "page": {"url": "http://localhost:8002/tracker/", "path": "/tracker/", "hash": "",
                                     "title": "My title",
                                     "referer": {"host": None, "query": None}, "history": {"length": 2}}},
                        },
            "properties": {"a": "tak"}, "type": "xxx1", "user": {"id": "user-id-2"}
        }

        event = Event(**payload)
        events = Events()
        events.append(event)
        events.append(event)
        session = Session(id="session-id")
        profile = Profile(id="profile-id")

        rules_engine = RulesEngine(
            session,
            profile,
            events)

        result = await rules_engine.execute(source.id)
        # print(stats.to_json())
        print(result)

        await rule.storage().delete()

    asyncio.run(main())


# -*- coding: utf-8 -*-
#
# Copyright (C) 2020 CERN.
# Copyright (C) 2020 Northwestern University.
#
# Invenio-Drafts-Resources is free software; you can redistribute it and/or
# modify it under the terms of the MIT License; see LICENSE file for more
# details.

"""RecordDraft Service API."""

from invenio_db import db
from invenio_records_resources.services import RecordService
from sqlalchemy.orm.exc import NoResultFound

from .config import RecordDraftServiceConfig

class RecordDraftService(RecordService):
    """Draft Service interface.

    This service provides an interface to business logic for
    published AND draft records. When creating a custom service
    for a specialized record (e.g. authors), consider if you need
    draft functionality or not. If you do, inherit from this service;
    otherwise, inherit from the RecordService directly.

    This service includes versioning.
    """

    default_config = RecordDraftServiceConfig

    # Draft attrs
    @property
    def draft_cls(self):
        """Factory for creating a record class."""
        return self.config.draft_cls

    # High-level API
    # Inherits record read, create, delete and update
    def search(self, identity, params=None, links_config=None,
               es_preference=None, **kwargs):
        """Search for records matching the querystring."""
        params = params or {}
        params.update(kwargs)

        status = params.pop("status", "published")
        record_cls = self.draft_cls if status == "draft" else self.record_cls

        return super().search(
            identity,
            params=params,
            links_config=links_config,
            es_preference=es_preference,
            record_cls=record_cls,
            # we don't pass kwargs, because they have already been merged.
        )

    def read_draft(self, id_, identity, links_config=None):
        """Retrieve a draft."""
        # Resolve and require permission
        # TODO must handle delete records and tombstone pages
        # TODO: Can we make in a similar fashion than "create"?
        draft = self.draft_cls.pid.resolve(id_, registered_only=False)
        self.require_permission(identity, "read_draft", record=draft)

        # Run components
        for component in self.components:
            if hasattr(component, 'read_draft'):
                component.read_draft(identity, draft=draft)

        return self.result_item(
            self, identity, draft, links_config=links_config)

    def update_draft(self, id_, identity, data, links_config=None,
                     revision_id=None):
        """Replace a draft."""
        draft = self.draft_cls.pid.resolve(id_, registered_only=False)

        self.check_revision_id(draft, revision_id)

        # Permissions
        self.require_permission(identity, "update_draft", record=draft)

        data, errors = self.schema.load(
            identity,
            data,
            pid=draft.pid,
            record=draft,
            # Saving a draft only saves valid metadata and reports
            # (doesn't raise) errors
            raise_errors=False
        )

        # Run components
        for component in self.components:
            if hasattr(component, 'update_draft'):
                component.update_draft(
                    identity, record=draft, data=data)

        # TODO: remove next two lines. (See also record-resources)
        draft.update(data)
        draft.clear_none()
        draft.commit()
        db.session.commit()
        self.indexer.index(draft)

        return self.result_item(
            self,
            identity,
            draft,
            links_config=links_config,
            errors=errors
        )

    def create(self, identity, data, links_config=None):
        """Create a draft for a new record.

        It does NOT eagerly create the associated record.
        """
        return self._create(
           self.draft_cls,
           identity,
           data,
           links_config=links_config,
           raise_errors=False
        )

    def edit(self, id_, identity, links_config=None):
        """Create a new revision or a draft for an existing record.

        Note: Because of the workflow, this method does not accept data.
        :param id_: record PID value.
        """
        self.require_permission(identity, "create")
        # Draft exists - return it
        try:
            draft = self.draft_cls.pid.resolve(id_, registered_only=False)
            return self.result_item(
                self, identity, draft, links_config=links_config)
        except NoResultFound:
            pass

        # Draft does not exists - create a new draft or get a soft deleted
        # draft.
        record = self.record_cls.pid.resolve(id_)
        try:
            # We soft-delete a draft once it has been published, in order to
            # keep the version_id counter around for optimistic concurrency
            # control (both for ES indexing and for REST API clients)
            draft = self.draft_cls.get_record(record.id, with_deleted=True)
            if draft.is_deleted:
                draft.undelete()
                draft.update(**record)
                draft.pid = record.pid
                draft.conceptpid = record.conceptpid
                draft.fork_version_id = record.revision_id
        except NoResultFound:
            # If a draft was ever force deleted, then we will create the draft.
            # This is a very exceptional case as normally, when we edit a
            # record then the soft-deleted draft exists and we are in above
            # case.
            draft = self.draft_cls.create(
                record, id_=record.id, fork_version_id=record.revision_id,
                pid=record.pid, conceptpid=record.conceptpid
            )

        # Run components
        for component in self.components:
            if hasattr(component, 'edit'):
                component.edit(identity, draft=draft)

        draft.commit()
        db.session.commit()
        self.indexer.index(draft)

        return self.result_item(
            self, identity, draft, links_config=links_config)

    def publish(self, id_, identity, links_config=None):
        """Publish a draft."""
        self.require_permission(identity, "publish")

        # Get draft
        draft = self.draft_cls.pid.resolve(id_, registered_only=False)

        data, _ = self.schema.load(
            identity,
            data=draft.dumps(),
            pid=draft.pid,
            record=draft,
            raise_errors=True  # this is the default, but might as well be
                               # explicit
        )

        record = self.record_cls.create_or_update_from(draft)

        # Run components
        for component in self.components:
            if hasattr(component, 'publish'):
                component.publish(draft=draft, record=record)

        # Register if needed
        if not record.is_published:
            record.register()

        record.commit()
        db.session.commit()

        draft.delete()
        db.session.commit()  # Persist DB
        self.indexer.delete(draft)
        self.indexer.index(record)

        return self.result_item(
            self, identity, record, links_config=links_config)

    def new_version(self, id_, identity, links_config=None):
        """Create a new version of a record."""
        self.require_permission(identity, "create")

        # Get record
        record = self.record_cls.pid.resolve(id_)

        # Create new record
        draft = self.draft_cls.create(
            {"metadata": record.metadata},  # FIXME: This doesnt look good
            conceptpid=record.conceptpid,
            fork_version_id=record.revision_id
        )

        # Run components
        for component in self.components:
            if hasattr(component, 'new_version'):
                component.new_version(identity, draft=draft, record=record)

        draft.commit()
        db.session.commit()
        self.indexer.index(draft)

        return self.result_item(
            self, identity, draft, links_config=links_config)

    def delete_draft(self, id_, identity, revision_id=None):
        """Delete a record from database and search indexes."""
        draft = self.draft_cls.pid.resolve(id_, registered_only=False)

        self.check_revision_id(draft, revision_id)

        # Permissions
        self.require_permission(identity, "delete_draft", record=draft)

        # Run components
        for component in self.components:
            if hasattr(component, 'delete_draft'):
                component.delete(identity, record=draft)

        try:
            record = self.record_cls.pid.resolve(id_, registered_only=False)
        except NoResultFound:  # FIXME: Should be at resolver level?
            record = None
        # If it is tight to a record soft remove, else wipe fully
        force = True if record else False
        draft.delete(force=force)
        db.session.commit()
        self.indexer.delete(draft)

        return True


## Code for computing the Legendre symbol (a/p), read "a on p," for any integer a and prime p.

import math
sqrt = math.sqrt

def sieve(n):
    '''wheel sieve with modulo 30.'''
    if not (type(n) == type(int())):
        raise TypeError
    if n < 0:
        raise ValueError
    if n < 2:
        return []
    prime = [True for x in range(2, n+1)]
    primes = []
    i = 0
    s = int(sqrt(n)-1)
    while i < n-1:
        if prime[i] == True:
            primes.append(i+2)
            if i < s:
                index = (i+2)**2 - 2
                while index < n-1:
                    prime[index] = False
                    index = index + i + 2
        if i > 4:
            if i > 28:
                if (i + 2)%30 == 1:
                    i = i + 6
                elif (i + 2)%30 == 7:
                    i = i + 4
                elif (i + 2)%30 == 11:
                    i = i + 2
                elif (i + 2)%30 == 13:
                    i = i + 4
                elif (i + 2)%30 == 17:
                    i = i + 2
                elif (i + 2)%30 == 19:
                    i = i + 4
                elif (i + 2)%30 == 23:
                    i = i + 6
                elif (i + 2)%30 == 29:
                    i = i + 2
            else:
                if (i + 2)%6 == 1:
                    i = i + 4
                elif (i + 2)%6 == 5:
                    i = i + 2
        else:
            i = i + 1
    return primes

def prime_test(n):
    '''A naïve deterministic primality test.'''
    if not (type(n) == type(int())):
        raise TypeError
    if not (n > 1):
        raise ValueError
    for prime in sieve(sqrt(n)):
        if n%prime == 0:
            return False
    return True

def Legendre(a, p):
    '''computes the Legendre symbol (a/p).'''
    if not ((type(a) == type(int())) and (type(p) == type(int()))):
        raise TypeError
    if not (p > 2):
        raise ValueError
    if not prime_test(p):
        raise ValueError
    return (a**((p-1)//2))%p


# _*_ coding: utf-8 _*_

"""
添加变更记录
"""
from src.head import *

class KaiTingGongGaoPlugin03:

    def __init__(self):
        pass

    def process(self, data_instance):
        """
        接口函数
        :param data_instance:
        :return:
        """
        if data_instance.dict["is_new"] and "related_companies" in data_instance.dict.keys():
            change_records = []
            related_companies = data_instance.dict["related_companies"]
            for comp in related_companies:
                self.__build_change_records(change_records,comp, data_instance)
            if len(change_records) > 0:
                now_time = datetime.datetime.utcnow()
                for i in change_records:
                    event_date = None
                    if int(i["date"].strftime("%Y%m%d")) <= int(now_time.strftime("%Y%m%d")):
                        event_date = i["date"]
                    else:
                        event_date = now_time
                    i["event_date"] = event_date
                MongoDBDAO.insert_many("CG", "changeitems", change_records)

    def __build_change_records(self, change_records, comp, data_instance):
        """
        构造变更记录数据
        :param change_records:
        :param comp:
        :param data_instance:
        :return:
        """
        if comp.get("item"):
            hearing_date = CommonUtils.MIN_DATE
            try:
                hearing_date = datetime.datetime.strptime(
                    CommonUtils.format_date(data_instance.dict["hearing_date"], "%Y-%m-%d"),
                    "%Y-%m-%d"
                )
            except Exception as e:
                logging.exception("Exception when parsing hearing date..")
                try:
                    hearing_date = datetime.datetime.strptime(
                        CommonUtils.format_date(data_instance.dict["release_time"][0:10], "%Y-%m-%d"),
                        "%Y-%m-%d"
                    )
                except Exception as e:
                    logging.exception("Exception when parsing release date substring..")
                    try:
                        hearing_date = datetime.datetime.strptime(
                            CommonUtils.format_date(data_instance.dict["release_time"], "%Y-%m-%d"),
                            "%Y-%m-%d"
                        )
                    except Exception as e:
                        logging.exception("Exception when parsing release date..")

            change_doc = {
                "status": 0,
                "ename": comp["name"],
                "eid": comp["item"],
                "ref_id": data_instance.dict["_id"],
                "items": [{
                    "field": "kaitinggongao",
                    "before": "",
                    "after": json.dumps(data_instance.dict, cls=CustomerJsonEncoder),
                    "desc": "开庭公告"
                }],
                "created_time": datetime.datetime.utcnow(),
                "last_updated_time": datetime.datetime.utcnow(),
                "info_type": 20,
                "date": hearing_date,
                "ops_flag": 0
            }
            change_records.append(change_doc)

    def __call__(self):
        pass