from casatools import logsink
setisimlog=logsink('setisim.log')
setisimlog.setlogfile='setisim.log'
setisimlog.setglobal(True)

from collections import defaultdict
from casatasks import tclean, split, listobs, plotants, imstat, clearcal, visstat, flagdata, flagmanager
from casatasks import uvcontsub, flagdata, gencal, plotweather, setjy, gaincal, bandpass, applycal, fluxscale, importgmrt
from casaviewer import imview
from casaplotms import plotms
from pathlib import Path
from astropy.wcs import WCS
from astropy.time import Time, TimeDelta
import matplotlib.pyplot as plt
import numpy as np
import io
import base64
from os import curdir, path, makedirs, remove, getcwd
import glob
from shutil import rmtree
from matplotlib import rcParams, rc
import os
from casatools import ms, msmetadata, table
from pyvirtualdisplay import Display

# casalog.filter('DEBUG1')
msmd = msmetadata()

from casatools import image as IA
from astropy.wcs import WCS
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.colors as colors
from mpl_toolkits.axes_grid1.anchored_artists import AnchoredSizeBar

def __clear_tmp(wd,cimagename):
    filelist = glob.glob(path.join(wd, cimagename+'*'))
    for f in filelist:
        try:
            rmtree(f)
        except OSError:
            remove(f)
def save_fig(plt, fig, kind='base64', output='output.jpg'):
    if kind == 'base64':
        buf = io.BytesIO()
        fig.savefig(buf, format='png', bbox_inches='tight',
                    transparent=True, pad_inches=0)
        buf.seek(0)
        string = base64.b64encode(buf.read())
        plt.close()
        return string
    elif kind == 'plot':
        plt.show()
        return 'plotted'
    else :
        if not path.exists('output'):
            makedirs('output')
        newPath = 'output/'+output
        opt = newPath
        if path.exists(newPath):
            numb = 1
            while path.exists(newPath):
                newPath = "{0}_{2}{1}".format(
                    *path.splitext(opt) + (numb,))
                try :
                    if path.exists(newPath):
                        numb += 1
                except:
                    pass
        fig.savefig(newPath, format=kind, bbox_inches='tight',
                    pad_inches=0)
        print("saved {}".format(newPath))
        plt.close()
        return newPath

def fetch_sources(vis):
    """
    fetch calibrator and target sources from the measurement set and return dictionary
    """
    pmcalf = '0'
    pmcals = '1'

    bpcalf = '0'
    bpcals = '3'

    fdcalf = '0'
    fdcals = '1'

    targets = '2'
    targetf = '1'

    return {'pmcalf':pmcalf, 'pmcals':pmcals,
            'bpcalf':bpcalf, 'bpcals':bpcals,
           'fdcalf':fdcalf, 'fdcals':fdcals,
           'targetf':targetf, 'targets':targets
           }

def genpng(cimg, chno=0, out='output.jpg', norm_max=None, **kwargs):
    ia = IA()
    ia.open(cimg)
    pix = ia.getchunk()[:,:,0,chno]
    csys = ia.coordsys()
    ia.close()

    rad_to_deg =  180/np.pi
    w = WCS(naxis=2)
    w.wcs.crpix = csys.referencepixel()['numeric'][0:2]
    w.wcs.cdelt = csys.increment()['numeric'][0:2]*rad_to_deg
    w.wcs.crval = csys.referencevalue()['numeric'][0:2]*rad_to_deg
    w.wcs.ctype = ['RA---SIN', 'DEC--SIN']
    fig = plt.figure(figsize=(20, 10))
    ax = plt.subplot(1, 1, 1, projection=w)
#     p1 = int(pix.shape[0]*0.25)
#     p2 = int(pix.shape[0]*0.75)

#     im = ax.imshow(pix[p1:p2,p1:p2].transpose(), origin='lower',  cmap=plt.cm.viridis)
    if norm_max:
        try:
            norm_max=float(norm_max)
            im = ax.imshow(pix.transpose(), origin='lower',
                           cmap=plt.cm.gist_heat, interpolation="lanczos",
                           norm=colors.CenteredNorm(norm_max/2.0)
            )
        except:
            im = ax.imshow(pix.transpose(), origin='lower',
                           cmap=plt.cm.gist_heat, interpolation="none",
                           norm=colors.CenteredNorm(0.0))

    else:
        im = ax.imshow(pix.transpose(), origin='lower',
                       cmap=plt.cm.gist_heat, interpolation="none",
                    #    norm=colors.CenteredNorm(0.0)
                       )

    plt.colorbar(im, ax=ax)
    ax.set_xlabel('Right Ascension')
    ax.set_ylabel('Declination')
    ax.set_title(str(out))
#     ax.add_size_bar(1/3600./f_hst[0].header["cdelt2"],
#                      r"$1^{\prime\prime}$", loc=3,
#                      borderpad=0.2)
    scalebar = AnchoredSizeBar(ax.transData,
                           5, r"$5^{\prime\prime}$", 'lower right',
                           pad=0.1,
                           color='white',
                           frameon=False,
                           size_vertical=1, borderpad=0.5, sep=5,
#                            fontproperties=fontprops
                   )
    ax.add_artist(scalebar)
    kind = kwargs['kind'] or 'jpg'
    save_fig(plt, fig, kind, output=out+f'{chno}.jpg')

def bandpass_fc(csource, suffix='_init',ref_ant='C00', plotfolder='bp_plots/',fspw = '0:101~1900',
                outvis='', **kwargs):
    cwd = getcwd()

    params={'reset':False,
            'quackinterval':10.0,
            'quackmode':'beg', 'bad_antenna':'','onvis':None,
                   }
#     params = defaultdict(lambda: -1, def_params)
    params.update(kwargs)
    if params['reset']:
        flagdata(vis=csource,mode='unflag', field='', spw='', antenna='', timerange='')
        clearcal(vis=csource)

    #### initialize ###########################
    s=fetch_sources(csource)
    csource_stem = Path(csource).stem

    ########## bandpass flagging ##############
    flagdata(vis=csource, mode='clip', clipzeros=True, flagbackup=False, action='apply') # clip zero amplitude
    flagdata(vis=csource, mode='shadow', tolerance=0.0, flagbackup=False, action='apply') # shadowing

    print(f' tfcrop on scan : {s["bpcals"]}')
    flagdata(vis=csource, mode='tfcrop', spw='0', scan=s["bpcals"], datacolumn='data', action='apply',
             flagbackup=True)
    print(f' tfcrop on scan : {s["pmcals"]}')
    flagdata(vis=csource, mode='tfcrop', spw='0', scan=s["pmcals"], datacolumn='data', action='apply',
             flagbackup=True)
    if params['bad_antenna']:
        flagdata(vis=csource, antenna=params['bad_antenna'],action='apply')

    ###### initial bandpass ##########
    csource_flagged = outvis or csource_stem+f'_scan{s["bpcals"]}_flagged.MS'
    bptab={}
    bptab['Ph_init'], bptab['BP_init']='bp_ph.initPh', 'bp_BP.initBP'

    os.system(f'rm -rf {bptab["Ph_init"]} {bptab["BP_init"]}')
    gaincal(vis=csource, caltable=bptab['Ph_init'],
            selectdata=True, scan=s['bpcals'],
            solint=' int ', spw=fspw, refant=ref_ant, minblperant=3,
            minsnr=3.0, calmode='p')
    bandpass(vis=csource, caltable=bptab['BP_init'],
            field=s['bpcalf'],
            solint=' inf ', combine='scan', refant=ref_ant,
            minblperant=3, minsnr=10.0, gaintable=bptab['Ph_init'],
    #          selectdata=True, scan=bpcals,
            interp='nearest', solnorm=False)

    if params['onvis']:
        applycal(vis=params['onvis'], field='', calwt=False, gaintable=bptab['BP_init'])
        print(f'splitting ms....{csource_flagged}')
        split(params['onvis'],csource_flagged)
    else:
        applycal(vis=csource, gaintable=bptab['BP_init'], calwt=False, scan=s['bpcals'])
        applycal(vis=csource, gaintable=bptab['BP_init'], calwt=False, scan=s['pmcals'])
        applycal(vis=csource, gaintable=bptab['BP_init'], calwt=False, scan=s['targets'])

        print(f'splitting ms....{csource_flagged}')
        split(csource,csource_flagged)

def rflag_pp(csource, scan, datacolumn='corrected'):
    csource_stem = Path(csource).stem
    outvis = csource_stem + 'rf.MS'
    flagdata(vis=csource, mode='rflag', spw='0', datacolumn=datacolumn, scan=scan,
            freqdevscale=2.5, timedevscale=3.5, extendflags=True, action='apply',
    #          display='both',
            timeavg=True , flagbackup=True
            )
    flagdata(csource, mode='extend', growfreq=80.0, growtime=60.0, action='apply', extendpols=True)
    try :
        print('...splitting corrected column')
        split(vis=csource, outputvis=outvis, datacolumn='corrected', keepflags=False)
    except:
        print('...splitting data column')
        split(vis=csource, outputvis=outvis, datacolumn='data', keepflags=False)

def dical0(csource, suffix='_init', ref_ant='C00', gspw='0:201~1800', field='0',):
    #### initialize ###########################
    s=fetch_sources(csource)
    csource_stem = Path(csource).stem

    gtab={}
    gtab['delay']=csource_stem+'.K'+suffix
    gtab['bph']=csource_stem+'.P.G'+suffix
    gtab['bp']=csource_stem+'.B'+suffix
    gtab['intph']=csource_stem+'.IP.G'+suffix
    gtab['scnph']=csource_stem+'.SP.G'+suffix
    gtab['amp']=csource_stem+'.A.G'+suffix

    os.system(f'rm -rf {gtab["delay"]}')
    gaincal(vis=csource, caltable=gtab['delay'],
            field=field,
            selectdata=True, scan=s['pmcals'],
            solint='inf', refant=ref_ant, gaintype='K')

    os.system(f'rm -rf {gtab["bph"]}')
    gaincal(vis=csource, caltable=gtab['bph'],
            field=field,
            spw=gspw, refant=ref_ant,
            calmode='p', solint='int',
            minsnr=3.0, gaintype='G',
            gaintable=[gtab['delay']])

    os.system(f'rm -rf {gtab["bp"]}')
    bandpass(vis=csource,
             caltable=gtab["bp"],
             spw ='0',
             field=field,
             solint='inf',
             refant=ref_ant,
             solnorm = True,bandtype='B', gaintable=[gtab['bph'], gtab['delay']],
        minsnr=3.0, fillgaps=8, parang = True,
             interp=['nearest,nearestflag','nearest,nearestflag'])

    # to fix Amp vs time
    os.system(f'rm -rf {gtab["intph"]}')
    gaincal(vis=csource,
            caltable=gtab["intph"],
            field=field,
            spw='0',
            refant=ref_ant,
            calmode='p',
            solint='int',
            minsnr=3.0,
            gaintable=[gtab['bph'], gtab['delay']]
           )

    # no enough for interpolating the phase solutions in time
    os.system(f'rm -rf {gtab["scnph"]}')
    gaincal(vis=csource,
            caltable=gtab['scnph'],
            field=field,
            spw='0',
            refant=ref_ant,
            calmode='p',
            solint='inf',
            combine='',
            minsnr=3.0,
            gaintable=[gtab['bph'], gtab['delay']]
           )

    # scaling amplitude gains skipping fluxscale as our flux-calibrator is also our
    # bandpass and gain calibrator, the absolute flux scale will already be ok
    os.system(f'rm -rf {gtab["amp"]}')
    gaincal(vis=csource,
            caltable=gtab['amp'],
            field=field,
            spw='0', selectdata=True, scan=s['pmcals'],
            solint='inf', combine='', refant=ref_ant,
            calmode='ap', minsnr=3.0,
            gaintable=[gtab['bph'], gtab['delay'], gtab['intph']])
    applycal(
        vis=csource,
        field=s['bpcalf'],
        gaintable=[gtab['delay'],gtab['bp'],gtab['scnph'],gtab['amp']],
        gainfield=[s['bpcalf'],s['bpcalf'],s['bpcalf'],s['bpcalf']],
        parang=True, calwt=False, applymode='calflagstrict', flagbackup=True)

    applycal(
        vis=csource,
        field=s['targetf'],
        gaintable=[gtab['delay'],gtab['bp'],gtab['scnph'],gtab['amp']],
        gainfield=[s['bpcalf'],s['bpcalf'],s['bpcalf'],s['bpcalf']],
        calwt=False, applymode='calflagstrict', flagbackup=True)
    return {'tables':gtab}

def dical(csource, suffix='_di', ref_ant='C00', gspw='0:201~1800', **kwargs):
    #### initialize ###########################
    params={'ph':False, 'bp' : False, 'ampvtime':False,
    'solint':'inf','field':'0', }
    params.update(kwargs)
    csource_stem=Path(csource).stem
    s=fetch_sources(csource)
    gtab={}
    gtab['delay']=csource_stem+'.K'+suffix
    gtab['bph']=csource_stem+'.P.G'+suffix
    gtab['bp']=csource_stem+'.B'+suffix
    gtab['intph']=csource_stem+'.IP.G'+suffix
    gtab['scnph']=csource_stem+'.SP.G'+suffix
    gtab['amp']=csource_stem+'.A.G'+suffix

    if params['ph']:
        os.system(f'rm -rf {gtab["intph"]}')
        gaincal(vis=csource,
            caltable=gtab['intph'],
            field=params['field'], spw='0', refant=ref_ant, calmode='p', solint=params['solint'],
            combine='',
            minsnr=3.0)
        applycal(vis=csource, field=params['field'], gaintable=[gtab['intph']], interp='linear')
        outvis=Path(csource).stem+suffix+'.MS'
        print(f'phase calibration of field:{params["field"]} on {csource_stem}')
        print(f'splitting: {outvis}, calibrated field="0"')
        os.system(f'rm -rf {outvis}')
        split(vis=csource, outputvis=outvis, datacolumn='corrected',
          field=params['field'], keepflags=False)

    if params['bp']:
        os.system(f'rm -rf {gtab["delay"]}')
        gaincal(vis=csource, caltable=gtab['delay'],
                field=params['field'],
                selectdata=True, scan=s['pmcals'],
                solint='inf', refant=ref_ant, gaintype='K')

        os.system(f'rm -rf {gtab["bph"]}')
        gaincal(vis=csource, caltable=gtab['bph'],
                field=params['field'],
                spw=gspw, refant=ref_ant,
                calmode='p', solint='int',
                minsnr=3.0, gaintype='G',
                gaintable=[gtab['delay']])

        os.system(f'rm -rf {gtab["bp"]}')
        bandpass(vis=csource,
                caltable=gtab["bp"],
                spw ='0',
                field=params['field'],
                solint='inf',
                refant=ref_ant,
                solnorm = True,bandtype='B', gaintable=[gtab['bph'], gtab['delay']],
            minsnr=3.0, fillgaps=8, parang = True,
                interp=['nearest,nearestflag','nearest,nearestflag'])
    if params['ampvtime']:
        # to fix Amp vs time
        os.system(f'rm -rf {gtab["intph"]}')
        gaincal(vis=csource,
                caltable=gtab["intph"],
                field=field,
                spw='0',
                refant=ref_ant,
                calmode='p',
                solint='int',
                minsnr=3.0,
                gaintable=[gtab['bph'], gtab['delay']]
            )

        # no enough for interpolating the phase solutions in time
        os.system(f'rm -rf {gtab["scnph"]}')
        gaincal(vis=csource,
                caltable=gtab['scnph'],
                field=field,
                spw='0',
                refant=ref_ant,
                calmode='p',
                solint='inf',
                combine='',
                minsnr=3.0,
                gaintable=[gtab['bph'], gtab['delay']]
            )

        # scaling amplitude gains skipping fluxscale as our flux-calibrator is also our
        # bandpass and gain calibrator, the absolute flux scale will already be ok
        os.system(f'rm -rf {gtab["amp"]}')
        gaincal(vis=csource,
                caltable=gtab['amp'],
                field=field,
                spw='0', selectdata=True, scan=s['pmcals'],
                solint='inf', combine='', refant=ref_ant,
                calmode='ap', minsnr=3.0,
                gaintable=[gtab['bph'], gtab['delay'], gtab['intph']])
        applycal(
            vis=csource,
            field=s['bpcalf'],
            gaintable=[gtab['delay'],gtab['bp'],gtab['scnph'],gtab['amp']],
            gainfield=[s['bpcalf'],s['bpcalf'],s['bpcalf'],s['bpcalf']],
            parang=True, calwt=False, applymode='calflagstrict', flagbackup=True)

        applycal(
            vis=csource,
            field=s['targetf'],
            gaintable=[gtab['delay'],gtab['bp'],gtab['scnph'],gtab['amp']],
            gainfield=[s['bpcalf'],s['bpcalf'],s['bpcalf'],s['bpcalf']],
            calwt=False, applymode='calflagstrict', flagbackup=True)
    return {'tables':gtab}

def init_flag(csource, **kwargs):

    params={'reset':False,
            'quackinterval':10.0,
            'quackmode':'beg', 'badant':'', 'badchan':'','clip':[0,50],'refant':'C00',
            'scan':'', 'extend':False
                   }
    params.update(kwargs)
    if params['reset']:
        flagdata(vis=csource,mode='unflag', field='', spw='', antenna='', timerange='')
        clearcal(vis=csource)

    s=fetch_sources(csource)
    csource_stem = Path(csource).stem

    flagdata(vis=csource, mode='clip', clipzeros=True, scan=params['scan'],clipminmax=params['clip'], flagbackup=False, action='apply') # clip zero amplitude
    flagdata(vis=csource, mode='shadow', tolerance=0.0, scan=params['scan'],flagbackup=False, action='apply') # shadowing
    if params['quackmode']:flagdata(vis=csource, mode='quack', scan=params['scan'],quackinterval=params['quackinterval'], quackmode=params['quackmode'],
             flagbackup=False, action='apply')
    if params['badant']: flagdata(vis=csource, antenna=str(params['badant']),action='apply')
#     flagdata(vis=csource, mode='tfcrop', flagbackup=True, action='apply', extendpols=True, extendflags=True)
    if params['extend']:flagdata(vis=csource, mode='extend', growfreq=80.0, growtime=60.0, action='apply', extendpols=True, extendflags=True)

def flagsummary(vis):
    fsummary = flagdata(vis, mode='summary')
    sof=fsummary.keys()
    fgen=['name','type', 'total', 'flagged']
    totflag=np.round((fsummary['flagged']/fsummary['total'])*100,2)
    print(f"{fsummary['name']}\n-----\nTotal flagged:{totflag}%")
    flaggable={}
    for s in sof:

        if s in fgen:
            pass
        else:
            print(f'\n{s}:\n')
            for k in fsummary[s]:
                v=fsummary[s][k]
                perc = np.round((fsummary[s][k]['flagged']/fsummary[s][k]['total'])*100,2)

                if perc>totflag and s=='antenna':
                    flaggable[k]=perc

                print(f"{k}:{perc}%")
    antmax = max(flaggable, key=flaggable.get)
    ant3sigma=np.round(float(totflag)+((float(flaggable[str(antmax)])-float(totflag))/3),2)
    print(f'antenna 3 sigma : {ant3sigma}')
    for ant,perc in flaggable.items():
        if perc>ant3sigma:
            print(f'flaggable antenna:{ant}')
    # print(f'flaggable antenna:{antmax}\n {flaggable}')
    return {'flaggable':flaggable}

def selfcal_model(calibrated_target, goodchans='0:500~600', ref_ant='C00'):
    ctarget_stem = str(Path(calibrated_target).stem)

    sctarget={}
    sctarget['sc1']=ctarget_stem+'.model1.MS'
    sctarget['sc2']=ctarget_stem+'.model2.MS'
    sctarget['sc3']=ctarget_stem+'.model3.MS'

    sgtab={}
    sgtab['sc1']='sc1.gcal'
    sgtab['sc2']='sc2.gcal'
    sgtab['sc3']='sc3.gcal'
    os.system(f"rm -rf {sctarget['sc1']}* {sctarget['sc2']}* {sctarget['sc3']}* {sgtab['sc1']}* {sgtab['sc2']}* {sgtab['sc3']}* {ctarget_stem}.model.cimage")

    tclean(vis=calibrated_target,
           datacolumn='data',
           imagename=ctarget_stem+'.model.cimage',
           spw=goodchans,
           specmode='mfs',
           niter=100,
           threshold='1.5Jy',
           imsize=[256,256], cell='1arcsec',
           weighting='briggs', robust=0.2,
           savemodel='modelcolumn')
    gaincal(vis=calibrated_target,
            caltable=sgtab['sc1'],
            field='0', spw='0', refant=ref_ant, calmode='p', solint='30s', combine='',
            minsnr=3.0)
    applycal(vis=calibrated_target, field='0', gaintable=[sgtab['sc1']], interp='linear')
    split(vis=calibrated_target, outputvis=sctarget['sc1'], datacolumn='corrected',
          field='0', keepflags=False)

    tclean(vis=sctarget['sc1'], datacolumn='data',
           imagename=sctarget['sc1']+'.cimage',
           spw=goodchans,
           specmode='mfs',
           niter=500,
           threshold='500mJy',
           imsize=[256,256],cell='1arcsec',
           weighting='briggs', robust=0.2,
           savemodel='modelcolumn')
    gaincal(vis=sctarget['sc1'], caltable=sgtab['sc2'],
            field='0', spw='0', refant=ref_ant, calmode='p', solint='10s', combine='',
            minsnr=3.0)
    applycal(vis=sctarget['sc1'], field='0', gaintable=[sgtab['sc2']], interp='linear')
    split(vis=sctarget['sc1'], outputvis=sctarget['sc2'], datacolumn='corrected',
          field='0', keepflags=False)

    tclean(vis=sctarget['sc2'], datacolumn='data',
           imagename=sctarget['sc2']+'.cimage', spw=goodchans,
           specmode='mfs',
           niter=1000, threshold='100mJy',
           imsize=[256,256], cell='1arcsec', weighting='briggs', robust=0.2,
           savemodel='modelcolumn')
    gaincal(vis=sctarget['sc2'], caltable=sgtab['sc3'],
            field='0', spw='0', refant=ref_ant, calmode='ap', solint='int', combine='',
            minsnr=3.0)
    applycal(vis=sctarget['sc2'], field='0', gaintable=[sgtab['sc3']], interp='linear')

    os.system(f'rm -rf {sctarget["sc3"]}.*')
    split(vis=sctarget['sc2'], outputvis=sctarget['sc3'], datacolumn='corrected',
          field='0', keepflags=False)

    tclean(vis=sctarget['sc3'], datacolumn='data',
           imagename=sctarget['sc3']+'.cimage', spw=goodchans,
           specmode='mfs',
           niter=1000, threshold='100mJy',
           imsize=[256,256], cell='1arcsec', weighting='briggs', robust=0.2,
           savemodel='modelcolumn')
    return {'model_imagename':sctarget['sc3']+'.cimage', 'model_vis':sctarget['sc3']}

def tclean_spectral_image(vis, imagename, f=None, suffix='', norm_max=None):
    os.system(f'rm -rf {imagename}.*')
    if f:
        rest_freq = str(f)+'MHz'
        f=float(f)
    tclean(vis=vis, datacolumn='data',
           imagename=imagename, spw=f'0:596~597.5MHz',#{rest_freq}',
           specmode='cube', outframe='bary', veltype='radio', restfreq=rest_freq,
           niter=10000000, threshold='0.05Jy',
           imsize=[256,256], cell='1arcsec',
        #    pbcor=True,
           weighting='briggs', robust=0.0,
           perchanweightdensity=True, savemodel='none',
           interpolation='nearest')
    msmd.open(vis)
    nchan = msmd.nchan(0)
    chanwidths = float(msmd.chanwidths(0, 'MHz')[0])
    freqstart = float(msmd.chanfreqs(0, 'MHz')[0])
    msmd.done()
    n = (f - freqstart)/chanwidths
    if not n == float(int(n)):
        plotchans = [int(n), int(n+1)]
    else:
        plotchans = [int(n)]
    for i in range(len(plotchans)):
        genpng(imagename+'.image',i,out=f"{imagename}_{plotchans[i]}.bary.cube", norm_max=norm_max, kind='jpg')
    #   imview(raster={'file':f"{imagename}"+'.image', 'colormap': 'Hot Metal 1'},
#        zoom={'channel': 0, },
#        axes={'z':'Frequency',},
#         out=f"{imagename}"+suffix+'.bary.cube.1'+'.png',
#       )
#     imview(raster={'file':f"{imagename}"+'.image', 'colormap': 'Hot Metal 1'},
#            zoom={'channel': 1, },
#            axes={'z':'Frequency',},
#             out=f"{imagename}"+suffix+'.bary.cube.2'+'.png',
#           )

def tclean_continuum_image(science_vis_cont, imagename, suffix='', norm_max=None):
    os.system(f'rm -rf {imagename} {imagename}.*')
    tclean(vis=science_vis_cont, datacolumn='data',
       imagename=imagename, spw='0',
       specmode='mfs',
       niter=1000, threshold='50mJy',
       imsize=[256,256], cell='1arcsec', weighting='briggs', robust=0.2,
#        savemodel='modelcolumn'
      )
    if norm_max is None:
        norm_max=imstat(imagename+'.image')['max'][0]
    genpng(imagename+'.image',0,out=f"{imagename}"+suffix, norm_max=norm_max, kind='jpg')
    return norm_max
#     imview(raster={'file':f"{imagename}"+'.image', 'colormap': 'Hot Metal 1'},
#             out=f"{imagename}"+suffix+'.cont'+'.png',

#           )

def fast_spectral_image(calvis, cvis, timerange='',f=None, suffix=''):
    os.system(f'rm -rf {cvis}*')
    rest_freq=None
    if f:
        rest_freq = str(f)+'MHz'
    # msmd.open(calvis)
    # msmd.chanwidths()
    fitspw=f'0:{rest_freq}'
    print(f"split({calvis}, {cvis}, timerange={timerange}, datacolumn='data')")
    split(calvis, cvis, timerange=timerange, datacolumn='data', )#spw='0:590~610MHz')
    if rest_freq is not None:
        uvcontsub(cvis, field='0',
            # fitspw=fitspw, #'0:0.566~0.575GHz',
            fitspw='0:596~597.5MHz',
            excludechans=True,
            fitorder=1,
            want_cont=False,
            )
        science_vis=cvis+'.contsub'
        # science_vis_cont=cvis+'.cont'
        # norm_max=tclean_continuum_image(science_vis_cont, science_vis_cont+'.cimage', suffix=suffix+'.cont')
        tclean_spectral_image(science_vis, science_vis+'.cimage',f=f, suffix=suffix+'.cube')
    else:
        tclean_continuum_image(cvis, cvis+'.cimage', suffix=suffix)

def plotd(visfile, **kwargs):
    display = Display(visible=0,size=(1024,768))
    display.start()

    plotfolder=f'plots/{Path(visfile).stem}/'
    if not path.exists(plotfolder):
        try:
            oumask = os.umask(0)
            makedirs(plotfolder)
        except:
            os.umask(oumask)
            makedirs(plotfolder, 775)
    print(f'plotting in folder: {plotfolder}')
    vis = visfile
    metadata=listobs(vis, verbose=True)
    msmd.open(vis)
    nchan = msmd.nchan(0)
    msmd.done()

    fieldl = {k.replace('field_', ''):{'name':v['name']} for k,v in metadata.items() if 'field_' in k}
    scanl = {k.replace('scan_', ''):v['0']['FieldName'] for k,v in metadata.items() if 'scan_' in k}
    for fk,fv in fieldl.items():
        fieldl[str(fk)]['scans']=[k.replace('scan_', '') for k,v in metadata.items() if ('scan_' in k) and (v['0']['FieldId']==int(fk))]
    metadata.update({'fieldl':fieldl, 'scanl':scanl})
    params={'spw':'',#f'0:{int(nchan*0.4)}~{int(nchan*0.6)}',
    'avgchannel' :str(nchan),
    'uvrange':'',
    'antenna':'C00', 'scan':'', 'corr':'',
    'xaxis':[''], 'yaxis':['']}
    params.update(kwargs)
    spw, avgchannel, uvrange, antenna = params['spw'], params['avgchannel'], params['uvrange'], params['antenna']
    print(f"{spw}, {avgchannel}, {uvrange}, {antenna}")
    # selectedfield=metadata['fieldl']['4']
    scanlist = params['scan'] or scanl.keys()
    comb={'amp':{'freq':{'averagedata':True,'avgtime':'1e9', 'avgbaseline':False, 'avgchannel':''},
             'time':{'averagedata':False,'avgtime':'', 'avgbaseline':False, 'avgchannel':''},
             'uvwave':{'averagedata':False,'avgtime':'', 'avgbaseline':False, 'avgchannel':''}
            },
      'phase':{'freq':{'averagedata':True,'avgtime':'1e9', 'avgbaseline':False, 'avgchannel':''},
               'time':{'averagedata':True,'avgtime':'', 'avgbaseline':True, 'avgchannel':nchan},
               'uvwave':{'averagedata':False,'avgtime':'', 'avgbaseline':False, 'avgchannel':nchan},
               'amp':{'averagedata':True,'avgtime':'1e9', 'avgbaseline':False, 'avgchannel':''}
              },
     }
    _pd={}
    for k,v in comb.items():
        for yi in params['yaxis']:
            if yi:
                if yi==k:
                    for xi in params['xaxis']:
                        for vi in v:
                            if xi and (xi==vi):
                                try:
                                    _pd[k][vi]=comb[k][vi]
                                except:
                                    _pd[k]={vi:comb[k][vi]}
                                break
                            else:
                                try:
                                    _pd[k][vi]=comb[k][vi]
                                except:
                                    _pd[k]={vi:comb[k][vi]}

            else:
                for xi in params['xaxis']:
                    for vi in v:
                        if xi:
                            if xi==vi:
                                try:
                                    _pd[k][vi]=comb[k][vi]
                                except:
                                    _pd[k]={vi:comb[k][vi]}
                                break
                        else:
                            try:
                                _pd[k][vi]=comb[k][vi]
                            except:
                                _pd[k]={vi:comb[k][vi]}
    try:
        for scan in scanlist:
            scan=str(scan)
            print(scan)
            # print(_pd)
            for yaxis,xaxes in _pd.items():
                for xaxis,avgdata in xaxes.items():
                    print(f'{yaxis} vs {xaxis}')
                    plotms(vis=vis, scan=scan, antenna=antenna, spw=spw, correlation=params['corr'],
                        xaxis=xaxis, yaxis=yaxis, coloraxis='corr', title=f'{scanl[str(scan)]}:{yaxis} vs {xaxis} - {scan}',
                        showgui=False,
                        averagedata=avgdata['averagedata'], avgtime=avgdata['avgtime'], avgbaseline=avgdata['avgbaseline'],
                        avgchannel=str(avgdata['avgchannel']),
                        uvrange=uvrange,
                        plotfile=plotfolder+f"{yaxis}_v_{xaxis}.{scanl[str(scan)]}.{scan}.png", overwrite=True)

    finally:
        display.stop()


from flask import Blueprint, jsonify, request
from api.middleware import login_required, read_token

from api.models.db import db
from api.models.issue import Issue

issues = Blueprint('issues', 'issues')

@issues.route('/', methods=["POST"])
@login_required
def create():
  data = request.get_json()
  profile = read_token(request)
  data["profile_id"] = profile["id"]

  issue = Issue(**data)
  db.session.add(issue)
  db.session.commit()
  return jsonify(issue.serialize()), 201

@issues.route('/', methods=["GET"])
def index():
  issues = Issue.query.all()
  return jsonify([issue.serialize() for issue in issues]), 201

@issues.route('/<id>', methods=["GET"])
def show(id):
  issue = Issue.query.filter_by(id=id).first()
  issue_data = issue.serialize()
  return jsonify(issue=issue_data), 200

@issues.route('/<id>', methods=["PUT"])
@login_required
def update(id):
  data = request.get_json()
  profile = read_token(request)
  issue = Issue.query.filter_by(id=id).first()

  if issue.profile_id != profile["id"]:
    return 'Forbidden', 403

  for key in data:
    setattr(issue, key, data[key])

  db.session.commit()
  return jsonify(issue.serialize()), 200

@issues.route('/<id>', methods=["DELETE"])
@login_required
def delete(id):
  profile = read_token(request)
  issue = Issue.query.filter_by(id=id).first()

  if issue.profile_id != profile["id"]:
    return 'Forbidden', 403

  db.session.delete(issue)
  db.session.commit()
  return jsonify(message="Success"), 200


import datetime
import json
import pymongo
import requests_oauthlib
import tqdm
import os

# API key
consumer_key = os.environ['twitter_consumer_key']
consumer_secret = os.environ['twitter_consumer_secret']
access_token_key = os.environ['twitter_access_token_key']
access_token_secret = os.environ['twitter_access_token_secret']
print("consumer_key",consumer_key,"consumer_secret",consumer_secret)
print("access_token_key",access_token_key,"access_token_secret",access_token_secret)

# Twitter Streaming API
twitter = requests_oauthlib.OAuth1Session(
	consumer_key, consumer_secret, access_token_key, access_token_secret)
uri = 'http://stream.twitter.com/1.1/statuses/sample.json'
r = twitter.get(uri, stream=True)
r.raise_for_status()

# Stored in mongoDB
mongo = pymongo.MongoClient()
for line in tqdm.tqdm(r.iter_lines(), unit='tweets', mininterval=1):
	if line:
		tweet = json.loads(line)
		tweet['_timestamp'] = datetime.datetime.utcnow().isoformat()
		mongo.twitter.sample.insert_one(tweet)


# Date: April 24th, 2018
# Author: Hasana Chaudry
# Class: Senior Colloquium
# Topic: Page Ranking
# Advisor: Kenneth Brakke
# Filename: pageRank.py

import numpy as np
import scipy as sc
#import pandas as pd
from fractions import Fraction

def float_format(vector, decimal):
    return np.round((vector).astype(np.float), decimals = decimal)

# Initializing three webpages. Each web page is assigned a probability of 1/3
n = Fraction(1, 3)

# Creates a 3x3 matrix that represents the transpose of an adjacency matrix
# Completely dense, rank-one teleportation matrix
E = np.matrix([[0, 0, 1], [Fraction(1,2), 0, 0], [Fraction(1,2), 1, 0]])

# Creates a 3x3 sparse matrix
# Sparse, stochastic matrix
S = np.zeros((3, 3))
S[:] = n

# Assigning the damping parameter (always between 0 and 1)
# The probability that when a user clicks on a link, they will continue browsing
a = 0.85

# The equation that gives the page rank values of links in the form of a matrix
# also called the Google Matrix
G = a * S + ((1 - a) * E)

# Creating a matrix, r, that holds the current PageRank values of the three pages
r = np.matrix([n, n, n])
r = np.transpose(r)

# Saving the initial r value
old_r = r

# Setting up a for loop that will go through a hundred iterations
for k in range(1,100):
    # Calculating the page rank value through each iteration
    r = G * r
    print(r)
    print("\n")

    # An if statement that breaks the loop when an iteration returns
    # the same page rank values as the previous iteration
    if(old_r == r).all():
        break
    old_r = r

# Outputting the results
print("Final Ranks: \n ")
print("Node A's rank: ", float_format(r, 6)[0])
print("Node B's rank: ", float_format(r, 6)[1])
print("Node C's rank: ", float_format(r, 6)[2])
print("\n")
print("Sum:", np.sum(r))


from django import forms
from .models import Review

class ReviewForm(forms.ModelForm):

    class Meta:
        model = Review
        exclude = ('user','name','event',)

        widgets = {
            'review' : forms.Textarea(attrs={
                'class': 'form-control',
                'placeholder': 'Review'
            }),

        }


from frontiers_yildizetal.ravaflow import Simulations
from frontiers_yildizetal.emulators import ScalarEmulators
from scipy.stats import skew
import numpy as np
from frontiers_yildizetal.utilities import data

class Moments:
    """
    A class to represent moments, i.e. mean, variance and skewness, for UQ analysis

    Arguments:
        name (str): name of the set. It can be either synth or acheron

    Attributes:
        locs (list): coordinates at which hmax and vmax are extracted.

    Methods:
        get_mcs: returns the moments calculated with Monte Carlo simulations
        get_pem: returns the moments calculated with Point Estimate Method

    Raises:
        TypeError: name must be a string
        ValueError: name must be either synth or acheron

    """
    funcs = {'mean':np.mean, 'var':np.var, 'skew':skew}

    def __init__(self, name:str):
        if not isinstance(name,str):
            raise TypeError('name must be a string')
        if name not in ['synth', 'acheron']:
            raise ValueError('name must be either synth or acheron')
        self.name = name
        loc_all = {'synth': [1000, 2000], 'acheron': [1490100, 5204100]}
        self.locs = loc_all[self.name]

    def get_mcs(self):
        """
        Calculates the three moments using Monte Carlo Simulations facilitated with Gaussian Process Emulation

        Returns:
            mcs_moments (dict): A dictionary storing the moments according to scalars
        """
        mcss = ['mcs' + str(i) for i in range(1, 4)]
        emulator = ScalarEmulators(self.name, 0.1,
                                   self.locs[0],
                                   self.locs[1])
        scalars = list(emulator.output.keys())

        mcs_moments = {}

        for n, f in self.funcs.items():
            mcs_moments[n] = {}
            for key in scalars:
                mcs_moments[n][key] = []
                for mcs in mcss:
                    input_test = data.load_input(name=emulator.name, analysis=mcs)
                    predicted = emulator.predict_scalar(key, input_test)[0]

                    if f is np.var:
                        val = f(predicted, ddof=1)
                        mcs_moments[n][key].append(round(val, 3))
                    else:
                        val = f(predicted)
                        mcs_moments[n][key].append(round(val, 3))
        return mcs_moments

    def get_pem(self):
        """
        Calculates the three moments using Point Estimate Method

        Returns:
            pem_moments (dict): A dictionary storing the moments according to scalars
        """
        pems = ['pem' + str(i) for i in range(1, 4)]
        scalars = Simulations((self.name + '_pem')).curate_scalars(0.1, self.locs[0], self.locs[1])

        pem_moments = {}

        for n, f in self.funcs.items():
            pem_moments[n] = {}
            for key in scalars:
                pem_moments[n][key] = []
                for j, pem in enumerate(pems):
                    if f is np.var:
                        val = f(scalars[key][8 * j : 8 * (j + 1)], ddof=1)
                        pem_moments[n][key].append(round(val, 3))
                    else:
                        val = f(scalars[key][8 * j : 8 * (j + 1)])
                        pem_moments[n][key].append(round(val, 3))
        return pem_moments


#!/usr/bin/python3
import json

from flask import Flask

app = Flask(__name__)

@app.route('/')
def task1():
    with open('task1-data.json', 'r') as f:
        return json.loads(f.read())

if __name__ == '__main__':
    app.run(debug=True)


# encoding=GBK
"""Conv2D

    - 和骨干网络的第0层的启动结构类似
        - 变化的只有不固定的卷积核与通道数
        - 用来适应3种情况下输出的内容

    - 封装步骤：
        1. conv
        2. batch_norm
        3. LeaKyRelU
"""
import torch.nn as nn
from collections import OrderedDict

class Conv2D:
    @staticmethod
    def conv2d(filter_in, filter_out, kernel_size):
        pad = (kernel_size - 1) // 2 if kernel_size else 0
        return nn.Sequential(OrderedDict([
            ("conv", nn.Conv2d(filter_in, filter_out, kernel_size=kernel_size, stride=1, padding=pad, bias=False)),
            ("bn", nn.BatchNorm2d(filter_out)),
            ("relu", nn.LeakyReLU(0.1)),
        ]))

if __name__ == '__main__':
    # img = torch.randn((1, 3, 200, 200))
    out = Conv2D.conv2d(3, 10, 3)
    print(out)


from trindikit import *  # update_rule, precondition, Speaker, ProgramState, Move, R, record, freetextquestion
from ibis_types import *  # Ask, Respond, Answer, Greet, Quit, If, YNQ, Findout, ICM, Raise, ConsultDB, Command, Imperative, Inform, State
import itertools
import ibis_generals
from trindikit import set

######################################################################
# IBIS update rules
######################################################################

# Grounding

@update_rule
def get_latest_moves(IS, LATEST_MOVES, LATEST_SPEAKER):
    """Copies the latest move(s) and speaker to the infostate.

    LATEST_MOVES and LATEST_SPEAKER are copied to /shared/lu.
    """

    @precondition
    def V():
        IS.shared.lu.moves = LATEST_MOVES
        yield LATEST_MOVES

    IS.shared.lu.speaker = LATEST_SPEAKER.get()

# Integrating utterances

@update_rule
def integrate_sys_ask(IS):
    """Integrate an Ask move by the system.

    The question is pushed onto /shared/qud.
    """

    @precondition
    def V():
        if IS.shared.lu.speaker == Speaker.SYS:
            for move in IS.shared.lu.moves:
                if isinstance(move, Ask):
                    yield R(move=move, que=move.content)

    IS.shared.qud.push(V.que)

@update_rule
def integrate_usr_ask(IS):
    """Integrate an Ask move by the user.

    The question is pushed onto /shared/qud, and
    a Respond move is pushed onto /private/agenda.
    """

    @precondition
    def V():
        if IS.shared.lu.speaker == Speaker.USR:
            for move in IS.shared.lu.moves:
                if isinstance(move, Ask):
                    yield R(move=move, que=move.content)

    IS.shared.qud.push(V.que)
    if isinstance(V.que, SecOrdQ):  # AND die frage steht noch offen, also V.que.whatever ist noch nicht vergeben
        IS.private.agenda.push(ClarifyPred2(V.que))
    else:
        IS.private.agenda.push(Respond(V.que))

@update_rule
def integrate_usr_impr(IS):
    """Integrate an Imperative move by the user.

    The question is pushed onto /shared/qud
    """

    @precondition
    def V():
        if IS.shared.lu.speaker == Speaker.USR:
            for move in IS.shared.lu.moves:
                if isinstance(move, Imperative):
                    yield R(move=move, que=move.content)

    IS.shared.qud.push(V.que)

@update_rule
def integrate_secordq_clarify(IS, DOMAIN, APICONNECTOR):
    @precondition
    def V():
        que = IS.shared.qud.top()
        if isinstance(que, SecOrdQ):
            # que ist bspw SecOrdQ(Pred2(WhenIs, ['semester', 'WhenSemester'])), arg1[0] ist dann semester. Wenn das im IS ist, dann mach draus arg1[1]
            for candidate in que.content.arg1:
                tmp = APICONNECTOR.getContext(IS, candidate[0])
                if tmp[0]:
                    yield R(que=que, firstarg=tmp[1], candidate=candidate)
                    break

    tmp = Pred1(V.candidate[1], V.firstarg, createdfrom=V.que)
    IS.shared.qud.pop()
    IS.shared.qud.push(WhQ(tmp))
    IS.private.agenda.push(Respond(WhQ(tmp)))

@update_rule
def integrate_answer(IS, DOMAIN, APICONNECTOR):
    """Integrate an Answer move.

    If the answer is relevant to the top question on the qud,
    the corresponding proposition is added to /shared/com.
    """

    @precondition
    def V():
        que = IS.shared.qud.top()
        for move in IS.shared.lu.moves:
            if isinstance(move, Answer):
                if DOMAIN.relevant(move.content, que, APICONNECTOR, IS=IS):
                    yield R(que=que, ans=move.content)

    prop = DOMAIN.combine(V.que, V.ans, APICONNECTOR, IS=IS)
    IS.shared.com.add(prop)
    IS.private.bel.remove(prop, silent=True)

@update_rule
def integrate_greet(IS):
    """Integrate a Greet move.

    Does nothing.
    """

    @precondition
    def V():
        for move in IS.shared.lu.moves:
            if isinstance(move, Greet):
                yield R(move=move)

    pass

@update_rule
def integrate_sys_quit(IS, PROGRAM_STATE):
    """Integrate a Quit move by the system.

    Sets the PROGRAM_STATE to QUIT.
    """

    @precondition
    def V():
        if IS.shared.lu.speaker == Speaker.SYS:
            for move in IS.shared.lu.moves:
                if isinstance(move, Quit):
                    yield R(move=move)

    PROGRAM_STATE.set(ProgramState.QUIT)

@update_rule
def integrate_usr_quit(IS):
    """Integrate a Quit move by the user.

    Pushes a Quit move onto /private/agenda.
    """

    @precondition
    def V():
        if IS.shared.lu.speaker == Speaker.USR:  # TODO figure out warum "Goodbye" zur Zeit noch kein Dialogue Move ist
            for move in IS.shared.lu.moves:
                if isinstance(move, Quit):
                    yield R(move=move)

    IS.private.agenda.push(Quit())

# Downdating the QUD

@update_rule
def downdate_qud(IS, DOMAIN, APICONNECTOR):
    """Downdate the QUD.

    If the topmost question on /shared/qud is resolved by
    a proposition in /shared/com, pop the question from the QUD.
    """

    @precondition
    def V():
        que = IS.shared.qud.top()
        for prop in IS.shared.com:
            if DOMAIN.resolves(prop, que, APICONNECTOR):
                yield R(que=que, prop=prop)

    IS.shared.qud.pop()

#TODO domain.resolves,combine,relevant sollten den APICONNECTOR als standard=None haben, sodass er ihn nur als arg nimmt wenn er ihn wirklich braucht
@update_rule
def downdate_qud_commands(IS, DOMAIN, APICONNECTOR):
    """Downdate the QUD.

    If the topmost question on /shared/qud is resolved by
    a proposition in /shared/com, pop the question from the QUD.
    """

    @precondition
    def V():
        que = IS.shared.qud.top()
        if isinstance(que, Command):
            cmdresolvees = DOMAIN.plans.get(que, False)
            if cmdresolvees:
                doesresolve = [[DOMAIN.resolves(prop, move.content, APICONNECTOR) for prop in IS.shared.com] for move in cmdresolvees
                               if isinstance(move, Question)]
                allresolved = all([any(i) for i in doesresolve])
                # print([i.content for i in cmdresolvees])
                # print(IS.shared.com)
                if allresolved:
                    yield

    IS.shared.qud.pop()

# @update_rule
# def downdate_qud_2(IS, DOMAIN):
#     @precondition
#     def V():
#         que = IS.shared.qud.top()
#         for issue in IS.shared.qud:
#             if issue != que and DOMAIN.resolves(que, issue):
#                 yield R(que=que, issue=issue)
#     IS.shared.qud.remove(V.issue)

# muss for find_plan kommen!
@update_rule
def clarify_pred2(IS, DOMAIN, NEXT_MOVES, APICONNECTOR):
    @precondition
    def V():
        move = IS.private.agenda.top()
        if isinstance(move, ClarifyPred2):
            que = "?x." + move.content.content.arg1[0][0] + "(x)"  # TODO - er fragt jetzt immer nach der ERSTEN möglichkeit das zu fulfillen, das kann unintended sein (wird besser mit besserem partial-parser)
            resolved = any(DOMAIN.resolves(prop, que, APICONNECTOR) for prop in IS.private.bel)
            if not resolved:
                yield R(move=move, question=que)

    IS.private.agenda.push(Ask(V.question))

# Finding plans

@update_rule
def find_plan1(IS, DOMAIN, NEXT_MOVES, APICONNECTOR):
    """Find a dialogue plan for resolving a question.

    If there is a Respond move first in /private/agenda, and
    the question is not resolved by any proposition in /private/bel,
    look for a matching dialogue plan in the domain. Put the plan
    in /private/plan, and pop the Respond move from /private/agenda.
    """

    @precondition
    def V():
        move = IS.private.agenda.top(soft=True)
        scnd = IS.private.agenda.penutop(soft=True)
        if isinstance(move, Ask) and isinstance(scnd, ClarifyPred2) and any(
                str(move.content.content) == candidate[0] for candidate in scnd.content.content.arg1):
            move = scnd

        if isinstance(move, (Respond, ClarifyPred2)):
            resolved = any(DOMAIN.resolves(prop, move.content, APICONNECTOR) for prop in IS.private.bel)
            if not resolved:
                plan = DOMAIN.get_plan(move.content, IS, APICONNECTOR)
                if plan[0] and plan[1]:
                    yield R(move=move, plan=plan[1], wasSecond=(move == scnd))
                elif not plan[0]:
                    string = ", ".join(["%s"] * len(plan[1]))
                    string = "The plan for this cannot be conducted yet, as the following Information is missing: " + string
                    IS.private.agenda.push(Inform(string, plan[1]))

    if not V.wasSecond:
        IS.private.agenda.pop()
    IS.private.plan = V.plan

@update_rule
def find_plan2(IS, DOMAIN, NEXT_MOVES, APICONNECTOR):
    """Find a dialogue plan for resolving a question.

    If there is a Respond move first in /private/agenda, and
    the question is not resolved by any proposition in /private/bel,
    look for a matching dialogue plan in the domain. Put the plan
    in /private/plan, and pop the Respond move from /private/agenda.
    """

    @precondition
    def V():
        move = IS.private.agenda.top(soft=True)
        scnd = IS.private.agenda.penutop(soft=True)
        if isinstance(move, Ask) and isinstance(scnd, ClarifyPred2) and any(
                str(move.content.content) == candidate[0] for candidate in scnd.content.content.arg1):
            move = scnd

        if isinstance(move, (Respond, ClarifyPred2)):
            resolved = any(DOMAIN.resolves(prop, move.content, APICONNECTOR) for prop in IS.private.bel)
            if resolved:
                for prop in IS.private.bel:
                    if DOMAIN.resolves(prop, move.content, APICONNECTOR):
                        yield R(move=move, answer=prop)

    IS.private.agenda.pop()
    NEXT_MOVES.push(Answer(V.answer))

# Executing plans

@update_rule
def execute_if(IS, DOMAIN, APICONNECTOR):
    """Execute an If(...) plan construct.

    If the topmost construct in /private/plan is an If,
    test if the condition is in /private/bel or /shared/com.
    If it is, add the iftrue plan to /private/plan,
    otherwise, add the iffalse plan to /private/plan.
    """

    @precondition
    def V():
        move = IS.private.plan.top()
        if isinstance(move, If):
            if isinstance(move.cond, YNQ):
                if move.cond.content in (IS.private.bel | IS.shared.com):
                    yield R(test=move.cond, success=True, subplan=move.iftrue)
                else:
                    yield R(test=move.cond, success=False, subplan=move.iffalse)
            elif isinstance(move.cond, WhQ):  # dann prüft er lediglich ob das wissen dazu da ist
                sources = list(IS.shared.com) + list(IS.private.bel) + list(find_knowledge_from_question(IS, DOMAIN, APICONNECTOR))
                if any(DOMAIN.resolves(candidate, move.cond, APICONNECTOR) for candidate in sources):
                    yield R(test=move.cond, success=True, subplan=move.iftrue)
                else:
                    yield R(test=move.cond, success=False, subplan=move.iffalse)

    IS.private.plan.pop()
    for move in reversed(V.subplan):
        IS.private.plan.push(move)

@update_rule
def remove_findout(IS, DOMAIN, APICONNECTOR):
    """Remove a resolved Findout from the current plan.

    If the topmost move in /private/plan is a Findout,
    and the question is resolved by some proposition
    in /shared/com, pop the Findout from /private/plan.
    """

    @precondition
    def V():
        move = IS.private.plan.top()
        if isinstance(move, Findout):
            for prop in IS.shared.com:
                if DOMAIN.resolves(prop, move.content, APICONNECTOR):
                    yield R(move=move, prop=prop)

    IS.private.plan.pop()

@update_rule
def exec_consultDB(IS, APICONNECTOR):
    """Consult the database for the answer to a question.

    If the topmost move in /private/plan is a ConsultDB,
    consult the DATABASE using /shared/com as context.
    The resulting proposition is added to /private/bel,
    and the ConsultDB move is popped from /private/plan.
    """

    @precondition
    def V():
        if isinstance(APICONNECTOR, ibis_generals.Database):
            move = IS.private.plan.top()
            if isinstance(move, ConsultDB):
                yield R(move=move)

    prop = APICONNECTOR.consultDB(V.move.content, IS, "com")
    IS.private.bel.add(prop)
    IS.private.plan.pop()

@update_rule
def recover_plan(IS, DOMAIN, APICONNECTOR):
    """Recover a plan matching the topmost question in the QUD.

    If both /private/agenda and /private/plan are empty,
    and there is a topmost question in /shared/qud,
    and there is a matching plan, then put the plan in
    /private/plan.
    """

    @precondition
    def V():
        if not IS.private.agenda and not IS.private.plan:
            que = IS.shared.qud.top()
            plan = DOMAIN.get_plan(que, IS, APICONNECTOR)
            if plan[0] and plan[1]:
                yield R(que=que, plan=plan[1])

    IS.private.plan = V.plan
    if isinstance(V.que, Command):
        IS.shared.qud.pop()

@update_rule
def remove_raise(IS, DOMAIN, APICONNECTOR):
    """Remove a resolved Raise move from the current plan.

    If the topmost move in /private/plan is a Raise,
    and the question is resolved by some proposition in
    /shared/com, pop the Raise from /private/plan.
    """

    @precondition
    def V():
        move = IS.private.plan.top()
        if isinstance(move, Raise):
            for prop in IS.shared.com:
                if DOMAIN.resolves(prop, move.content, APICONNECTOR):
                    yield R(move=move, prop=prop)

    IS.private.plan.pop()

######################################################################
# IBIS selection rules
######################################################################

# Selecting actions

@update_rule
def select_from_plan(IS):
    """Select a move from the current plan.

    If /private/agenda is empty, but there is a topmost move in
    /private/plan, push the move onto /private/agenda.
    """

    @precondition
    def V():
        if not IS.private.agenda:
            move = IS.private.plan.top()
            yield R(move=move)

    IS.private.agenda.push(V.move)

@update_rule
def select_respond(IS, DOMAIN, APICONNECTOR):
    """Answer a question on the QUD.

    If both /private/agenda and /private/plan are empty, and there
    is a topmost question on /shared/qud for which there is a
    relevant proposition in /private/bel, push a Respond move
    onto /private/agenda.
    """

    @precondition
    def V():
        if not IS.private.agenda and not IS.private.plan:
            que = IS.shared.qud.top()
            for prop in IS.private.bel:
                # if prop not in IS.shared.com: #I get warum das nicht so sein sollte, aber wenn das hier ist fragt er halt mich sachen die ich ihn frage, weil statt select_respond halt reraise_issue triggert
                if DOMAIN.relevant(prop, que, APICONNECTOR):
                    yield R(que=que, prop=prop)

    IS.private.agenda.push(Respond(V.que))

@update_rule
def reraise_issue(IS, DOMAIN, APICONNECTOR):
    """Reraise the topmost question on the QUD.

    If there is no dialogue plan for the topmost question on
    /shared/qud, reraise the question by pushing a Raise move
    onto /private/agenda.
    """

    @precondition
    def V():
        que = IS.shared.qud.top()
        tmp = DOMAIN.get_plan(que, IS, APICONNECTOR)
        if tmp[0] and not tmp[1] and isinstance(que,
                                                Question):  # vorher hat get_plan nur den planstack returned, jetzt muss sowohl die erste True sein als auch der planstack != None für not(tmp)
            yield R(que=que)

    IS.private.agenda.push(Raise(V.que))

# Selecting dialogue moves

@update_rule
def select_icm_sem_neg(IS, INPUT, NEXT_MOVES):
    """If interpretation failed, select ICM for negative
    semantic understanding."""

    @precondition
    def V():
        if len(IS.shared.lu.moves) == 0:
            if INPUT.value:
                if INPUT.value != '':
                    if IS.shared.lu.speaker == Speaker.USR:
                        yield True

    NEXT_MOVES.push(ICM('per', 'pos', INPUT.value))  # perception is positive "I heard you say XYZ" (with string as arg)
    NEXT_MOVES.push(ICM('sem',
                        'neg'))  # semantic understanding is negative "I don't understand" (would have the move as arg, unrecognized here)
    # Quote from https://pdfs.semanticscholar.org/0066/b5c5b49e1a7eb4ea95ee22984b695ec5d2c5.pdf:
    # A general strategy used by GoDiS in ICM selection is that if negative or checking feedback on some level is provided,
    # the system should also provide positive feedback on the level below

@update_rule
def select_ask(IS, NEXT_MOVES):
    """Select an Ask move from the agenda.

    If the topmost move in /private/agenda is a Findout or a Raise,
    add an Ask move to NEXT_MOVES. Also, if the topmost move in
    /private/plan is the same Raise move, pop it from /private/plan.
    """

    @precondition
    def V():
        move = IS.private.agenda.top()
        if isinstance(move, Findout) or isinstance(move, Raise) or isinstance(move, Inform):
            yield R(move=move, que=move.content)

    if isinstance(V.move, Inform):  # TODO diesen teil vom code muss Statement/State haben god damn it
        string = V.move.content
        index = -1
        while string.find("%s") > 0:
            index += 1
            string = string.replace("%s", str(V.move.replacers[index])[4:-1], 1)
        NEXT_MOVES.push(State(string))
    else:
        NEXT_MOVES.push(Ask(V.que))
    if IS.private.plan:
        move = IS.private.plan.top()
        if isinstance(move, Raise) and move.content == V.que:
            IS.private.plan.pop()

@update_rule
def reselect_ask(IS, NEXT_MOVES):
    """this one cames after all the other select_... rules. select_ask for example re-asks a question originally
    from the plan (at first select_from_plan, then select_ask). If however a 2-place-pred was not correctly answered,
    the system doesn't know at all what to ask anymore --> this will look if something was not correctly answered, there
    is no other question to be asked, and if so, re-asks the top-qud-question"""

    @precondition
    def V():
        qud = IS.shared.qud.top(soft=True)
        if qud and NEXT_MOVES:
            if all(isinstance(i, ICM) for i in NEXT_MOVES):
                yield R(qud=qud)

    NEXT_MOVES.push(Ask(V.qud))

@update_rule
def select_answer(IS, DOMAIN, NEXT_MOVES, APICONNECTOR):
    """Select an Answer move from the agenda.

    If the topmost move in /private/agenda is a Respond, and there
    is a relevant proposition in /private/bel which is not in
    /shared/com, add an Answer move to NEXT_MOVES.
    """

    # V = precondition(lambda:
    #                  (R(prop=prop)
    #                   for move in [IS.private.agenda.top()]
    #                   if isinstance(move, Respond)
    #                   for prop in IS.private.bel
    #                   if prop not in IS.shared.com
    #                   if DOMAIN.relevant(prop, move.content)))

    @precondition
    def V():
        move = IS.private.agenda.top()
        if isinstance(move, Respond):
            for prop in IS.private.bel:
                # if prop not in IS.shared.com: #siehe select_respond
                if DOMAIN.relevant(prop, move.content, APICONNECTOR):
                    yield R(prop=prop)

    NEXT_MOVES.push(Answer(V.prop))

@update_rule
def select_other(IS, NEXT_MOVES):
    """Select any dialogue move from the agenda.

    If the topmost move in /private/agenda is a Move,
    add it as it is to NEXT_MOVES.
    """

    @precondition
    def V():
        move = IS.private.agenda.top()
        if isinstance(move, Move):
            yield R(move=move)

    NEXT_MOVES.push(V.move)

######################################################################
# Other rules
######################################################################

@update_rule
def handle_empty_plan_agenda_qud(IS, PROGRAM_STATE):
    """Handles the end of a conversation.

    pushes saying goodbye onto the agenda
    """

    @precondition
    def V():
        if len(IS.shared.qud.elements) == 0 and len(IS.private.plan.elements) == 0 and len(
                IS.private.agenda.elements) == 0:
            no_greet = True
            for move in IS.shared.lu.moves:
                if isinstance(move, Greet):
                    no_greet = False
            if no_greet:
                yield R(move=None)

    # IS.private.agenda.push(Quit())
    # import os
    # os.remove("CurrState.pkl")
    # Alternatively:
    # PROGRAM_STATE.set(ProgramState.QUIT)

@update_rule
def exec_inform(IS, NEXT_MOVES):
    @precondition
    def V():
        move = IS.private.plan.top()
        if isinstance(move, Inform):
            if not any([i == move.content for i in IS.private.bel]):
                if len(move.replacers) == 0:
                    yield R(move=move)
                else:
                    mustbe = [False] * len(move.replacers)
                    relevants = {}
                    index = -1
                    for i in move.replacers:
                        index += 1
                        if i.startswith("bel("): #TODO - hier APICONNECTOR und getContext verwenden!
                            relevantpart = i[4:-1]
                            for j in IS.private.bel:
                                if isinstance(j, Prop):
                                    # print("----------")
                                    # print(j.content[0])
                                    # print(relevantpart)
                                    if str(j.content[0]) == relevantpart:
                                        mustbe[index] = True
                                        relevants[i] = j.content[1]
                        elif i.startswith("com("):
                            relevantpart = i[4:-1]
                            for j in IS.shared.com:
                                if isinstance(j, Prop):
                                    if str(j.content[0]) == relevantpart:
                                        mustbe[index] = True
                                        relevants[i] = j.content[1]
                    if all(mustbe):
                        yield R(move=move, toreplace=relevants)

    string = V.move.content
    index = -1
    while string.find("%s") > 0:
        index += 1
        string = string.replace("%s", str(V.toreplace[V.move.replacers[index]]), 1)

    NEXT_MOVES.push(State(string))
    IS.private.plan.pop()
    IS.private.bel.add(V.move.content)

flatten = lambda l: [item for sublist in l for item in sublist]

def powerset(L, fixedLen=False, incShuffles=True):
    pset = set()
    for n in range(len(L) + 1):
        for sset in itertools.combinations(L, n):
            pset.add(sset)
    if fixedLen:
        pset = [i for i in pset if len(i) == fixedLen]
    if not incShuffles:
        return pset
    else:
        return flatten([list(itertools.permutations(i)) for i in pset])

def find_knowledge_from_question(IS, DOMAIN, APICONNECTOR):
    prop = []
    try:
        if isinstance(IS.shared.qud.top(), WhQ):
            topqud = IS.shared.qud.top().content
            if isinstance(topqud, Pred1) and hasattr(topqud, "createdfrom") and isinstance(topqud.createdfrom, str):
                for candidate in DOMAIN.preds2[str(topqud.createdfrom)]:
                    anotherquestion = Question("?x." + candidate[0] + "(x)")
                    if DOMAIN.resolves(Answer(str(topqud.arg2)).content, anotherquestion, APICONNECTOR, IS=IS):
                        prop.append(DOMAIN.combine(anotherquestion, Answer(str(topqud.arg2)).content, APICONNECTOR, IS=IS))
    except:
        pass
    return prop

@update_rule
def exec_func(IS, DOMAIN, NEXT_MOVES, DM, APICONNECTOR):
    @precondition
    def V():
        move = IS.private.plan.top()
        if isinstance(move, ExecuteFunc):
            kwparams = {key: val for key, val in move.kwparams.items() if key != "optionals"}
            mustknow = [Question(i) for i in move.params] + [Question(i) for i in kwparams.values()]

            prop = find_knowledge_from_question(IS, DOMAIN, APICONNECTOR)
            sources = list(IS.shared.com) + list(IS.private.bel) + list(prop)
            knowledgecombos = powerset(sources, fixedLen=len(mustknow), incShuffles=True)
            for knowledge in knowledgecombos:
                alls = [False] * len(mustknow)
                for i in range(len(mustknow)):
                    if DOMAIN.resolves(knowledge[i], mustknow[i], APICONNECTOR):
                        alls[i] = True
                if all(alls):
                    if len(knowledge) > len(move.params):  # dann sind die hinteren nämlich kw-args
                        kw = dict(zip(kwparams.keys(), knowledge[len(move.params):]))
                        if "optionals" in move.kwparams.keys():
                            optknowledge = {}
                            optionals = move.kwparams["optionals"]
                            for key, val in optionals.items():
                                for knw in sources:
                                    if DOMAIN.resolves(knw, Question(val), APICONNECTOR):
                                        optknowledge[key] = knw
                            yield R(knowledge=knowledge[:len(move.params)], move=move, kwknowledge=kw,
                                    optknowledge=optknowledge)
                        else:
                            yield R(knowledge=knowledge[:len(move.params)], move=move, kwknowledge=kw)
                    else:
                        yield R(knowledge=knowledge, move=move)

    if "kwknowledge" in V._typedict:
        kwknowledge = {key: val.ind.content for key, val in V.kwknowledge.items()}
        if "optknowledge" in V._typedict: kwknowledge["optionals"] = V.optknowledge
        res = V.move.content(DM, *[i.ind.content for i in V.knowledge], **kwknowledge)  # executes it!
    else:
        res = V.move.content(DM, *[i.ind.content for i in V.knowledge])  # executes it!
    if res and hasattr(res, "__len__") and len(res) >= 2:
        if len(res) == 3:
            prop, place, cleanup = res
        else:
            prop, place = res
            cleanup = None
        place(prop)
        if cleanup:
            kwparams = {key: val for key, val in V.move.kwparams.items() if key != "optionals"}
            if "optionals" in V.move.kwparams.keys(): kwparams = {**kwparams, **V.move.kwparams["optionals"]}
            cleanup = [kwparams[i] for i in cleanup]
            for i in cleanup:
                IS.shared.com.remove(i, silent=True)
                IS.private.bel.remove(i, silent=True)
    if res:
        IS.private.plan.pop()

@update_rule
def mention_command_conditions(IS, DOMAIN, APICONNECTOR):
    @precondition
    def V():
        cmd = IS.shared.qud.top()
        if len(DOMAIN.check_for_plan(cmd, IS, APICONNECTOR)) > 0 and isinstance(cmd, Command) and cmd.new:
            yield R(cmd=cmd)

    missings = DOMAIN.check_for_plan(V.cmd, IS, APICONNECTOR)
    string = ", ".join(["%s"] * len(missings))
    string = "The plan for this cannot be conducted yet, as the following Information is missing: " + string
    IS.private.agenda.push(Inform(string, missings))
    V.cmd.new = False

@update_rule
def make_command_old(IS):
    @precondition
    def V():
        cmd = IS.shared.qud.top()
        if isinstance(cmd, Command) and cmd.new:
            yield R(cmd=cmd)

    V.cmd.new = False

@update_rule
def expire_expirables(IS):
    @precondition
    def V():
        for i in IS.private.bel:
            if isinstance(i, (Prop, Knowledge)):
                if i.expires and i.expires < round(time.time()):
                    yield R(prop=i, pos=IS.private.bel)
        for i in IS.shared.com:
            if isinstance(i, (Prop, Knowledge)):
                if i.expires and i.expires < round(time.time()):
                    yield R(prop=i, pos=IS.shared.com)

    V.pos.remove(V.prop)


import serial.tools.list_ports

def get_ports():
    ports = list(serial.tools.list_ports.comports())
    for port in ports:
        if port.manufacturer.startswith("STM"):
            port_number = port.device
    return port_number

def receive_data():
    try:
        value = iot_device.readline()
        print(value)
    except:
        print("Nothing to print")

def send_data(data):
    try:
        iot_device.write(data.encode())
    except:
        print("Nothing to send")

#iot_device = serial.Serial(port=get_ports(),baudrate=115200, bytesize=serial.EIGHTBITS,parity=serial.PARITY_NONE,timeout=1)
#while True:
#   try:
#        value = iot_device.readline()
#        print(value)
#    except:
#        print("Nothing to print")

while True:
    receive_data()
    send_data('A45')


# -*- coding=utf-8
import sys
import logging
import json
import time
import os

def write_content_to_file(file_name, content):
    file_ob = open(file_name, 'w')
    file_ob.write(content)
    file_ob.close()

def read_file_content(file_name):
    file_ob = open(file_name, 'r')
    content = file_ob.read()
    file_ob.close()
    return content

def main():
    result = {"items": []}

    content = read_file_content('history.log')
    if len(content) == 0:
        result["items"].append({
            "title":'已上传 0 张图片',
            "subtitle": "移动到指定图片，按下 Cmd + Y 可查看图片",
            "arg": "noPic",
            "icon": "5E995555-07AF-4314-B882-C89ABAF2ED7A.png"})
        print(json.dumps(result))
        return -1
    answers = content.split("\n")

    if not answers:
        result["items"].append({
            "title":'已上传 0 张图片',
            "subtitle": "移动到指定图片，按下 Cmd + Y 可查看图片",
            "arg": "noPic",
            "icon": "5E995555-07AF-4314-B882-C89ABAF2ED7A.png"})
        print(json.dumps(result))
        return -1

    result["items"].append({
            "title":'已上传 ' + str(len(answers) / 2) + ' 张图片，移动后 Cmd + Y 可查看图片',
            "subtitle": "移动后 Enter 可复制对于图片链接",
            "arg": "noPic",
            "icon": "5E995555-07AF-4314-B882-C89ABAF2ED7A.png"})
    i = 0
    while(i < len(answers) - 1):
        result["items"].append({
            "title": answers[i],
            "subtitle": answers[i + 1],
            "arg": answers[i + 1],
            "autocomplete": answers[i],
            "icon": "5E995555-07AF-4314-B882-C89ABAF2ED7A.png"
        })
        i = i + 2
    print(json.dumps(result))

if __name__ == '__main__':
    main()


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation

fig = plt.figure()

def make_noise_wave(data):
    # 現在描写されているグラフを消去
    plt.cla()
    rand = np.random.randn(50)
    # y軸固定
    plt.ylim(-4, 4)
    im = plt.plot(rand)

ani = animation.FuncAnimation(fig, make_noise_wave, interval=100)
plt.show()


import io
import os
import re

from setuptools import find_packages
from setuptools import setup

def read(filename):
    filename = os.path.join(os.path.dirname(__file__), filename)
    text_type = type(u"")
    with io.open(filename, mode="r", encoding="utf-8") as fd:
        return re.sub(text_type(r":[a-z]+:`~?(.*?)`"), text_type(r"``\1``"), fd.read())

requirements = [
    # use environment.yml
    "numpy"
]

setup(
    name="signal_propagation_plot",
    version="0.0.1",
    url="https://github.com/mehdidc/signal_propagation_plot",
    author="Mehdi Cherti",
    author_email="m.cherti@fz-juelich.de",
    description="Short description",
    long_description=read("README.md"),
    packages=find_packages(exclude=("tests",)),
    install_requires=requirements,
    classifiers=[
        "Programming Language :: Python",
        "Programming Language :: Python :: 3.8",
    ],
)


from airflow.hooks.postgres_hook import PostgresHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults

class LoadFactOperator(BaseOperator):
    """
    Defines loading class for fact tabels
    """

    ui_color = '#F98866'

    @apply_defaults
    def __init__(self,
                 redshift_conn_id="",
                 table="",
                 sql="",
                 *args, **kwargs):

        super(LoadFactOperator, self).__init__(*args, **kwargs)
        self.redshift_conn_id=redshift_conn_id
        self.table=table
        self.sql=sql

    def execute(self, context):
        """
        Connection to redshift cluster is performed and then
        data are transformed and loaded into fact table
        """

        redshift = PostgresHook(postgres_conn_id=self.redshift_conn_id)

        self.log.info("Transforming and loading data from staging tables to fact table")
        redshift.run(self.sql)


from __future__ import annotations
from copy import deepcopy
from exceptions import ConstraintViolationError
from typing import List

class SolverState(object):
    """
    This class resembles the scratch pad that a human would use when solving Sudoku. It tracks what is assigned in
    every cell, what is still possible for each cell, and what is still possible in each row and column. Most of the
    methods in here are related to manipulating that information and accessing it in different ways.

    This allows the algorithm to be separate from the data.
    """
    def __init__(self, board):
        self.board = board

        # This gives the remaining choices for cells in a row. When this is empty the row is solved.
        self.row_remaining = [[i for i in range(1, 10)] for _ in range(1, 10)]

        # This gives the remaining choices for cells in a column. When this is empty the column is solved.
        self.col_remaining = [[i for i in range(1, 10)] for _ in range(1, 10)]

        # cell_possibles[row][col] gives the possible values for each cell
        self.cell_possible = [[[i for i in range(1, 10)] for _ in range(1, 10)] for _ in range(1, 10)]

    def copy(self) -> SolverState:
        """ Returns a deep copy of this SolverState. Useful for undoing changes. """
        new_state = SolverState(deepcopy(self.board))
        new_state.cell_possible = deepcopy(self.cell_possible)
        new_state.row_remaining = deepcopy(self.row_remaining)
        new_state.col_remaining = deepcopy(self.col_remaining)
        return new_state

    def get_choices_for_cell(self, row: int, column: int) -> List[int]:
        """ Returns the list of remaining choices possible for the given cell """
        return self.cell_possible[row][column]

    def get_choices_for_cells_in_row(self, row) -> set[int]:
        """
        Returns the union of remaining possible choices for all the cells in the given row. This only includes
        unsolved cells because solved ones are no longer possible to assign.
        """
        choices = set()
        for col in range(9):
            # cells with one possibility are known
            if len(self.cell_possible[row][col]) != 1:
                choices.update(self.cell_possible[row][col])
        return choices

    def get_choices_for_cells_in_col(self, col: int) -> set[int]:
        """
        Returns the union of remaining possible choices for all the cells in the given column. This only includes
        unsolved cells because solved ones are no longer possible to assign.
        """
        # building a set explicitly is much faster than using itertools.chain(filter()))
        choices = set()
        for row in range(9):
            if len(self.cell_possible[row][col]) != 1:
                choices.update(self.cell_possible[row][col])
        return choices

    def get_unique_values_in_section(self, start_cell: (int, int), end_cell: (int, int)) -> list[int]:
        """
        Returns the unique values that are known within a given section, bounded by start_cell and end_cell.
        :param (int, int) start_cell: The upper left corner of the range (inclusive)
        :param (int, int) end_cell: The lower right corner of the range (exclusive)
        """
        # building a set explicitly is much faster than using itertools.chain(filter()))
        values_in_section = []
        for r in range(start_cell[0], end_cell[0]):
            for c in range(start_cell[1], end_cell[1]):
                if self.board[r][c] is not None:
                    values_in_section.append(self.board[r][c])
        return values_in_section

    def get_choices_for_row(self, row: int) -> list[int]:
        """ Returns the remaining possible choices for unsolved cells in the given row """
        return self.row_remaining[row]

    def get_choices_for_col(self, col: int) -> list[int]:
        """ Returns the remaining possible choices for unsolved cells in the given row """
        return self.col_remaining[col]

    def mark_impossible_in_cell(self, row, column, to_remove) -> bool:
        """
        Remove the given value from the list of remaining possibilities for the given cell
        :return bool True if anything was changed, False if the value was already impossible
        """
        if to_remove in self.cell_possible[row][column]:
            self.cell_possible[row][column].remove(to_remove)
            return True
        else:
            return False

    def mark_impossible_in_row(self, row: int, value: int):
        """ Remove the given value from the list of remaining possibilities for a given row """
        if value in self.row_remaining[row]:
            self.row_remaining[row].remove(value)

    def mark_impossible_in_col(self, col: int, value: int):
        """ Remove the given value from the list of remaining possibilities for a given row """
        if value in self.col_remaining[col]:
            self.col_remaining[col].remove(value)

    def set_choices_for_cell(self, row: int, column: int, value: list[int]):
        """ Sets the list of remaining possibilities for a given cell """
        self.cell_possible[row][column] = value

    def set_choices_for_row(self, row, new_choices: List[int]):
        """ Sets the list of remaining possibilities for a given row """
        self.row_remaining[row] = new_choices

    def set_choices_for_col(self, col, new_choices: List[int]):
        """ Sets the list of remaining possibilities for a given column """
        self.col_remaining[col] = new_choices

    def board_at(self, row: int, column: int) -> int:
        """ Returns the value of the given cell. Returns None if it does not have a known value. """
        return self.board[row][column]

    def is_cell_solved(self, row: int, column: int) -> bool:
        """ Returns whether the given cell has a known value """
        return self.board[row][column] is not None

    def get_row(self, row: int):
        """ Returns the given row of the board, including both solved and unsovled cells """
        return self.board[row]

    def get_column(self, col: int):
        """ Returns the given column of the board, including both solved and unsovled cells """
        return [v[col] for v in self.board]

    def update_board(self, row: int, column: int, value: int):
        """ Sets a value for the given cell in the board and updates the other possibility lists to match """
        self.board[row][column] = value

        # update all derived information
        self.set_choices_for_cell(row, column, [value])
        if value in self.row_remaining:
            self.row_remaining[row].remove(value)
        if value in self.col_remaining:
            self.col_remaining[column].remove(value)

    def is_solved(self) -> bool:
        """ Returns True if every cell in the board has a known value """
        for row in self.board:
            for cell in row:
                if not cell:
                    return False
        try:
            self.assert_still_valid()
        except ConstraintViolationError:
            return False

        return True

    def matches_expected(self, expected_solution) -> bool:
        """ Returns true if the current board matches the given expectation """
        matched = True
        for r in range(9):
            for c in range(9):
                if self.board[r][c] != expected_solution[r][c]:
                    return False
        return matched

    def assert_still_valid(self):
        """
        Ensures that all of the possiblity lists are consistent with each other. If they are not, it either
        indicates the solver has guessed an incorrect value (raises ConstraintViolationError) or there is a bug in the
        code (assertion failure)
        """
        # no two values in the same column:
        self._check_rows()

        # no two values in the same row:
        self._check_columns()

        # no two values in the same section
        for start_row in range(0, 7, 3):
            for start_col in range(0, 7, 3):
                self._check_section(start_row, start_row + 3, start_col, start_col + 3)

        self._assert_internal_consistency()

    def _check_rows(self):
        for r in range(9):
            row_values = set()
            for c in range(9):
                value = self.board[r][c]
                if value:
                    if value in row_values:
                        raise ConstraintViolationError(f"Cell {r, c} value {value} is duplicatedd in row")
                    row_values.add(value)

    def _check_columns(self):
        for c in range(9):
            col_values = set()
            for r in range(9):
                value = self.board[r][c]
                if value:
                    if self.board[r][c] in col_values:
                        raise ConstraintViolationError(f"Cell {r, c} is duplicated in column")
                    col_values.add(value)

    def _check_section(self, start_row, end_row, start_col, end_col):
        nums_in_section = set()
        for r in range(start_row, end_row):
            for c in range(start_col, end_col):
                if self.board[r][c] in nums_in_section:
                    raise ConstraintViolationError(
                        f"Duplicate value {self.board[r][c]} in section {(start_row, end_row), (start_col, end_col)}")
                if self.board[r][c] is not None:
                    nums_in_section.add(self.board[r][c])

    def _assert_internal_consistency(self):
        # any of these assertions that fail indicate a bug in the code
        for r in range(9):
            for c in range(9):
                assert(self.get_choices_for_cell(r, c) != 0)

                # if there are no more possibiliites, yet this cell has not been solved, there is a bug
                if len(self.get_choices_for_cell(r, c)) == 1:
                    assert(self.board[r][c] is not None)

                if self.board[r][c] is not None:
                    assert(self.board[r][c] not in self.row_remaining)
                    assert(self.board[r][c] not in self.col_remaining)
                    assert(self.board[r][c] == self.get_choices_for_cell(r, c)[0])
                    assert(len(self.get_choices_for_cell(r, c)) == 1)


import matplotlib.pyplot as plt
import numpy as np
from scipy.linalg import sqrtm

from problemSetting import *

def plotAction(initState, dyn_u):

    x = initState
    traj = []
    for u in dyn_u.reshape(-1,2):
        x = sys_A @ x + sys_B @ u
        traj.append(x)
    traj = np.array(traj)
    plt.plot(traj[:,0],traj[:,1],"-","x")

    N = 36
    for c in collisionList:
        plt.plot(c.x + np.sqrt(c.r2)*np.cos(np.arange(0,(2+1./N)*3.14,2*3.14/N)),
                 c.y + np.sqrt(c.r2)*np.sin(np.arange(0,(2+1./N)*3.14,2*3.14/N)))
    plt.show()

def drawEclips(A,b,c,ax = None):
    if(ax is None):
        ax = plt.gca()

    A,b,c = np.array(A)[:2,:2],np.array(b)[:2],np.array(c)
    sign = np.sign(A[0][0]) # assume A is either pos definate or neg definate
    A,b,c = sign * A, sign * b, sign * c
    A_inv = np.linalg.inv(A)
    c = c - 1/4 * b.T @ A_inv @ b
    b = - 1/2 * A_inv @ b
    R = sqrtm(A_inv)

    N = 36
    points = np.array([np.sqrt(-c) * R @ np.array([np.cos(th),np.sin(th)]) + b for th in np.arange(0,(2+1./N)*3.14,2*3.14/N)])
    ax.plot(points[:,0],points[:,1], "--", label = "CBF boundary")


from django.core.paginator import Paginator
from django.contrib.auth.decorators import login_required
from django.shortcuts import redirect, render, get_object_or_404
from .forms import PostForm, CommentForm
from .models import Post, Group, User, Follow
from django.views.decorators.cache import cache_page
from django.core.cache import cache
from django.conf import settings
from django.http import HttpResponseRedirect

@cache_page(20)
def index(request):
    posts = cache.get('index_page')
    if posts is None:
        posts = Post.objects.all()
        cache.set('index_page', posts, timeout=20)
    paginator = Paginator(posts, 10)
    page_number = request.GET.get('page')
    page = paginator.get_page(page_number)
    return render(request,
                  'posts/index.html',
                  {'page': page, 'post': posts})

def group_posts(request, slug):
    group = get_object_or_404(Group, slug=slug)
    post = group.posts.all()

    paginator = Paginator(post, 10)
    page_number = request.GET.get('page')
    page = paginator.get_page(page_number)
    return render(request,
                  'posts/group.html',
                  {'page': page, 'post': post, 'group': group})

def profile(request, username):
    author = get_object_or_404(User, username=username)
    posts = author.posts.all()

    paginator = Paginator(posts, 10)
    page_num = request.GET.get('page')
    page = paginator.get_page(page_num)

    following = False
    if (not request.user.is_anonymous
        and Follow.objects.filter(
            user=request.user, author=author).exists()):
        following = True

    return render(request,
                  'posts/profile.html', {
                      'author': author,
                      'page': page,
                      'following': following})

def post_view(request, username, post_id):
    author = get_object_or_404(User, username=username)
    post = author.posts.get(pk=post_id)
    form = CommentForm()
    following = False
    if (not
        request.user.is_anonymous and Follow.objects.filter(
            user=request.user, author=author).exists()):
        following = True

    return render(request,
                  'posts/post.html', {
                      'author': author,
                      'post': post,
                      'form': form,
                      'following': following})

@login_required
def new_post(request):
    form = PostForm(request.POST, files=request.FILES or None)
    if form.is_valid():
        post = form.save(commit=False)
        post.author = request.user
        post.save()
        return redirect('/')
    return render(request, 'posts/new_post.html', {'form': form})

@login_required
def post_edit(request, username, post_id):
    post = get_object_or_404(Post, author__username=username, pk=post_id)
    author = post.author
    if request.user != author:
        return redirect("post", username=username, post_id=post_id)
    form = PostForm(
        request.POST or None, files=request.FILES or None, instance=post)
    if form.is_valid():
        post = form.save(commit=False)
        post.save()
        return redirect("post", username=username, post_id=post_id)
    return render(request,
                  'posts/new_post.html',
                  {'form': form,
                   'post': post,
                   'is_edit': True})

def page_not_found(request, exception):
    return render(
        request,
        "misc/404.html",
        {"path": request.path},
        status=404
    )

def server_error(request):
    return render(request, "misc/500.html", status=500)

@login_required
def add_comment(request, username, post_id):
    post = get_object_or_404(Post, pk=post_id)
    form = CommentForm(request.POST or None)
    if form.is_valid():
        comment = form.save(commit=False)
        comment.post = post
        comment.author = request.user
        comment.save()
        return redirect("post", username=username, post_id=post_id)
    return render(
        request, 'includes/comments.html', {
            'form': form,
            'username': request.user,
            'post_id': post_id})

@login_required
def follow_index(request):
    posts = Post.objects.filter(author__following__user=request.user)

    paginator = Paginator(posts, 10)
    page_num = request.GET.get('page')
    page = paginator.get_page(page_num)

    return render(request, "posts/follow.html", {'page': page})

@login_required
def profile_follow(request, username):
    author = get_object_or_404(User, username=username)
    if request.user != author and (
        not Follow.objects.filter(
            user=request.user, author=author).exists()):
        Follow.objects.create(user=request.user, author=author)
    return redirect('profile', username)

@login_required
def profile_unfollow(request, username):
    author = get_object_or_404(User, username=username)
    post_delete = Follow.objects.get(user=request.user, author=author)
    post_delete.delete()
    return redirect('profile', username)

def set_language(request):
    lang = request.GET.get('l', 'en')
    request.session[settings.LANGUAGE_SESSION_KEY] = lang
    response = HttpResponseRedirect(request.META.get('HTTP_REFERER', '/'))
    response.set_cookie(settings.LANGUAGE_COOKIE_NAME, lang)
    return response


import numpy as np, pandas as pd
import patsy
from drdid import drdid, reg_did

from csdid.utils.bmisc import panel2cs2

fml = patsy.dmatrices

# data Struct
# output = {
#   "att" : []
#   'group': []
#   'year': []
#   'post ': []
# }

def compute_att_gt(dp, est_method = "dr", base_period = 'varying'):
  yname = dp['yname']
  tname = dp['tname']
  idname = dp['idname']
  xformla = dp['xformla']
  data = dp['data']
  weights_name = dp['weights_name']
  # base_period = dp['base_period']
  panel = dp['panel']
  true_rep_cross_section = dp['true_rep_cross_section']
  control_group = dp['control_group']
  anticipation = dp['anticipation']
  gname = dp['gname']
  n = dp['n']
  nT = dp['nT']
  nG = dp['nG']
  tlist = dp['tlist']
  glist = dp['glist']
  tlist_len, tfac = len(tlist), 0
  if base_period != 'universal':
    tlist_len = tlist_len - 1
    tfac = 1

  inf_func = []

  att_est, group, year, post_array = [], [], [], []

  def add_att_data(att = 0, pst = 0, inf_f = []):
    inf_func.append(inf_f)
    att_est.append(att)
    group.append(g)
    year.append(tn)
    post_array.append(pst)

  never_treated = control_group == 'nevertreated'
  if never_treated:
    data = data.assign(C = 1 * (data[gname] == 0))
  data = data.assign(y_main = data[yname])

  # g, t = glist[0], tlist[0]

  for _, g, in enumerate(glist):
    # g = glist[1]
    G_main = (data[gname] == g)
    data = data.assign(G_m = 1 * G_main)

    for t_i in range(tlist_len):
      pret = t_i
      tn = tlist[t_i + tfac]
      if base_period == 'universal' or g < tn:
        try:
          pret = np.where(tlist + anticipation < g)[0][-1]
        except:
          raise f"There are no pre-treatment periods for the group first treated at {g}\nUnits from this group are dropped"
          # break

      if base_period == 'universal':
        if pret == tn:
          add_att_data()

      if not never_treated:
        n1 = data[gname] == 0
        n2 = (data[gname] > (tlist[np.max([t_i, pret]) + tfac]) + anticipation)
        n3 = np.where(data[gname] != glist[g], True, False)
        row_eval = n1 | n2 & n3
        data = data.assign(C = 1 * row_eval)

      post_treat = 1 * (g <= tn)
      disdat = data[(data[tname] == tn) | (data[tname] == tlist[pret])]

      if panel:
        disdat = panel2cs2(disdat, yname, idname, tname)
        disdat = disdat.dropna()
        n = len(disdat)
        dis_idx = np.array(disdat.G_m == 1) | np.array(disdat.C == 1)
        disdat = disdat.loc[dis_idx, :]
        n1 = len(disdat)
        G = disdat.G_m
        C = disdat.C
        w = disdat.w

        ypre = disdat.y0 if tn > pret else disdat.y1
        ypost = disdat.y0 if tn < pret else disdat.y1
        _, covariates = fml(xformla, data = disdat, return_type = 'dataframe')

        G, C, w, ypre = map(np.array, [G, C, w, ypre])
        ypost, covariates = map(np.array, [ypost, covariates])

        if callable(est_method):
          est_att_f = est_method
        elif est_method == "reg":
          est_att_f = reg_did.reg_did_panel
        elif est_method == "dr":
          est_att_f = drdid.drdid_panel

        att_gt, att_inf_func = est_att_f(ypost, ypre, G, i_weights=w, covariates=covariates)

        inf_zeros = np.zeros(n)
        att_inf = n / n1 * att_inf_func
        inf_zeros[dis_idx] = att_inf

        add_att_data(att_gt, inf_f=inf_zeros)

      if not panel:
        right_ids = np.array(disdat.query('(G_m == 1) or (C == 1)').rowid.to_numpy())
        dis_idx = (data['rowid'].isin(right_ids)) &\
          ((data[tname] == tlist[t_i + tfac]) |\
            (data[tname] == tlist[pret]))

        disdat = data.loc[dis_idx]

        G = disdat.G_m.to_numpy()
        C = disdat.C.to_numpy()
        Y = disdat[yname].to_numpy()
        post = 1 * (disdat[tname] == tlist[t_i + tfac]).to_numpy()
        w = disdat.w.to_numpy()

        # G, C, Y, post, w = map(np.array, [G, C, Y, post, w])

        n1 = sum(G + C)

        skip_this_att_gt = False
        if np.sum(G * post) == 0:
          print(f"No units in group {g} in time period {tn}")
          skip_this_att_gt = True

        if np.sum(G * (1 - post)) == 0:
          print(f"No units in group {g} in time period {t}")
          skip_this_att_gt = True

        if np.sum(C * post) == 0:
          print(f"No available control units for group {g} in time period {tn}")
          skip_this_att_gt = True

        if np.sum(C * (1 - post)) == 0:
          print(f"No available control units for group {g} in time period {t}")
          skip_this_att_gt = True

        if skip_this_att_gt:
          add_att_data()

        try:
          _, covariates = fml(xformla, data = disdat, return_type = 'dataframe')
          covariates = np.array(covariates)
        except:
          y_str, x_str = xformla.split("~")
          xs1 = x_str.split('+')
          xs1_col_names = [x.strip() for x in xs1 if x.strip() != '1']
          n_dis = len(disdat)
          ones = np.ones((n_dis, 1))
          try:
            covariates = disdat[xs1_col_names].to_numpy()
            covariates = np.append(covariates, ones, axis=1)
          except:
            covariates = ones

        if callable(est_method):
          est_att_f = est_method
        elif est_method == "reg":
          est_att_f = reg_did.reg_did_rc
        elif est_method == "dr":
          est_att_f = drdid.drdid_rc
        att_gt, att_inf_func = est_att_f(y=Y, post=post, D = G, i_weights=w, covariates=covariates)

        inf_func_df = pd.DataFrame(
          {
            "inf_func": att_inf_func,
            "right_ids": right_ids
          }
        )
        inf_zeros = np.zeros(n)
        aggte_infffuc = inf_func_df.groupby('right_ids').inf_func.sum()
        try:
          dis_idx1 = np.isin(data['rowid'].unique(), aggte_infffuc.index.to_numpy())
        except:
          dis_idx1 = np.isin(data['rowid'].unique().to_numpy(), aggte_infffuc.index.to_numpy())
        inf_zeros[dis_idx1] = np.array(aggte_infffuc)

        add_att_data(att_gt, pst = post_treat, inf_f=inf_zeros)
        # print(att_est)

  output = {
    'group': group ,
    'year': year,
    "att" : att_est,
    'post ': post_array
  }
  return (output, np.array(inf_func))