"""

GOL: Graphics in One Line

To make matplotlib plots in a single command line

(I am tired of polluting my code with matplotlib instructions)

"""

__author__ = "Manuel Sanchez del Rio"
__contact__ = "srio@esrf.eu"
__copyright = "ESRF, 2016"

import numpy as np
try:
    import matplotlib.pylab as plt
except:
    raise ImportError("Please install matplotlib to allow graphics")

def set_qt():
    try:
        plt.switch_backend("Qt5Agg")
    except:
        raise Exception("Failed to set matplotlib backend to Qt5Agg")

def plot_show():
    plt.show()

def plot_image(*positional_parameters,title="TITLE",xtitle=r"X",ytitle=r"Y",
               xrange=None, yrange=None,
               cmap=None,aspect=None,show=1,
               add_colorbar=True,figsize=None):

    n_arguments = len(positional_parameters)
    if n_arguments == 1:
        z = positional_parameters[0]
        x = np.arange(0,z.shape[0])
        y = np.arange(0,z.shape[1])
    elif n_arguments == 2:
        z = positional_parameters[0]
        x = positional_parameters[1]
        y = positional_parameters[1]
    elif n_arguments == 3:
        z = positional_parameters[0]
        x = positional_parameters[1]
        y = positional_parameters[2]
    else:
        raise Exception("Bad number of inputs")

    fig = plt.figure(figsize=figsize)

    # cmap = plt.cm.Greys
    plt.imshow(z.T,origin='lower',extent=[x[0],x[-1],y[0],y[-1]],cmap=cmap,aspect=aspect)
    if add_colorbar:
        plt.colorbar()
    ax = fig.gca()
    ax.set_xlabel(xtitle)
    ax.set_ylabel(ytitle)

    plt.title(title)

    plt.xlim( xrange )
    plt.ylim( yrange )

    if show:
        plt.show()

    return fig,ax

def plot_image_with_histograms(*positional_parameters,
            title="",xtitle=r"X",ytitle=r"Y",
            xrange=None, yrange=None,
            cmap=None,aspect_ratio=None,show=True,
            add_colorbar=False,figsize=(8,8)
            ):

    n_arguments = len(positional_parameters)
    if n_arguments == 1:
        z = positional_parameters[0]
        x = np.arange(0,z.shape[0])
        y = np.arange(0,z.shape[1])
    elif n_arguments == 2:
        z = positional_parameters[0]
        x = positional_parameters[1]
        y = positional_parameters[1]
    elif n_arguments == 3:
        z = positional_parameters[0]
        x = positional_parameters[1]
        y = positional_parameters[2]
    else:
        raise Exception("Bad number of inputs")

    if xrange is None:
        xrange = [x.min(),x.max()]

    if yrange is None:
        yrange = [y.min(),y.max()]

    figure = plt.figure(figsize=figsize)

    hfactor = 1.0
    vfactor = 1.0

    left, width = 0.1, 0.6
    bottom, height = 0.1, 0.6
    bottom_h = left_h = left + width + 0.02
    rect_scatter = [left, bottom, width, height]
    rect_histx = [left, bottom_h, width, 0.2]
    rect_histy = [left_h, bottom, 0.2, height]

    #
    #main plot
    #
    axScatter = figure.add_axes(rect_scatter)

    axScatter.set_xlabel(xtitle)
    axScatter.set_ylabel(ytitle)

    axScatter.axis(xmin=hfactor*xrange[0],xmax=xrange[1])
    axScatter.axis(ymin=vfactor*yrange[0],ymax=yrange[1])

    if aspect_ratio is not None:
        axScatter.set_aspect(aspect_ratio)

    axs = axScatter.pcolormesh(x,y,z,cmap=cmap)

    #
    #histograms
    #
    axHistx = figure.add_axes(rect_histx, sharex=axScatter)
    axHisty = figure.add_axes(rect_histy, sharey=axScatter)

    hx = z.sum(axis=0)
    hy = z.sum(axis=1)
    axHistx.plot(x,hx)
    axHisty.plot(hy,y)

    # tt = np.where(hx >= hx.max() * 0.5)
    # if hx[tt].size > 1:
    #     binSize = x[1] - x[0]
    #     print("FWHM X: ",binSize * (tt[0][-1] - tt[0][0]))
    #
    #
    # tt = np.where(hy >= hy.max() * 0.5)
    # if hx[tt].size > 1:
    #     binSize = y[1] - y[0]
    #     print("FWHM Y: ",binSize * (tt[0][-1] - tt[0][0]))

    # supress ordinates labels ans ticks
    axHistx.get_yaxis().set_visible(False)
    axHisty.get_xaxis().set_visible(False)

    # supress abscissas labels (keep ticks)
    for tl in axHistx.get_xticklabels(): tl.set_visible(False)
    for tl in axHisty.get_yticklabels(): tl.set_visible(False)

    if title != "":
        axHistx.set_title(title)

    if add_colorbar:
        plt.colorbar(axs)

    if show:
        plt.show()

    return figure #,ax

def plot(*positional_parameters,title="",xtitle="",ytitle="",
         xrange=None,yrange=None,show=1,legend=None,legend_position=None,color=None,marker=None,linestyle=None,
         xlog=False,ylog=False,figsize=None):

    if isinstance(positional_parameters,tuple):
        if len(positional_parameters) == 1: # in the cvase that input is a tuple with all curves
            positional_parameters = positional_parameters[0]

    n_arguments = len(positional_parameters)
    if n_arguments == 0:
        return

    fig = plt.figure(figsize=figsize)
    if n_arguments == 1:
        y = positional_parameters[0]
        x = np.arange(y.size)
        if linestyle == None:
            linestyle = '-'
        plt.plot(x,y,label=legend,marker=marker,color=color,linestyle=linestyle)
    elif n_arguments == 2:
        x = positional_parameters[0]
        y = positional_parameters[1]
        if linestyle == None:
            linestyle = '-'
        plt.plot(x, y, label=legend, color=color, marker=marker, linestyle=linestyle)
    elif n_arguments % 2 == 0:
        if legend is None:
            legend = [None] * (n_arguments // 2)
        if color is None:
            color = [None] * (n_arguments // 2)

        if marker is None:
            marker = [None] * (n_arguments // 2)

        if linestyle is None:
            linestyle = ['-'] * (n_arguments // 2)

        for i in range(n_arguments // 2):
            plt.plot(positional_parameters[2*i],positional_parameters[2*i+1],label=legend[i],marker=marker[i],linestyle=linestyle[i],color=color[i])
    else:
        raise Exception("Incorrect number of arguments: found an odd number of data sets")
        # x = positional_parameters[0]
        # y = positional_parameters[1]
        # plt.plot(x,y,label=legend)

    ax = plt.subplot(111)
    if legend is not None:
        ax.legend(bbox_to_anchor=legend_position)

    if xlog:
        ax.set_xscale("log")

    if ylog:
        ax.set_yscale("log")

    plt.xlim( xrange )
    plt.ylim( yrange )

    plt.title(title)
    plt.xlabel(xtitle)
    plt.ylabel(ytitle)

    if show:
        plt.show()

    return fig,ax

def plot_table(*positional_parameters,errorbars=None,xrange=None,yrange=None,
               title="",xtitle="",ytitle="",show=1,
               legend=None,legend_position=None,color=None,
               xlog=False,ylog=False,figsize=None):

    n_arguments = len(positional_parameters)
    if n_arguments == 0:
        return

    fig = plt.figure(figsize=figsize)

    if n_arguments == 1:
        y = positional_parameters[0]
        x = np.arange(y.size)
        plt.plot(x,y,label=legend)
    elif n_arguments == 2:
        x = positional_parameters[0]
        y = positional_parameters[1]

        if len(y.shape) == 1:
            y = np.reshape(y,(1,y.size))
            if isinstance(legend,str):
                legend = [legend]
            if isinstance(color,str):
                color = [color]

        for i in range(y.shape[0]):
            if legend is None:
                ilegend = None
            else:
                ilegend = legend[i]

            if color is None:
                icolor = None
            else:
                icolor = color[i]

            if errorbars is None:
                plt.plot(x,y[i],label=ilegend,color=icolor)
            else:
                plt.errorbar(x,y[i],yerr=errorbars[i],label=ilegend,color=icolor)
    else:
        raise Exception("Incorrect number of arguments")

    ax = plt.subplot(111)

    if xlog:
        ax.set_xscale("log")

    if ylog:
        ax.set_yscale("log")

    if legend is not None:
        if legend_position is None:
            legend_position = (1.1, 1.05)
        ax.legend(bbox_to_anchor=legend_position)

    plt.xlim( xrange )
    plt.ylim( yrange )

    plt.title(title)
    plt.xlabel(xtitle)
    plt.ylabel(ytitle)

    if show:
        plt.show()

    return fig,ax
def four_plots(x1,y1,x2,y2,x3,y3,x4,y4,title="",xtitle="",ytitle="",xrange=None,yrange=None,show=True):
    """
    Creates four plots in a window

    :param x1: abscissas for plot 1
    :param y1: ordinates for plot 1
    :param x2: abscissas for plot 2
    :param y2: ordinates for plot 2
    :param x3: abscissas for plot 3
    :param y3: ordinates for plot 3
    :param x4: abscissas for plot 4
    :param y4: ordinates for plot 4
    :param title: a string or list of 4 strings with title
    :param xtitle: a string or list of 4 strings with title for X
    :param ytitle: a string or list of 4 strings with title for Y
    :param xrange: the X range for all plots
    :param yrange: the Y range for all plots
    :param show:
    :return:
    """

    if isinstance(title,list):
        Title = title
    else:
        Title = [title,title,title,title]

    if isinstance(xtitle,list):
        Xtitle = xtitle
    else:
        Xtitle = [xtitle,xtitle,xtitle,xtitle]

    if isinstance(ytitle,list):
        Ytitle = ytitle
    else:
        Ytitle = [ytitle,ytitle,ytitle,ytitle]

    # Create subplots.
    f, ((ax00, ax01), (ax10, ax11)) = plt.subplots(2, 2, sharex="all", sharey="all")

    ax00.plot(x1,y1, "-")
    ax00.set_title(  Title[0])
    ax00.set_xlabel(Xtitle[0])
    ax00.set_ylabel(Ytitle[0])
    ax00.set_xlim(xrange)
    ax00.set_ylim(yrange)

    ax01.plot(x2,y2, "-")
    ax01.set_title(  Title[1])
    ax01.set_xlabel(Xtitle[1])
    ax01.set_ylabel(Ytitle[1])
    ax01.set_xlim(xrange)
    ax01.set_ylim(yrange)

    ax10.plot(x3,y3, "-")
    ax10.set_title(  Title[2])
    ax10.set_xlabel(Xtitle[2])
    ax10.set_ylabel(Ytitle[2])
    ax10.set_xlim(xrange)
    ax10.set_ylim(yrange)

    ax11.plot(x4,y4, "-")
    ax11.set_title(  Title[3])
    ax11.set_xlabel(Xtitle[3])
    ax11.set_ylabel(Ytitle[3])
    ax11.set_xlim(xrange)
    ax11.set_ylim(yrange)

    if show: plt.show()

    return f,ax00,ax01,ax10,ax11

def plot_surface(mymode,theta,psi,title="TITLE",xtitle="",ytitle="",ztitle="",legend=None,cmap=None,
                 figsize=None,show=1):

    from matplotlib import cm
    from matplotlib.ticker import LinearLocator, FormatStrFormatter
    from mpl_toolkits.mplot3d import Axes3D

    ftheta, fpsi = np.meshgrid(theta, psi)
    fig = plt.figure(figsize=figsize)
    ax = fig.gca(projection='3d')

    II0 = mymode.T

    if cmap == None:
        cmap = cm.coolwarm

    print(II0.shape,ftheta.shape,fpsi.shape)
    surf = ax.plot_surface(ftheta, fpsi, II0, rstride=1, cstride=1, cmap=cmap,
                           linewidth=0, antialiased=False)

    ax.set_zlim(II0.min(),II0.max())
    ax.zaxis.set_major_locator(LinearLocator(10))
    ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))
    fig.colorbar(surf, shrink=0.5, aspect=5)
    plt.title(title)
    ax.set_xlabel(xtitle)
    ax.set_ylabel(ytitle)
    ax.set_zlabel(ztitle)

    if show:
        plt.show()

    return fig,ax

def plot_scatter(x,y,show=1,nbins=100,xrange=None,yrange=None,plot_histograms=True,title="",xtitle="",ytitle=""):
    """

    makes a scatter plot with histograms

    :param x: x data array
    :param y: y data arrayif False the plot is not shown (use  not show
    :param show: if False the plot is not shown (use  plot_show() later on)
    :param nbins: number of bins for plots
    :param xrange: [xmin,xmax] range for abscissas
    :param yrange: [ymin,ymax] range for ordinates
    :param plot_histograms: Flag to plot:
            False or 0: plot no histograms
            True or 1: plot both histograms
            2: plot histograms vs abscissas only
            3: plot histogram vs ordinates only
    :param title: string with a title
    :param xtitle: string with abscissas label
    :param ytitle: string with ordinates label
    :return: the matplotlib objects with the elements created
    """

    from matplotlib.ticker import NullFormatter

    # the random data

    nullfmt   = NullFormatter()         # no labels

    # definitions for the axes

    if plot_histograms:
        left, width    = 0.1, 0.65
        bottom, height = 0.1, 0.65
        bottom_h = left_h = left+width+0.02
        rect_scatter = [left, bottom, width, height]
        rect_histx   = [left, bottom_h, width, 0.2]
        rect_histy   = [left_h, bottom, 0.2, height]
    else:
        left, width    = 0.1, 0.8
        bottom, height = 0.1, 0.8
        rect_scatter = [left, bottom, width, height]

    # start with a rectangular Figure
    fig = plt.figure(figsize=(8,8))

    axScatter = plt.axes(rect_scatter)
    if plot_histograms:
        if plot_histograms == 2:
            axHistx = plt.axes(rect_histx)
            axHistx.xaxis.set_major_formatter(nullfmt)
        elif plot_histograms == 3:
            axHisty = plt.axes(rect_histy)
            axHisty.yaxis.set_major_formatter(nullfmt)
        else:
            axHistx = plt.axes(rect_histx)
            axHisty = plt.axes(rect_histy)
            # no labels
            axHistx.xaxis.set_major_formatter(nullfmt)
            axHisty.yaxis.set_major_formatter(nullfmt)

    # now determine nice limits by hand:
    binwidth = np.array([x.max() - x.min(), y.max() - y.min()]).max() / nbins

    if xrange == None:
        xrange = np.array((x.min(), x.max()))
    if yrange == None:
        yrange = np.array((y.min(), y.max()))

    # the scatter plot:
    axScatter.scatter(x, y, marker='.', edgecolor='b', s=.1)

    axScatter.set_xlabel(xtitle)
    axScatter.set_ylabel(ytitle)
    if not plot_histograms:
        axScatter.set_title(title)

    axScatter.set_xlim( xrange )
    axScatter.set_ylim( yrange )

    if plot_histograms:
        bins_x = np.arange(xrange[0], xrange[1] + binwidth, binwidth)
        if plot_histograms == 2:
            axHistx.hist(x, bins=nbins)
            axHistx.set_xlim( axScatter.get_xlim() )
            axHistx.set_title(title)
        elif plot_histograms == 3:
            axHisty.hist(y, bins=nbins, orientation='horizontal')
            axHisty.set_ylim( axScatter.get_ylim() )
        else:
            axHistx.hist(x, bins=nbins, range=xrange)
            axHisty.hist(y, bins=nbins, range=yrange, orientation='horizontal')

            axHistx.set_xlim( axScatter.get_xlim() )

            axHistx.set_title(title)
            axHisty.set_ylim( axScatter.get_ylim() )

    if show: plt.show()

    if plot_histograms:
        if plot_histograms == 2:
            return fig,axScatter,axHistx
        elif plot_histograms == 3:
            return fig,axScatter,axHisty
        else:
            return fig,axScatter,axHistx,axHisty
    else:
        return fig,axScatter

def plot_contour(z,x,y,title="TITLE",xtitle="",ytitle="",xrange=None,yrange=None,plot_points=0,contour_levels=20,
                 cmap=None,cbar=True,fill=False,cbar_title="",figsize=None,show=1):

    fig = plt.figure(figsize=figsize)

    if fill:
        fig = plt.contourf(x, y, z.T, contour_levels, cmap=cmap, origin='lower')
    else:
        fig = plt.contour( x, y, z.T, contour_levels, cmap=cmap, origin='lower')

    if cbar:
        cbar = plt.colorbar(fig)
        cbar.ax.set_ylabel(cbar_title)

    plt.title(title)
    plt.xlabel(xtitle)
    plt.ylabel(ytitle)

    # the scatter plot:
    if plot_points:
        axScatter = plt.subplot(111)
        axScatter.scatter( np.outer(x,np.ones_like(y)), np.outer(np.ones_like(x),y))

    # set axes range
    plt.xlim(xrange)
    plt.ylim(yrange)

    if show:
        plt.show()

    return fig
#
# examples
#

def example_plot_image():
    x = np.linspace(-4, 4, 90)
    y = np.linspace(-4, 4, 90)
    print('Size %d pixels' % (len(x) * len(y)))
    z = np.sqrt(x[np.newaxis, :]**2 + y[:, np.newaxis]**2)
    plot_image(z,x,y,title="example_plot_image",xtitle=r"X [$\mu m$]",ytitle=r"Y [$\mu m$]",cmap=None,show=1)

def example_plot_image_with_histograms():
    x = np.linspace(-4, 4, 200)
    y = np.linspace(-4, 4, 90)
    print('Size %d pixels' % (len(x) * len(y)))
    z = -np.sqrt(x[np.newaxis, :]**2 + y[:, np.newaxis]**2)
    plot_image_with_histograms(z,x,y,title="example_plot_image",xtitle=r"X [$\mu m$]",ytitle=r"Y [$\mu m$]",
                               cmap=None,show=1,figsize=(8,8),add_colorbar=True)

def example_plot_surface():
    x = np.linspace(-4, 4, 20)
    y = np.linspace(-4, 4, 20)
    print('Size %d pixels' % (len(x) * len(y)))
    z = np.sqrt(x[np.newaxis, :]**2 + y[:, np.newaxis]**2)
    plot_surface(z,x,y,title="example_plot_surface",xtitle=r"X [$\mu m$]",ytitle=r"Y [$\mu m$]",cmap=None,show=1)

def example_plot_scatter():
    #example motivated by http://www.ster.kuleuven.be/~pieterd/python/html/core/scipystats.html
    from scipy import stats
    # x = np.random.rand(1000)
    # y = np.random.rand(1000)
    x = stats.norm.rvs(size=2000)
    y = stats.norm.rvs(scale=0.5, size=2000)
    data = np.vstack([x+y, x-y])
    f = plot_scatter(data[0],data[1],xrange=[-10,10],title="example_plot_scatter",
                     xtitle=r"X [$\mu m$]",ytitle=r"Y [$\mu m$]",plot_histograms=2,show=0)
    f[1].plot(data[0],data[0]) # use directly matplotlib to overplot
    plot_show()

def example_plot_contour():
    # deprecated in matplotlib. Copied from" https://github.com/matplotlib/matplotlib/blob/81e8154dbba54ac1607b21b22984cabf7a6598fa/lib/matplotlib/mlab.py#L1866
    def bivariate_normal(X, Y, sigmax=1.0, sigmay=1.0,
                         mux=0.0, muy=0.0, sigmaxy=0.0):
        """
        Bivariate Gaussian distribution for equal shape *X*, *Y*.
        See `bivariate normal
        <http://mathworld.wolfram.com/BivariateNormalDistribution.html>`_
        at mathworld.
        """
        Xmu = X - mux
        Ymu = Y - muy

        rho = sigmaxy / (sigmax * sigmay)
        z = Xmu ** 2 / sigmax ** 2 + Ymu ** 2 / sigmay ** 2 - 2 * rho * Xmu * Ymu / (sigmax * sigmay)
        denom = 2 * np.pi * sigmax * sigmay * np.sqrt(1 - rho ** 2)
        return np.exp(-z / (2 * (1 - rho ** 2))) / denom

    # inspired by http://stackoverflow.com/questions/10291221/axis-limits-for-scatter-plot-not-holding-in-matplotlib
    # random data
    x = np.random.randn(50)
    y = np.random.randn(100)

    X, Y = np.meshgrid(y, x)
    Z1 = bivariate_normal(X, Y, 1.0, 1.0, 0.0, 0.0)
    Z2 = bivariate_normal(X, Y, 1.5, 0.5, 1, 1)
    Z = 10 * (Z1 - Z2)

    plot_contour(Z,x,y,title='example_plot_contour',xtitle='x-stuff',ytitle='y-stuff',plot_points=1,show=1)

def example_plot_one_curve():
    x = np.linspace(-100,100,10)
    y = x**2
    plot(x,y,xtitle=r'$x$',title="example_plot_one_curve",
         ytitle=r'$y=f(x)=x^2$',legend="Example 1",color='pink',marker='o',linestyle=None,
         figsize=(4,8),show=1)

def example_plot_one_curve_log():
    x = np.linspace(-100,100,10)
    y = x**2
    plot(x,y,xtitle=r'$x$',title="example_plot_one_curve",
         ytitle=r'$y=f(x)=x^2$',legend="Example 1",color='pink',marker='o',linestyle=None,xlog=1,ylog=1,show=1)

def example_plot_two_curves():
    x1 = np.linspace(-100,100,1000)
    y1 = x1**2
    x2 = np.linspace(0,200,700)
    y2 = x2**2.1
    plot(x1,y1,x2,y2,xtitle=r'$x$',title="example_plot_two_curves",
         ytitle=r'$y=f(x)$',legend=[r"$x^2$",r"$x^{2.1}$"],color=['green','blue'],marker=[' ','o'],linestyle=['-',' '],show=1)

def example_plot_table():
    x1 = np.linspace(0,100,100)
    out = np.zeros((6,x1.size))
    out[0,:] = x1**2
    out[1,:] = x1**2.1
    out[2,:] = x1**2.2
    out[3,:] = x1**2.3
    out[4,:] = x1**2.4
    out[5,:] = x1**2.5
    # another way
    # out = np.vstack( (
    #     x1**2,
    #     x1**2.1,
    #     x1**2.2,
    #     x1**2.3,
    #     x1**2.4,
    #     x1**2.5 ))
    legend=np.arange(out.shape[0]).astype("str")
    plot_table(x1,out,xtitle=r'$x$',ytitle=r'$y=f(x)$',title="example_plot_table",legend=legend,show=1)

def example_plot_table_one_curve():
    x1 = np.linspace(-100,100,1000)
    out = x1**2
    plot_table(x1,out,title="example_plot_table_one_curve",xtitle=r'$x$',ytitle=r'$y=f(x)$',legend="Example 1",color='pink',show=1)

def example_plot_table_with_errorbars():
    x = np.linspace(0,100,30)
    out = np.zeros((2,x.size))
    out[0,:] = 1e-3 * x**2
    out[1,:] = 5 + 1e-3 * x**2
    yerr = np.sqrt(out)
    yerr[1,:] = 1.0
    plot_table(x,out,errorbars=yerr,title="example_plot_table_with_errorbars",xtitle=r'$x$',ytitle=r'$y=f(x)=x^2$',xrange=[20,80],
               legend=["Statistical error","Constant error"],color=['black','magenta'],show=1)

def example_plot_image_ascent():
    from scipy.misc import ascent

    ascent = np.rot90(ascent(),-1)
    plot_image(ascent,np.arange(0,ascent.shape[0]),np.arange(0,ascent.shape[1]),cmap='gray' )
#
# main
#
if __name__ == "__main__":
    pass
    # example_plot_one_curve()
    # example_plot_two_curves()
    # example_plot_one_curve_log()
    # example_plot_table()
    # example_plot_table_one_curve()
    # example_plot_table_with_errorbars()
    # example_plot_image()
    # example_plot_image_with_histograms()
    # example_plot_surface()
    # example_plot_contour()
    # example_plot_scatter()
    # example_plot_image_ascent()


# coding: utf-8

#
Table of Contents

#

1  Use pyresample to plot channel 30 Tbright

2  Copy your working files into the generic files

3  Calculate chan 30 brightness temperature

4  resample the brightness temperatures

5  replace missing values with floating point nan

6  Plot the image using cartopy

# # Use pyresample to plot channel 30 Tbright
#
# This notebook uses a MYD03 file and a modis_chans.hdf file to resample the channel 30 radiance
# from your granule onto a laea projection.
#
# I've deleted two cells below (sections 5 and 6).  Using the cartopy_resample_ch30.ipynb and
# the assign4_solution.ipynb notebooks as guides, fill those cells in with the code that plots the channel 30 brightness temperature for your granule.
#
#

# In[ ]:

import a301
import json
from a301.utils.data_read import download
import a301
import pprint
import shutil
from pyhdf.SD import SD, SDC
import json
from pyresample import kd_tree
from a301.scripts.modismeta_read import parseMeta

# In[ ]:

import cartopy.crs as ccrs
import matplotlib.pyplot as plt
import cartopy
from pathlib import Path
import pprint
import numpy as np
import pdb
import shutil
import sys
#

# First run the modis_multichannel.ipynb notebook for your granule and get a
# calibrated ch30,ch31 radiance file out.  Copy tha file into your a301_code/data directory.
#
#

# # Copy your working files into the generic files
#
# I've wrapped the cell below in a "try/except" block so that you can run it to
# copy your own files into the generic files, but the notebook will also run on
# my computer, which will only have my own generic files to test with.
#
# By catching the general exception called Exception, I make this cell succeed
# regardless of error.
#
#

# In[ ]:

try:
    #
    # this cell copies your files to the following generic files if they don't exist
    #
    generic_rad = a301.data_dir / Path('rad_file_2018_10_1.hdf')
    generic_m3 = a301.data_dir / Path('m3_file_2018_10_1.hdf')
    #
    # put your MYD03 file and you modis radiance file names here instead of my files
    #
    m3_file = a301.data_dir / Path('MYD03.A2013222.2105.006.2013223155808.hdf')
    rad_file = a301.data_dir / Path('modis_chans_2018_9_24.hdf')
    #
    # do the copies  (could also do this by hand)
    #
    shutil.copy(m3_file,generic_m3)
    shutil.copy(rad_file,generic_rad)
    #
    # test to make sure we have the right files
    #
    rad_file = SD(str(generic_rad), SDC.READ)
    rad_filename = rad_file.filename
    rad_file.end()
    print(f"\nworking with radiance file {generic_rad}\n"
          f"with original data {rad_filename}\n")
    m3_metadata=parseMeta(generic_m3)
    print(f"\nworking with m3_file {generic_m3} \n"
          f"with original data {m3_metadata['filename']}")
except Exception as ex:
    ex_type, ex_val, tb = sys.exc_info()
    print(f'caught {ex_type}, {ex_val} but ignoring it')

# In[ ]:

# Read the lats and lons from the MYD03 file
generic_rad = a301.data_dir / Path('rad_file_2018_10_1.hdf')
generic_m3 = a301.data_dir / Path('m3_file_2018_10_1.hdf')
print(f'reading {generic_m3}')
m3_file = SD(str(generic_m3), SDC.READ)
lats = m3_file.select('Latitude').get()
lons = m3_file.select('Longitude').get()
m3_file.end()

# In[ ]:

#Read ch30 from the generic_rad file
rad_file = SD(str(generic_rad), SDC.READ)
ch30 = rad_file.select('ch30').get()
rad_file.end()

# # Calculate chan 30 brightness temperature
#
# copy code from the assignment4 solution using planck_invert for channel 30

# In[ ]:

# YOUR CODE HERE
raise NotImplementedError()

# # resample the brightness temperatures
#
# now put the brightness temperatures on the grid following cartopy_resample_ch30.ipynb
#
#
# Note that you don't need to be exact about your center lat/lon points
# below -- just get them within a degree or so.

# In[ ]:

proj4_params = {'datum': 'WGS84',
                  'ellps': 'WGS84',
                  'lat_0': "your center lat here",
                  'lon_0': "your center lon here",
                  'proj': 'laea',
                  'x_0': 0.0,
                  'y_0': 0.0}

# In[ ]:

# YOUR CODE HERE
raise NotImplementedError()

# # replace missing values with floating point nan

# In[ ]:

nan_value = np.array([np.nan],dtype=np.float32)[0]
image_30[image_30< -9000]=nan_value

# # Plot the image using cartopy

# In[ ]:

crs = area_def.to_cartopy_crs()
fig, ax = plt.subplots(1, 1, figsize=(10,10),
                          subplot_kw={'projection': crs})
ax.gridlines(linewidth=2)
ax.add_feature(cartopy.feature.GSHHSFeature(scale='coarse', levels=[1,2,3]));
ax.set_extent(crs.bounds,crs)
cs=ax.imshow(image_30, transform=crs, extent=crs.bounds, origin='upper',alpha=0.8)
fig.colorbar(cs);


"""
Creates a small GCToo instance (with representative examples of typically found fields); can use for testing.

ex:
    import mini_gctoo_for testing
    my_mini_gctoo = mini_gctoo_for_testing.make()
"""
import logging
import pandas
import numpy
import cmapPy.pandasGEXpress.GCToo as GCToo
import cmapPy.pandasGEXpress.setup_GCToo_logger as setup_logger

__author__ = 'Oana Enache'
__email__ = 'oana@broadinstitute.org'

logger = logging.getLogger(setup_logger.LOGGER_NAME)

def make(convert_neg_666=True):
    """
    Creates a small GCToo instance (with representative examples of typically found fields); can use for testing.
    """
    # metadata examples; should be one of each type reasonable to find
    id_vals = ["LJP007_MCF10A_24H:TRT_CP:BRD-K93918653:3.33", "MISC003_A375_24H:TRT_CP:BRD-K93918653:3.33",
               "LJP007_MCF7_24H:TRT_POSCON:BRD-K81418486:10", "LJP007_MCF7_24H:TRT_POSCON:BRD-A61304759:10",
               "LJP007_MCF7_24H:CTL_VEHICLE:DMSO:-666", "LJP007_MCF7_24H:TRT_CP:BRD-K64857848:10"]
    count_cv = ["14|15|14", "13|14|13",
                "13|15|14|14|15|14|14|13|14|15|15|14|14|15|14|15|14|14|15|14|15|14|14|14|14|14|14|15|14|14|15|14|14|14|14|13|14|14|14|14|14|14|15|14|13|13|15|14|14|15|14|14|14|15|13|13|15|13|14|13|13|14|14|14|14|13",
                "13", "13", "14"]
    distil_ss = [9.822065353, 6.8915205, 1.35840559, 5.548898697, 3.355231762, 4.837643147]
    zmad_ref = ["population", "population", "population", "population", "population", "population"]
    distil_nsample = [3, 3, 66, 2, 9, 111111]
    mfc_plate_id = ["-666", "-666", "-666", "-666", "-666", "-666"]

    # build metadata dataframe
    mini_meta_dict = {}
    mini_meta_dict["id"] = id_vals
    mini_meta_dict["count_cv"] = count_cv
    mini_meta_dict["distil_ss"] = distil_ss
    mini_meta_dict["zmad_ref"] = zmad_ref
    mini_meta_dict["distil_nsample"] = distil_nsample
    mini_meta_dict["mfc_plate_id"] = mfc_plate_id
    mini_row_metadata = pandas.DataFrame(mini_meta_dict)

    if convert_neg_666:
        mini_row_metadata = mini_row_metadata.replace([-666, "-666", -666.0], [numpy.nan, numpy.nan, numpy.nan])
    else:
        mini_row_metadata = mini_row_metadata.replace([-666, -666.0], ["-666", "-666"])

    # for now (at least) col and row metadata are the same
    mini_col_metadata = mini_row_metadata.copy()

    # data example values
    r1 = [1, 2, 3, 4, 5, 6]
    r2 = [4.3, 4.5, 4.3, 4.3, 4.3, 4.3]
    r3 = [7, 8, 9, 0, 1.23476, 9.758320]
    r4 = [0.11, 3.3456356, 2.345667, 9.822065353, 4.78865099, 4.7886]
    r5 = [-0.11, -3.3456356, -2.345667, -9.822065353, -4.78865099, -4.7886]
    r6 = [1, -2, 3, -4, 5, -6]

    # build data dataframe
    mini_data_mat = pandas.DataFrame([r1, r2, r3, r4, r5, r6], dtype=numpy.float32)
    mini_data_mat.index = id_vals
    mini_data_mat.columns = id_vals

    # instantiate & assign attributes of GCToo instance
    mini_version = "GCTX1.0"
    mini_src = "mini_gctoo.gctx"

    mini_row_metadata_df = mini_row_metadata
    mini_row_metadata_df.set_index("id", inplace=True)
    mini_row_metadata.index.name = "rid"
    mini_row_metadata_df.columns.name = "rhd"

    mini_col_metadata_df = mini_col_metadata
    mini_col_metadata_df.set_index("id", inplace=True)
    mini_col_metadata.index.name = "cid"
    mini_col_metadata_df.columns.name = "chd"

    mini_data_df = mini_data_mat
    mini_data_df.index.name = "rid"
    mini_data_df.columns.name = "cid"

    logger.debug("Making mini_gctoo instance...")
    mini_gctoo = GCToo.GCToo(data_df=mini_data_df, row_metadata_df=mini_row_metadata_df,
                             col_metadata_df=mini_col_metadata_df, src=mini_src, version=mini_version)

    return mini_gctoo


from numba import jit, int32

@jit(int32(int32, int32))
def f(x, y):
    # A somewhat trivial example
    return x + y

print(f)

# print(f(123, 123**30))

@jit(nopython=True)
def f(x, y):
    return x + y


"""
Module for reading files, more in style of IDL's "readcol" procedure.

Author:

    C.M. Gosmeyer

"""

from __future__ import print_function
from __future__ import with_statement
import numpy as np
import os
import re
import sys

#-----------------------------------------------------------------------------#

def readcol(filename, headerstart=0, datastart=1, comment=' ',
    delimiter="\s+"):
    """ Reads the columns of a text file. Returns the columns as a
    list of list and the header as a list.

    Parameters
    ----------
    filename : str
        Name of the text file.
    headerstart : int
        The row of the header. By default, row=0.
    datastart : int
         The row the data begins. By default, row=1
    comment : str
         The character denoting a comment line, to be ignored when reading
         columns. By default nothing.
    delimiter : str
         The column divider. By default, spaces or tabs.

    Returns
    -------
    header : list of strings
        List of the column names if they exist in row specified by
        'headerstart'. If first column name contains a '#' it will
        be removed. If no column names exist, then returns a list
        of string numbers, starting at '0'.
    cols : list of lists
        List of the columns, each their own list. If no columns found,
        returns an empty list.

    future improvements:
    - option to return a dictionary??

    """

    # Try and Except to exit gracefully and return empty lists
    # if the file does not exist.
    # (but if end up not returning empty lists, could try to
    # re-write the with loop into the try-with)
    try:
        # Open using with so that if error occurs the file will
        # be closed properly.
        with open(filename, 'r') as f:
            print("reading {}".format(filename))

            # Initialize counts and lists.
            cols = []
            row_count = 0

            # Make sure that \t, \n, etc will still be split out.
            if delimiter != "\s+":
                delimiter = '[' + delimiter + '\s\+]'

            # not too happy with the with then for structure
            # but better to use with so that file always is closed.
            # how to return custom output if exception occurs?
            for line in f:

                # Change line into list of entires, split by the
                # delimiter.
                linestrip = re.split(delimiter, line)

                # Remove all empty strings.
                while '' in linestrip:
                    linestrip.remove('')

                # First test that line is not empty.
                # Second test whether the first line is in fact
                # a custom comment.
                # If the user chooses to have a comment above the header,
                # and that header is denoted by the same comment marker,
                # it is on the user to choose correctly the 'headerstart'.
                if linestrip != [] and linestrip[0][0] != comment:

                    # If there is a header, retrieve the column names.
                    if row_count == headerstart and headerstart != datastart:
                        # Figure out how many columns and their names.
                        if linestrip[0] == '#':
                            ncols = len(linestrip)-1
                            linestrip.remove('#')
                        else:
                            ncols = len(linestrip)
                        print('ncols: {}'.format(ncols))
                        header = linestrip
                        # Initilize empty list for each column.
                        cols = [[] for row in range(ncols)]

                        # Remove comment if first character.
                        if header[0][0] == '#':
                            header[0] = header[0][1:]

                    # If there is no header, just name the columns
                    # by number.
                    elif row_count == datastart and datastart == headerstart:
                        ncols = len(linestrip)
                        header = list(map(str, np.arange(ncols)))
                        # Initilize empty list for each column.
                        cols = [[] for row in range(ncols)]

                    # For all rows not in header, not empty, and not
                    # a comment, append to column list.
                    # This is an if so that it can execute should
                    # headerstart = datastart = 0.
                    if row_count >= datastart and linestrip != []:
                        for item, row in zip(linestrip, range(ncols)):
                            cols[row].append(item)
                    row_count+=1

        # Check whether the file was empty or no valid columns read.
        if cols == []:
            print("No valid columns found for file {}.".format(filename))
            return [], []
        else:
            return header, cols

    except IOError:
        print("File {} does not exist.".format(filename))
        return [], []


"""
Tets the :mod:`fatf.transparency.models.submodular_pick` module.
"""
# Author: Alex Hepburn <ah13558@bristol.ac.uk>
#         Kacper Sokol <k.sokol@bristol.ac.uk>
# License: new BSD

import pytest

import numpy as np

from fatf.exceptions import IncorrectShapeError
from fatf.utils.testing.arrays import NOT_BASE_NP_ARRAY

import fatf

import fatf.transparency.models.submodular_pick as ftms

# yapf: disable
NUMERICAL_NP_ARRAY = np.array([
    [0, 0, 0.08, 0.69],
    [1, 0, 0.03, 0.29],
    [0, 1, 0.99, 0.82],
    [0, 1, 0.07, 0.21]])

CATEGORICAL_STRUCT_ARRAY = np.array(
    [('a', 'b', 'c'),
     ('a', 'f', 'g'),
     ('b', 'c', 'c')],
    dtype=[('a', 'U1'), ('b', 'U1'), ('c', 'U1')])

EXPLAINERS = [
    {'a': 0.1, 'b': 0.1, 'c': 0.1, 'd': 0.1},
    {'a': 0.1, 'b': 0.1},
    {'e': 0.1, 'd': 0.1},
    {'a': 0.9, 'b': 0.9, 'c': 0.9, 'd': 0.9}]
# yapf: enable

def explain_instance_a(instance):
    ind = np.where((NUMERICAL_NP_ARRAY == instance).all(axis=1))
    i = ind[0][0]
    return EXPLAINERS[i]

def explain_instance_b(instance):
    ind = np.where((NUMERICAL_NP_ARRAY == instance).all(axis=1))
    i = ind[0][0]
    return EXPLAINERS[::-1][i]

def test_validate_input():
    """
    Tests :func:`fatf.transparency.models.submodular_pick._validate_input`.
    """
    explain_instance = lambda x: x + 1  # noqa: E731

    msg = 'The input data set must be a 2-dimensional array.'
    with pytest.raises(IncorrectShapeError) as exin:
        ftms._validate_input(
            np.array([0, ]), None, None, None)  # yapf: disable
    assert str(exin.value) == msg

    msg = ('The input data set must only contain base types (strings and '
           'numbers).')
    with pytest.raises(ValueError) as exin:
        ftms._validate_input(NOT_BASE_NP_ARRAY, None, None, None)
    assert str(exin.value) == msg

    msg = 'sample_size must be an integer.'
    with pytest.raises(TypeError) as exin:
        ftms._validate_input(NUMERICAL_NP_ARRAY, explain_instance, 'int', None)
    assert str(exin.value) == msg

    msg = 'sample_size must be a non-negative integer.'
    with pytest.raises(ValueError) as exin:
        ftms._validate_input(NUMERICAL_NP_ARRAY, explain_instance, -1, None)
    assert str(exin.value) == msg

    msg = 'explanations_number must be an integer.'
    with pytest.raises(TypeError) as exin:
        ftms._validate_input(NUMERICAL_NP_ARRAY, explain_instance, 1, 'a')
    assert str(exin.value) == msg

    msg = 'explanations_number must be a non-negative integer.'
    with pytest.raises(ValueError) as exin:
        ftms._validate_input(NUMERICAL_NP_ARRAY, explain_instance, 1, -1)
    assert str(exin.value) == msg

    msg = ('The explain_instance should be a Python callable '
           '(function or method).')
    with pytest.raises(TypeError) as exin:
        ftms._validate_input(NUMERICAL_NP_ARRAY, None, 1, 1)
    assert str(exin.value) == msg

    msg = ('The explain_instance callable must accept '
           'exactly one required parameter.')
    cal = lambda x, y: x + y  # noqa: E731
    with pytest.raises(RuntimeError) as exin:
        ftms._validate_input(NUMERICAL_NP_ARRAY, cal, 1, 1)
    assert str(exin.value) == msg

    msg = ('The number of explanations cannot be larger than '
           'the number of samples.')
    with pytest.raises(ValueError) as exin:
        ftms._validate_input(NUMERICAL_NP_ARRAY, explain_instance, 1, 2)
    assert str(exin.value) == msg

    # All OK
    assert ftms._validate_input(NUMERICAL_NP_ARRAY, explain_instance, 1, 1)
    assert ftms._validate_input(
        CATEGORICAL_STRUCT_ARRAY, explain_instance, 0, 0)  # yapf: disable
    assert ftms._validate_input(NUMERICAL_NP_ARRAY, explain_instance, 0, 1)

def test_submodular_pick():
    """Tests :func:`fatf.transparency.models.submodular_pick`."""
    fatf.setup_random_seed()

    explanations, explanation_ind = ftms.submodular_pick(
        NUMERICAL_NP_ARRAY, explain_instance_a, explanations_number=2)
    assert explanation_ind == [0, 2]
    assert explanations == [EXPLAINERS[0], EXPLAINERS[2]]

    explanations, explanation_ind = ftms.submodular_pick(
        NUMERICAL_NP_ARRAY, explain_instance_b, explanations_number=2)
    assert explanation_ind == [0, 1]
    assert explanations == [EXPLAINERS[3], EXPLAINERS[2]]

    msg = ('sample_size is larger than the number of samples in the data set. '
           'The whole dataset will be used.')
    with pytest.warns(UserWarning) as warning:
        explanations, explanation_ind = ftms.submodular_pick(
            NUMERICAL_NP_ARRAY,
            explain_instance_a,
            sample_size=100,
            explanations_number=1)
    assert len(warning) == 1
    assert str(warning[0].message) == msg
    assert explanation_ind == [0]
    assert explanations == [EXPLAINERS[0]]

    explanations, explanation_ind = ftms.submodular_pick(
        NUMERICAL_NP_ARRAY,
        explain_instance_a,
        sample_size=1,
        explanations_number=1)
    assert explanation_ind == [1]
    assert explanations == [EXPLAINERS[1]]

    explanations, explanation_ind = ftms.submodular_pick(
        NUMERICAL_NP_ARRAY,
        explain_instance_a,
        sample_size=0,
        explanations_number=0)
    assert explanation_ind == [0, 2, 1, 3]
    assert explanations == [
        EXPLAINERS[0], EXPLAINERS[2], EXPLAINERS[1], EXPLAINERS[3]
    ]

    explanations, explanation_ind = ftms.submodular_pick(
        NUMERICAL_NP_ARRAY,
        explain_instance_a,
        sample_size=2,
        explanations_number=0)
    assert explanation_ind == [3, 1]
    assert explanations == [EXPLAINERS[3], EXPLAINERS[1]]

    msg = ('The number of explanations cannot be larger than '
           'the number of instances (rows) in the data set.')
    with pytest.warns(UserWarning) as warning:
        explanations, explanation_ind = ftms.submodular_pick(
            NUMERICAL_NP_ARRAY, explain_instance_a, 0, 222)
    assert len(warning) == 1
    assert str(warning[0].message) == msg
    assert explanation_ind == [0, 2, 1, 3]
    assert explanations == [
        EXPLAINERS[0], EXPLAINERS[2], EXPLAINERS[1], EXPLAINERS[3]
    ]


#!/usr/bin/env python
# coding: utf-8

# ## 1.项目背景介绍

# 我国白菜种植面积大，2015年，白菜种植面积261.8万公顷，2016年白菜种植面积262.73万公顷。
# 大白菜产量大，且多采用大面积种植方案，但由于人力资源及成本问题难以对大规模菜叶一一进行大白菜叶子的健康和潜在感染检测判断。
# 随着计算机视觉技术的发展，以及无人化、自动化农田运营理念的提出，利用图像识别技术及目标检测技术实现农产品的自动检测的需求呼之欲出，可通过计算机视觉的方法对菜叶情况进行分析，可以实现早期病害预警，减少人力资源成本以及降低财产损失。
#
# ![](https://ai-studio-static-online.cdn.bcebos.com/70b4e5785ed04fb686f8645edab87144db0cb961e5ce4ff0aac063e88700e011)
# 本项目基于PaddleX，并使用MobileNetV2网络进行训练。

# ## 2.数据介绍

# ### 2.1 数据集介绍

# 该项目包含两份数据集。
# 其中[大白菜病害数据集](https://aistudio.baidu.com/aistudio/datasetdetail/107285)为母体，该数据集包含大白菜叶子的健康和潜在感染图片，目的是将其用于由拉古纳马来亚大学学生进行的有关机器学习和疾病检测的本科论文研究。参与的学生是 Giane Apuada、JanPeter Virtucio 和 Dante Parra。
# 数据由训练数据集和测试数据集组成。训练 csv 数据集已被标记为包含疾病类别，例如背蛾、潜叶虫和霉菌。相应的图像也已正确命名以正确反映它们所包含的疾病类别。如果它们接触 x 种疾病，那么它们将在数据集上标记为“1”。否则标记为“0”。
# 而[白菜数据集](https://aistudio.baidu.com/aistudio/datasetdetail/129132)为以大白菜数据集为母体，经过加工分割出训练集，测试集等产生的子训练集。

# ### 2.2 数据集处理

# 使用PaddleX进行训练对数据集的划分和格式具有一定要求，需要进行预处理，在这里我将母体数据集直接导入PaddleX的可视化模型训练客户端（由于我的python语言水平比较差），再将其自动处理好的数据打包成数据集子体上传。
# 下图为自动生成的数据：![](https://ai-studio-static-online.cdn.bcebos.com/08a1d17570774e0e863ca362267c4ed658432b7a159244c19351e8249a1f20c6)
#

# ### 2.3 数据集展示
# 展示数据集中图像的数量，类别，以及随机一张图像。

# In[1]:

get_ipython().system("unzip 'data/data129132/baicai.zip' -d 'work'      #解压数据库")
get_ipython().system("tree 'work' -d                                    #列出目录结构")

# In[8]:

import cv2    #导入库
import numpy as np
import matplotlib.pylab as plt

with open("work/train_list.txt", "r") as trainList:
    trainDatas = trainList.readlines()
    print('训练集图片数量: {}'.format(len(trainDatas)))

with open("work/test_list.txt", "r") as testList:
    testDatas = testList.readlines()
    print('测试集图片数量: {}'.format(len(testDatas)))

with open("work/val_list.txt", "r") as valList:
    valDatas = valList.readlines()
    print('验证集图片数量: {}'.format(len(valDatas)))

# 从三种病害种类中各抽取一张图像进行可视化
img1_bgr = cv2.imread('work/backmoth/Backmoth1729.jpg')    #读取图像（opencv读取的图像格式为BGR）
img2_bgr = cv2.imread('work/leafminer/Leafminer58.jpg')
img3_bgr = cv2.imread('work/mildew/Mildew126.jpg')

img1_rgb = cv2.cvtColor(img1_bgr,cv2.COLOR_BGR2RGB)         #将BGR格式转换为RGB格式
img2_rgb = cv2.cvtColor(img2_bgr,cv2.COLOR_BGR2RGB)
img3_rgb = cv2.cvtColor(img3_bgr,cv2.COLOR_BGR2RGB)
plt.imshow(img1_rgb)          #根据数组绘制图像
plt.show()                    #显示图像
plt.imshow(img2_rgb)
plt.show()
plt.imshow(img3_rgb)
plt.show()

# ## 3.模型介绍
# MobileNetV2是一个图像分类模型。
# （参考论文[ "MobileNetV2: Inverted Residuals and Linear Bottlenecks" ](https://arxiv.org/abs/1801.04381)）
# MobileNetV2 架构基于倒置残差结构，其中残差块的输入和输出是与传统残差模型相反的薄瓶颈层，后者在输入中使用扩展表示，而 MobileNetV2 使用轻量级深度卷积来过滤中间扩展层中的特征。
# MobileNetV2能更好地匹配移动和嵌入式设备，符合将来部署在移动端的需求。
# ![](https://ai-studio-static-online.cdn.bcebos.com/ea477df3a6394374bb38c06c7475aab0d62513adba9c48d0b50b045b08bef0a0)
#
#

# ## 4.模型训练

# ### 4.1 安装PaddleX

# In[9]:

get_ipython().system(' pip install paddlex==2.0.0')
#安装PaddleX

# ### 4.2 配置GPU

# In[10]:

#设置使用0号GPU卡（如无GPU，执行此代码后仍然会使用CPU训练模型）
import matplotlib
matplotlib.use('Agg')
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
import paddlex as pdx

# ### 4.3 定义数据处理流程

# In[11]:

from paddlex import transforms as T

train_transforms = T.Compose(
    [T.RandomCrop(crop_size=224), T.RandomHorizontalFlip(), T.Normalize()])
#RandomCrop：剪裁图像并调整剪裁后的图像的大小到crop_size*crop_size
#RandomHorizontalFlip：以一定的概率对图像进行随机水平翻转
eval_transforms = T.Compose([
    T.ResizeByShort(short_size=256), T.CenterCrop(crop_size=224), T.Normalize()
])

# ### 4.4 定义数据集

# In[12]:

#采用pdx.datasets.ImageNet来加载数据集
train_dataset = pdx.datasets.ImageNet(
    data_dir='work',
    file_list='work/train_list.txt',
    label_list='work/labels.txt',
    transforms=train_transforms,
    shuffle=True)
eval_dataset = pdx.datasets.ImageNet(
    data_dir='work',
    file_list='work/val_list.txt',
    label_list='work/labels.txt',
    transforms=eval_transforms)

# ### 4.5 开始训练

# In[13]:

num_classes = len(train_dataset.labels)
model = pdx.cls.MobileNetV2(num_classes=num_classes)
model.train(num_epochs=10,                #模型训练迭代的总轮数为10
            train_dataset=train_dataset,  #设置训练集
            train_batch_size=32,          #样本数量为32
            eval_dataset=eval_dataset,    #设置测试集
            lr_decay_epochs=[4, 6, 8], #学习率在第4个epoch时衰减一次，第6个epoch时再衰减一次，第8个epoch时再衰减一次
            save_interval_epochs=1,  #每间隔一轮进行评估和保存
            learning_rate=0.025,
            save_dir='output/mobilenetv2',     #保存目录
            use_vdl=True)   #通过VisualDL对训练过程中的指标进行可视化
#模型在训练过程中，会在save_dir下生成vdl_log目录，通过在命令行终端执行以下命令，启动VisualDL。
#visualdl --logdir=output/vdl_log --port=8008
#在浏览器打开 http://0.0.0.0:8008 便可直接查看随训练迭代动态变化的各个指标

# ## 5.模型评估

# In[14]:

import paddlex as pdx
model = pdx.load_model('output/mobilenetv2/best_model')
image_name = 'work/backmoth/Backmoth425.jpg'      #从验证集中抽取一张图片进行预测
result = model.predict(image_name)
print("预测结果:", result)      #展示预测结果

# ## 6.总结与升华

# 本项目基于PaddleX组件进行开发，实现简单，但预测结果的分数一般，还需要进一步的优化数据库，挑选更合适的模型，以求达到更好的效果。

# ## 7.个人总结

# 来自东北大学秦皇岛分校机械专业，是个菜鸡，但是也欢迎大家找我交流学习。希望以后在AI方面有更深的研究。
# [这里是我的主页链接](https://aistudio.baidu.com/aistudio/usercenter)


from packaging import version

def test_numpy():
    import numpy
    print(numpy.__version__)

def test_scipy():
    import scipy
    print(scipy.__version__)
    ## on 10/28/2020 Devin discovered that CMSE201 needs version 1.5 or grater
    assert(version.parse("1.5.2") > version.parse(scipy.__version__))

def test_matplotlib():
    import matplotlib
    print(matplotlib.__version__)

def test_sympy():
    import sympy
    print(sympy.__version__)

def test_skimage():
    import skimage
    print(skimage.__version__)

# def test_scikit-learn:
#     import scikit-learn
#     print(sscikit.__version__)


from __future__ import print_function

import matplotlib
import matplotlib.animation as animation
import matplotlib.pyplot as plt
import numpy as np
from mpl_toolkits.axes_grid1 import make_axes_locatable

class plotProj:
    # NOTE: type help(plotImg) after importing in order to get a readable manual.
    (
        "\n"
        "plotProj(proj, dim) \n"
        "    plots figure \n"
        "default: progressive in slices following\n"
        "    axis (dim)\n"
        "Parameters \n"
        "---------- \n"
        "proj : Any 3D numpy array \n"
        "\n"
        'dim : ("U","V","T","u","v","t"), optional \n'
        '       default is "T"\n'
        "       NOTE: string arguments!"
        "\n"
        "angles: Any 1D numpy array. \n"
        "        Its length must be the same as proj.shape[0].\n"
        '        Works only when dim is "T" or "t"\n'
        "slice: int, optional\n"
        "     returns page of matrix according to index\n"
        "step: int, optional\n"
        "      Sets the step size between slice and slice."
        "      Step is 1 by default.\n"
        "savegif: string, optional\n"
        "         Saves the image as .gif with the file name\n"
        "show_plot: bool, optional\n"
        "           Sets whether to show the plot.\n"
        "           Default is None, automatically detects matplotlib backend\n"
        "           and decides whether to call plt.show.\n"
        "Examples:\n"
        "---------\n"
        "a=np.ones([3,3,3])\n"
        "plotImg(a)\n"
        ">>>returns plot along dim T\n"
        'plotImg(a,dim="v")\n'
        ">>>returns plot along dim V\n"
    )

    def __init__(
        self,
        proj,
        angles=None,
        dim=None,
        slice=None,
        step=1,
        savegif=None,
        colormap="gray",
        clims=None,
        show_plot=None,
    ):
        self.proj = proj
        self.dim = dim
        self.slice = slice
        self.dimint = None  # keeps track of what dim
        self.dimlist = ["U", "V", "T", "u", "v", "t", None]  # accepted parameters for dim
        self.step = step
        self.savegif = savegif
        self.angles = angles
        self.colormap = colormap
        if clims is None:
            self.min_val = np.amin(self.proj)
            self.max_val = np.amax(self.proj)
        else:
            self.min_val = clims[0]
            self.max_val = clims[1]
        if show_plot is None:
            # https://matplotlib.org/stable/tutorials/introductory/usage.html#backends
            backend = matplotlib.get_backend()
            if backend in [
                "GTK3Agg",
                "GTK3Cairo",
                "MacOSX",
                "nbAgg",
                "Qt4Agg",
                "Qt4Cairo",
                "Qt5Agg",
                "Qt5Cairo",
                "TkAgg",
                "TkCairo",
                "WebAgg",
                "WX",
                "WXAgg",
                "WXCairo",
                "module://ipykernel.pylab.backend_inline",
            ]:
                self.show_plot = True
            elif backend in ["agg", "cairo", "pdf", "pgf", "ps", "svg", "template"]:
                self.show_plot = False
            else:
                self.show_plot = True
        if self.step is None or self.step == 0:
            self.step = 1
        if self.savegif == "":
            self.savegif == None
        if self.slice is None:
            self.run()
        if self.slice is not None:
            self.slicer()

    def run(self):
        if self.dim not in self.dimlist and self.dim is not None:
            raise NameError("check inputs for dim, should be string.")
        if self.angles is not None and self.angles.shape[0] != self.proj.shape[0]:
            raise NameError("check inputs for angles, should be size of proj.shape[0]")

        if self.dim in ["U", "u"]:
            self.dimint = 2
            self.dimlist = ["->V", "->T", "U"]
            self.run_plot()
        if self.dim in ["V", "v"]:
            self.dimint = 1
            self.dimlist = ["->U", "->T", "V"]
            self.run_plot()
        if self.dim in [None, "T", "a"]:
            self.dimint = 0
            self.dimlist = ["->U", "->V", "T"]
            self.run_plot()

    def update_frame(self, it, fig, min_val, max_val):
        i = range(0, self.proj.shape[self.dimint])[:: self.step][it]
        fig.clf()
        axis = fig.add_subplot(1, 1, 1)
        if self.dimint == 2:
            mappable = axis.imshow(
                np.squeeze(self.proj[:, :, i]),
                cmap=self.colormap,
                origin="lower",
                vmin=self.min_val,
                vmax=self.max_val,
            )
        if self.dimint == 1:
            mappable = axis.imshow(
                np.squeeze(self.proj[:, i]),
                cmap=self.colormap,
                origin="lower",
                vmin=self.min_val,
                vmax=self.max_val,
            )
        if self.dimint == 0:
            mappable = axis.imshow(
                np.squeeze(self.proj[i]),
                cmap=self.colormap,
                origin="lower",
                vmin=self.min_val,
                vmax=self.max_val,
            )
        # axis.get_xaxis().set_ticks([])
        # axis.get_yaxis().set_ticks([])
        axis.set_xlabel(self.dimlist[0])
        axis.set_ylabel(self.dimlist[1])
        if self.angles is not None:
            axis.set_title(
                "{}:{}, alpha={:+.3f} pi".format(self.dimlist[2], i, self.angles[i] / np.pi)
            )
        else:
            axis.set_title(self.dimlist[2] + ":" + str(i))
        divider = make_axes_locatable(axis)
        cax = divider.append_axes("right", size="5%", pad=0.05)
        fig.colorbar(mappable, cax=cax)
        # plt.pause(0.01)

    def run_plot(self):

        dim = self.proj.shape

        fig = plt.figure()
        ani = animation.FuncAnimation(
            fig,
            self.update_frame,
            fargs=(fig, self.min_val, self.max_val),
            interval=100,
            repeat_delay=1000,
            frames=len(range(0, dim[self.dimint])[:: self.step]),
        )
        if self.savegif is not None:
            ani.save(self.savegif, writer="pillow")
            self._show()
        else:
            self._show()

    def slicer(self):

        if self.dim in ["U", "u"]:
            plt.xlabel("V")
            plt.ylabel("T")
            plt.imshow(
                np.squeeze(self.proj[:, :, self.slice]),
                cmap=self.colormap,
                origin="lower",
                vmin=self.min_val,
                vmax=self.max_val,
            )
        if self.dim in ["V", "v"]:
            plt.xlabel("U")
            plt.ylabel("T")
            plt.imshow(
                np.squeeze(self.proj[:, self.slice]),
                cmap=self.colormap,
                origin="lower",
                vmin=self.min_val,
                vmax=self.max_val,
            )
        if self.dim in [None, "T", "t"]:
            if self.angles is not None:
                plt.title("alpha={:+.3f} pi".format(self.angles[self.slice] / np.pi))
            plt.xlabel("U")
            plt.ylabel("V")
            plt.imshow(
                np.squeeze(self.proj[self.slice]),
                cmap=self.colormap,
                origin="lower",
                vmin=self.min_val,
                vmax=self.max_val,
            )
        self._show()

    def _show(self):
        if self.show_plot:
            plt.show()

def plotSinogram(proj, posV, show_plot=None):  # noqa: N803
    """
    plotSinogram(proj, posV)
        plots sinogram at V=posV

    Parameters
    ----------
    proj : Any 3D numpy array
    posV : integer. in range of 0:proj.shape[1].
    """
    plotProj(proj, dim="V", slice=posV, show_plot=show_plot)

plotproj = plotProj


"""
This module contains some tools to perform parallel procedures using the python
multiprocessing package

"""
import multiprocessing as mp, time as tm
import numpy as np
from datetime import timedelta

def loop(func, pars, *args, ntasks = 4, verbose = True, **kwargs):
    """
    Perform a parallel loop over the values of the pars array and compute the
    values of the function func, using ntasks parallel processes

    Args:
        func (function) : a function that returns a value for each element of pars
        pars (:py:class:`array`) : array with the values iterate by the loop
        ntask (:py:class:`int`) : number of parallel tasks
        verbose (:py:class:`bool`) : determine the amount of information provided on terminal
        args, kwargs : arguments and keyword arguments passed to func

     """
    def func_loop(func,pars_subset,task,output,*args,**kwargs):
        """
        Evaluate the function func for all the values inside a single task.
        Add the dictionary with the results of the task to the queue of the multiprocess

        """
        results = []
        for p in pars_subset:
            results.append(func(p,*args,**kwargs))
        output.put({task:np.array(results)})

    pars_split = np.array_split(pars,ntasks)
    if verbose : print('Run a parallel loop with %s tasks...'%ntasks)
    t0 = tm.time()
    output = mp.Queue()
    tasks = [mp.Process(target=func_loop, args=(func,pars_split[task],task,output,*args,), kwargs=kwargs) for task in range(ntasks)]
    for p in tasks:
        p.start()
    results_dict = {}
    for p in tasks:
        results_dict.update(output.get())
    results = np.concatenate([results_dict[i] for i in range(ntasks)])
    if verbose :
        deltaTime = int(tm.time()-t0)
        dT_str = "{:0>8}".format(str(timedelta(seconds=deltaTime)))
        print('Loop executed in',dT_str)
    return results


# encoding: UTF-8

'''
本文件中包含的是遗传算法参数优化器的实现
华富资产 李来佳
'''

from typing import Callable, List
from itertools import product
from functools import lru_cache
from time import time
from datetime import  datetime
import multiprocessing
import random
import traceback
from copy import copy, deepcopy
from uuid import uuid1
import logging
import os
import numpy as np

# deap是遗传算法的实现工具
from deap import creator, base, tools, algorithms

from vnpy.app.cta_strategy_pro.portfolio_testing import PortfolioTestingEngine
from vnpy.trader.util_logger import setup_logger

class OptimizationSetting:
    """
    Setting for runnning optimization.
    设置参数优化
    """

    def __init__(self):
        """"""
        self.params = {}
        self.target_name = ""

    def add_parameter(
            self, name: str,
            start: float,
            end: float = None,
            step: float = None
    ):
        """添加int/float类型参数优化"""
        if not end and not step:
            self.params[name] = [start]
            return

        if start >= end:
            print("参数优化起始点必须小于终止点")
            return

        if step <= 0:
            print("参数优化步进必须大于0")
            return

        value = start
        value_list = []

        while value <= end:
            value_list.append(value)
            value += step

        # win: [20,22,24]
        self.params[name] = value_list

    def add_parameters(self,
                       name: str,
                       values: List = []):
        """添加参数清单"""
        self.params[name] = values

    def set_target(self, target_name: str):
        """"""
        self.target_name = target_name

    def generate_setting(self) -> List:
        """"""
        keys = self.params.keys()
        values = self.params.values()
        products = list(product(*values))

        settings = []
        for p in products:
            setting = dict(zip(keys, p))
            settings.append(setting)

        return settings

    def generate_setting_ga(self):
        """"""
        settings_ga = []  # [ [(key1:value1), (key2:value2)], [],,,]
        settings = self.generate_setting()
        for d in settings:
            # dict => [(key1:value1), (key2:value2)]
            param = [tuple(i) for i in d.items()]
            settings_ga.append(param)
        return settings_ga

class GeneticOptimize(object):
    """
    遗传算法优化器
    """
    def __init__(self):

        self.s = OptimizationSetting()  # 参数生成器
        self.be = PortfolioTestingEngine()  # (组合)回测引擎

        self.settings_for_ga = []  # 供遗传算法进行优化的参数列表
        self.settings_for_env = {}  # 供回测使用的参数配置，如合约、资金账号、回测时间等
        self.settings_for_strategies = {}  # cta_strategy_settings， dict格式：{ "strategy_instance_name": {策略配置}}

        self.target_names = []  # 回测结果目标值名称列表，如最大化收益回撤比，最大化夏普比率
        self.logger = None

    creator.create("FitnessMulti", base.Fitness, weights=(1.0, 1.0))
    creator.create("Individual", list, fitness=creator.FitnessMulti)

    def write_log(self,msg: str, level: int = logging.DEBUG):

        if self.logger:
            self.logger.log(msg=msg, level=level)

    def init_setting(self,
                     test_settings,
                     strategy_settings,
                     target_names):

        # 回测引擎所需的所有基本配置
        self.settings_for_env = deepcopy(test_settings)

        if not self.logger:
            logs_folder = os.path.abspath(os.path.join(os.getcwd(), 'log'))
            filename = os.path.abspath(os.path.join(logs_folder, '{}'.format(test_settings['name'])))
            self.logger = setup_logger(file_name=filename,
                                       name="go_{}".format(datetime.now().strftime("%Y%m%d_%H%M%S")),
                                       log_level=logging.DEBUG,
                                       backtesing=True)

        # 回测实例及配置
        self.settings_for_strategies = deepcopy(strategy_settings)

        # 优化的目标值清单
        self.target_names = deepcopy(target_names)

        return self.settings_for_env

    def add_parameter(self, name, start, end, step):
        """
        添加参数
        :param name: 参数名称
        :param start: 开始数字
        :param end: 结束数字
        :param step: 步进
        :return:
        """
        self.s.add_parameter(name, start, end, step)

    def add_parameters(self, name, values):
        """
        添加参数
        :param name: 参数名
        :param values: 参数值列表
        :return:
        """
        self.s.add_parameters(name, deepcopy(values))

    def generate_setting_for_ga(self):
        """
        产生遗传算法的参数列表
        :return:
        """
        settings = self.s.generate_setting()
        for d in settings:
            param = [tuple(i) for i in d.items()]
            self.settings_for_ga.append(param)
        return self.settings_for_ga

    def generate_parameter(self):
        """"""
        return random.choice(self.settings_for_ga)

    def mutArrayGroup(self, individual, indpb):
        size = len(individual)
        paralist = self.generate_parameter()
        for i in range(size):
            if random.random() < indpb:
                individual[i] = paralist[i]
        return individual,

    def object_func(self, strategy_avg):
        """
        遗传优化的目标执行函数
        :param strategy_avg:
        :return:
        """
        return self._object_func(tuple(strategy_avg))

    def optimize(self):
        """
        运行优化
        :return:
        """
        start = time()
        toolbox = base.Toolbox()

        # 使用多进程
        pool = multiprocessing.Pool(multiprocessing.cpu_count())
        toolbox.register("map", pool.map)
        # 初始化
        toolbox.register("individual", tools.initIterate, creator.Individual, self.generate_parameter)
        toolbox.register("population", tools.initRepeat, list, toolbox.individual)
        toolbox.register("mate", tools.cxTwoPoint)
        toolbox.register("mutate", self.mutArrayGroup, indpb=1)
        toolbox.register("evaluate", self.object_func)
        toolbox.register("select", tools.selNSGA2)

        MU = 16
        LAMBDA = 20
        POP = 20
        pop = toolbox.population(POP)
        CXPB, MUTPB, NGEN = 0.95, 0.05, 4
        hof = tools.ParetoFront()

        stats = tools.Statistics(lambda ind: ind.fitness.values)
        np.set_printoptions(suppress=True)
        stats.register("mean", np.mean, axis=0)
        stats.register("std", np.std, axis=0)
        stats.register("min", np.min, axis=0)
        stats.register("max", np.max, axis=0)

        self.write_log("开始运行遗传算法，每代族群总数：%s, 优良品种筛选个数：%s，迭代次数：%s，交叉概率：%s，突变概率：%s" % (POP, MU, NGEN, CXPB, MUTPB))
        algorithms.eaMuPlusLambda(pop, toolbox, MU, LAMBDA, CXPB, MUTPB, NGEN, stats, halloffame=hof, verbose=True)
        end = time()
        cost = int((end - start))

        self.write_log("遗传算法优化完成，耗时%s秒" % (cost))
        self.write_log("----------输出帕累托前沿解集,解集数量%s----------" % (len(hof)))
        # return hof
        for i in range(len(hof)):
            solution = hof[i]
            self.write_log(solution)

    @lru_cache(maxsize=1000000)
    def _object_func(self, strategy_avg):
        """
        使用了缓存，可以减少重复参数的运行
        :param strategy_avg: 选取的策略参数
        :return:
        """
        engine = self.be
        self.settings_for_env['name'] = self.settings_for_env['name'] + '_' + str(uuid1())
        engine.prepare_env(self.settings_for_env)
        # 选取参数 => dict 结构
        ga_setting = dict(strategy_avg)
        settings_for_strategies = deepcopy(self.settings_for_strategies)

        # 策略实例名
        for ins_name in list(settings_for_strategies.keys()):
            # 策略实例配置
            ins_config = settings_for_strategies[ins_name]
            # 策略实例的策略参数
            strategy_setting = ins_config['setting']
            # 候选参数值 => 更新 => 策略参数值
            for k, v in ga_setting.items():
                if k in strategy_setting:
                    strategy_setting.update({k: v})
            # 更新回测实例配置
            settings_for_strategies.update({ins_name: ins_config})

        try:
            # 运行（组合）回测
            engine.run_portfolio_test(settings_for_strategies)
            # 回测结果，保存
            result = engine.show_backtesting_result()

            # 保存策略得内部数据
            engine.save_strategy_data()

            # 根据target_names => (value1, value2)
            return tuple([round(result.get(k, 0), 2) for k in self.target_names])

        except Exception as ex:
            self.write_log('组合回测异常{}'.format(str(ex)))
            traceback.print_exc()
            engine.save_fail_to_mongo(f'回测异常{str(ex)}')
            return tuple([0 for k in self.target_names])

def run_go(test_setting: dict, strategy_setting: dict, ga_setting: dict, target_names: List):
    """
    遗传算法优化+回测
    : test_setting, 组合回测所需的配置，包括合约信息，数据bar信息，回测时间，资金等。
    ：strategy_setting, dict, 一个或多个策略配置
    : ga_setting,dict, 代优化参数的设定： name: (开始值，结束值，步进）, name: [value1, value2,value3,,,]

    : return 定义的
    """
    # 创建遗传优化器
    GO = GeneticOptimize()

    # 初始化 环境参数、策略参数，优化目标清单
    GO.init_setting(
        test_settings=test_setting,
        strategy_settings=strategy_setting,
        target_names=target_names
    )

    # 添加代优化的参数
    for k, v in ga_setting.items():
        if isinstance(v, tuple) and len(v) == 3:
            GO.add_parameter(k, v[0], v[1], v[2])

        if isinstance(v, list):
            GO.add_parameters(k, v)

    GO.generate_setting_for_ga()

    GO.generate_parameter()

    GO.optimize()


# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.11.3
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
#

# + [markdown] id="_Qd2diMQZlQL"
# # Superimport demo
#
# The [superimport library](https://github.com/probml/superimport), written by [Mahmoud Soliman](https://github.com/mjsML), takes care of installing missing python packages for you. All you have to do is type `pip install superimport` (once per colab session), and then add `import superimport` to the top of any of your python files; then, when you run those files, superimport will read the source code, figure out any missing dependencies, install them for you automagically, and then run the rest of your code as usual. We illustrate this below.
#
#

# + id="1G16DyRzi6kK" colab={"base_uri": "https://localhost:8080/"} outputId="b5baeea5-861e-4436-d126-0d6275a254b4"
# !pip install superimport -qqq
# !pip install deimport -qqq

# + id="iiC72KXh5jae"
import superimport

def try_deimport():
  try:
    from deimport.deimport import deimport
    deimport(superimport,verbose=False)
  except Exception as e:
    print(e)

# + [markdown] id="V_82RC0lahoP"
# # An example with PgmPy
#
# Colab has most popular ML packages already installed. However, there are a few missing ones, such as [PgmPy](https://github.com/pgmpy/pgmpy). Below we create a short file, called `test.py`, that relies on that missing library. We then show what happens if we try to run the script  without first installing the library.

# + colab={"base_uri": "https://localhost:8080/"} id="UuJYZu_1kP1B" outputId="1e0b7fc1-98a6-4d54-a92c-ecc378ea54cb"
# %%file test.py
import pgmpy
import numpy
import matplotlib
print('pgmpy ', pgmpy.__version__)

# + [markdown] id="mkwHEjfwknJt"
# Without importing superimport, if you have a missing package your script will fail.

# + colab={"base_uri": "https://localhost:8080/", "height": 215} id="6T6BDgaRkSdM" outputId="437179d6-4d6b-4f40-d570-1283f5108549"
# %run test.py

# + [markdown] id="cZd4ITLYa33l"
#
#
# Now we add one new line to our file: `import superimport`

# + colab={"base_uri": "https://localhost:8080/"} id="fMPOfeIHi9i3" outputId="b78a00c1-cd7b-41da-9287-cd86ea719b4e"
# %%file test.py
import superimport
import pgmpy
import numpy
import matplotlib
print('pgmpy ', pgmpy.__version__)

# + [markdown] id="aIFQaI6pkxIL"
# We can now successfully the script, and it will install any missing packages.
#
#
# Note, however, that we have to deimport the `superimport` symbol before running any code that uses superimport, to force the package to be reloaded (and hence re-executed), otherwise colab will use the cached version (if available) of superimport, which may be stale.

# + colab={"base_uri": "https://localhost:8080/"} id="tL4_358-jJD8" outputId="a8fe754e-af56-400a-c663-39695ad4862b"
try_deimport()
# %run -n test.py

# + [markdown] id="-IDlmbuCgDMx"
# # An example with NumPyro
#
# This time we make a demo that uses numpyro, that is not installed in colab by default.

# + id="SuSqhaqEgQ3z" colab={"base_uri": "https://localhost:8080/"} outputId="682be66a-a277-44e9-b9b9-23c06c9975d1"
# %%file test.py
import superimport
import numpyro
print('numpyro version ', numpyro.__version__)

# + id="yHihouB-gUJK" colab={"base_uri": "https://localhost:8080/"} outputId="8787fa90-4332-42fd-da3c-6c4ce3657df0"

try_deimport()
# %run -n test.py

# + [markdown] id="KJSga2iNeauy"
# # An example with Pyro
#
# This time we make a demo that uses pyro, that is not installed in colab by default. Furthermore, its package name (pyro-ppl) does not match its import name (pyro).

# + id="Sy7eFOQxfQB6" colab={"base_uri": "https://localhost:8080/"} outputId="b16bde2c-d2ef-4cb2-9292-e2f99f70a3d9"
# %%file test.py
import superimport
import pyro
print('pyro version ', pyro.__version__)

# + id="wMgsJq1ieoeH" colab={"base_uri": "https://localhost:8080/"} outputId="9fb807c0-999f-41b8-aaae-4efdde5fe571"

try_deimport()
# %run -n test.py

# + [markdown] id="KFogkyP8gWZ5"
# # An example from the book

# + id="4DvNikcygYlC"
# !git clone --depth 1 https://github.com/probml/pyprobml  /pyprobml &> /dev/null
# %cd -q /pyprobml/scripts

# + colab={"base_uri": "https://localhost:8080/", "height": 550} id="aOl69rgCgvVX" outputId="139d0c93-446e-4c9e-a264-11cd353fa1fe"

try_deimport()
# %run -n linreg_residuals_plot.py

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="HpDO7eE1gz-Y" outputId="3e2299dc-01c9-4738-b301-66a05458d3c5"

try_deimport()
# %run -n linreg_poly_vs_degree.py

# + colab={"base_uri": "https://localhost:8080/", "height": 553} id="UnKMPSBOh16k" outputId="5b77a9d2-c1cb-47a3-ea47-c899aea0f19f"

try_deimport()
# %run -n iris_kmeans.py

# + [markdown] id="gERNSvUpcru-"
# # Sharp edges
#
# * There are some packages whose install names differ from their import names  (eg we type `pip install pyro-ppl` but `import pyro`). There is a [public mapping file](https://github.com/bndr/pipreqs/blob/master/pipreqs/mapping) stored by pipreqs. However, this is missing some entries (such as pyro).  These must be manually added to the [mapping2 file](https://github.com/probml/superimport/blob/main/superimport/mapping2). If your favorite package is missing, open a PR on the superimport repo.
#
# * There are some packages that do not list of all of their requirements.txt (eg GPyOpt depends on matplotlib, but does not mention this). If this 'hidden requirement' is missing, superimport cannot find it either. If it is not already installed in colab, then your script will fail, even with superimport.

# + id="GUWz9Gr-d5SO"