from PyQt5.QtWidgets import (QWidget, QLabel, QListWidget, QPushButton,
                             QVBoxLayout, QSplitter, QGroupBox, QRadioButton,
                             QAbstractItemView, QPlainTextEdit)
from PyQt5.QtCore import pyqtSignal, Qt
from mooda import WaterFrame
from mooda.access import EGIM
from mooda.app.mooda_gui.widgets import (DropWidget, QCWidget, RenameWidget,
                                         ResampleWidget, SliceWidget,
                                         ScatterMatrixPlotWidget, QCPlotWidget,
                                         TSPlotWidget, QCBarPlotWidget,
                                         SpectrogramPlotWidget)
from mooda.app.mooda_gui.widgets.histoplotwidget import HistoPlotWidget

class PlotSplitter(QSplitter):
    """Pyqt5 widget to show data in plots and lists."""

    # Signals
    msg2Statusbar = pyqtSignal(str)
    msg2TextArea = pyqtSignal(str)

    def __init__(self):
        """
        Constructor
        """
        super().__init__()

        # Instance variables
        self.wf = WaterFrame()
        # List of PlotWidget, to control them in any case
        self.plotWidgetList = []

        self.initUI()

    def initUI(self):
        """UI creator"""

        # Lists
        self.dataList = QListWidget(self)
        self.dataList.setSelectionMode(QAbstractItemView.ExtendedSelection)
        self.dataList.itemClicked.connect(self.dataListClick)
        self.graphList = QListWidget(self)
        self.graphList.itemClicked.connect(self.graphClick)

        # PlainTextEdit
        self.otherInfoPlainText = QPlainTextEdit(self)
        self.metadataPlainText = QPlainTextEdit(self)

        # Labels
        metadataLabel = QLabel("Metadata")
        dataLabel = QLabel("Data")
        graphLabel = QLabel("Graph")
        infoLabel = QLabel("Other information")

        # DropWidget
        self.dropWidget = DropWidget()
        self.dropWidget.list2drop[list, list, bool].connect(self.dropData)
        self.dropWidget.hide()

        # Group Box
        plotGroupBox = QGroupBox("Plot properties", self)
        vPlotGroupBox = QVBoxLayout()
        # - RadioButton
        self.autoPlotRadioButton = QRadioButton("Time series plot", self)
        self.autoPlotRadioButton.setChecked(True)
        self.multipleParameterRadioButton = QRadioButton("Multiparameter", self)
        self.correlationRadioButton = QRadioButton("Correlation", self)
        self.histogramRadioButton = QRadioButton("Histogram", self)
        self.parameterQCRadioButton = QRadioButton("QC of the parameter", self)
        vPlotGroupBox.addWidget(self.autoPlotRadioButton)
        vPlotGroupBox.addWidget(self.histogramRadioButton)
        vPlotGroupBox.addWidget(self.multipleParameterRadioButton)
        vPlotGroupBox.addWidget(self.correlationRadioButton)
        vPlotGroupBox.addWidget(self.parameterQCRadioButton)
        plotGroupBox.setLayout(vPlotGroupBox)

        # QCWidget
        self.qcWidget = QCWidget()
        self.qcWidget.list2qc[list].connect(self.applyQC)
        self.qcWidget.hide()

        # RenameWidget
        self.renameWidget = RenameWidget()
        self.renameWidget.key2change[str, str].connect(self.applyRename)
        self.renameWidget.hide()

        # ResampleWidget
        self.resampleWidget = ResampleWidget()
        self.resampleWidget.resampleRule[str].connect(self.applyResample)
        self.resampleWidget.hide()

        # SliceWidget
        self.sliceWidget = SliceWidget()
        self.sliceWidget.sliceTimes[str, str].connect(self.applySlice)
        self.sliceWidget.hide()

        # Splitters
        self.vDataSplitter = QSplitter(Qt.Vertical)
        # Custom Widget Metadata
        self.vMetadataWidget = QWidget()

        # Buttons
        # - For metadata area
        hideMetadataButton = QPushButton("Hide")
        hideMetadataButton.clicked.connect(self.vMetadataWidget.hide)
        # - For data Area
        plotButton = QPushButton("Plot")
        plotButton.clicked.connect(self.addPlot)
        hideDataButton = QPushButton("Hide")
        hideDataButton.clicked.connect(self.vDataSplitter.hide)

        # Custom Widget Data
        # - Data Widget
        parameterWidget = QWidget()
        # -- Layout
        vData = QVBoxLayout()
        vData.addWidget(dataLabel)
        vData.addWidget(self.dataList)
        vData.addWidget(plotGroupBox)
        vData.addWidget(plotButton)
        parameterWidget.setLayout(vData)
        # - Graph Widget
        graphWidget = QWidget()
        # -- Layout
        vGraph = QVBoxLayout()
        vGraph.addWidget(graphLabel)
        vGraph.addWidget(self.graphList)
        vGraph.addWidget(hideDataButton)
        graphWidget.setLayout(vGraph)
        # - Data splitter
        self.vDataSplitter.addWidget(parameterWidget)
        self.vDataSplitter.addWidget(graphWidget)

        # Layouts
        # - Metadata -
        vMetadata = QVBoxLayout()
        vMetadata.addWidget(metadataLabel)
        vMetadata.addWidget(self.metadataPlainText)
        vMetadata.addWidget(infoLabel)
        vMetadata.addWidget(self.otherInfoPlainText)
        vMetadata.addWidget(hideMetadataButton)
        self.vMetadataWidget.setLayout(vMetadata)

        # Splitter (self)
        # - Layout for actions
        vActionsWidget = QWidget()
        vActions = QVBoxLayout()
        vActions.addWidget(self.renameWidget)
        vActions.addWidget(self.resampleWidget)
        vActions.addWidget(self.dropWidget)
        vActions.addWidget(self.sliceWidget)
        vActions.addStretch()
        vActionsWidget.setLayout(vActions)
        # - Add to self (splitter)
        self.addWidget(self.qcWidget)
        self.addWidget(vActionsWidget)
        self.addWidget(self.vMetadataWidget)
        self.addWidget(self.vDataSplitter)

    def dataListClick(self):
        if (self.autoPlotRadioButton.isChecked() or
        self.histogramRadioButton.isChecked()):
            self.addPlot()

    def addPlot(self):
        """It creates a FigureCanvas with the input figure"""

        self.msg2Statusbar.emit("Making the figure")

        # Create key list
        keys = [item.text() for item in self.dataList.selectedItems()]

        # If nothing is selected, go out
        if not keys:
            self.msg2Statusbar.emit("Ready")
            return

        # Check if it is a QC plot
        if self.parameterQCRadioButton.isChecked():
            keys = [keys[0] + "_QC"]

        # Check if it is a correlation
        if self.correlationRadioButton.isChecked():
            plotWidget = ScatterMatrixPlotWidget(wf=self.wf, keys=keys)
            self.addWidget(plotWidget)
            return

        # Create name of the plot
        name = '_'.join(keys)

        if self.histogramRadioButton.isChecked():
            name = "hist_" + name

        # Check if plot is done
        new = True
        for plotWidget in self.plotWidgetList:
            if plotWidget.name == name:
                if ~plotWidget.isVisible():
                    plotWidget.refreshPlot()
                    plotWidget.show()
                new = False
                break

        # Create the plot if is new
        if new:
            if len(keys) == 1 and "_QC" in keys[0]:
                plotWidget = QCPlotWidget(wf=self.wf, key=keys[0])
            else:
                if self.histogramRadioButton.isChecked():
                    plotWidget = HistoPlotWidget(wf=self.wf, keys=keys)
                else:
                    plotWidget = TSPlotWidget(wf=self.wf, keys=keys)
                plotWidget.msg2Statusbar[str].connect(self.msg2Statusbar.emit)
            self.addWidget(plotWidget)
            # Add the widget to the list
            self.plotWidgetList.append(plotWidget)

        self.msg2Statusbar.emit("Ready")

    def addQCBarPlot(self):
        """
        It creates a FigureCanvas with the input figure (QC)
        """
        self.msg2Statusbar.emit("Making the figure")
        # Check if the plot exists
        plotWidget = None
        for plotWidget_ in self.plotWidgetList:
            if plotWidget_.name == "QC":
                plotWidget = plotWidget_
                plotWidget.wf = self.wf
                plotWidget.refreshPlot()
                plotWidget.show()
                break
        if plotWidget is None:

            plotWidget = QCBarPlotWidget(wf=self.wf)
            self.addWidget(plotWidget)
            # Add the widget to the list
            self.plotWidgetList.append(plotWidget)
        self.msg2Statusbar.emit("Ready")

    def addSpectroPlot(self):
        """It ads the SpectroPlotWidget to the screen"""

        self.msg2Statusbar.emit("Making the figure")
        # Check if the plot exists
        plotWidget = None
        for plotWidget_ in self.plotWidgetList:
            if plotWidget_.name == "Spectrogram":
                plotWidget = plotWidget_
                plotWidget.wf = self.wf
                plotWidget.refreshPlot()
                plotWidget.show()
                break
        if plotWidget is None:
            plotWidget = SpectrogramPlotWidget(wf=self.wf)
            self.addWidget(plotWidget)
            # Add the widget to the list
            self.plotWidgetList.append(plotWidget)
        self.msg2Statusbar.emit("Ready")

    def openData(self, path, concat=False):
        """
        It opens the netcdf of the path.

        Parameters
        ----------
            path: str or WaterFrame
                Path where the file is or WaterFrame object.
            concat: bool, optional (concat = False)
                It adds the new dataframe to the current dataframe.

        Returns
        -------
            True/False: bool
                It indicates if the operation is ok.
        """
        self.msg2Statusbar.emit("Opening data")
        if isinstance(path, str):
            # Obtain type of file
            extension = path.split(".")[-1]
            # Init ok
            ok = False
            wf_new = WaterFrame()
            if extension == "nc":
                ok = wf_new.from_netcdf(path)
            elif extension == "pkl":
                ok = wf_new.from_pickle(path)
            elif extension == "csv":
                try:
                    wf_new = EGIM.from_raw_csv("EMSO-Azores", path)
                    ok = True
                except:
                    ok = False
            if ok:
                # Check if we want actual data
                if not concat:
                    self.newWaterFrame()
                self.wf.concat(wf_new)

                self.msg2TextArea.emit("Working with file {}".format(path))
                # Add metadata information into metadataList
                self.addMetadata(self.wf.metadata)
                # Add other information
                self.otherInfoPlainText.setPlainText(repr(self.wf))
                # Add data information into dataList
                self.addData(self.wf.data)
                # Plot QC
                self.addQCBarPlot()
                self.msg2Statusbar.emit("Ready")
                return True
            else:
                self.msg2Statusbar.emit("Error opening data")
                return False
        else:
            # Path is a WaterFrame

            # Check if there is no data in the waterframe
            if path.data.empty:
                return False

            # Check if it is a dataframe of an acoustic data.
            # In this case, we are going to delete the previous dataframe.
            if "Sequence" in self.wf.data.keys():
                self.wf.clear()
            self.wf.concat(path)

            # Add data information into dataList
            self.addData(self.wf.data)
            self.addMetadata(self.wf.metadata)
            # Plot QC
            self.addQCBarPlot()
            return True

    def addMetadata(self, metadataDict):
        """
        Add Metadata information into self.metadataPlainText
        :param metadataDict: WaterFrame Metadata Dictionary
        """
        # Clear the list
        self.metadataPlainText.clear()

        items = []
        msg = "\nMetadata:"
        for key, value in metadataDict.items():
            items.append("{}: {}".format(key, value))
            msg += "\n- {}: {}".format(key, value)
        self.metadataPlainText.setPlainText(msg[11:])
        # Send a message to the text area
        self.msg2TextArea.emit(msg)

    def addData(self, data):
        """
        Add data names into self.dataList
        :param data: WaterFrame data variable
        """

        def is_acoustic_data(a):
            # will be True also for 'NaN'
            if a == "Sequence":
                return True
            elif a == "Data Points":
                return True
            try:
                float(a)
                return True
            except ValueError:
                return False

        # Clear the list
        self.dataList.clear()
        # Parameter keys (without QC)
        # NO DEPTH in nc files, NO TIME
        keys_to_work = [key for key in data.keys() if 'TIME' not in key
                        if not is_acoustic_data(key)
                        if key + "_QC" in data.keys()]
        self.dataList.addItems(keys_to_work)
        # Add graphs
        self.graphList.clear()
        self.graphList.addItem("QC")
        # Check if we have acoustic data
        for key in data.keys():
            if is_acoustic_data(key):
                self.graphList.addItem("Spectrogram")
                break

        # Add tooltip
        msg = "\nData:"
        for i in range(self.dataList.count()):
            if "_QC" in self.dataList.item(i).text():
                self.dataList.item(i).setToolTip(
                    'QC flags of {}'.format(self.dataList.item(i).text()[:-3]))
            else:
                try:
                    self.dataList.item(i).setToolTip('{} ({})'.format(
                        self.wf.meaning[
                            self.dataList.item(i).text()]['long_name'],
                        self.wf.meaning[
                            self.dataList.item(i).text()]['units']))
                    msg += "\n- {}: {} ({})".format(
                        self.dataList.item(i).text(),
                        self.wf.meaning[
                            self.dataList.item(i).text()]['long_name'],
                        self.wf.meaning[
                            self.dataList.item(i).text()]['units'])
                except KeyError:
                    pass
        # Send a message to the text area
        self.msg2TextArea.emit(msg)
        # Send the labels to the dropWidget
        self.dropWidget.addLabels(keys_to_work)
        self.qcWidget.addLabels(keys_to_work)
        self.renameWidget.addLabels(keys_to_work)
        self.sliceWidget.refresh(self.wf.data.index[0], self.wf.data.index[-1])

    def saveData(self, path):
        """
        Save current data into a pickle file
        :param path: File path
        :return: Bool
        """
        self.msg2Statusbar.emit("Saving data")
        extension = path.split(".")[-1]
        # Init ok
        ok = False
        wf_new = WaterFrame()
        if extension == "nc":
            pass
        elif extension == "pkl":
            ok = self.wf.to_pickle(path)
        elif extension == "csv":
            ok = self.wf.to_csv(path)

        if ok:
            self.msg2TextArea.emit("Data saved on file {}".format(path))
            self.msg2Statusbar.emit("Ready")
        return ok

    def dropData(self, labels, flagList, dropNaN):
        """
        Delete some parameters from self.wf and refresh the lists
        :param labels: list of labels to drop
        :param flagList: list of flags to drop
        :return:
        """
        self.msg2TextArea.emit("Deleting data")

        # This is a trick, delete the list if is a list of None
        if flagList[0] is None:
            flagList = None

        if flagList:
            self.wf.use_only(parameters=labels, flags=[0, 1], dropnan=dropNaN)
        else:
            # Delete the parameters
            self.wf.drop(keys=labels, flags=flagList)
        # Refresh the lists
        self.addData(self.wf.data)

        # Delete plots with the key
        for label in labels:
            for plotWidget in self.plotWidgetList:
                if plotWidget.name == "QC":
                    plotWidget.refreshPlot()
                if label == plotWidget.name or label+"_" in \
                   plotWidget.name or "_"+label in plotWidget.name:
                    plotWidget.deleteLater()
                    self.plotWidgetList.remove(plotWidget)

        # Send message
        msg = ""
        if flagList is None:
            for label in labels:
                if '_QC' in label:
                    continue
                msg += "{} ".format(label)
            msg += "deleted"
        else:
            msg += "Data with QC Flags "
            for flag in flagList:
                msg += "{}, ".format(flag)
            msg += "from "
            for label in labels:
                if '_QC' in label:
                    continue
                msg += "{}, ".format(label)
            msg += "deleted"
        self.msg2TextArea.emit("\n{}".format(msg))
        self.msg2Statusbar.emit(msg)

    def applyQC(self, listQC):
        """
        Apply the QC procedures
        :param listQC:
        :return:
        """

        def doIt(key_in):
            """
            Common part, to not repeat code
            :param key_in: key to apply QC tests
            """
            if '_QC' in key_in:
                return
            if listQC[0]:
                #  Reset flags
                self.msg2Statusbar.emit(
                    "Setting flags from {} to {}".format(key_in, listQC[0]))
                self.wf.reset_flag(key=key_in, flag=int(listQC[0]))
                self.msg2Statusbar.emit("Ready")
            if listQC[3]:
                # Spike test
                threshold = float(listQC[4].replace(',', '.'))
                self.msg2Statusbar.emit(
                    "Applying spike test to "
                    "{}, with rolling window {} and threshold {}".format(
                        key_in, listQC[5], threshold))
                self.wf.spike_test(key=key_in, window=int(listQC[5]),
                                   threshold=threshold, flag=int(listQC[3]))
                self.msg2Statusbar.emit("Ready")
            if listQC[1]:
                # Range test
                self.msg2Statusbar.emit(
                    "Applying range test to {}".format(key_in))
                self.wf.range_test(key=key_in, flag=int(listQC[1]))
                self.msg2Statusbar.emit("Ready")
            if listQC[2]:
                # Flat test
                self.msg2Statusbar.emit(
                    "Applying flat test to"
                    " {}, with rolling window {}".format(key_in, listQC[5]))
                self.wf.flat_test(key=key_in, window=int(listQC[5]),
                                  flag=int(listQC[2]))
                self.msg2Statusbar.emit("Ready")
            if listQC[6]:
                # Flag to flag
                self.msg2Statusbar.emit(
                    "Changing flags of "
                    "{} from {} to {}".format(key_in, listQC[6], listQC[7]))
                self.wf.flag2flag(key=key_in, original_flag=int(listQC[6]),
                                  translated_flag=int(listQC[7]))
                self.msg2Statusbar.emit("Ready")

        self.msg2Statusbar.emit("Creating QC flags")

        if listQC[8] == 'all':
            for key in self.wf.parameters():
                doIt(key_in=key)
        else:
            for i in range(8, len(listQC)):
                key = listQC[i]
                doIt(key_in=key)

        self.msg2Statusbar.emit("Updating graphs")
        # Refresh the QC graph
        for plotWidget in self.plotWidgetList:
            if "QC" in plotWidget.name:

                if plotWidget.isVisible():
                    plotWidget.refreshPlot()
                # Show the QCBarPlot
                elif plotWidget.name == "QC":
                    plotWidget.refreshPlot()
                    plotWidget.show()
        self.msg2Statusbar.emit("Ready")

    def applyRename(self, original_key, new_key):
        """It renames keys from a WaterFrame"""
        # Rename key from the Waterframe
        self.msg2Statusbar.emit(
            "Changing name {} to {}".format(original_key, new_key))
        self.wf.rename(original_key, new_key)
        # Rename the key of the plotWidgets if it process
        for plotWidget in self.plotWidgetList:
            if isinstance(plotWidget.key, list):
                for i, key in enumerate(plotWidget.key):
                    if key == original_key:
                        plotWidget.key[i] = new_key
                        plotWidget.name = plotWidget.name.replace(original_key,
                                                                  new_key)
                        plotWidget.refreshPlot()
            else:
                if plotWidget.name == "QC":
                    plotWidget.refreshPlot()
                elif plotWidget.key == original_key:
                    plotWidget.key = new_key
                    plotWidget.name = new_key
                    plotWidget.refreshPlot()
        # Add data information into dataList
        self.addData(self.wf.data)
        self.msg2Statusbar.emit("Ready")
        self.msg2TextArea.emit(
            "Key name {} changed to {}.".format(original_key, new_key))

    def applyResample(self, rule):
        """
        It applies the resample function to  self.waterframe
        :param rule: Rule to resample.
        """
        self.msg2Statusbar.emit("Resampling data")
        self.wf.resample(rule)
        self.msg2Statusbar.emit("Ready")

    def applySlice(self, start, stop):
        """
        It applies the resample function to  self.waterframe
        :param start: Start time.
        :param stop: Stop time
        """
        self.msg2Statusbar.emit("Slicing data")

        self.wf.slice_time(start, stop)

        self.addData(self.wf.data)
        self.refreshPlots()

        self.msg2Statusbar.emit("Ready")
        self.msg2TextArea.emit(
            "Dataframe sliced from {} to {}.".format(start, stop))

    def graphClick(self, item):
        if item.text() == "QC":
            self.addQCBarPlot()
        elif item.text() == "Spectrogram":
            self.addSpectroPlot()

    def newWaterFrame(self):
        """Create a new WaterFrame object and clean all screens."""
        self.wf = WaterFrame()
        # Delete all plots
        for plotWidget in self.plotWidgetList:
            plotWidget.deleteLater()
        self.plotWidgetList.clear()
        # Hide the widget
        self.hide()

    def refreshPlots(self):
        """It refresh all plots"""
        self.msg2Statusbar.emit("Refreshing plots")

        for plotWidget in self.plotWidgetList:
            try:
                plotWidget.refreshPlot()
            except KeyError:
                self.plotWidgetList.remove(plotWidget)
                plotWidget.hide()
                plotWidget.deleteLater()

        self.msg2Statusbar.emit("Ready")


from flask import redirect, url_for, render_template
from flask_login import login_required

from . import meetings_bp
from .forms import MeetingForm, ActivityForm
from ..models import Meetings, Activity
from datetime import datetime, date

@meetings_bp.route('/addclass')
def addclass():
    activity_a = Activity(topic="taiwan", lead_by="person_taiwan", avenue="Lims", date=datetime.today())
    activity_b = Activity(topic="singapore", lead_by="person_singapore", avenue="Tans", date=datetime.today())
    category = "saturday"
    season = "winter 2018"
    remark = ""
    # meet_a = Meetings(category=category, activities=[activity_a, activity_b])
    # meet_a.save()

    meet_a = Meetings(category=category, season=season, remark=remark)
    meet_a.activities.append(activity_a)
    meet_a.activities.append(activity_b)
    meet_a.save()

    return redirect(url_for('justhello.withmsg', inmsg="add meeting success"))

@meetings_bp.route('/class_category', methods=['GET', 'POST'])
@login_required
def add_class_category():
    form = MeetingForm()
    # user =
    if form.validate_on_submit():
        meet = Meetings(category=form.category.data, season=form.season.data, remark=form.remark.data)
        is_success = meet.save()
        return redirect(url_for('justhello.withmsg', inmsg="add meeting success"))

    return render_template('meetings/class_category.html', title="add category", form=form)

@meetings_bp.route('/class_activities', methods=['GET', 'POST'])
@login_required
def add_activities():
    form = ActivityForm()
    if form.validate_on_submit():
        #for now just pick any category
        category_picked = Meetings.objects.first()
        # from datetime.date to datetime.datetime
        form_date_str = form.date.data.strftime('%Y-%m-%d')  # 2002-03-11
        date_picked = datetime.strptime(form_date_str, '%Y-%m-%d')
        activity = Activity(topic=form.topic.data, lead_by=form.lead_by.data, avenue=form.avenue.data, date=date_picked)

        category_picked.activities.append(activity)
        is_success = category_picked.save()
        return render_template('meetings/class_category.html', title="add category", form=form)
    return render_template('meetings/class_activities.html', title="add activities", form=form)


#!/usr/bin/python3

import os
import xarray
import pandas as pd
import glob
import numpy as np
from datetime import datetime
from datetime import timedelta

binDir = '/home/disk/bob/impacts/bin'
ldmDirBase = '/home/disk/data/albany/profiler'
siteNcDirBase = '/home/disk/bob/impacts/raw/nys_profiler_2023'
ncPrefix = 'nysm_profiler'
floatMissingValue = -996.
ubyteMissingValue = 255

# get dates for today and yesterday
#current_dt = datetime.utcnow()
#current_date_str = datetime.strftime(current_dt,'%Y%m%d')
#current_hour_str = datetime.strftime(current_dt,'%H')
#current_date_obj = datetime.strptime(current_date_str,'%Y%m%d')
#yesterday_date_obj = current_date_obj - timedelta(hours=24)
#yesterday_date_str = datetime.strftime(yesterday_date_obj,'%Y%m%d')
#if current_hour_str == '00':
#    date_list = [yesterday_date_str,current_date_str]
#else:
#    date_list = [current_date_str]
date_list = ['20221207','20221208','20221211','20221212',
             '20221214','20221215','20221216']

for date in date_list:

    print(date)

    # read nc file into dataframe & save attribute info
    ldmDir = ldmDirBase
    ncFile = glob.glob(ldmDir+'/*'+date+'*.nc')[0]
    #ncFile = ldmDir+'/2358-netcdf.proc.20220106.nc'
    print('ncFile = ',ncFile)
    ds = xarray.open_dataset(ncFile)
    # save attributes in dict
    globalAttrDict = ds.attrs
    # ADD SITE LONG/SHORT NAMES AND LAT/LON/ALT TO GLOBAL ATTS
    #varAttrDict = {'time_5M':{'units':'seconds','long_name':'seconds since 1970-01-01 00:00:00'}}
    varAttrDict = {}
    for var in ds.data_vars.keys():
        #print(var)
        dict = ds[var].attrs
        varAttrDict[var] = dict
    df = ds.to_dataframe()

    # get site list from file
    site_list = []
    for ind in df.index.values:
        site_list.append(ind[1])
    site_list = list(set(site_list))

    # create netcdf files by site
    for site in site_list:
        #print(site)
        (prof,siteShort) = site.split('_')

        df_site = df.xs((slice(None),site))
        df_site = df_site.swaplevel(0)
        df_site.reset_index(inplace=True)

        # Convert times from yyyy-mm-ddThh:mm:ss.ssssss' to seconds since 01-01-1970
        #df_site.reset_index(inplace=True)
        # getting warnings on next two lines
        # A value is trying to be set on a copy of a slice from a DataFrame.
        # Try using .loc[row_indexer,col_indexer] = value instead
        df_site['time'] = pd.to_datetime(df_site['time'])
        df_site['time'] = df_site['time'].map(lambda time: int( (time - datetime(1970,1,1)).total_seconds() ) )
        df_site.set_index(['time','range'],inplace=True)
        df_site = df_site.sort_index(level=0)

        # Change df_site column datatypes to original datatypes
        #df_site = df_site.astype(typeDict)

        # convert to xarray
        xr = xarray.Dataset.from_dataframe(df_site)
        for var in varAttrDict.keys():
            xr[var].attrs = varAttrDict[var]
        for attr in globalAttrDict.keys():
            xr.attrs[attr] = globalAttrDict[attr]

        # convert to netcdf
        siteNcDir = siteNcDirBase+'/'+date
        if not os.path.exists(siteNcDir):
            os.makedirs(siteNcDir)
        ncFile = siteNcDir+'/'+ncPrefix+'.'+date+'.'+siteShort.lower()+'.nc'

        encoding = {}
        for var in varAttrDict.keys():
            #print('var =',var)
            if var != 'time':
                if 'qc' not in var:
                    encoding[var] = {'_FillValue':floatMissingValue}
                else:
                    encoding[var] = {'_FillValue':ubyteMissingValue}
        xr.to_netcdf(ncFile,encoding=encoding)

        command = binDir+'/NYS_profiler_chgDataType.csh '+ncFile
        os.system(command)


#!/usr/bin/env python3
from collections import deque
import sys
try:
    from typing import List
except ImportError:
    pass

def solve(N: int, u: "List[int]", v: "List[int]", w: "List[int]"):
    G = [[] for _ in range(N)]
    for ui, vi, wi in zip(u, v, w):
        ui -= 1
        vi -= 1
        G[ui].append((vi, wi))
        G[vi].append((ui, wi))
    color = [-1] * N
    color[0] = 0
    q = deque([0])
    while q:
        s = q.popleft()
        for t, wi in G[s]:
            if color[t] >= 0:
                continue
            color[t] = (color[s] + wi) % 2
            q.append(t)
    for colori in color:
        print(colori)

def main():
    def iterate_tokens():
        for line in sys.stdin:
            for word in line.split():
                yield word
    tokens = iterate_tokens()
    N = int(next(tokens))  # type: int
    u = [int()] * (N - 1)  # type: "List[int]"
    v = [int()] * (N - 1)  # type: "List[int]"
    w = [int()] * (N - 1)  # type: "List[int]"
    for i in range(N - 1):
        u[i] = int(next(tokens))
        v[i] = int(next(tokens))
        w[i] = int(next(tokens))
    solve(N, u, v, w)

if __name__ == '__main__':
    main()


import time
import requests

def grab_image(url, img_name):
	path = "Images/" + img_name
	img_data = requests.get(url).content
	with open(path, 'wb') as handler:
		handler.write(img_data)

def grab_image_1min(url, img_name):
	frame = 0
	while True:
		image_name = str(frame) +  "_" + img_name
		grab_image(url, image_name)
		time.sleep(60)
		frame = frame + 1

def main():
	url = "http://www.dot.ca.gov/cwwp2/data/d4/cctv/image/TV388_N1PRESIDIO.jpg?1378316522948"
	img_name = "TV388_N1PRESIDIO.jpg"
	grab_image_1min(url, img_name)

if __name__ == "__main__":
	main()


import sys
import time
import numpy as np
from pathlib import Path
import json
import b2sdk.v1 as b2
import aljpy
import multiprocessing
from pavlov import runs, files, storage
from tqdm.auto import tqdm

@aljpy.memcache()
def api(bucket):
    # Keys are in 1Password
    api = b2.B2Api()
    keys = json.loads(Path('credentials.json').read_text())['backblaze'][bucket]
    api.authorize_account('production', **keys)
    return api

def sync_up(local, remote):
    bucket, path = remote.split(':')
    workers = multiprocessing.cpu_count()
    syncer = b2.Synchronizer(workers)
    with b2.SyncReport(sys.stdout, False) as reporter:
        syncer.sync_folders(
            source_folder=b2.parse_sync_folder(local, api(bucket)),
            dest_folder=b2.parse_sync_folder(f'b2://{bucket}/{path}', api(bucket)),
            now_millis=int(round(time.time() * 1000)),
            reporter=reporter)

def sync_down(local, remote, **kwargs):
    bucket, path = str(remote).split(':')
    workers = multiprocessing.cpu_count()
    syncer = b2.Synchronizer(workers, **kwargs)
    with b2.SyncReport(sys.stdout, False) as reporter:
        syncer.sync_folders(
            source_folder=b2.parse_sync_folder(f'b2://{bucket}/{path}', api(bucket)),
            dest_folder=b2.parse_sync_folder(str(local), api(bucket)),
            now_millis=int(round(time.time() * 1000)),
            reporter=reporter)

def upload(local, remote):
    bucket, path = remote.split(':')
    bucket = api(bucket).get_bucket_by_name(remote)
    bucket.upload_local_file(
        local_file=local,
        file_name=path)

def ablate_run_snapshots(run):
    for i, info in tqdm(storage.snapshots(run).items()):
        if (i == 0) or (np.log2(i) % 1 == 0):
            pass
        else:
            print('Removing', run, info['path'].name)
            files.remove(run, info['path'].name)

def ablate_snapshots():
    for run, info in runs.runs().items():
        ablate_run_snapshots(run)

def download(local, remote):
    bucket, path = remote.split(':')
    b2.get_bucket_by_name()
    download_dest = b2.DownloadDestLocalFile(local)
    bucket.download_file_by_name(path, download_dest)

def backup():
    sync_up('./output/pavlov', 'boardlaw:output/pavlov')
    sync_up('./output/experiments/architecture/results', 'boardlaw:output/experiments/architecture/results')

def fetch():
    sync_down('./output/pavlov', 'boardlaw:output/pavlov')

if __name__ == '__main__':
    backup()


# from https://www.rototron.info/raspberry-pi-esp32-micropython-web-server-tutorial/
# modified for MicroWebSrv2

from time import sleep
from MicroWebSrv2 import *
from machine import Pin
from neopixel import NeoPixel
import json

np = NeoPixel(Pin(13), 1)

@WebRoute(POST, '/led')
def RequestLedGet(microWebSrv2, request):
    content = request.Content  # Read JSON color data
    colors = json.loads(content)
    blue, green, red = [colors[k] for k in sorted(colors.keys())]
    np[0] = (red, green, blue)
    np.write()
    response = request.Response
    response.ReturnOk()

mws2 = MicroWebSrv2()

# For embedded MicroPython, use a very light configuration,
mws2.SetEmbeddedConfig()

# Blake 2020-08-09 set root path for ESP32
mws2.RootPath = '/www/'

mws2.StartManaged()

# Main program loop until keyboard interrupt,
try:
    while mws2.IsRunning:
        sleep(1)
except KeyboardInterrupt:
    pass

# End,
print()
mws2.Stop()
print('Bye')
print()


# coding=utf-8

import pytest
from test_module.models.tool import getResults
from test_module.models.TDB import *

class TestOrderdetail():
    """
    订单详情测试
    """
    test_data=[
        ('/order/',"SELECT A.id FROM Order_baseorder A WHERE A.`status`='CONFIRMED' AND A.product_id in (1,7) AND A.contract_id IS NULL LIMIT 1",200),
        ('/order/terminalorder/',"SELECT A.id FROM Order_baseorder A  WHERE A.`status`='CONFIRMED' AND A.product_id in (2,3,4,5,6,8) AND A.contract_id IS NULL LIMIT 3",200),
        ('/order/contract_order/',"SELECT A.id FROM Order_baseorder A  WHERE A.`status`='CONFIRMED'AND A.product_id in (16,17,32,33,48,49) AND A.contract_id IS NOT NULL LIMIT 1",200)
    ]
    #凯励程订单详情
    @pytest.mark.parametrize('url,sql,result1',test_data)
    def test_orderdetail1(self,url,sql,result1,connectDB):
        sql=sql
        cur = connectDB
        data = searchDB(cur, sql)
        if data is None:
            pytest.skip('无数据，用例无效')
        else:
            result = getResults(url+'%s/' % (str(data['id'])))
            assert result.status_code == result1
            response = result.json()
            try:
                assert response['id']==data['id']
            except AssertionError:
                raise AssertionError('订单详情接口失败')


from django.urls import path, include
from django.contrib import admin
from . import views

urlpatterns = [
    path('', views.courses, name = 'Courses'),
    path('new_account/', views.new_account, name = 'Create an account'),
    path('create_account/', views.create_account, name = 'Create an account - step 2'),
    path('user/', views.user_page, name = 'User page'),
    path('delete_account/', views.delete_account, name = 'Delete user account'),
    path('course/<course_id>/', views.course_page, name = 'Course page'),
    path('private_courses/', views.private_courses, name = 'Private courses'),
    path('create/', views.create_course, name = 'Create a new course'),
    path('new_course/', views.create_course_follow_up, name = 'Create a new course - step 2'),
    path('edit/<course_id>/', views.edit_course, name = 'Edit course'),
    path('edit/<course_id>/modify_data/', views.modify_course_data, name = 'Modify course data'),
    path('edit/<course_id>/add_level', views.add_level_to_course, name = 'Add a level to a course'),
    path('add_unit/<course_id>/<level_num>/', views.add_unit_to_level, name = 'Add a unit to an existing level of a course'),
    path('edit_new_unit/', views.edit_new_unit, name = 'Edit a newly created unit'),
    path('edit_unit/<unit_id>/', views.edit_unit, name = 'Edit a unit'),
    path('edit_unit/<unit_id>/add_lesson/', views.add_lesson, name = 'Add a lesson to a unit'),
    path('edit_unit/<unit_id>/<lesson_id>/', views.edit_lesson, name = 'Edit a lesson of a unit'),
    path('exercise_types/', views.exercise_types, name = 'A listing of the available sorts of exercises'),
    path('edit_unit/<unit_id>/<lesson_id>/add_exercise/', views.add_exercise, name = 'Add an exercise to a lesson'),
    path('edit_unit/<unit_id>/<lesson_id>/exercise_created/', views.exercise_created, name = 'Used right after a new exercise is created'),
    path('edit_unit/<unit_id>/<lesson_id>/<exercise_id>/', views.edit_exercise, name = 'Edit an existing exercise'),
    path('edit_unit/<unit_id>/<lesson_id>/<exercise_id>/delete/', views.delete_exercise, name = 'Delete an exercise'),
    path('edit/<course_id>/edit_glosses/', views.edit_glosses, name = 'Edit glosses used in the course'),
    path('unit/<unit_id>/', views.view_unit, name = 'View unit'),
    path('unit/<unit_id>/<lesson_id>/', views.do_lesson, name = 'Do a lesson'),
    path('submit_ticket/', views.submit_ticket, name = 'Submit an error ticket'),
    path('course/<course_id>/submit_ticket/', views.submit_ticket_to_course, name = 'Submit an error ticket for a particular course'),
    path('comments/', views.see_messages, name = 'See comments, report tickets, etc.'),
    path('comment/<exercise_id>/', views.view_exercise_comments, name = 'Read and write comments for an exercise'),
    path('add_new_UI/', views.add_ui_language, name = 'Add an interface language'),
    path('edit_UI/<ui_lang_id>/', views.edit_ui_language, name = 'Edit an interface language'),
    path('special/', views.special, name = 'Used for development purposes'),
    #path('accounts/', include('django.contrib.auth.urls')),
]

admin.site.site_header = 'Erkonlang admin'
admin.site.site_title = 'Erkonlang admin'


from GoCoach import Coach
from go.GoGame import GoGame as Game
from go.pytorch.NNet import NNetWrapper as nn
from utils import *
import os,sys
from enum import Enum

sys.setrecursionlimit(5000)

class Display(Enum):
    NO_DISPLAY = 0
    DISPLAY_BAR = 1
    DISPLAY_BOARD = 2

BoardSize=7
NetType='CNN' # or 'RES'
tag='MCTS_SimModified'

args = dotdict({
    'numIters': 1000,
    'numEps': 100,
    'tempThreshold': 15,
    'updateThreshold': 0.54,
    'maxlenOfQueue': 200000,
    'numMCTSSims': 200,
    'arenaCompare': 50,
    'cpuct': 3,

    'checkpoint': './logs/go/{}_checkpoint/{}/'.format(NetType + '_' + tag, BoardSize),
    'load_model': True,
    'numItersForTrainExamplesHistory': 25,
    'display': Display.NO_DISPLAY
})

if __name__=="__main__":

    g = Game(BoardSize)
    nnet = nn(g, t=NetType)
    logPath='./logs/go/{}_log/{}/'.format(NetType + '_' + tag, BoardSize)
    try:
        os.makedirs(logPath)
    except:
        pass

    if args.load_model:
        nnet.load_checkpoint(args.checkpoint, 'best.pth.tar')

    c = Coach(g, nnet, args, log=True, logPath=logPath)
    if args.load_model:
        print("Loading trainExamples from file")
        c.loadTrainExamples()
    c.learn()


from django.test import TestCase
import json
# Create your tests here.

 ##########################################################
  #Test Cases
  ##########################################################
class TestAPI(TestCase):

    def test_fact_API(self):
        num = 5;

        response = self.client.get('/mobile/api/fact/?n1=%s' %(num))
        data = (response.content).decode('utf-8')

        data = json.loads(data)
        fact = data.get('FACTORIAL')
        status = data.get('status')

        self.assertEquals(120,fact)
        self.assertEquals('Success',status)

    def test_add_API(self):
        a = 5; b = 3;

        response = self.client.get('/mobile/api/add/?n1=%s&n2=%s' %(a,b))
        data = (response.content).decode('utf-8')

        data = json.loads(data)
        sum = data.get('Addition_result')
        status = data.get('status')

        self.assertEquals(8,sum)
        self.assertEquals('Success',status);

    '''
	def test_mobileCSVInventory(self):
		response = self.client.get('/mobile/api/mobileCSVInventory/')
		status = json.loads((response.content).decode('utf-8')).get('status')
		self.assertEquals('Success',status)

	def test_mobileDBInventory(self):
		response = self.client.get('/mobile/api/mobileDBInventory/')
		status = json.loads((response.content).decode('utf-8')).get('status')
		self.assertEquals('Success',status)

	def test_mobileInventoryUI(self):
		response = self.client.get('/mobile/api/mobileInventory.html')
		status = response.status_code
		self.assertEquals(200,status)
		'''


#!/usr/bin/env python3

from argparse import ArgumentParser
from operator import itemgetter
from collections import OrderedDict
import pathlib

def loaddata(filepath):
    data = {}
    fpath = pathlib.Path(filepath).resolve()
    with open(fpath) as fh:
        for line in fh:
            cols = line.split()
            target = cols[0].split('_')[0]
            try:
                data[target].append((cols[0], float(cols[1])))
            except KeyError:
                data[target] = [(cols[0], float(cols[1]))]
    return data

def output(data):
    text = []
    for target in data:
        text.append('Target: {}'.format(target))
        subdata = data[target]
        text.append('\tHighest GDT score: '
                    '\n\t\t{}, {}'.format(subdata[0][0],
                                          subdata[0][1]))
        text.append('\tHighest alignment score:')
        for cand in subdata[1:]:
            text.append('\t\t{}, {}, GDT={}, '
                        'diff={:.2f}, rank={}'.format(*cand))
        text.append('\n')
    print('\n'.join(text))

def compare(gdt, other, howmany=1):
    #Ensure gdt and other share a common target set
    target1 = set(gdt.keys())
    target2 = set(other.keys())
    for key in target1 - target2:
        gdt.pop(key)
    for key in target2 - target1:
        other.pop(key)

    results = {}
    for target in gdt:
        first = sorted(gdt[target], key=itemgetter(1), reverse=True)
        firstod = OrderedDict(first)
        second = sorted(other[target], key=itemgetter(1), reverse=True)
        best1 = first[0]
        data = [(best1[0], best1[1], best1[1], 0 , 1)]
        i = 0
        for cand in second:
            candname = cand[0]
            candscore = cand[1]
            try:
                gdtscore = firstod[candname]
                diff = best1[1] - firstod[candname]
                rank = first.index((candname, gdtscore)) + 1
            except KeyError:
                continue
            data.append((
                candname, candscore, gdtscore, diff, rank
            ))
            i += 1
            if i == howmany:
                break
        results[target] = data

    return results

def run(gdtfile, otherfile, howmany=1):
    data = compare(loaddata(gdtfile), loaddata(otherfile), howmany)
    output(data)

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('gdtfile', help='GDT score file')
    parser.add_argument('otherfile',
                        help='File to be compared to GDT file')
    parser.add_argument('-m', '--howmany', type=int, default=1,
                        help='How many resuts from the other file '
                        'to display')
    args = parser.parse_args()
    run(args.gdtfile, args.otherfile, args.howmany)


import os
import sys
from datetime import datetime

import tensorflow.contrib.summary as tfs
#import tensorflow.contrib.summary.summary as tfs
from tensorflow.python.keras.callbacks import History
from tensorflow.python.ops.summary_ops_v2 import SummaryWriter

from utils.summary_callback import SummaryCallback

def save_model(model, hist: History, time):
    now = time.strftime("%m-%d_%H-%M")
    save_dir = "save/" + os.path.basename(sys.argv[0]) + "_" + now + "/"
    a = []

    def returnMe(y):
        a.append(y)

    model.summary(print_fn=returnMe)

    model.save(filepath=save_dir + "model.h5", overwrite=True, include_optimizer=False)
    file = open(save_dir + now + ".info.txt", mode='w')
    file.write(os.path.basename(sys.argv[0]) + "\r\n")
    file.write("in:" + str(model.input_shape) + "\r\n"
               + "\r\n".join(a) + "\r\n"
               + "out:" + str(model.output_shape) + "\r\n"
               + "time:" + datetime.now().strftime("%m-%d_%H-%M"))
    file.close()
    file = open(save_dir + now + ".history.txt", mode='w')
    file.write(os.path.basename(sys.argv[0]) + "\r\n")
    for k, v in hist.history.items():
        file.write("k:{}=v:{}\r\n".format(k, v))
    file.close()

def get_callbacks(model, time):
    now = time.strftime("%m-%d_%H-%M")
    logdir = "save/" + os.path.basename(sys.argv[0]) + "_" + now + "/logs"
    os.makedirs(logdir, exist_ok=True)
    file_writer: SummaryWriter = tfs.create_file_writer(logdir, flush_millis=1000)
    file_writer.set_as_default()
    km = SummaryCallback(file_writer, model, logdir)
    return [km]


import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk

from webbrowser import open as open_browser
import requests
from crypto_helper import CryptoHelper

OAUTH_HOST = "https://www.dropbox.com/oauth2/authorize"
TOKEN_HOST = "https://api.dropbox.com/oauth2/token"
API_HOST = "https://api.dropboxapi.com/2/files/list_folder"
CLIENT_ID = "tpr1swzr3p44n26"
OAUTH_URL = "{0}?client_id={1}&response_type=code&code_challenge_method=S256&code_challenge={2}"
TOKEN_URL = "{0}?client_id={1}&grant_type=authorization_code&code={2}&code_verifier={3}"

class OauthWindow(Gtk.Window):
  def __init__(self):
    Gtk.Window.__init__(self, title="OAuth authenticator")

    self.code_verifier = ""
    self.token = ""
    self.crypto_helper = CryptoHelper()

    builder = Gtk.Builder()
    builder.add_from_file("./main.glade")
    container = builder.get_object("container")
    btn_grantapp = builder.get_object("btn_grantapp")
    btn_grantapp.connect("clicked", self.grant_app)
    btn_gettoken = builder.get_object("btn_gettoken")
    btn_gettoken.connect("clicked", self.generate_token)
    btn_callapi = builder.get_object("btn_callapi")
    btn_callapi.connect("clicked", self.call_api)
    self.txt_authcode = builder.get_object("txt_auth_code")
    self.txt_response = builder.get_object("txt_response")
    self.add(container)

  def grant_app(self, widget):
    code = self.crypto_helper.generate_verifier_code()
    self.code_verifier = code["verifier"]
    auth_url = OAUTH_URL.format(OAUTH_HOST, CLIENT_ID, code["challenge"])
    open_browser(auth_url)

  def generate_token(self, widget):
    oauth_code = self.txt_authcode.get_text()
    params = {
      "client_id": CLIENT_ID,
      "grant_type": "authorization_code",
      "code": oauth_code,
      "code_verifier": self.code_verifier
    }
    response = requests.post(TOKEN_HOST, params)
    json = response.json()
    self.token = json["access_token"]

  def call_api(self, widget):
    json = {
      "path": ""
    }
    headers = {
      "Authorization": "Bearer {0}".format(self.token)
    }
    response = requests.post(API_HOST,headers=headers,json=json)
    json = response.text
    buffer = self.txt_response.get_buffer()
    buffer.set_text(json.replace("{", "{\n").replace("}", "}\n").replace(",", ",\n"))


import logging
import os
import random
import datetime
import time
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, BaseFilter
from telegram import Message, User, Chat, MessageEntity, Document
from random import randint
import duel

games = ['R6','R6','R6', 'RL', 'RL', 'RL', 'Apex']
stickerCount = {}
englishCount = {}
baseClock = time.time()
msgCount = {}
floodStat = {}
floodClock = time.time()

def start(bot, update):
    update.message.reply_text("GG ?")

def randomgame(bot, update):
    update.message.reply_text("GG "+ random.choice(games))

def roll(bot, update):
    update.message.reply_text("GG "+ str(randint(0,100)))

def isEnglish(text):
    countEnglishLetters = 0
    for c in text:
        if ord(c) < 128 and c.isalpha():
            countEnglishLetters += 1
    return countEnglishLetters/len(text) >0.5

def processText(bot, update):
    global englishCount
    global baseClock
    print("new Text Upate : "+str(update))

    user = update.effective_user
    username = user['username']
    print('user name'+ str(username))
    print('update.message.text '+ update.message.text + 'isenglish: '+str(isEnglish(update.message.text)))
    if isEnglish(update.message.text):
        englishCount[username] = englishCount.setdefault(username, 0) + 1

    timenow = time.time()
    temp = timenow-baseClock
    hours = temp//3600
    if(hours>5):
        baseClock = timenow
        englishCount = {}

    if(englishCount.setdefault(username, 0)>10) and isEnglish(update.message.text):
        bot.delete_message(update.message.chat.id, update.message.message_id)

def processSticker(bot, update):
    global stickerCount
    global baseClock
    print("sticker update"+str(update))

    user = update.effective_user
    username = user['username']

    stickerCount[username] = stickerCount.setdefault(username, 0) + 1

    timenow = time.time()
    temp = timenow-baseClock
    hours = temp//3600
    if(hours>5):
        baseClock = timenow
        stickerCount = {}

    if(englishCount.setdefault(username, 0)>5):
        bot.delete_message(update.message.chat.id, update.message.message_id)

def antiFlood(bot, update):
    global msgCount
    global floodClock

    print("All Filter update"+str(update))

    user = update.effective_user
    username = user['username']
    msgCount[username] = msgCount.setdefault(username, 0) + 1

    timenow = time.time()

    temp = timenow - floodClock
    if(temp < 10 and msgCount.setdefault(username, 0) > 10 ):
        bot.delete_message(update.message.chat.id, update.message.message_id)
        floodStat[username] = True

    if(temp > 10):
        floodClock = time.time()
        floodStat[username] = False
        msgCount[username] = 0

def error(bot, update, error):
    logger.warning('Update "%s" caused error "%s"', update, error)

if __name__ == "__main__":
    # Set these variable to the appropriate values
    TOKEN = os.environ.get('TOKEN')
    NAME = "gg-manager-bot"

    # Port is given by Heroku
    PORT = os.environ.get('PORT')

    # Enable logging
    logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',level=logging.INFO)
    logger = logging.getLogger(__name__)

    # Set up the Updater
    updater = Updater(TOKEN)
    dp = updater.dispatcher
    # Add handlers
    dp.add_handler(CommandHandler('start', start))
    dp.add_handler(CommandHandler('roll', roll))
    dp.add_handler(CommandHandler('randomgame', randomgame))
    dp.add_handler(CommandHandler('start_duel', duel.start_duel))
    dp.add_handler(CommandHandler('shot!', duel.shot))
    dp.add_handler(MessageHandler((Filters.text & (~ Filters.entity(MessageEntity.MENTION))), processText))
    dp.add_handler(MessageHandler((Filters.sticker | Filters.animation), processSticker))
    dp.add_handler(MessageHandler(Filters.all, antiFlood))

    dp.add_error_handler(error)

    # Start the webhook
    updater.start_webhook(listen="0.0.0.0",port=int(PORT),url_path=TOKEN)
    updater.bot.setWebhook("https://{}.herokuapp.com/{}".format(NAME, TOKEN))

    updater.idle()


# -*- coding: utf-8 -*-
"""
Created on Sat Sep 16 14:47:26 2017

@author: 王
"""

#encoding=utf-8
import os
import tensorflow as tf
from PIL import Image
import numpy as np
import time
start=time.time()

def weight_variable(shape):
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)

def bias_variable(shape):
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)

def conv2d(x, W):
    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
    #x是输入，W是卷积的参数，如[5,5,1,32]:前面两个数字代表卷积核的尺寸，第三个表示channel
    #灰度图就是1，RGB就是3，最后一个数字代表卷积核的数量，就是卷积层提取多少类的特征。
    #Strides代表卷积模板移动的步长，[1,1,1,1]代表不遗漏每个点
    #padding代表处理的方式，AAME代表给边界上加上Padding让卷积的输出和输入保持同样尺寸
    #因此卷积前后图像尺寸不变，而池化时候减小
def max_pool_2x2(x):
    return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1], padding='SAME')
    #最大池化函数，对k*k，是ksize=[1, k, k, 1],strides=[1, k, k, 1]

cwd = os.getcwd()

classes = {'classOne','classTwo','classThree'}
#制作二进制数据
def create_record():
    writer = tf.python_io.TFRecordWriter("train.tfrecords")
    for index, name in enumerate(classes):
        class_path = cwd +"/"+ name+"/"
        for img_name in os.listdir(class_path):
            img_path = class_path + img_name
            img = Image.open(img_path)
            img = img.resize((64, 64))
            img_raw = img.tobytes() #将图片转化为原生bytes
            print(index,img_raw)
            example = tf.train.Example(
               features=tf.train.Features(feature={
                    "label": tf.train.Feature(int64_list=tf.train.Int64List(value=[index])),
                    'img_raw': tf.train.Feature(bytes_list=tf.train.BytesList(value=[img_raw]))
               }))
            writer.write(example.SerializeToString())
    writer.close()

data = create_record()

#读取二进制数据
def read_and_decode(filename):
    # 创建文件队列,不限读取的数量
    filename_queue = tf.train.string_input_producer([filename])
    # create a reader from file queue
    reader = tf.TFRecordReader()
    # reader从文件队列中读入一个序列化的样本
    _, serialized_example = reader.read(filename_queue)
    # get feature from serialized example
    # 解析符号化的样本
    features = tf.parse_single_example(
        serialized_example,
        features={
            'label': tf.FixedLenFeature([], tf.int64),
            'img_raw': tf.FixedLenFeature([], tf.string)
        }
    )
    label = features['label']
    img = features['img_raw']
    img = tf.decode_raw(img, tf.uint8)
    #############################################################################
    img = tf.reshape(img, [64, 64, 3])
    img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
    label = tf.cast(label, tf.int32)
    return img, label

if __name__ == '__main__':
    if 0:
        data = create_record("train.tfrecords")
    else:
        img, label = read_and_decode("train.tfrecords")
        print(img,label)
        #使用shuffle_batch可以随机打乱输入 next_batch挨着往下取
        # shuffle_batch才能实现[img,label]的同步,也即特征和label的同步,不然可能输入的特征和label不匹配
        # 比如只有这样使用,才能使img和label一一对应,每次提取一个image和对应的label
        # shuffle_batch返回的值就是RandomShuffleQueue.dequeue_many()的结果
        # Shuffle_batch构建了一个RandomShuffleQueue，并不断地把单个的[img,label],送入队列中
        img_batch, label_batch = tf.train.shuffle_batch([img, label],
                                                    batch_size=4, capacity=2000,
                                                    min_after_dequeue=1000)

        '''
        # 初始化所有的op
        init =tf.global_variables_initializer()

        sess = tf.Session()
        sess.run(init)
        # 启动队列
        threads = tf.train.start_queue_runners(sess=sess)
        for i in range(5):
            print(img_batch.get_shape(),label_batch)
            #修改img_batch.shape为img_batch.get_shape()
            val, l = sess.run([img_batch, label_batch])
            # l = to_categorical(l, 12)
            print(val.shape, l)
        '''
    print('开始构建图......')
    sess = tf.InteractiveSession()
    x = tf.placeholder(tf.float32, shape=[None, 64*64])#tf.float32, [None, w*h] x为特征
    y_ = tf.placeholder(tf.float32, shape=[None, 3])#Y_是真实的label
    # [None, ysize]) y的数目个数 比如3个类 就是3
    x_image = tf.reshape(x, [-1,64,64,3])
    #将输入的1*784形式转为图片原始的28*28结构 tf.reshape
    #第2、第3维对应图片的宽、高，最后一维代表图片的颜色通道数
    #(因为是灰度图所以这里的通道数为1，如果是rgb彩色图，则为3)
    #-1代表样本数量不顾的
    ############# 第一层卷积 #################
    W_conv1 = weight_variable([5, 5, 3, 32])
    #5*5的卷积核(patch)，1是输入的通道数目，32是32个不同的卷积核，算出32个特征
    b_conv1 = bias_variable([32])
    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
    #卷积
    h_pool1 = max_pool_2x2(h_conv1)
    #池化
    ############# 第二层卷积 ################
    W_conv2 = weight_variable([5, 5, 32, 64])
    #32个输入，64个输出
    b_conv2 = bias_variable([64])
    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
    #第一层卷积所得的h_pool1作为输入
    h_pool2 = max_pool_2x2(h_conv2)

    ############ 密集连接层 #################
    W_fc1 = weight_variable([7 * 7 * 64, 1024])
    b_fc1 = bias_variable([1024])
    h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])
    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
    ########### Dropout层 ##################
    keep_prob = tf.placeholder(tf.float32)
    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
    ########### Softmax层 (输出) ############
    W_fc2 = weight_variable([1024, 3])
    b_fc2 = bias_variable([3])
    y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
    #W = tf.Variable(tf.zeros([784,10]))
    #b = tf.Variable(tf.zeros([10]))

    #训练和评估模型
    #定义损失函数和优化器
    cross_entropy = -tf.reduce_sum(y_*tf.log(y_conv))
    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
    #定义评测准确率的操作
    correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    print('开始执行op......')
    sess.run(tf.global_variables_initializer())

    threads = tf.train.start_queue_runners(sess=sess)

    for i in range(1000):
        batch = img_batch
        #if i%10 == 0:
        #   train_accuracy = accuracy.eval(feed_dict={x:batch[0], y_: batch[1], keep_prob: 1.0})
        #    print("step %d, training accuracy %g"%(i, train_accuracy))
        train_step.run(feed_dict={x: batch[0].eval(), y_: batch[1].eval(), keep_prob: 0.5})

    #print("test accuracy %g"%accuracy.eval(feed_dict={
    #    x: img_batch, y_: label_batch, keep_prob: 1.0}))

    end=time.time()
    print('运行时长：',(end-start)/60,'分钟')


from Utils import read_csv
import csv
import itertools
import numpy as np
from pandas import DataFrame, get_dummies
from collections import defaultdict
import codecs
import re
import xml.etree.ElementTree as ET

class DatasheetReader:
    # This class reads csv data files and wrap them in dataframe
    # and output features as options for Dataset class

    def __init__(self, filename, features):
        self.filename = filename
        self.load()
        #selected = ['date_submitted', 'faq_id', 'product_code', 'product_name', 'source', 'sr', 'sr_type', 'title', 'urgency']
        if not features:
            self.features = []
        else:
            self.features = features
        self.subset(self.features + ["faq_id"])
        self.separate_faq_id()
        #self.split_data_labels()
        #self.normalize_separators()
        #self.process_problem_code()

    def getUniqueClasses(self, column):
        return self.df[column].unique().tolist()

    def load(self):
        reader = read_csv(self.filename)
        line = next(reader)
        self.n_fields = len(line)
        self.fieldnames = line

        # fix fieldnames
        self.normalize_names()
        print("Loading csv file...")
        print("Column names:")
        print(", ".join([ str(i) + "-" + item for i, item in enumerate(line)]))
        print("Number of columns: ", len(line))

        # to dataframe
        data = defaultdict(list)
        for line in reader:
            #print line
            for i, x in enumerate(line):
                data[self.fieldnames[i]].append(x)

        self.df = DataFrame(data)
        self.fieldnames = self.df.columns

    def normalize_names(self):
        old_names = self.fieldnames
        #old_new = {}
        new_names = []
        for old in old_names:
            new = old.replace(" ", "_").replace(".", "").lower()
            #old_new[old] = new
            new_names.append(new)
        self.fieldnames =  new_names

    def subset(self, s):
        if 'sr' not in s:
            s += ['sr']

        if len(s) == 2:
            s = self.df.columns
        self.df = self.df[s]

    def process_problem_code(self):
        # split problem codes to 3 Cats
        self.df.loc[:,'cat1'] = self.df['problem_code'].str[0:2]
        self.df.loc[:,'cat2'] = self.df['problem_code'].str[2:4]
        self.df.loc[:,'cat3'] = self.df['problem_code'].str[4:6]
        self.df.drop('problem_code', 1, inplace=True)

    def binarize_labels(self):
        labels = list()
        for _cls in ('cat1', 'cat2', 'cat3'):
            for val in set(self.df[_cls]):
                name = _cls + "=" + val
                typ  = ['0', '1']
                labels.append((name, typ))
        self._label_classes = labels

    def normalize_separators(self):
        def f(ids):
            sep = r"\W*[\u3001-\u303f;/；:：.,&　，＆]"
            labels = re.split(sep, ids)
            #labels = ";".join([l.strip() for l in labels if l.strip() != ""])
            return labels
        self.df["faq_id"] = self.df["faq_id"].apply(f)

    def separate_faq_id(self):
        sep = r"\W*[\u3001-\u303f;/；:：.,&　，＆]"
        s = self.df["faq_id"].str.split(sep, expand=True).stack()
        i = s.index.get_level_values(0)
        df2 = self.df.loc[i].copy()
        df2["faq_id"] = s.values
        self.df = df2#.reset_index()

    def split_data_labels(self):
        self.labels = self.df["faq_id"]
        self.data   = self.df.drop('faq_id', 1)

    def to_dict(self):
        data = []
        labels = self.labels.tolist()
        for row in self.data.to_dict('records'):
            new_row = {}
            for k, v in row.items():
                name = k
                val  = v
                new_row.update({name:  val})
            data.append(new_row)
        return data, labels

    def get_data(self):
        data = defaultdict(dict)
        for row in self.df.to_dict('records'):
            new_row = {}
            sr = row['sr']
            #data[sr]['data'] = []
            if 'label' not in data[sr]:
                data[sr]['label'] = set()
            data[sr]['label'].add(row['faq_id'])
            for k, v in row.items():
                if k == 'faq_id':
                    continue
                name = k
                val  = v
                new_row.update({name:  val})
            data[sr]['data'] = new_row
        return data

if __name__ == "__main__":
    d = DatasheetReader("../dataset/Query1.csv")


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^$', views.index),
	url(r'^index$', views.index),
	url(r'^oauth_info$', views.oauth_info),
	url(r'^submitmobile$', views.submitmobile, name='submitmobile'),
	url(r'^validsmscode$', views.validsmscode, name='validsmscode'),
	url(r'^login/(?P<openid>.*)/$', views.login),
	url(r'^mycenter$', views.mycenter, name='mycenter'),
	url(r'^logout$', views.logout),
	url(r'^editprofile$', views.editprofile, name='editprofile'),
]


# pylint: disable=import-error
import logging
from importlib import import_module
import uuid

from flask import Flask, jsonify
from flask_cors import CORS

# pylint: disable=no-name-in-module
from app.factories.api_factory import create_api
from utils.configmanager import ConfigManager
from utils.common import make_url

logging.getLogger("flask_cors").level = logging.DEBUG

def make_flask():
    flask_app = Flask(__name__)
    flask_app.config["SECRET_KEY"] = str(uuid.uuid4())
    flask_app.config["MAX_CONTENT_LENGTH"] = 8 * 1024 * 1024

    if "database" in ConfigManager.get_config():
        config_database(flask_app, ConfigManager.get_config_value("database"))

    apis_config = ConfigManager.get_config_value("apis")

    for api_config in apis_config.values():
        api = create_api(api_config)
        api.init_app(flask_app)

    # pylint: disable=unused-variable
    @flask_app.errorhandler(422)
    def handle_error(err):
        headers = err.data.get("headers", None)
        messages = err.data.get("messages", ["Invalid request."])
        if headers:
            return jsonify({"errors": messages}), err.code, headers
        return jsonify({"errors": messages}), err.code

    CORS(flask_app)
    return flask_app

sqlalchemy_backends = ["postgresql", "sqlite"]

def config_database(flask_app, config):
    module_name = "database." + next(iter(config.keys()))
    module = import_module(module_name)
    db = getattr(module, "db")
    db_config = getattr(module, "db_config")
    if db_config["type"] in sqlalchemy_backends:
        flask_app.config["SQLALCHEMY_DATABASE_URI"] = make_url(db_config)
        flask_app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = True
        db.init_app(flask_app)


# Deep Neural Network Classifier
# Author: Louis Heery

import sys
import time

import pandas as pd
from keras.layers import Dense
from keras.models import Sequential
from sklearn.preprocessing import scale
import tensorflow as tf

mirrored_strategy = tf.distribute.MirroredStrategy()

#tfg.config.gpu.set_per_process_memory_fraction(0.4)

#config = tf.ConfigProto()
#config.gpu_options.allow_growth=True
#sess = tf.Session(config=config)

sys.path.append("../dataset-and-plotting")
from nnPlotting import *

def totalSensitivity(A,B,errorA,errorB):
    totalSensitivity = np.sqrt(A**2 + B**2)
    totalError = np.sqrt(((A*errorA)/np.sqrt(A**2 + B**2))**2 + ((B*errorB)/np.sqrt(A**2 + B**2))**2)

    return (totalSensitivity,totalError)

print("STARTED")
start = time.time()

for nJets in [2,3]:

    print("STARTED TRAINING " + str(nJets) + " Jet Neural Network")

    if nJets == 2:
        variables = ['dRBB','mBB','pTB1', 'pTB2', 'MET','dPhiVBB','dPhiLBmin','Mtop','dYWH', 'mTW', 'pTV', 'MV1cB1_cont', 'MV1cB2_cont', 'nTags', 'nTrackJetsOR']

    else:
        variables = ['mBB', 'dRBB', 'pTB1', 'pTB2', 'MET', 'dPhiVBB', 'dPhiLBmin', 'Mtop', 'dYWH', 'mTW', 'pTV', 'mBBJ', 'pTJ3', 'MV1cB1_cont', 'MV1cB2_cont', 'MV1cJ3_cont', 'nTags', 'nTrackJetsOR']

    # Read in Data
    if nJets == 2:
        dfEven = pd.read_csv('../dataset-and-plotting/CSV/VHbb_data_2jet_even.csv')
        dfOdd = pd.read_csv('../dataset-and-plotting/CSV/VHbb_data_2jet_odd.csv')

    else:
        dfEven = pd.read_csv('../dataset-and-plotting/CSV/VHbb_data_3jet_even.csv')
        dfOdd = pd.read_csv('../dataset-and-plotting/CSV/VHbb_data_3jet_odd.csv')

    # Process Even Events
    xEven = scale(dfEven[variables].to_numpy())
    yEven = dfEven['Class'].to_numpy()
    wEven = dfEven['training_weight'].to_numpy()

    # Process Odd Events
    xOdd = scale(dfOdd[variables].to_numpy())
    yOdd = dfOdd['Class'].to_numpy()
    wOdd = dfOdd['training_weight'].to_numpy()

    # Define Architecture
    def DNNClassifier():

        model = Sequential()

        # Add Layers
        model.add(Dense(units=14, input_shape=(xEven.shape[1],), activation='relu')) # 1st layer
        model.add(Dense(14, init='uniform', activation='relu')) # hidden layer
        model.add(Dense(14, init='uniform', activation='relu')) # hidden layer
        model.add(Dense(1,activation='sigmoid')) # output layer
        model.compile(loss='binary_crossentropy', optimizer='SGD', metrics=['accuracy'])
        return model

    # Create and Compile models
    modelEven = DNNClassifier()
    modelOdd = DNNClassifier()

    # Set these parameters
    epochs = 200
    batchSize = 100

    # Train Model
    modelEven.fit(xEven,yEven, sample_weight = wEven, epochs=epochs, batch_size=batchSize, verbose = 1)
    modelOdd.fit(xOdd,yOdd, sample_weight = wOdd, epochs=epochs, batch_size=batchSize, verbose = 1)

    print("Time Taken = " + str(round(time.time()-start,2))+"s")

    ## EVALUATION DNN & Plots
    dfOdd['decision_value'] = modelEven.predict_proba(xOdd)
    dfEven['decision_value'] = modelOdd.predict_proba(xEven)
    df = pd.concat([dfOdd,dfEven])

    figureName = "DNN_" + str(nJets) + "Jets"+ ".pdf"
    nn_output_plot(df, figureName)

    if nJets == 2:
        df.to_csv("DNNOPTIMALALL_2jet_200epoch.csv")
        sensitivity2Jet = sensitivity_NN(df)

    else:
        df.to_csv("DNNOPTIMALALL_3jet_200epoch.csv")
        sensitivity3Jet = sensitivity_NN(df)

    print("FINISHED TRAINING " + str(nJets) + " Jet Neural Network")

print("2 Jet Sensitivity: " + str(sensitivity2Jet[0]) + " ± "+ str(sensitivity2Jet[1]))
print("3 Jet Sensitivity: "+ str(sensitivity3Jet[0]) + " ± "+ str(sensitivity3Jet[1]))

sensitivityCombined = totalSensitivity(sensitivity2Jet[0],sensitivity3Jet[0],sensitivity2Jet[1],sensitivity3Jet[1])

print("Combined Sensitivity", sensitivityCombined[0], "±",sensitivityCombined[1])
print("Total Time Taken", time.time() - start)
print("FINISHED")


from django.shortcuts import render

from .models import Website
from .forms import AddSiteForm

def index(request):
    empty_form = AddSiteForm()

    if request.method == "POST":
        form = AddSiteForm(request.POST)
        if form.is_valid():
            form.save()

    if request.GET.get('remove_site'):
        w = Website.objects.get(url=request.GET.get('remove_site'))
        w.delete()

    if request.GET.get('ping_site'):
        w = Website.objects.get(url=request.GET.get('ping_site'))
        w.is_up = Website.is_site_up(w.url)
        w.save()

    return render(request, "index.html",
                  {"sites": Website.objects.all(), "form": empty_form})


import ROOT
import AbstractClasses
import os, json

class SetEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, set):
            return list(obj)
        return json.JSONEncoder.default(self, obj)

class ProductionOverview(AbstractClasses.GeneralProductionOverview.GeneralProductionOverview):

    def CustomInit(self, TestResultEnvironmentObject = 0):
        if TestResultEnvironmentObject:
            self.TestResultEnvironmentObject = TestResultEnvironmentObject
        self.HTMLFileName = 'ProductionOverview.html'
        self.ImportPath = 'OverviewClasses.CMSPixel.ProductionOverview.ProductionOverviewPage'

        self.NameSingle = 'ProductionOverviewPage'
        self.Name = 'CMSPixel_ProductionOverview_%s'%self.NameSingle

        self.SaveHTML = True

        Rows = self.FetchData()
        ModuleIDsList = self.GetModuleIDsList(Rows)
        NumModules = len(ModuleIDsList)
        NumModulesMaxPerList = 50

        TestsList = ['m20_1', 'p17_1']
        ReadbackParameters = [
            {
                'Parameter': 'par0vd',
                'Xmin': -30,
                'Xmax': 30
            },
            {
                'Parameter': 'par1vd',
                'Xmin': 30,
                'Xmax': 100
            },
            {
                'Parameter': 'par0va',
                'Xmin': -30,
                'Xmax': 30
            },
            {
                'Parameter': 'par1va',
                'Xmin': 0,
                'Xmax': 100
            },
            {
                'Parameter': 'par0rbia',
                'Xmin': -20,
                'Xmax': 50
            },
            {
                'Parameter': 'par1rbia',
                'Xmin': -1,
                'Xmax': 3,
            },
            {
                'Parameter': 'par0tbia',
                'Xmin': 0,
                'Xmax': 10
            },
            {
                'Parameter': 'par1tbia',
                'Xmin': 0,
                'Xmax': 1
            },
            {
                'Parameter': 'par2tbia',
                'Xmin': -0.001,
                'Xmax': 0.001
            },
        ]

        self.SubPages.append({
            "InitialAttributes" : {
                "Sections": ["BumpBonding", "DeadPixel", "PerformanceParameters", "DACs", "IVCurves", "Readback", "HighRate", "VcalCalibration"],
                "DateBegin": self.Attributes['DateBegin'],
                "DateEnd": self.Attributes['DateEnd'],
            },
            "Key": "Section",
            "Module": "SectionNavigation"
        })

        self.SubPages.append(
            {
                "Key": "GradingOverview",
                "Module": "GradingOverview",
                "InitialAttributes" : {
                    "StorageKey" : "GradingOverview",
                    "DateBegin": self.Attributes['DateBegin'],
                    "DateEnd": self.Attributes['DateEnd'],
                },
            }
        )

        if self.Attributes['ShowWeeklyPlots']:
            self.SubPages.append(
                {
                    "Key": "WeeklyProduction",
                    "Module": "WeeklyProduction",
                    "InitialAttributes" : {
                        "StorageKey" : "WeeklyProduction",
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    },
                }
            )

            self.SubPages.append(
                {
                    "Key": "CumulativeProductionGraph",
                    "Module": "CumulativeProductionGraph",
                    "InitialAttributes" : {
                        "StorageKey" : "CumulativeProductionGraph",
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    },
                }
            )

            self.SubPages.append(
                {
                    "Key": "ModuleList",
                    "Module": "ModuleList",
                    "InitialAttributes" : {
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    },
                }
            )

            Offset = 0
            NumModulesToShow = NumModules

            while (NumModulesToShow > 0):
                self.SubPages.append(
                    {
                        "Key": "ModuleFailuresOverview",
                        "Module": "ModuleFailuresOverview",
                        "InitialAttributes" : {
                            "StorageKey" : "ModuleFailuresOverview_%d"%Offset,
                            "DateBegin": self.Attributes['DateBegin'],
                            "DateEnd": self.Attributes['DateEnd'],
                            "NumModules": NumModulesMaxPerList,
                            "Offset": Offset,
                        },
                    }
                )
                NumModulesToShow -= NumModulesMaxPerList
                Offset += NumModulesMaxPerList
        else:
            self.SubPages.append(
                {
                    "Key": "ModuleFailuresOverview",
                    "Module": "ModuleFailuresOverview",
                    "InitialAttributes" : {
                        "StorageKey" : "ModuleFailuresOverview",
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                        "Width": 4,
                    },
                }
            )

            self.SubPages.append(
                {
                    "Key": "ModuleList",
                    "Module": "ModuleList",
                    "InitialAttributes" : {
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    },
                }
            )

        ### bump bonding ###
        self.SubPages.append({"InitialAttributes" : {"Anchor": "BumpBonding", "Title": "BumpBonding Defects"}, "Key": "Section","Module": "Section"})
        for Grade in ['All','A', 'B', 'C']:
            self.SubPages.append(
                {
                    "Key": "BumpBondingOverlay_{Grade}".format(Grade = Grade),
                    "Module": "BumpBondingOverlay",
                    "InitialAttributes" : {
                        "Test": "m20_1",
                        "Grade": "{Grade}".format(Grade = Grade),
                        "StorageKey" : "BumpBondingOverlay_{Grade}".format(Grade = Grade),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )
        self.SubPages.append(
            {
                "Key": "BumpBondingVsId",
                "Module": "BumpBondingVsId",
                "InitialAttributes" : {
                    "StorageKey" : "BumpBondingVsId",
                    "DateBegin": self.Attributes['DateBegin'],
                    "DateEnd": self.Attributes['DateEnd'],
                }
            }
        )
        ### dead pixels ###
        self.SubPages.append({"InitialAttributes" : {"Anchor": "DeadPixel", "Title": "Dead Pixels"}, "Key": "Section","Module": "Section"})
        for Grade in ['All','A', 'B', 'C']:
            self.SubPages.append(
                {
                    "Key": "DeadPixelOverlay_{Grade}".format(Grade = Grade),
                    "Module": "DeadPixelOverlay",
                    "InitialAttributes" : {
                        "Test": "m20_1",
                        "Grade": "{Grade}".format(Grade = Grade),
                        "StorageKey" : "DeadPixelOverlay_{Grade}".format(Grade = Grade),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )

        ### pixels with too high or low gain ###
        self.SubPages.append({"InitialAttributes" : {"Anchor": "BadGain", "Title": "Pixels with bad gain"}, "Key": "Section","Module": "Section"})
        for Grade in ['All','A', 'B', 'C']:
            self.SubPages.append(
                {
                    "Key": "GainOverlay_{Grade}".format(Grade = Grade),
                    "Module": "GainOverlay",
                    "InitialAttributes" : {
                        "Test": "m20_1",
                        "Grade": "{Grade}".format(Grade = Grade),
                        "StorageKey" : "GainOverlay_{Grade}".format(Grade = Grade),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )

        ### performance parameters ###
        self.SubPages.append({"InitialAttributes" : {"Anchor": "PerformanceParameters", "Title": "Performance Parameters per ROC"}, "Key": "Section","Module": "Section"})

        for Test in TestsList:
            self.SubPages.append(
                {
                    "Key": "MeanNoise_{Test}".format(Test = Test),
                    "Module": "MeanNoise",
                    "InitialAttributes" : {
                        "Test": "{Test}".format(Test = Test),
                        "StorageKey" : "MeanNoise_{Test}".format(Test = Test),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )

        #self.SubPages.append({"Key": "Dummy","Module": "Dummy"})

        for Test in TestsList:
            self.SubPages.append(
                {
                    "Key": "RelativeGainWidth_{Test}".format(Test = Test),
                    "Module": "RelativeGainWidth",
                    "InitialAttributes" : {
                        "Test": "{Test}".format(Test = Test),
                        "StorageKey" : "RelativeGainWidth_{Test}".format(Test = Test),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )

        #self.SubPages.append({"Key": "Dummy","Module": "Dummy"})

        for Test in TestsList:
            self.SubPages.append(
                {
                    "Key": "PedestalSpread_{Test}".format(Test = Test),
                    "Module": "PedestalSpread",
                    "InitialAttributes" : {
                        "Test": "{Test}".format(Test = Test),
                        "StorageKey" : "PedestalSpread_{Test}".format(Test = Test),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )

        #self.SubPages.append({"Key": "Dummy","Module": "Dummy"})

        for Test in TestsList:
            self.SubPages.append(
                {
                    "Key": "VcalThresholdWidth_{Test}".format(Test = Test),
                    "Module": "VcalThresholdWidth",
                    "InitialAttributes" : {
                        "Test": "{Test}".format(Test = Test),
                        "StorageKey" : "VcalThresholdWidth_{Test}".format(Test = Test),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )

        ### per pixel ###
        self.SubPages.append({"InitialAttributes" : {"Anchor": "PerPixel", "Title": "Pixel-based quantities"}, "Key": "Section","Module": "Section"})
        for Test in TestsList:
            self.SubPages.append(
                {
                    "Key": "GainPerPixel_{Test}".format(Test = Test),
                    "Module": "GainPerPixel",
                    "InitialAttributes" : {
                        "Test": "{Test}".format(Test = Test),
                        "StorageKey" : "GainPerPixel__{Test}".format(Test = Test),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )
        for Test in TestsList:
            self.SubPages.append(
                {
                    "Key": "PedestalPerPixel_{Test}".format(Test = Test),
                    "Module": "PedestalPerPixel",
                    "InitialAttributes" : {
                        "Test": "{Test}".format(Test = Test),
                        "StorageKey" : "PedestalPerPixel__{Test}".format(Test = Test),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )
        for Test in TestsList:
            self.SubPages.append(
                {
                    "Key": "SCurveWidthsPerPixel_{Test}".format(Test = Test),
                    "Module": "SCurveWidthsPerPixel",
                    "InitialAttributes" : {
                        "Test": "{Test}".format(Test = Test),
                        "StorageKey" : "SCurveWidthsPerPixel__{Test}".format(Test = Test),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )
        for Test in TestsList:
            self.SubPages.append(
                {
                    "Key": "VcalThresholdTrimmedPerPixel_{Test}".format(Test = Test),
                    "Module": "VcalThresholdTrimmedPerPixel",
                    "InitialAttributes" : {
                        "Test": "{Test}".format(Test = Test),
                        "StorageKey" : "VcalThresholdTrimmedPerPixel__{Test}".format(Test = Test),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )

        ### DACs ###
        self.SubPages.append({"InitialAttributes" : {"Anchor": "DACs", "Title": "DAC Parameters"}, "Key": "Section","Module": "Section"})
        TrimThresholds = ['', '35']
        DACs = ['caldel', 'phoffset', 'phscale', 'vana', 'vthrcomp', 'vtrim']
        for Test in TestsList:
            for Trim in TrimThresholds:
                for DAC in DACs:
                    self.SubPages.append(
                        {
                            "Key": "DACDistribution_{Test}".format(Test = Test),
                            "Module": "DACDistribution",
                            "InitialAttributes" : {
                                "Test": "{Test}".format(Test = Test),
                                "Trim": "{Trim}".format(Trim = Trim),
                                "DAC": "{DAC}".format(DAC = DAC),
                                "Maximum": 256,
                                "NBins": 256,
                                "StorageKey" : "DACDistribution_{Test}_{DAC}_{Trim}".format(Test=Test, DAC=DAC,Trim=Trim),
                                "DateBegin": self.Attributes['DateBegin'],
                                "DateEnd": self.Attributes['DateEnd'],
                            }
                        }
                    )

        ### TrimBits ###
        for Test in TestsList:
            for Trim in TrimThresholds:
                self.SubPages.append(
                    {
                        "Key": "DACDistribution_{Test}".format(Test = Test),
                        "Module": "DACDistribution",
                        "InitialAttributes" : {
                            "Test": "{Test}".format(Test = Test),
                            "Trim": "{Trim}".format(Trim = Trim),
                            "DAC": "{DAC}".format(DAC = "TrimBits_mu"),
                            "Maximum": 16,
                            "NBins": 64,
                            "StorageKey" : "DACDistribution_{Test}_{DAC}_{Trim}".format(Test=Test, DAC="TrimBits_mu",Trim=Trim),
                            "DateBegin": self.Attributes['DateBegin'],
                            "DateEnd": self.Attributes['DateEnd'],
                        }
                    }
                )
                self.SubPages.append(
                    {
                        "Key": "DACDistribution_{Test}".format(Test = Test),
                        "Module": "DACDistribution",
                        "InitialAttributes" : {
                            "Test": "{Test}".format(Test = Test),
                            "Trim": "{Trim}".format(Trim = Trim),
                            "DAC": "{DAC}".format(DAC = "TrimBits_sigma"),
                            "Maximum": 5,
                            "NBins": 64,
                            "StorageKey" : "DACDistribution_{Test}_{DAC}_{Trim}".format(Test=Test, DAC="TrimBits_sigma",Trim=Trim),
                            "DateBegin": self.Attributes['DateBegin'],
                            "DateEnd": self.Attributes['DateEnd'],
                        }
                    }
                )

        ### Full test duration ###
        self.SubPages.append(
            {
                "Key": "Duration",
                "Module": "Duration",
                "InitialAttributes" : {
                    "DateBegin": self.Attributes['DateBegin'],
                    "DateEnd": self.Attributes['DateEnd'],
                }
            }
        )

        ### IV ###
        self.SubPages.append({"InitialAttributes" : {"Anchor": "IVCurves", "Title": "IV Curves"}, "Key": "Section","Module": "Section"})
        for Test in TestsList:
            self.SubPages.append(
                {
                    "Key": "IVCurveOverlay_{Test}".format(Test = Test),
                    "Module": "IVCurveOverlay",
                    "InitialAttributes" : {
                        "Test": "{Test}".format(Test = Test),
                        "StorageKey" : "IVCurveOverlay_{Test}".format(Test = Test),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )

        for Test in TestsList:
            self.SubPages.append(
                {
                    "Key": "LeakageCurrent_{Test}".format(Test = Test),
                    "Module": "LeakageCurrent",
                    "InitialAttributes" : {
                        "Test": "{Test}".format(Test = Test),
                        "StorageKey" : "LeakageCurrent_{Test}".format(Test = Test),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )

        for Test in TestsList:
            self.SubPages.append(
                {
                    "Key": "LeakageCurrentSlope_{Test}".format(Test = Test),
                    "Module": "LeakageCurrentSlope",
                    "InitialAttributes" : {
                        "Test": "{Test}".format(Test = Test),
                        "StorageKey" : "LeakageCurrentSlope_{Test}".format(Test = Test),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )

        ### Readback ###
        #self.SubPages.append({"InitialAttributes" : {"Anchor": "Readback", "Title": "Readback"}, "Key": "Section","Module": "Section"})
        #for Test in TestsList:
        #    for Parameter in ReadbackParameters:
        #        self.SubPages.append(
        #            {
        #                "Key": "ReadbackParameter_{Test}_{Parameter}".format(Test = Test, Parameter = Parameter['Parameter']),
        #                "Module": "ReadbackParameter",
        #                "InitialAttributes" : {
        #                    "Test": Test,
        #                    "Parameter": Parameter['Parameter'],
        #                    "Xmin": Parameter['Xmin'],
        #                    "Xmax": Parameter['Xmax'],
        #                    "StorageKey" : "ReadbackParameter_{Test}_{Parameter}".format(Test = Test, Parameter = Parameter['Parameter']),
        #                    "DateBegin": self.Attributes['DateBegin'],
        #                    "DateEnd": self.Attributes['DateEnd'],
        #                }
        #            }
        #        )

        ### HR ###
        self.SubPages.append({"InitialAttributes" : {"Anchor": "HighRate", "Title": "HighRateTests"}, "Key": "Section","Module": "Section"})
        for Rate in [20,50,80]:
            self.SubPages.append(
                {
                    "Key": "Efficiency_{Rate}".format(Rate = Rate),
                    "Module": "Efficiency",
                    "InitialAttributes" : {
                        "Rate": "{Rate}".format(Rate = Rate),
                        "StorageKey" : "Efficiency_{Rate}".format(Rate = Rate),
                        "DateBegin": self.Attributes['DateBegin'],
                        "DateEnd": self.Attributes['DateEnd'],
                    }
                }
            )
        #self.SubPages.append(
        #    {
        #        "Key": "XrayNoisePerPixel",
        #        "Module": "XrayNoisePerPixel",
        #        "InitialAttributes" : {
        #            "StorageKey" : "XrayNoisePerPixel",
        #            "DateBegin": self.Attributes['DateBegin'],
        #            "DateEnd": self.Attributes['DateEnd'],
        #        }
        #    }
        #)

        ### Vcal Calibration ###
        #self.SubPages.append({"InitialAttributes" : {"Anchor": "VcalCalibration", "Title": "Vcal Calibration"}, "Key": "Section","Module": "Section"})
        #self.SubPages.append(
        #    {
        #        "Key": "VcalSlope",
        #        "Module": "VcalSlope",
        #        "InitialAttributes" : {
        #            "StorageKey" : "VcalSlope_Spectrum",
        #            "DateBegin": self.Attributes['DateBegin'],
        #            "DateEnd": self.Attributes['DateEnd'],
        #        }
        #    }
        #)
        #self.SubPages.append(
        #    {
        #        "Key": "VcalOffset",
        #        "Module": "VcalOffset",
        #        "InitialAttributes" : {
        #            "StorageKey" : "VcalOffset_Spectrum",
        #            "DateBegin": self.Attributes['DateBegin'],
        #            "DateEnd": self.Attributes['DateEnd'],
        #        }
        #    }
        #)

    def GenerateOverview(self):
        AbstractClasses.GeneralProductionOverview.GeneralProductionOverview.GenerateOverview(self)

        # for all plots that have been split into several boxes
        # merge the JSON files to simplify further processing of the data
        SumJSONFilesModules = ['ModuleFailuresOverview']
        for SumJSONFilesModule in SumJSONFilesModules:
            print("merge JSON files for '%s'..."%SumJSONFilesModule)
            TotalJSONDict = {}
            for Page in [x for x in self.SubPages if x['Key'] == SumJSONFilesModule]:
                Path = self.GlobalOverviewPath + '/' + self.Attributes['BasePath'] + '/' + Page['InitialAttributes']['StorageKey'] + "/KeyValueDictPairs.json"
                with open(Path) as data_file:
                    JSONData = json.load(data_file)
                    TotalJSONDict.update(JSONData)
                    print("add file: %s"%Path)

            # create directory
            JsonFileDir = self.GlobalOverviewPath + '/' + self.Attributes['BasePath'] + '/' + SumJSONFilesModule
            JsonFileName = JsonFileDir + '/KeyValueDictPairs.json'
            try:
                os.mkdir(JsonFileDir)
            except:
                pass

            # save file
            try:
                f = open(JsonFileName, 'w')
                f.write(json.dumps(TotalJSONDict, sort_keys=True, indent=4, separators=(',', ': '), cls=SetEncoder))
                f.close()
                print("-"*100)
                print("-> written to %s"%JsonFileName)
            except:
                print("could not write json file: '%s'!"%JsonFileName)

        HTML = "<a href='%s'>%s</a><br />"%(self.GetStorageKey()+'/'+self.HTMLFileName, self.Attributes['Title'])
        return HTML


import json
import time
from enum import Enum

def current_time():
    return int(round(time.time() * 1000))

def add_leading_zero(value):
    if value >= 10:
        return str(value)
    return "0" + str(value)

def to_JSON(obj, sort_keys=True):
    return json.dumps(obj, default=default_serializer,
                      sort_keys=sort_keys, indent=4)

def default_serializer(obj):
    """JSON serializer for objects not serializable by default json code"""

    if isinstance(obj, Enum):
        serial = obj.value
        return serial

    return {x: obj.__dict__[x] for x in obj.__dict__ if not x.startswith("_")}

def write_size(data):
    if data < 1100:
        return str(data) + 'b'
    if data < 1100000:
        return str(round(data / 1000, 0)) + 'kb'
    if data < 1100000000:
        return str(round(data / 1000000, 2)) + 'mb'
    if data < 1100000000000:
        return str(round(data / 1000000000, 2)) + 'gb'
    else:
        return str(round(data / 1000000000, 2)) + 'tb'

class Singleton(type):
    _instances = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
        return cls._instances[cls]

headers = {
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
    'Accept-Encoding': 'none',
    'Accept-Language': 'en-US,en;q=0.8',
    'Connection': 'keep-alive'}