#!/usr/bin/python
# -*- coding: <<encoding>> -*-
#-------------------------------------------------------------------------------
#   <<project>>
#
#-------------------------------------------------------------------------------

# The program tries to find the relevant data pages (containing e.g. fuel consumption info) in the *PDFs listed in 'files_to_process'.
# When a relevant page is found, it is saved as a *.png and a *.html.
# All data lines in the relevant pages are printed to a file, from where the Wx UI extracts the data to show it.

# import wxversion
# wxversion.select("2.8")
import wx, wx.html
import sys
import webbrowser
import fitz
import os
import html
import unicodedata
import string
import json

aboutText = """
Sorry, there is no information about this program. It is
running on version %(wxpy)s of wxPython and %(python)s of Python.
See wxPython Wiki
"""

resultsFolder1 = "Phase_1_Docs_Pages"

# We have 8 groups of keywords and other words, see about line 330
        # Fuel consump: key = g/kWh OR kJ/kWh, other = fuel and consumption
        # Fuel SFOC: key = g/kWh OR kJ/kWh, other = sfoc
        # Fuel BSFC: key = g/kWh OR kJ/kWh, other = bsfc
        # Charge air: key = kg/s OR kg/kWh OR m3/h, other = charge air flow
        # Combust air: key = kg/s OR kg/kWh OR m3/h, other = combustion air flow
        # Exhaust gas: key = kg/s OR kg/kWh OR m3/h, other = exhaust gas flow
        # Heat balance: key = kW, other = heat balance
        # Heat quantity: key = kW, other = heat quantity

key_words_to_find = [
    "g/kWh kJ/kWh",
    "g/kWh kJ/kWh",
    "g/kWh kJ/kWh",
    "kg/s kg/kWh",
    "kg/s kg/kWh",
    "kg/s kg/kWh",
    "kW"
    ]
other_words_to_find = [
    "fuel consumption",
    "sfoc",
    "bsfc",
    "charge air flow",
    "combustion air flow",
    "exhaust gas flow",
    "heat balance"
    ]

files_to_process = [
    "MAN/man-175d-imo-tier-ii-imo-tier-iii-marine.pdf",
    "MAN/man-32-40-imo-tier-ii-marine.pdf",
    "MAN/man-32-40-imo-tier-iii-marine.pdf",
    "MAN/man-32-44cr-imo-tier-ii-marine.pdf",
    "MAN/man-32-44cr-imo-tier-iii-marine.pdf",
    "MAN/man-48-60cr-imo-tier-ii-marine.pdf",
    "MAN/man-48-60cr-imo-tier-iii-marine.pdf",
    "MAN/man-51-60df-imo-tier-ii-imo-tier-iii-marine.pdf",
    "MAN/man-51-60df-imo-tier-iii-marine.pdf",
    "MAN/man-es_l3_factsheets_4p_man_175d_genset_n5_201119_low_new.pdf",
    "MAN/man-l32-44-genset-imo-tier-ii-marine.pdf",
    "MAN/man-l32-44-genset-imo-tier-iii-marine.pdf",
    "MAN/man-l35-44df-imo-tier-ii-marine.pdf",
    "MAN/man-l35-44df-imo-tier-iii-marine.pdf",
    "MAN/man-v28-33d-stc-imo-tier-ii-marine.pdf",
    "MAN/man-v28-33d-stc-imo-tier-iii-marine.pdf",
    "MAN/PG_M-III_L2131.pdf",
    "MAN/PG_M-II_L1624.pdf",
    "MAN/PG_P-III_L2131.pdf",
    "MAN/PG_P-II_L2131.pdf",
    "MAN/PG_P-I_L2131.pdf",

    "Wartsila/brochure-o-e-w31.pdf",
    "Wartsila/brochure-o-e-wartsila-dual-fual-low-speed.pdf",
    "Wartsila/leaflet-w14-2018.pdf",
    "Wartsila/marine-power-catalogue.pdf",
    "Wartsila/product-guide-o-e-w26.pdf",
    "Wartsila/product-guide-o-e-w31df.pdf",
    "Wartsila/product-guide-o-e-w32.pdf",
    "Wartsila/product-guide-o-e-w34df.pdf",
    "Wartsila/product-guide-o-e-w46df.pdf",
    "Wartsila/product-guide-o-e-w46f.pdf",
    "Wartsila/wartsila-14-product-guide.pdf",
    "Wartsila/wartsila-31sg-product-guide.pdf",
    "Wartsila/wartsila-46df.pdf",
    "Wartsila/wartsila-o-e-ls-x92.pdf",

    "Hyundai_HHI/2016_HHI_en.pdf",
    "Hyundai_HHI/2017_HHI_en.pdf",
    "Hyundai_HHI/2018_HHI_en.pdf",
    "Hyundai_HHI/2019_HHI_en.pdf",
    "Hyundai_HHI/2020_KSOE_IR_EN.pdf",
    "Hyundai_HHI/2021_KSOE_IR_EN.pdf",
    "Hyundai_HHI/HHI_EMD_brochure2017_1.pdf",
    "Hyundai_HHI/HHI_EMD_brochure2017_2.pdf",
    "Hyundai_HHI/HHI_EMD_brochure2017_3.pdf",
    "Hyundai_HHI/HHI_EMD_brochure2019_1.pdf",
    "Hyundai_HHI/speclal_NSD2020.pdf",

    "WinGD/16-06-pres-kit_x92.pdf",
    "WinGD/cimac2016_120_kyrtatos_paper_thedevelopmentofthemodern2strokemarinedieselengine.pdf",
    "WinGD/cimac2016_173_brueckl_paper_virtualdesign-and-simulation-in2strokemarineenginedevelopment.pdf",
    "WinGD/cimac2016_233_ott_paper_x-df.pdf",
    "WinGD/dominikschneiter_tieriii-programme.pdf",
    "WinGD/Fuel-Flexible-Injection-System-How-to-Handle-a-Fuel-Specturm-from-Diesel_CIMAC2019_paper_404_Andreas-Schmid.pdf",
    "WinGD/marcspahni_generation-x-engines.pdf",
    "WinGD/MIM_WinGD-RT-flex48T-D.pdf",
    "WinGD/MIM_WinGD-X72.pdf",
    "WinGD/MM_WinGD-RT-flex58T-D.pdf",
    "WinGD/MM_WinGD-X35-B_2021-09.pdf",
    "WinGD/MM_WinGD-X52.pdf",
    "WinGD/MM_WinGD-X72.pdf",
    "WinGD/MM_WinGD-X72DF.pdf",
    "WinGD/MM_WinGD-X82-B.pdf",
    "WinGD/Motorship-May-2018-VP-R-D-Leader-Brief.pdf",
    "WinGD/OM_WinGD-X62_2021-09.pdf",
    "WinGD/OM_WinGD-X72DF_2021-09.pdf",
    "WinGD/OM_WinGD-X82-B.pdf",
    "WinGD/OM_WinGD_RT-flex50DF.pdf",
    "WinGD/WinGD-12X92DF-Development-of-the-Most-Powerful-Otto-Engine-Ever_CIMAC2019_paper_425_Dominik-Schneiter.pdf",
    "WinGD/wingd-paper_engine_selection_for_very_large_container_vessels_201609.pdf",
    "WinGD/WinGD-WiDE-Brochure.pdf",
    "WinGD/WinGD_Engine-Booklet_2021.pdf",
    "WinGD/wingd_moving-inlet-ports-concept-for-optimization-of-2-stroke-uni-flow-engines_patrick-rebecchi.pdf",
    "WinGD/X-DF-Engines-by-WinGD.pdf",
    "WinGD/X-DF-FAQ.pdf"
    ]

# The following 'global' functions have been copied from file 'ExtractInfoPages_AllLines.py'.
# That program just processed the input files ('files_to_process') and extracted the relevant pages.
def findTocItems(doc, page):
    toc = doc.get_toc()
    if not toc:
        return []

    prev_page = 0
    items = []
    for level in range(1,6):
        curr_page = 0
        curr_text = ""
        for t in toc:
            if t[0]==level and t[2]>=prev_page and t[2]<=page:
                curr_page = t[2]
                curr_text = html.unescape(unicodedata.normalize('NFKC', t[1]))
            elif t[0]==level and t[2]>page:
                if curr_text:
                    items.append(curr_text)
                    prev_page = curr_page
                break
    return items

def findLineInfo(ln):
    ispan = ln.find("<span")
    if ispan<0:
        return [-1, -1, ""]
    iend = ln.find(">", ispan)
    iesp = ln.find("</span", iend)
    text = ln[iend+1:iesp]
    itop = ln.find("top:")
    ietp = ln.find("pt", itop)
    ileft = ln.find("left:")
    ielf = ln.find("pt", ileft)
    top = int(float(ln[itop+4:ietp]))
    left = int(float(ln[ileft+5:ielf]))
    return [top,left,text]

def findPageInfo(pg):
    idiv = pg.find("<div")
    if idiv<0:
        return [-1, -1]
    iend = pg.find(">", idiv)
    iwid = pg.find("width:", idiv)
    iewd = pg.find("pt", iwid)
    wid = int(float(pg[iwid+6:iewd]))
    ihei = pg.find("height:", idiv)
    iehe = pg.find("pt", ihei)
    hei = int(float(pg[ihei+7:iehe]))
    return [wid,hei]

def findMatchingLines(pageHtml, xwid, line_tops, nwid, nhei):
    spaces200 = " " * nwid
    linesToPrint = [spaces200] * nhei
    lines = pageHtml.split('\n')
    minleft = nwid*xwid
    for ln in lines:
        [top, left, text] = findLineInfo(ln)
        if top<0 or left<0:
            continue
        if left<minleft:
            minleft = left
    minleft /= xwid
    minleft -= 1

    for ln in lines:
        [top, left, text] = findLineInfo(ln)
        if top<0 or left<0:
            continue
        y = line_tops.index(top)
        x = int(round(left/xwid)-minleft)
        try:
            text = html.unescape(unicodedata.normalize('NFKC', text))
        except:
            pass
        linesToPrint[y] = linesToPrint[y][0:x] + text + linesToPrint[y][x+len(text):]

    return linesToPrint

# Find out whether 'str' is a number (int or float) or an int with commas (as thousands separator) or <something>: number
def isNumber(str):
    str1 = str[:]
    # Find out whether the string contains a colon; if it does, only consider the text after the colon
    idx = str.find(': ')
    if idx>=0:
        str1 = str[idx+2:]
    else:
        idx = str.find(':')
        if idx>=0:
            str1 = str[idx+1:]

    # If there are commas or spaces as thousands separator, remove them
    str1 = str1.replace(',', '')
    str1 = str1.replace(' ', '')

    # And now finally: try to convert the string into a floating-point number; if this succeeds, return True; else False
    try:
        float(str1)
        return True
    except ValueError:
        return False

def countLinesInPage(pageHtml):
    tops = set()
    lines = pageHtml.split('\n')
    for ln in lines:
        [top, left, text] = findLineInfo(ln)
        if top<0 or left<0:
            continue
        tops.add(top)
    s_tops = sorted(tops)
    return s_tops

class HtmlWindow(wx.html.HtmlWindow):
    def __init__(self, parent, id, size=(600,400)):
        wx.html.HtmlWindow.__init__(self,parent, id, size=size)
        if "gtk2" in wx.PlatformInfo:
            self.SetStandardFonts()

    def OnLinkClicked(self, link):
        wx.LaunchDefaultBrowser(link.GetHref())

class AboutBox(wx.Dialog):
    def __init__(self):
        wx.Dialog.__init__(self, None, -1, "About VesselAI_ExtractData_Phase_1_KeyWords",
            style=wx.DEFAULT_DIALOG_STYLE|wx.RESIZE_BORDER|
                wx.TAB_TRAVERSAL)
        hwin = HtmlWindow(self, -1, size=(400,200))
        vers = {}
        vers["python"] = sys.version.split()[0]
        vers["wxpy"] = wx.VERSION_STRING
        hwin.SetPage(aboutText % vers)
        btn = hwin.FindWindowById(wx.ID_OK)
        irep = hwin.GetInternalRepresentation()
        hwin.SetSize((irep.GetWidth()+25, irep.GetHeight()+10))
        self.SetClientSize(hwin.GetSize())
        self.CentreOnParent(wx.BOTH)
        self.SetFocus()

class Frame(wx.Frame):
    def __init__(self, title):
        wx.Frame.__init__(self, None, title=title, pos=(150,150), size=(1200,800))
        self.Bind(wx.EVT_CLOSE, self.OnClose)

        menuBar = wx.MenuBar()
        menu = wx.Menu()
        m_exit = menu.Append(wx.ID_EXIT, "E&xit\tAlt-X", "Close window and exit program.")
        self.Bind(wx.EVT_MENU, self.OnClose, m_exit)
        menuBar.Append(menu, "&File")
        menu = wx.Menu()
        m_about = menu.Append(wx.ID_ABOUT, "&About", "Information about this program")
        self.Bind(wx.EVT_MENU, self.OnAbout, m_about)
        menuBar.Append(menu, "&Help")
        self.SetMenuBar(menuBar)

        self.statusbar = self.CreateStatusBar()

        splitter = wx.SplitterWindow(self, style=wx.SP_NO_XP_THEME | wx.SP_3D | wx.SP_LIVE_UPDATE)
        panel = wx.Panel(splitter)
        vert_box = wx.BoxSizer(wx.VERTICAL)
        hor_box_search = wx.BoxSizer(wx.HORIZONTAL)
        hor_box_dataSets = wx.BoxSizer(wx.HORIZONTAL)
        hor_box_refresh = wx.BoxSizer(wx.HORIZONTAL)
        hor_box_exclude = wx.BoxSizer(wx.HORIZONTAL)

        self.m_docPage = ""
        # file:///<file_path>.pdf#page=<page_number> entries will be added to this array for those pages where 'Exclude This Page from Results' check box is set.
        self.m_excludedPages = []
        input_file = resultsFolder1 + "/Phase1_excluded.txt"
        if os.path.exists(input_file):
            with open(input_file, 'r') as fp:
                self.m_excludedPages = fp.readlines()

        m_text0 = wx.StaticText(panel, -1, "  Enter Keywords and Other words. Add/Mod: Adds a new item to the end of the list, if Name is new; otherwise, it Modifies existing item.\n"+
                                "  Press 'Refresh' to see what data pages match these criteria => they appear in the drop-down below. Select the desired data page from the drop-down to see its contents.")
        vert_box.Add(m_text0, 0, wx.ALL, 10)

        vert_box_DS = wx.BoxSizer(wx.VERTICAL)
        vert_box_DS_Name = wx.BoxSizer(wx.VERTICAL)
        vert_box_DS_Keyw = wx.BoxSizer(wx.VERTICAL)
        vert_box_DS_Othr = wx.BoxSizer(wx.VERTICAL)
        m_textD1 = wx.StaticText(panel, -1, "Data Sets:")
        vert_box_DS.Add(m_textD1, 0, wx.ALL, 4)
        self.m_addDS = wx.Button(panel, -1, "Add/Mod")
        self.m_addDS.Bind(wx.EVT_BUTTON, self.OnAddDataSet)
        vert_box_DS.Add(self.m_addDS, 0, wx.ALL, 4)
        self.m_delDS = wx.Button(panel, -1, "Delete")
        self.m_delDS.Bind(wx.EVT_BUTTON, self.OnDelDataSet)
        vert_box_DS.Add(self.m_delDS, 0, wx.ALL, 4)
        hor_box_dataSets.Add(vert_box_DS, 0, wx.ALL, 10)

        # Fuel consump: key = g/kWh OR kJ/kWh, other = fuel consumption
        # Fuel SFOC: key = g/kWh OR kJ/kWh, other = sfoc
        # Fuel BSFC: key = g/kWh OR kJ/kWh, other = bsfc
        # Charge air: key = kg/s OR kg/kWh OR m3/h, other = charge air flow
        # Combust air: key = kg/s OR kg/kWh OR m3/h, other = combustion air flow
        # Exhaust gas: key = kg/s OR kg/kWh OR m3/h, other = exhaust gas flow
        # Heat balance: key = kW, other = heat balance
        # Heat quantity: key = kW, other = heat quantity
        # Heat rad engine: key = kW, other = heat radiation engine
        # Charge air cool: key = kW, other = charge air cooler
        # Lube oil cool: key = kW, other = lube oil cooler
        # Jacket cool: key = kW, other = jacket cooling
        self.m_dataSets = [
            { "name": "Fuel consump", "keywords": "g/kWh kJ/kWh", "other_words": "fuel consumption"},
            { "name": "Fuel SFOC", "keywords": "g/kWh kJ/kWh", "other_words": "sfoc"},
            { "name": "Fuel BSFC", "keywords": "g/kWh kJ/kWh", "other_words": "bsfc"},
            { "name": "Charge air", "keywords": "kg/s kg/kWh m3/h", "other_words": "charge air flow"},
            { "name": "Combust air", "keywords": "kg/s kg/kWh m3/h", "other_words": "combustion air flow"},
            { "name": "Exhaust gas", "keywords": "kg/s kg/kWh m3/h", "other_words": "exhaust gas flow"},
            { "name": "Heat balance", "keywords": "kW", "other_words": "heat balance"},
            { "name": "Heat quantity", "keywords": "kW", "other_words": "heat quantity"},
            { "name": "Heat rad engine", "keywords": "kW", "other_words": "heat radiation engine"},
            { "name": "Charge air cool", "keywords": "kW", "other_words": "charge air cooler"},
            { "name": "Lube oil cool", "keywords": "kW", "other_words": "lube oil cooler"},
            { "name": "Jacket cool", "keywords": "kW", "other_words": "jacket cooling"},
            ]

        input_file_j = resultsFolder1 + "/Phase1_dataSets.json"
        try:
            fileJson = open(input_file_j, "r")
            jsonContent = fileJson.read()
            self.m_dataSets = json.loads(jsonContent)
        except IOError:
            pass # using the default self.m_dataSets data defined above...

        ds_names = [i["name"] for i in self.m_dataSets]
        self.m_dataSetsList = wx.ListBox(panel, choices = ds_names, size=(100,100))
        self.m_dataSetsList.Bind(wx.EVT_LISTBOX, self.OnSelectDataSet)
        hor_box_dataSets.Add(self.m_dataSetsList, 0, wx.ALL, 10)

        m_textD2 = wx.StaticText(panel, -1, "Selected\nData Set:")
        hor_box_dataSets.Add(m_textD2, 0, wx.ALL, 16)
        m_textD3 = wx.StaticText(panel, -1, "Name")
        vert_box_DS_Name.Add(m_textD3, 0, wx.ALL, 6)
        self.m_dataSetName = wx.TextCtrl(panel, -1, "(name)", size=(100,-1))
        vert_box_DS_Name.Add(self.m_dataSetName, 0, wx.ALL, 2)
        hor_box_dataSets.Add(vert_box_DS_Name, 0, wx.ALL, 10)
        m_textD4 = wx.StaticText(panel, -1, "Keywords")
        vert_box_DS_Keyw.Add(m_textD4, 0, wx.ALL, 6)
        self.m_dataSetKeywords = wx.TextCtrl(panel, -1, "(keywords)", size=(150,-1))
        vert_box_DS_Keyw.Add(self.m_dataSetKeywords, 0, wx.ALL, 2)
        hor_box_dataSets.Add(vert_box_DS_Keyw, 0, wx.ALL, 10)
        m_textD5 = wx.StaticText(panel, -1, "Other words")
        vert_box_DS_Othr.Add(m_textD5, 0, wx.ALL, 6)
        self.m_dataSetOtherWords = wx.TextCtrl(panel, -1, "(other words)", size=(200,-1))
        vert_box_DS_Othr.Add(self.m_dataSetOtherWords, 0, wx.ALL, 2)
        hor_box_dataSets.Add(vert_box_DS_Othr, 0, wx.ALL, 10)
        m_textD6 = wx.StaticText(panel, -1, "Name is an ID for the Data set.\nKeywords are OR'ed, but must be whole table cell.\nOther words are AND'ed, they all must appear.\n"+
                                 "Case is respected for Keywords.\nCase is ignored for Other words.")
        hor_box_dataSets.Add(m_textD6, 0, wx.ALL, 10)

        vert_box.Add(hor_box_dataSets, 0, wx.ALL, 4)

        m_textR0 = wx.StaticText(panel, -1, "After you have entered the desired criteria to 'Data Sets' above, press button 'Refresh', " +
                                "and the drop-down below is filled with matching data pages.\n" +
                                "Check that the desired data pages are indeed in the drop-down (use check box Exclude...), then press button 'Next >>', " +
                                "and the Phase 2 window is opened.")
        vert_box.Add(m_textR0, 0, wx.ALL, 6)

        m_textR1 = wx.StaticText(panel, -1, "Good data pages usually contain numbers;\nto enforce this use the parameter.")
        hor_box_refresh.Add(m_textR1, 0, wx.ALL, 4)
        m_textR2 = wx.StaticText(panel, -1, "Min Nbrs in Page\nto accept page")
        hor_box_refresh.Add(m_textR2, 0, wx.ALL, 4)
        self.m_numbersInPage = wx.TextCtrl(panel, -1, "0", size=(50,-1))
        hor_box_refresh.Add(self.m_numbersInPage, 0, wx.ALL, 4)

        self.m_refresh = wx.Button(panel, -1, "Refresh")
        self.m_refresh.Bind(wx.EVT_BUTTON, self.OnRefresh)
        hor_box_refresh.Add(self.m_refresh, 0, wx.ALL, 6)
        self.m_nextPhase = wx.Button(panel, -1, "Next >>")
        self.m_nextPhase.Bind(wx.EVT_BUTTON, self.OnNextPhase)
        hor_box_refresh.Add(self.m_nextPhase, 0, wx.ALL, 6)
        vert_box.Add(hor_box_refresh, 0, wx.ALL, 6)

        m_textE1 = wx.StaticText(panel, -1, "You may wish to eliminate some pages from the results. Select result page from the drop-down below, and check\n'Exclude This Page...'. "+
                                 "If you intend to exclude most of the result pages, button 'Exclude All' checks it for all pages.")
        hor_box_exclude.Add(m_textE1, 0, wx.ALL, 6)
        self.m_excludePage = wx.CheckBox(panel, label="Exclude This Page\nfrom Results")
        self.Bind(wx.EVT_CHECKBOX, self.OnExcludePage)
        hor_box_exclude.Add(self.m_excludePage, 0, wx.ALL, 6)
        self.m_excludeAll = wx.Button(panel, wx.ID_OPEN, "Exclude All")
        self.m_excludeAll.Bind(wx.EVT_BUTTON, self.OnExcludeAll)
        hor_box_exclude.Add(self.m_excludeAll, 0, wx.ALL, 10)
        self.m_includeAll = wx.Button(panel, wx.ID_OPEN, "Include All")
        self.m_includeAll.Bind(wx.EVT_BUTTON, self.OnIncludeAll)
        hor_box_exclude.Add(self.m_includeAll, 0, wx.ALL, 10)
        vert_box.Add(hor_box_exclude, 0, wx.ALL, 6)

        self.m_choice = wx.Choice(panel, choices = [], size=(1800,-1))
        self.m_choice.Bind(wx.EVT_CHOICE, self.OnChoice)
        vert_box.Add(self.m_choice, 0, wx.ALL, 10)

        self.m_searchText = wx.TextCtrl(panel, -1, "", size=(200,-1))
        hor_box_search.Add(self.m_searchText, 0, wx.ALL, 10)
        m_textS1 = wx.StaticText(panel, -1, "Search for text\nin all data:")
        hor_box_search.Add(m_textS1, 0, wx.ALL, 6)
        self.m_search = wx.Button(panel, -1, "Search")
        self.m_search.Bind(wx.EVT_BUTTON, self.OnSearchText)
        hor_box_search.Add(self.m_search, 0, wx.ALL, 10)

        m_textC1 = wx.StaticText(panel, -1, "   ")
        hor_box_search.Add(m_textC1, 0, wx.ALL, 10)
        m_textC2 = wx.StaticText(panel, -1, "Current\nPage:")
        hor_box_search.Add(m_textC2, 0, wx.ALL, 6)
        self.m_openLink = wx.Button(panel, wx.ID_OPEN, "Open PDF")
        self.m_openLink.Bind(wx.EVT_BUTTON, self.OnOpenLink)
        hor_box_search.Add(self.m_openLink, 0, wx.ALL, 10)
        self.m_copyLink = wx.Button(panel, wx.ID_OPEN, "Copy Link")
        self.m_copyLink.Bind(wx.EVT_BUTTON, self.OnCopyLink)
        hor_box_search.Add(self.m_copyLink, 0, wx.ALL, 10)
        self.m_openHtml = wx.Button(panel, wx.ID_OPEN, "Open Html")
        self.m_openHtml.Bind(wx.EVT_BUTTON, self.OnOpenHtml)
        hor_box_search.Add(self.m_openHtml, 0, wx.ALL, 10)
        self.m_openPng = wx.Button(panel, wx.ID_OPEN, "Refresh Png")
        self.m_openPng.Bind(wx.EVT_BUTTON, self.OnShowPng)
        hor_box_search.Add(self.m_openPng, 0, wx.ALL, 10)

        vert_box.Add(hor_box_search, 0, wx.ALL, 10)

        self.m_context = wx.TextCtrl(panel, -1, "Context = headers", size=(1800,-1), style=wx.TE_READONLY)   # style=wx.TE_MULTILINE|wx.TE_READONLY
        vert_box.Add(self.m_context, 0, wx.ALL, 10)

        self.m_extracted = wx.TextCtrl(panel, -1, "Extracted data", size=(1800,1500), style=wx.TE_MULTILINE|wx.TE_READONLY|wx.TE_RICH2)
        tcFont = self.m_extracted.GetFont()
        tcFont.SetFamily( wx.FONTFAMILY_TELETYPE )
        self.m_extracted.SetFont( tcFont )
        vert_box.Add(self.m_extracted, 0, wx.ALL, 10)

        self.m_bmPanel = wx.Panel(splitter)
        self.m_bitmap = wx.StaticBitmap(self.m_bmPanel, pos=(5, 5))

        self.m_choice.SetSelection(0)
        self.OnChoice(None)
        self.OnShowPng(None)

        panel.SetSizer(vert_box)

        splitter.SplitVertically(panel, self.m_bmPanel)
        splitter.SetMinimumPaneSize(600)

        panel.Layout()

        # Load whatever we have in the Results folder, i.e. results from the previous run...
        self.LoadResults()

    def computeResults(self):
        global key_words_to_find
        global other_words_to_find
        key_words_to_find = [i["keywords"] for i in self.m_dataSets]
        other_words_to_find = [i["other_words"] for i in self.m_dataSets]
        if os.path.isdir(resultsFolder1):
            for i in os.listdir(resultsFolder1):
                os.remove(os.path.join(resultsFolder1, i))
        else:
            os.makedirs(resultsFolder1)

        min_nbrs_in_page = float(self.m_numbersInPage.GetValue())

        output_file = resultsFolder1 + "/Phase1_results.txt"
        with open(output_file, 'w') as fp:

            for docNumber, filename in enumerate(files_to_process):

                # creating a pdf file object
                doc = fitz.open(filename)

                self.m_extracted.AppendText(filename+'\n')
                self.m_extracted.Refresh()
                self.m_extracted.Update()

                prev_lines = []
                # creating a page object
                for pageNumber, page in enumerate(doc):
                    # extracting text from page
                    pageText = page.get_text('text')
                    lines = pageText.split('\n')
                    foundKeys = [False]*len(key_words_to_find)
                    foundOthers = [False]*len(other_words_to_find)
                    linesWithOth = []
                    linesWithKey = []
                    # Other words are AND'ed (they must all appear), keywords are OR'ed (one appearing is enough)
                    for ioth,oth in enumerate(other_words_to_find):
                        oths = oth.split(' ')
                        foundAll = True
                        for o in oths:
                            foundOne = False
                            # OPTION: We accept this page, if other words are found in the previous page; but keywords must be found in this page!
                            for ln in lines: # +prev_lines:
                                if o in ln.lower():
                                    foundOne = True
                            if not foundOne:
                                foundAll = False
                        if foundAll:
                            foundOthers[ioth] = True
                    for ln in lines:
                        for ikey,key in enumerate(key_words_to_find):
                            keys = key.split(' ')
                            for k in keys:
                                if k==ln or ('('+k+')') in ln or ('['+k+']') in ln or ('in '+k) in ln :
                                    foundKeys[ikey] = True

                    nbrOfNbrs = 0
                    if min_nbrs_in_page>0:
                        for ln in lines:
                            if isNumber(ln):
                                nbrOfNbrs += 1

                    # Keyword and other word must match...
                    match = False
                    for ikey,key in enumerate(foundKeys):
                        if foundKeys[ikey] and foundOthers[ikey]:
                            match = True
                    if match and nbrOfNbrs>=min_nbrs_in_page:
                        pix = page.get_pixmap()
                        png_filename = resultsFolder1 + "/Doc" + str(docNumber) + "_page" + str(pageNumber+1) + ".png"
                        pix.save(png_filename)
                        url = "file:///" + os.path.abspath(filename) + "#page=" + str(pageNumber+1)
                        print(url, file=fp)
                        # Occasionally headers contain non-printing characters; print what can be printed, skip the rest
                        try:
                            print(findTocItems(doc, pageNumber+1), file=fp)
                        except:
                            toc_list = findTocItems(doc, pageNumber+1)
                            print("['", end='', file=fp)
                            for toc in toc_list:
                                try:
                                    print(toc, end="', '", file=fp)
                                except:
                                    for t in toc:
                                        try:
                                            print(t, end='', file=fp)
                                        except:
                                            pass
                                    print("', '", end='', file=fp)
                            print("']", file=fp)

                        # extracting html from page
                        pageHtml = page.get_text('html', sort=True)
                        html_filename = resultsFolder1 + "/Doc" + str(docNumber) + "_page" + str(pageNumber+1) + ".html"
                        html_file = open(html_filename, "w")
                        html_file.write(pageHtml)
                        html_file.close()

                        # 200 characters per line AND separate text line for each different "top" coord in HTML,
                        # If we make value 200 (chars) much smaller, columns are no longer separated by enough space chars.
                        (wid, hei) = findPageInfo(pageHtml)
                        xwid = int(round(wid/200))
                        tops = countLinesInPage(pageHtml)
                        nwid = int(round(wid/xwid))
                        nhei = len(tops)
                        linesToPrint = findMatchingLines(pageHtml, xwid, tops, nwid, nhei)

                        self.m_extracted.AppendText("   Doc:"+str(docNumber)+", Page:"+str(pageNumber+1)+"\n")
                        print("Doc:"+str(docNumber)+", Page:"+str(pageNumber+1)+" => ", file=fp)

                        for ltp in linesToPrint:
                            try:
                                print(ltp.rstrip(), file=fp)
                            except:
                                pass

                    prev_lines = lines[:]

    def OnSelectDataSet(self, event):
        idx = self.m_dataSetsList.GetSelection()
        if idx>=0:
            self.m_dataSetName.SetValue(self.m_dataSets[idx]["name"])
            self.m_dataSetKeywords.SetValue(self.m_dataSets[idx]["keywords"])
            self.m_dataSetOtherWords.SetValue(self.m_dataSets[idx]["other_words"])
        else:
            self.m_dataSetName.SetValue("")
            self.m_dataSetKeywords.SetValue("")
            self.m_dataSetOtherWords.SetValue("")

    def OnAddDataSet(self, event):
        name = self.m_dataSetName.GetValue()
        keyw = self.m_dataSetKeywords.GetValue()
        othw = self.m_dataSetOtherWords.GetValue()
        if not name.strip():
            resp = wx.MessageBox('No Name', 'Operation cancelled', wx.OK | wx.ICON_WARNING)
            return
        if not keyw.strip():
            resp = wx.MessageBox('No keyword(s)', 'Operation cancelled', wx.OK | wx.ICON_WARNING)
            return
        idx = -1
        for ids, ds in enumerate(self.m_dataSets):
            if ds["name"]==name:
                idx = ids
                break
        if idx>=0:
            self.m_dataSets[idx]["keywords"] = keyw.strip()
            self.m_dataSets[idx]["other_words"] = othw.strip().lower()
        else:
            newitem = { "name": name, "keywords": keyw.strip(), "other_words": othw.strip().lower() }
            self.m_dataSets.append(newitem)

        ds_names = [i["name"] for i in self.m_dataSets]
        self.m_dataSetsList.Clear()
        self.m_dataSetsList.AppendItems(ds_names)

    def OnDelDataSet(self, event):
        idx = self.m_dataSetsList.GetSelection()
        if idx<0:
            resp = wx.MessageBox('No list item selected', 'Operation cancelled', wx.OK | wx.ICON_WARNING)
            return
        self.m_dataSets.pop(idx)
        ds_names = [i["name"] for i in self.m_dataSets]
        self.m_dataSetsList.Clear()
        self.m_dataSetsList.AppendItems(ds_names)

    def OnRefresh(self, event):
        self.m_extracted.SetLabel("")
        self.m_extracted.Refresh()
        self.m_extracted.Update()

        self.computeResults()
        self.LoadResults()

    def LoadResults(self):
        input_file = resultsFolder1 + "/Phase1_results.txt"
        if not os.path.exists(input_file):
            return
        with open(input_file, "r") as file:
            self.m_data = file.readlines()

        self.m_links = []
        for d in self.m_data:
            if "file:///" in d:
                self.m_links.append(d)

        self.m_choice.Clear()
        self.m_choice.AppendItems(self.m_links)
        if self.m_choice.GetCount()>0:
            self.m_choice.SetSelection(0)
            self.OnChoice(None)

    def OnNextPhase(self, event):
        output_file = resultsFolder1 + "/Phase1_excluded.txt"
        with open(output_file, 'w') as fp:
            for pg in self.m_excludedPages:
                print(pg, end='', file=fp) # The file-page strings contain a \n already...

        output_file = resultsFolder1 + "/Phase1_dataSets.json"
        jsonString = json.dumps(self.m_dataSets)
        jsonFile = open(output_file, "w")
        jsonFile.write(jsonString)
        jsonFile.close()

        os.system('python VesselAI_extractData_2.py')

    def OnExcludePage(self, event):
        fpage = self.m_choice.GetString(self.m_choice.GetSelection());
        if not fpage:
            return
        check = self.m_excludePage.IsChecked()
        if check:
            if not fpage in self.m_excludedPages:
                self.m_excludedPages.append(fpage)
        else:
            if fpage in self.m_excludedPages:
                self.m_excludedPages.remove(fpage)

    def OnExcludeAll(self, event):
        self.m_excludedPages = []
        for i in range(self.m_choice.GetCount()):
            self.m_excludedPages.append(self.m_choice.GetString(i))
        self.m_excludePage.SetValue(True)

    def OnIncludeAll(self, event):
        self.m_excludedPages = []
        self.m_excludePage.SetValue(False)

    def OnOpenLink(self, event):
        url = self.m_choice.GetString(self.m_choice.GetSelection())
        webbrowser.open(url)

    def OnCopyLink(self, event):
        url = self.m_choice.GetString(self.m_choice.GetSelection())
        if wx.TheClipboard.Open():
            wx.TheClipboard.SetData(wx.TextDataObject(url))
            wx.TheClipboard.Close()

    def OnShowPng(self, event):
        if self.m_docPage:
            self.m_bmPanel.Refresh()
            imageFile = self.m_docPage
            imageFile = imageFile.replace(':', '')
            imageFile = imageFile.replace(' =>', '')
            imageFile = imageFile.replace(', ', '_')
            imageFile = imageFile.replace(' \n', '')
            imageFile = os.path.join(resultsFolder1, imageFile + '.png')
            png = wx.Image(imageFile, wx.BITMAP_TYPE_ANY).ConvertToBitmap()
            self.m_bitmap.SetBitmap(png) # size=(png.GetWidth(), png.GetHeight()))

    def OnOpenHtml(self, event):
        if self.m_docPage:
            htmlFile = self.m_docPage
            htmlFile = htmlFile.replace(':', '')
            htmlFile = htmlFile.replace(' =>', '')
            htmlFile = htmlFile.replace(', ', '_')
            htmlFile = htmlFile.replace(' \n', '')
            htmlFile = os.path.join(resultsFolder1, htmlFile + '.html')
            wx.LaunchDefaultBrowser(htmlFile)

    def OnSearchText(self, event):
        searchText = self.m_searchText.GetValue()
        if searchText:
            currentLink = self.m_choice.GetStringSelection();
            firstLink = ""
            firstText = ""
            lastLink = ""
            afterCurrent = False
            for d in self.m_data:
                if "file:///" in d:
                    lastLink = d
                if searchText in d:
                    if not afterCurrent:
                        if not firstLink:
                            firstLink = lastLink
                            firstText = d
                    elif lastLink:
                        self.m_choice.SetStringSelection(lastLink)
                        self.OnChoice(event)
                        '''
                        dlg = wx.MessageDialog(self, d,
                            "Text Found: "+searchText, wx.OK|wx.ICON_INFORMATION)
                        dlg.ShowModal()
                        dlg.Destroy()
                        '''
                        break
                if d==currentLink:
                    afterCurrent = True
                    lastLink = ""
            else:
                if firstLink:
                    self.m_choice.SetStringSelection(firstLink)
                    self.OnChoice(event)
                    dlg = wx.MessageDialog(self, firstText,
                        "Text Found: "+searchText + ' (from beginning)', wx.OK|wx.ICON_INFORMATION)
                    dlg.ShowModal()
                    dlg.Destroy()
                else:
                    dlg = wx.MessageDialog(self,
                        "No such text in data",
                        searchText, wx.OK|wx.ICON_INFORMATION)
                    dlg.ShowModal()
                    dlg.Destroy()

    def OnClose(self, event):
        '''
        dlg = wx.MessageDialog(self,
            "Do you really want to close this application?",
            "Confirm Exit", wx.OK|wx.CANCEL|wx.ICON_QUESTION)
        result = dlg.ShowModal()
        dlg.Destroy()
        if result == wx.ID_OK:
        '''
        self.Destroy()

    def OnAbout(self, event):
        dlg = AboutBox()
        dlg.ShowModal()
        dlg.Destroy()

    def OnChoice(self, event):
        idx = self.m_choice.GetSelection()
        if idx<0:
            return
        line = self.m_choice.GetString(idx);
        if not line:
            return
        foundStart = False
        foundTOC = False
        foundDoc = False
        extrText = ""
        self.m_docPage = ""
        for ln in self.m_data:
            if foundStart and foundTOC:
                if "file:///" in ln:
                    break
                else:
                    if not foundDoc:
                        self.m_docPage = ln
                        foundDoc = True
                    if ln.strip():
                        extrText += ln
            if foundStart and not foundTOC:
                self.m_context.SetLabel(ln)
                self.m_extracted.SetLabel('')
                foundTOC = True
            if line==ln:
                foundStart = True
        self.m_extracted.SetLabel(extrText)
        if self.m_docPage:
            self.OnShowPng(event)
        if line in self.m_excludedPages:
            self.m_excludePage.SetValue(True)
        else:
            self.m_excludePage.SetValue(False)

app = wx.App(redirect=True)   # Error messages go to popup window
top = Frame("VesselAI_ExtractData_Phase_1")
top.Show()
app.MainLoop()


from django.shortcuts import render,redirect
from post.models import Post
from math import ceil

def create(request):
	if request.method == 'POST':
		title = request.POST.get('title')
		content = request.POST.get('content')
		post = Post.objects.create(title=title, content=content)
		return redirect('/post/read/?post_id=%d'%post.id)
	else:
		return render(request,'create.html')

def edit(request):
	if request.method == 'POST':
		# 取出原文章
		post_id = request.POST.get('post_id')
		post = Post.objects.get(id=post_id)
		# 修改文章
		post.title = title
		post.content = content
		post.save()
		return redirect('/post/read/?post_id=%d' % post.id)
	else:
		# get不修改
		post_id = request.GET.get('post_id')
		post = Post.objects.get(id=post_id)
		return render(request,'edit.html',{'post': post})

def read(request):
	post_id = int(request.GET.get('post_id'))
	try:
		post = Post.objects.get(id=post_id)
		return render(request,'read.html',{'post': post})
	except Post.DoesNotExist:
		return redirect('/')

def list(request):
	page = int(request.GET.get('page',1)) # 当前页码，默认为-1
	p = 5
	# 计算页数
	total = Post.objects.count()
	pages = ceil(total / p)

	# 取出本页需要实现的文章
	start = (page - 1) * p
	end = start + 5
	posts = Post.objects.all()[start: end]
	return render(request,'post_list.html', { "posts": posts, 'pages': range(pages) })

def search(request):
	keyword = request.POST.get('keyword')
	posts = Post.objects.filter(content__contains=keyword)
	return render(request,'search.html',{'posts': posts})


import pytest

#### writing tests for the main function ####

DATA = 'data'
def test_load_data(tmp_path):
    """Test for the game to be loading a file from temp directory"""
    d = tmp_path / "dict"
    d.mkdir()
    p = d / "test data"
    p.write_text(DATA)
    assert p.read_text() == DATA
    assert len(list(tmp_path.iterdir())) == 1

def test_load_data_missing_file():
    """Test for when the loading fails on the game"""
    file_name = "art_database/game_data_clean.json"
    if file_name == FileNotFoundError:
        with pytest.raises(FileNotFoundError):
            file_name = "art_database/game_data.json"  # replace with another file


from django.conf import settings
from django.apps import apps

REFERRAL_MODEL = getattr(settings, "REFERRAL_CODE_MODEL", "referrals.ReferralCode")
REFERRAL_PARAM_KEY = getattr(settings, "REFERRAL_PARAM_KEY", "ref_code")
REFERRAL_COOKIE_KEY = getattr(settings, "REFERRAL_COOKIE_KEY", "ref_code")
REFERRAL_COOKIE_AGE = getattr(settings, "REFERRAL_COOKIE_AGE", 60 * 60 * 24)

class ReferralLinkMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        ReferralCode = apps.get_model(REFERRAL_MODEL, require_ready=True)
        cookie_age = REFERRAL_COOKIE_AGE

        # Get user referrer from cookie referral code
        cookie_referral_code = request.COOKIES.get(REFERRAL_COOKIE_KEY, None)
        cookie_referrer = ReferralCode.get_user_from_code(cookie_referral_code)
        param_referrer = None
        request.referrer = cookie_referrer
        if cookie_referrer is None:
            # try get referral code from query parameter
            param_referral_code = request.GET.get(REFERRAL_PARAM_KEY, None)
            param_referrer = ReferralCode.get_user_from_code(param_referral_code)
            request.referrer = param_referrer
        response = self.get_response(request)
        if param_referrer is not None:
            response.set_cookie(
                REFERRAL_PARAM_KEY,
                str(param_referral_code),
                max_age=cookie_age,
            )
        return response


from django.shortcuts import render,redirect
from django.db import connections
import uuid
from django.views.decorators.csrf import csrf_exempt

from django.contrib import messages
import sqlite3,os,datetime,random,string
from pathlib import Path
BASE_DIR = Path(__file__).resolve().parent.parent
# Create your views here.
def index(request):
    return render(request,'index.html')
    # return HttpResponse('ths is')
# dashboard
def dashboard(request):
     return render(request,'dashboard.html')
# product
def get_length(request):
    with sqlite3.connect(BASE_DIR/'data.db') as db:
        cursor=db.cursor()
        query = "SELECT * FROM o_product"
        cursor.execute(query)

        rows = cursor.fetchall()
        column_names = [description[0] for description in cursor.description]

        product_count = len(rows)

    # Render the dashboard template with the product count
    return render(request, 'dashboard.html', {'product_count': product_count})
def product(request):
    with sqlite3.connect(BASE_DIR/'data.db') as db:
        cursor=db.cursor()
        query = "SELECT * FROM o_product"
        cursor.execute(query)

        rows = cursor.fetchall()
        column_names = [description[0] for description in cursor.description]

        # Prepare the data as a list of dictionaries
        products = []
        for row in rows:
            product = dict(zip(column_names, row))
            products.append(product)
        print(products)

        return render(request,'product.html',{'product':products})
def delete_product(request,pro_id):
    with sqlite3.connect(BASE_DIR / 'data.db') as db:
        cursor = db.cursor()
        query = "DELETE FROM o_product WHERE pro_id= ?"
        cursor.execute(query, (pro_id,))
        messages.success(request, 'Product deleted successfully.')

        return redirect('/product')

def save_product(request):
    if request.method == 'POST':
        pro_id =  random.randint(1, 100)
        ar_name = request.POST.get('ar_name')
        brand = request.POST.get('brand')
        color = request.POST.get('color')
        size = int(request.POST.get('size'))
        cate = request.POST.get('cate')

        unique_id = uuid.uuid4()
        with sqlite3.connect(BASE_DIR/'data.db') as db:
            cursor=db.cursor()
            # query = 'INSERT INTO "axha"("ID","aname","brand","color","size","cate","price") VALUES (?,?,?,?,?,?,?)'
            query = 'INSERT INTO "o_product"("pro_id","ar_name","brand","color","size","cate") VALUES (?,?,?,?,?,?)'
            values = (pro_id,ar_name, brand, color, size, cate)

            cursor.execute(query, values)

    return redirect('/product')
def getproduct(request):

    with sqlite3.connect(BASE_DIR/'data.db') as db:
        cursor = db.cursor()
        query = "SELECT * FROM o_product"
        cursor.execute(query)

        rows = cursor.fetchall()
        column_names = [description[0] for description in cursor.description]

        # Prepare the data as a list of dictionaries
        products = []
        for row in rows:
            product = dict(zip(column_names, row))
            products.append(product)

        # Count the total number of products
        total_products = len(products)

        return render(request, 'product.html', {'product': products, 'total_product': total_products})
def editproduct(request):
    if request.method == 'POST':
        pro_id = request.POST.get('pro_id')
        ar_name = request.POST.get('ar_name')
        brand = request.POST.get('brand')
        color = request.POST.get('color')
        size = request.POST.get('size')
        cate = request.POST.get('cate')

        with sqlite3.connect(BASE_DIR / 'data.db') as db:
            cursor = db.cursor()

            query = """
                UPDATE o_product
                SET ar_name = ?,
                    brand = ?,
                    color = ?,
                    size = ?,
                    cate = ?
                WHERE pro_id = ?
            """
            cursor.execute(query, (ar_name, brand, color, size, cate, pro_id))

    return redirect('/product')

def update_product(request, pro_id):
    # Process the form submission and update the product information in the database

    with sqlite3.connect(BASE_DIR / 'data.db') as db:
        cursor = db.cursor()
        query = """
            SELECT * FROM o_product
            WHERE pro_id = ?
        """
        cursor.execute(query, (pro_id,))
        rows = cursor.fetchone()

        # rows = cursor.fetchall()
        column_names = [description[0] for description in cursor.description]

        # Prepare the data as a list of dictionaries
        # Prepare the data as a dictionary
        product = dict(zip(column_names, rows))

    return render(request, 'update_prod.html', {"product": product})

def product_count(request):
    with sqlite3.connect(BASE_DIR / 'data.db') as db:
        cursor =db.cursor()
        cursor.execute("SELECT COUNT(*) FROM o_product")
        product_count = cursor.fetchone()[0]

        count = len(product_count)

        query = 'INSERT INTO "c_product"("co_id","count") VALUES (?,?)'
        values = (co_id,count)

        cursor.execute(query, values)
# suppliers function
def supplier(request):
     with sqlite3.connect(BASE_DIR/'data.db') as db:
        cursor=db.cursor()
        query = "SELECT * FROM o_supplier"
        cursor.execute(query)

        rows = cursor.fetchall()
        column_names = [description[0] for description in cursor.description]

        # Prepare the data as a list of dictionaries
        suppliers = []
        for row in rows:
            supplier = dict(zip(column_names, row))
            suppliers.append(supplier)
        print(suppliers)
        return render(request,'supplier.html',{'supplier':suppliers})

def delete_supplier(request,sup_id):
    with sqlite3.connect(BASE_DIR / 'data.db') as db:
        cursor = db.cursor()
        query = "DELETE FROM o_supplier WHERE sup_id= ?"
        cursor.execute(query, (sup_id,))
        messages.success(request, 'Supplier deleted successfully.')

        return redirect('/supplier')

def save_supplier(request):
    if request.method == 'POST':
        sup_id = random.randint(1, 100)
        sup_name = request.POST.get('sup_name')

        sup_ph=request.POST.get('sup_ph')
        unique_id = uuid.uuid4()
        with sqlite3.connect(BASE_DIR/'data.db') as db:
            cursor=db.cursor()
            # query = 'INSERT INTO "axha"("ID","aname","brand","color","size","cate","price") VALUES (?,?,?,?,?,?,?)'
            query = 'INSERT INTO "o_supplier"("sup_id","sup_name","sup_ph") VALUES (?,?,?)'
            values = (sup_id, sup_name,sup_ph)
            cursor.execute(query, values)

    return redirect('/supplier')
def editsupplier(request):
    if request.method == 'POST':
        sup_id= request.POST.get("sup_id")
        sup_name = request.POST.get('sup_name')

        sup_ph=request.POST.get('sup_ph')

        with sqlite3.connect(BASE_DIR / 'data.db') as db:
            cursor = db.cursor()

            query = """
                UPDATE o_supplier
                SET sup_name = ?,
                    sup_ph = ?

                WHERE sup_id = ?
            """
            cursor.execute(query, (sup_name, sup_ph,  sup_id))

    return redirect('/supplier')

def update_supplier(request, sup_id):
    # Process the form submission and update the product information in the database

    with sqlite3.connect(BASE_DIR / 'data.db') as db:
        cursor = db.cursor()
        query = """
            SELECT * FROM o_supplier
            WHERE sup_id = ?
        """
        cursor.execute(query, (sup_id,))
        rows = cursor.fetchone()

        # rows = cursor.fetchall()
        column_names = [description[0] for description in cursor.description]

        # Prepare the data as a list of dictionaries
        # Prepare the data as a dictionary
        supplier = dict(zip(column_names, rows))

    return render(request, 'upd_supplier.html', {"supplier": supplier})
# purchase
def add_to_cart(request):
    if request.method == 'POST':
        product_id = request.POST.get('pro_id')
        # Perform the desired logic with the product ID
        # For example, add the product to the user's cart
        request.session['pro_id'] = product_id
        # Redirect to the appropriate page
        return redirect('/purchase')
    else:
        # Handle the case when the request method is not POST
        # You can redirect to an error page or display an error message
        return redirect('Invalid request method')

def purchase(request):
    with sqlite3.connect(BASE_DIR/'data.db') as db:

        cursor=db.cursor()
        query = "SELECT * FROM o_purchase"
        cursor.execute(query)

        rows = cursor.fetchall()
        column_names = [description[0] for description in cursor.description]
        # total amount

        # Prepare the data as a list of dictionaries
        purchases = []
        for row in rows:
            purchase = dict(zip(column_names, row))

            purchases.append(purchase)
        print(purchases)
    #    product
        query2 = "SELECT * FROM o_product"
        cursor.execute(query2)
        rows2 = cursor.fetchall()
        column_names2 = [description[0] for description in cursor.description]
        # total amount
        # Prepare the data as a list of dictionaries
        products = []
        for row2 in rows2:
            product = dict(zip(column_names2, row2))
            products.append(product)

        print(products)

        return render(request, 'purchase.html', {'product': products,'purchase': purchases})

def delete_purchase(request,pur_id):
    with sqlite3.connect(BASE_DIR / 'data.db') as db:
        cursor = db.cursor()
        query = "DELETE FROM o_purchase WHERE pur_id= ?"
        cursor.execute(query, (pur_id,))
        messages.success(request, 'Purchase deleted successfully.')

        return redirect('/purchase')

def save_purchase(request):

    if request.method == 'POST':
        pur_id = random.randint(1, 100)
        product_id = request.session.get('pro_id')
        sup_id = request.POST.get('sup_id')
        pur_quan = int(request.POST.get('pur_quan'))
        pur_price = float(request.POST.get('pur_price'))
        pur_total = pur_price* pur_quan
        #we  inventory
        in_id = random.randint(1, 100)
        sal_quan = None
        # inserting in purchase
        with sqlite3.connect(BASE_DIR / 'data.db') as db:
            cursor = db.cursor()

            # Check if the supplier ID exists
            query_check_supplier = 'SELECT * FROM o_supplier WHERE sup_id = ?;'
            cursor.execute(query_check_supplier, (sup_id,))
            result_supplier = cursor.fetchone()

            if result_supplier:
                # Supplier ID exists, proceed with saving the purchase
                query_insert_purchase = 'INSERT INTO o_purchase (pur_id, pro_id, sup_id, pur_quan, pur_price, pur_total) VALUES (?, ?, ?, ?, ?, ?);'
                values_insert_purchase = (pur_id, product_id, sup_id, pur_quan, pur_price, pur_total)
                cursor.execute(query_insert_purchase, values_insert_purchase)

                # Check if the product already exists in the inventory
                query_check_inventory = 'SELECT * FROM o_inventory WHERE pro_id = ?;'
                cursor.execute(query_check_inventory, (product_id,))
                result_inventory = cursor.fetchone()

                if result_inventory:
                    # If the product already exists, update the quantity
                    current_quantity = result_inventory[2]
                    new_quantity = int(current_quantity) + int(pur_quan)
                    query_update_inventory = 'UPDATE o_inventory SET pur_quan = ? WHERE pro_id = ?;'
                    cursor.execute(query_update_inventory, (new_quantity, product_id))
                else:
                    # If the product doesn't exist, insert a new record
                    query_insert_inventory = 'INSERT INTO o_inventory (in_id, pro_id, pur_quan, sal_quan) VALUES (?, ?, ?, ?);'
                    values_insert_inventory = (in_id, product_id, pur_quan, sal_quan)
                    cursor.execute(query_insert_inventory, values_insert_inventory)

                db.commit()
                return redirect('/purchase')
            else:
                # Supplier ID does not exist, generate an error
                error_message = 'Supplier ID does not match any existing suppliers.'
                return render(request, 'error.html', {'error_message': error_message})

    return redirect('/purchase')
    #     with sqlite3.connect(BASE_DIR/'data.db') as db:

    #         cursor=db.cursor()
    #         query = 'INSERT INTO "o_purchase"("pur_id","pro_id","sup_id","pur_quan","pur_price","pur_total") VALUES (?,?,?,?,?,?);'
    #         values = (pur_id,product_id,sup_id,pur_quan,pur_price,pur_total)
    #         cursor.execute(query, values)
    #         db.commit()
    #           # Insert into the purchase table using the foreign key

    #     with sqlite3.connect(BASE_DIR/'data.db') as db:
    #         cursor = db.cursor()

    #         # Check if the product already exists in the inventory
    #         query_check = 'SELECT * FROM o_inventory WHERE pro_id = ?;'
    #         cursor.execute(query_check, (product_id,))
    #         result = cursor.fetchone()

    #         if result:
    #             # If the product already exists, update the quantity
    #             current_quantity = result[2]
    #             new_quantity =int(current_quantity) + int(pur_quan)

    #             query_update = 'UPDATE o_inventory SET pur_quan = ? WHERE pro_id = ?;'
    #             cursor.execute(query_update, (new_quantity, product_id))
    #         else:
    #             # If the product doesn't exist, insert a new record
    #             in_id = random.randint(1, 100)
    #             query_insert = 'INSERT INTO o_inventory (in_id, pro_id, pur_quan, sal_quan) VALUES (?, ?, ?, ?);'
    #             values_insert = (in_id, product_id, pur_quan, sal_quan)
    #             cursor.execute(query_insert, values_insert)

    #  return redirect('/purchase')

        # Retrieve the patient and doctor objects using their IDs
        # with sqlite3.connect(BASE_DIR/'data.db') as db:
        #     cursor=db.cursor()
        #     cursor.execute("SELECT * FROM Patient WHERE id_p=?", (patient_id,))
        #     patient_result = cursor.fetchone()
        #     cursor.execute("SELECT * FROM Doctor WHERE id_d=?", (doctor_id,))
        #     doctor_result = cursor.fetchone()
        #     appointment_id = "".join(random.choices(string.ascii_letters + string.digits, k=4))

        #     # Create a new Appointment object and save it to the database
        #     cursor.execute("INSERT INTO Appointment (id_a, patient, doctor,date) VALUES (?, ?, ?,?)", (appointment_id, patient_result[0], doctor_result[0],datetime.date.today()))
def editpurchase(request):
    if request.method == 'POST':
        in_id = request.POST.get('in_id')
        pur_id = request.POST.get('pur_id')
        product_id = request.session.get('pro_id')
        sup_id = request.POST.get('sup_id')
        pur_quan = int(request.POST.get('pur_quan'))
        pur_price = float(request.POST.get('pur_price'))
        pur_total = pur_price * pur_quan
        sal_quan = None

        with sqlite3.connect(BASE_DIR / 'data.db') as db:
            cursor = db.cursor()

            # Get the original purchase quantity
            query_original_purchase = """
                SELECT pur_quan
                FROM o_purchase
                WHERE pur_id = ?
            """
            cursor.execute(query_original_purchase, (pur_id,))
            original_purchase_quan = cursor.fetchone()[0]

            # Calculate the quantity difference
            purchase_quantity_diff = int(pur_quan) - int(original_purchase_quan)

            # Update the purchase
            query_purchase = """
                UPDATE o_purchase
                SET pro_id = ?,
                    sup_id = ?,
                    pur_quan = ?,
                    pur_price = ?,
                    pur_total = ?
                WHERE pur_id = ?
            """
            cursor.execute(query_purchase, (product_id, sup_id, pur_quan, pur_price, pur_total, pur_id))

            if int(pur_quan) > int(original_purchase_quan):
                # Update the inventory
                query_update_inventory = """
                    UPDATE o_inventory
                    SET pur_quan = pur_quan + ?
                    WHERE pro_id = ?
                """
                cursor.execute(query_update_inventory, (purchase_quantity_diff, product_id))

    return redirect('/purchase')

def update_purchase(request, pur_id):
    # Process the form submission and update the product information in the database

    with sqlite3.connect(BASE_DIR / 'data.db') as db:
        cursor = db.cursor()
        query = """
            SELECT * FROM o_purchase
            WHERE pur_id = ?
        """
        cursor.execute(query, (pur_id,))
        rows = cursor.fetchone()

        # rows = cursor.fetchall()
        column_names = [description[0] for description in cursor.description]

        # Prepare the data as a list of dictionaries
        # Prepare the data as a dictionary
        purchase = dict(zip(column_names, rows))

    return render(request, 'upd_purchase.html', {"purchase": purchase})

# customer

def customer(request):
    with sqlite3.connect(BASE_DIR/'data.db') as db:
        cursor=db.cursor()
        query = "SELECT * FROM o_customer"
        cursor.execute(query)

        rows = cursor.fetchall()
        column_names = [description[0] for description in cursor.description]
        # total amount

        # Prepare the data as a list of dictionaries
        customers = []
        for row in rows:
            customer = dict(zip(column_names, row))
            customers.append(customer)
        print(customers)

        return render(request, 'customer.html', {'customer': customers})
def delete_customer(request,cus_id):
    with sqlite3.connect(BASE_DIR / 'data.db') as db:
        cursor = db.cursor()
        query = "DELETE FROM o_customer WHERE cus_id= ?"
        cursor.execute(query, (cus_id,))
        messages.success(request, 'Customer deleted successfully.')

        return redirect('/customer')
def save_customer(request):
     if request.method == 'POST':
        cus_id =random.randint(1, 100)
        c_name = request.POST.get('c_name')
        c_ph = int(request.POST.get('c_ph'))

        with sqlite3.connect(BASE_DIR/'data.db') as db:
            cursor=db.cursor()

            # query = 'INSERT INTO "axha"("ID","aname","brand","color","size","cate","price") VALUES (?,?,?,?,?,?,?)'
            query = 'INSERT INTO "o_customer"("cus_id","c_name","c_ph") VALUES (?,?,?);'
            values = (cus_id,c_name,c_ph)
            cursor.execute(query, values)

     return redirect('/customer')
# sale
def add_to_cart_sale(request):
    if request.method == 'POST':
        product_id = request.POST.get('pro_id')
        # Perform the desired logic with the product ID
        # For example, add the product to the user's cart
        request.session['pro_id'] = product_id
        # Redirect to the appropriate page
        return redirect('/sale')
    else:
        # Handle the case when the request method is not POST
        # You can redirect to an error page or display an error message
        return redirect('Invalid request method')
def sale(request):

    with sqlite3.connect(BASE_DIR/'data.db') as db:
        cursor=db.cursor()
        query = "SELECT * FROM o_sale"
        cursor.execute(query)

        rows = cursor.fetchall()
        column_names = [description[0] for description in cursor.description]
        # total amount

        # Prepare the data as a list of dictionaries
        sales = []
        for row in rows:
            sale = dict(zip(column_names, row))
            sales.append(sale)
        print(sales)
        # products
        query2 = "SELECT * FROM o_product"
        cursor.execute(query2)
        rows2 = cursor.fetchall()
        column_names2 = [description[0] for description in cursor.description]
        # total amount
        # Prepare the data as a list of dictionaries
        products = []
        for row2 in rows2:
            product = dict(zip(column_names2, row2))
            products.append(product)

        print(products)
        return render(request, 'sale.html', {'sale': sales,'product':products})

def delete_sale(request,sal_id):
    with sqlite3.connect(BASE_DIR / 'data.db') as db:
        cursor = db.cursor()
        query = "DELETE FROM o_sale WHERE sal_id= ?"
        cursor.execute(query, (sal_id,))
        messages.success(request, 'sales deleted successfully.')

        return redirect('/sale')
def save_sale(request):
    if request.method == 'POST':
        sal_id = random.randint(1, 100)
        product_id = request.session.get('pro_id')
        sal_price = float(request.POST.get('sal_price'))
        sal_quan = int(request.POST.get('sal_quan'))
        t_amtt = sal_price * sal_quan

        with sqlite3.connect(BASE_DIR/'data.db') as db:
            cursor = db.cursor()

            # Check if the product exists in the inventory and has sufficient quantity
            query_check = 'SELECT * FROM o_inventory WHERE pro_id = ?;'
            cursor.execute(query_check, (product_id,))
            result = cursor.fetchone()

            if result:
                current_quantity = result[2]

                if int(current_quantity) >= sal_quan:
                    # Sufficient quantity available, proceed with the sale

                    # Insert sale record
                    query_insert_sale = 'INSERT INTO "o_sale"("sal_id","pro_id","sal_price","sal_quan","sal_total") VALUES (?,?,?,?,?);'
                    values_sale = (sal_id, product_id, sal_price, sal_quan, t_amtt)
                    cursor.execute(query_insert_sale, values_sale)

                    # Update inventory and sales quantity
                    new_quantity = int(current_quantity) - int(sal_quan)
                    query_update_inventory = 'UPDATE o_inventory SET pur_quan = ?, sal_quan = ? WHERE pro_id = ?;'
                    values_update_inventory = (new_quantity, sal_quan, product_id)
                    cursor.execute(query_update_inventory, values_update_inventory)
                else:
                    # Insufficient quantity, display error message or redirect
                    error_message = "Insufficient quantity available for sale."
                    return render(request, 'error.html', {'error_message': error_message})
                    # You can choose to render an error page or redirect with the error message
                    # Example: return render(request, 'error.html', {'message': error_message})
                    # Example: return redirect('/error/?message=' + error_message)

    return redirect('/sale')

# INVENTORY
def editsale(request):
    if request.method == 'POST':
        sal_id = request.POST.get('sal_id')
        product_id = request.session.get('pro_id')
        sal_quan = int(request.POST.get('sal_quan'))
        sal_price = float(request.POST.get('sal_price'))
        sal_total = sal_price * sal_quan

        with sqlite3.connect(BASE_DIR / 'data.db') as db:
            cursor = db.cursor()

            # Get the original sale details
            query_original_sale = """
                SELECT sal_quan
                FROM o_sale
                WHERE sal_id = ?
            """
            cursor.execute(query_original_sale, (sal_id,))
            original_sale_quan = cursor.fetchone()[0]

            # Update the sale
            query_update_sale = """
                UPDATE o_sale
                SET pro_id = ?,
                    sal_quan = ?,
                    sal_price = ?,
                    sal_total = ?
                WHERE sal_id = ?
            """
            cursor.execute(query_update_sale, (product_id, sal_quan, sal_price, sal_total, sal_id))

            # Check if the new quantity is less than the original quantity
            if int(sal_quan) < int(original_sale_quan):
                # Calculate the remaining quantity
                remaining_quan = int(original_sale_quan) - (sal_quan)
                remaining_sale = int(sal_quan)
                # Update the inventory
                query_update_inventory = """
                    UPDATE o_inventory
                    SET pur_quan = pur_quan + ?,
                    sal_quan = sal_quan -?
                    WHERE pro_id = ?
                """
                cursor.execute(query_update_inventory, (remaining_quan,remaining_sale,product_id))
            else:
                remaining_quan = int(original_sale_quan) + (sal_quan)
                remaining_sale = int(sal_quan)
                # Update the inventory
                query_update_inventory = """
                    UPDATE o_inventory
                    SET pur_quan = pur_quan - ?,
                    sal_quan = sal_quan +?
                    WHERE pro_id = ?
                """
                cursor.execute(query_update_inventory, (remaining_quan,remaining_sale,product_id))

    return redirect('/sale')

def update_sale(request, sal_id):
    # Process the form submission and update the product information in the database

    with sqlite3.connect(BASE_DIR / 'data.db') as db:
        cursor = db.cursor()
        query = """
            SELECT * FROM o_sale
            WHERE sal_id = ?
        """
        cursor.execute(query, (sal_id,))
        rows = cursor.fetchone()

        # rows = cursor.fetchall()
        column_names = [description[0] for description in cursor.description]

        # Prepare the data as a list of dictionaries
        # Prepare the data as a dictionary
        sale = dict(zip(column_names, rows))

    return render(request, 'upd_sale.html', {"sale": sale})

def display_inventory(request):

    with sqlite3.connect(BASE_DIR/'data.db') as db:

        cursor=db.cursor()
        query = "SELECT * FROM o_inventory"
        cursor.execute(query)

        rows = cursor.fetchall()
        column_names = [description[0] for description in cursor.description]
        # total amount

        # Prepare the data as a list of dictionaries
        invent = []
        for row in rows:
            inventory = dict(zip(column_names, row))
            invent.append(inventory)
        print(invent)
        return render(request, 'inventory.html',{'inventory': invent})
def delete_inventory(request,in_id):
    with sqlite3.connect(BASE_DIR / 'data.db') as db:
        cursor = db.cursor()
        query = "DELETE FROM o_inventory WHERE in_id= ?"
        cursor.execute(query, (in_id,))
        messages.success(request, 'Inventory deleted successfully.')

        return redirect('/inventory')


from django.urls import path

from . views import  drivers, create_driver, single_driver,rate

urlpatterns = [
    # path('drivers_dash/', drivers_dash, name='drivers_dash'),
    path("create_driver", create_driver, name="create_driver"),
    path('drivers/', drivers, name='drivers'),
    path('drivers/<str:name>', single_driver, name='single_driver'),
    path("rate/<int:id>", rate, name='rate'),

]


### This script calculates semantic closeness ###

#usr/bin/env python3

HAPPY = True

import glob, os, io, argparse, csv, pandas, string, math
import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity

punctuation = list(string.punctuation)

def total(wc_dict):

	total_c = 0

	for w, c in wc_dict.items():
		total_c += int(c)

	return total_c

def lexical_frequency(phrase, wc_dict):

	toks = phrase.split()

#	total_c = total(wc_dict)

	lf = 1
	length = 0

	for w in toks:
		if w not in punctuation:
			length += 1

			if w in wc_dict:
				p = wc_dict[w]
				if p != '0.0':
					lf = lf * float(p)
				else:
					lf = lf * (1 / (10 ** 10))

			if w not in wc_dict:
				p = 1 / (1 / (10 ** 10))
				lf = lf * p

#	lf = lf ** (-1 / length)

	return lf

def pmi(np, v, jc_dict, wc_dict):

	wc_total = total(wc_dict)

	total_c = 0
	for p, c in jc_dict.items():
		total_c += int(c)

	v_p = 1 / wc_total
	if v in wc_dict:
		v_p = wc_dict[v] / wc_total

	np_p = 1 / wc_total
	if np in wc_dict:
		np_p = wc_dict[np] / wc_total

	pmi_v = math.log2((jc_dict[v + ' ' + np] / total_c) / (v_p * np_p))

	return pmi_v

if __name__ == '__main__':

	parser = argparse.ArgumentParser()
	parser.add_argument('--pp', type = str, help = 'input path to initial *_pp.csv data')
	parser.add_argument('--language', type = str, help = 'language being computed')
	parser.add_argument('--size', action='store_true', help = 'whether contextual predictability exists')

	args = parser.parse_args()

	path = args.pp
	os.chdir(path)

	language = args.language

	pp_data = pd.read_csv(path + language + '_pp.csv', encoding = 'utf-8')

	verbs = pp_data['Verb']
	Adp1 = pp_data['Adp1']
	Adp2 = pp_data['Adp2']
	NP1 = pp_data['NP1']
	NP2 = pp_data['NP2']
	PP1 = pp_data['PP1_simple']
	PP2 = pp_data['PP2_simple']
	PP1_len = pp_data['PP1_len']
	PP2_len = pp_data['PP2_len']
	PP1_Pronom = pp_data['PP1_Pronom']
	PP2_Pronom = pp_data['PP2_Pronom']
	position = pp_data['Position']

	#### Calculating semantic closenes ####

	embeddings = {}

	with io.open(path + language + '_em', encoding = 'utf-8') as f:
		for line in f:
			toks = line.split()
			embeddings[toks[0]] = [float(x) for x in toks[1 : ]]

	PP1_SC = []
	PP2_SC = []

	for i in range(len(verbs)):
		v = verbs[i]
		adp1 = Adp1[i]
		adp2 = Adp2[i]
		np1 = NP1[i]
		np2 = NP2[i]

		v_em = 'NONE'
		if v in embeddings:
			v_em = embeddings[v]

		adp1_em = 'NONE'
		if adp1 in embeddings:
			adp1_em = embeddings[adp1]

		np1_em = 'NONE'
		if np1 in embeddings:
			np1_em = embeddings[np1]

		adp2_em = 'NONE'
		if adp2 in embeddings:
			adp2_em = embeddings[adp2]

		np2_em = 'NONE'
		if np2 in embeddings:
			np2_em = embeddings[np2]

		pp1_sc = 'NONE'
		if v_em != 'NONE' and np1_em != 'NONE' and adp1_em != 'NONE':
			pp1_em = [(x + y) / 2 for x, y in zip(np1_em, adp1_em)]
			pp1_sc = cosine_similarity([v_em], [pp1_em])[0][0]

		if v_em != 'NONE' and np1_em != 'NONE' and adp1_em == 'NONE':
			pp1_sc = cosine_similarity([v_em], [np1_em])[0][0]

		if v_em != 'NONE' and np1_em == 'NONE' and adp1_em != 'NONE':
			pp1_sc = cosine_similarity([v_em], [adp1_em])[0][0]

		pp2_sc = 'NONE'
		if v_em != 'NONE' and np2_em != 'NONE' and adp2_em != 'NONE':
			pp2_em = [(x + y) / 2 for x, y in zip(np2_em, adp2_em)]
			pp2_sc = cosine_similarity([v_em], [pp2_em])[0][0]

		if v_em != 'NONE' and np2_em != 'NONE' and adp2_em == 'NONE':
			pp2_sc = cosine_similarity([v_em], [np2_em])[0][0]

		if v_em != 'NONE' and np2_em == 'NONE' and adp2_em != 'NONE':
			pp2_sc = cosine_similarity([v_em], [adp2_em])[0][0]

		PP1_SC.append(pp1_sc)
		PP2_SC.append(pp2_sc)

	print(len(PP1_SC))
	print(len(PP2_SC))

	print(PP1_SC)

	#### Calculating lexical frequency ####

	wc = {}
	with io.open(path + language + '_wc', encoding = 'utf-8') as f:
		for line in f:
			toks = line.split()
			if toks[0] not in punctuation:
				wc[toks[0]] = toks[1]

	PP1_LF = []
	PP2_LF = []

	for i in range(len(verbs)):
		pp1_lf = lexical_frequency(PP1[i], wc)
		pp2_lf = lexical_frequency(PP2[i], wc)

		PP1_LF.append(pp1_lf)
		PP2_LF.append(pp2_lf)

	#### Calculating PMI ####

#	joint_c = {}
#	with io.open(path + language + '_jc', encoding = 'utf-8') as f:
#		for line in f:
#			toks = line.split()
#			if toks[1] not in punctuation and toks[2] not in punctuation:
#				joint_c[toks[1] + ' ' + toks[2]] = int(toks[0])

#	PP1_PMI = []
#	PP2_PMI = []

#	for i in range(len(verbs)):
#		pp1_pmi = pmi(NP1[i], verbs[i], joint_c, wc)
#		pp2_pmi = pmi(NP2[i], verbs[i], joint_c, wc)

	#### Getting predictability ####

	if args.size:
		PP1_simple_Pred = []
		PP2_simple_Pred = []

		PP1_Pred = []
		PP2_Pred = []

		with io.open(path + language + '_pp1_pred', encoding = 'utf-8') as f:
			for line in f:
				toks = line.split()
				PP1_simple_Pred.append(float(toks[0]))
				PP1_Pred.append(float(toks[1]))

		with io.open(path + language + '_pp2_pred', encoding = 'utf-8') as f:
			for line in f:
				toks = line.split()
				PP2_simple_Pred.append(float(toks[0]))
				PP2_Pred.append(float(toks[1]))

	#### Generating data set for regression ####

#	new_data = pp_data.copy()

#	new_data['PP1_SC'] = PP1_SC
#	new_data['PP2_SC'] = PP2_SC
#	new_data['PP1_PMI'] = PP1_PMI
#	new_data['PP2_PMI'] = PP2_PMI
#	new_data['PP1_simple_Pred'] = PP1_simple_Pred
#	new_data['PP2_simple_Pred'] = PP2_simple_Pred
#	new_data['PP1_Pred'] = PP1_Pred
#	new_data['PP2_Pred'] = PP2_Pred

#	print('Generating new PP data')

#	outfile = io.open(path + language + '_regression.csv', 'w', encoding = 'utf-8')
#	outfile.write(new_data.to_csv(index = False, encoding = 'utf-8'))

	pre_features = []
	post_features = []
	a=0

	for i in range(len(verbs)):
		print(PP1_SC[i] - PP2_SC[i])
	for i in range(len(verbs)):
		feature = []

		if position[i] == 'Postverbal':
			len_diff = int(PP2_len[i]) - int(PP1_len[i])
			len_c = ''
			if len_diff > 0:
				len_c = 1
			if len_diff < 0:
				len_c = -1
			if len_diff == 0:
				len_c = 0
			feature.append(len_diff)
			feature.append(len_c)

		#	feature.append(PP1_PMI[i] - PP2_PMI[i])

			sc_diff = ''
			try:
				sc_diff = PP1_SC[i] - PP2_SC[i]
				a += 1
				print(a)
			except:
				sc_diff = 'NONE'

			sc_c = ''
			try:
				if sc_diff > 0:
					sc_c = 1
				if sc_diff < 0:
					sc_c = -1
				if sc_diff == 0:
					sc_c = 0
			except:
				sc_c = 'NONE'

			feature.append(sc_diff)
			feature.append(sc_c)

		else:
			len_diff = int(PP1_len[i]) - int(PP2_len[i])
			len_c = ''
			if len_diff > 0:
				len_c = 1
			if len_diff < 0:
				len_c = -1
			if len_diff == 0:
				len_c = 0
			feature.append(len_diff)
			feature.append(len_c)

		#	feature.append(PP2_PMI[i] - PP1_PMI[i])

			sc_diff = ''
			try:
				sc_diff = PP2_SC[i] - PP1_SC[i]
				a+=1
				print(a)
			except:
				sc_diff = 'NONE'

			sc_c = ''
			try:
				if sc_diff > 0:
					sc_c = 1
				if sc_diff < 0:
					sc_c = -1
				if sc_diff == 0:
					sc_c = 0
			except:
				sc_c = 'NONE'

			feature.append(sc_diff)
			feature.append(sc_c)

	#	lf_diff = PP1_LF[i] - PP2_LF[i]
		lf_diff = PP2_LF[i] - PP1_LF[i]  ## forgot to change this earlier; so in analysis.R, multiply coefficient of lexical frequency by (-1)
		lf_c = ''
		if lf_diff > 0:
			lf_c = 1
		if lf_diff < 0:
			lf_c = -1
		if lf_diff == 0:
			lf_c = 0
		feature.append(lf_diff)
		feature.append(lf_c)

		pred_diff = ''
		pred_c = ''
		if args.size:
			pred_diff = PP1_simple_Pred[i] ** (-1 / len(PP1[i].split())) - PP2_simple_Pred[i] ** (-1 / len(PP2[i].split()))
		#	pred_diff = PP2_simple_Pred[i] - PP1_simple_Pred[i]
			if pred_diff > 0:
				pred_c = 1
			if pred_diff < 0:
				pred_c = -1
			if pred_diff == 0:
				pred_c = 0
		feature.append(pred_diff)
		feature.append(pred_c)

		if PP1_Pronom[i] in ['PRON'] and PP2_Pronom[i] not in ['PRON']:
			feature.append(1)
		if PP1_Pronom[i] in ['PRON'] and PP2_Pronom[i] in ['PRON']:
			feature.append(0)
		if PP1_Pronom[i] not in ['PRON'] and PP2_Pronom[i] not in ['PRON']:
			feature.append(0)
		if PP1_Pronom[i] not in ['PRON'] and PP2_Pronom[i] in ['PRON']:
			feature.append(-1)

		feature.append(verbs[i])
		feature.append(position[i])

		if position[i] == 'Preverbal':
			pre_features.append(feature)

		if position[i] == 'Postverbal':
			post_features.append(feature)

	print('Generating regression data')

	pre_split = round(len(pre_features) / 2) + 1
	post_split = round(len(post_features) / 2) + 1

	c = 0

	with io.open(path + language + '_regression.csv', 'w', newline = '', encoding = 'utf-8') as f:
		writer = csv.writer(f)
		writer.writerow(['Order', 'Len', 'Len_c', 'Semantic_closeness', 'Semantic_closeness_c', 'Lexical_frequency', 'Lexical_frequency_c', 'Predictability', 'Predictability_c', 'Pronominality', 'Verb', 'Position'])

			#### Preverbal data ####

		if len(pre_features) != 0:

			for i in range(pre_split):
				if pre_features[i][2] != 'NONE':
					c += 1

				writer.writerow([1, pre_features[i][0], pre_features[i][1], pre_features[i][2], pre_features[i][3], pre_features[i][4], pre_features[i][5], pre_features[i][6], pre_features[i][7], pre_features[i][8], pre_features[i][9], pre_features[i][10]])

			for i in range(pre_split, len(pre_features)):
				if pre_features[i][2] != 'NONE':
					c += 1

				writer.writerow([0, -1 * pre_features[i][0], -1 * pre_features[i][1], -1 * pre_features[i][2], -1 * pre_features[i][3], -1 * pre_features[i][4], -1 * pre_features[i][5], -1 * pre_features[i][6], -1 * pre_features[i][7], -1 * pre_features[i][8], pre_features[i][9], pre_features[i][10]])

		#### Postverbal data ####

		if len(post_features) != 0:

			for i in range(post_split):
				if post_features[i][2] != 'NONE':
					c += 1

					writer.writerow([1, post_features[i][0], post_features[i][1], post_features[i][2], post_features[i][3], post_features[i][4], post_features[i][5], post_features[i][6], post_features[i][7], post_features[i][8], post_features[i][9], post_features[i][10]])

			for i in range(post_split, len(post_features)):
				if post_features[i][2] != 'NONE':
					c += 1

					writer.writerow([0, -1 * post_features[i][0], -1 * post_features[i][1], -1 * post_features[i][2], -1 * post_features[i][3], -1 * post_features[i][4], -1 * post_features[i][5], -1 * post_features[i][6], -1 * post_features[i][7], -1 * post_features[i][8], post_features[i][9], post_features[i][10]])

	print(len(verbs))
	print(c)


from collections import List

class Solution:
    def twoSum(self, nums: List[int], target: int) -> List[int]:
        dict={} #Empty dictionary to hold index and value in a K,V manner
        result=[]
        for i, j in enumerate(nums):
            result= target - j # result is subtraction of target - value J at index i

            #If the result is in the dict return the value of the key result in the dict and the index i
            if result in dict:
                return [dict[result], i]

            dict[j]= i # element j will be key and index i will be value for the dict


import tensorflow as tf
from Data_Loader import DataLoader
import os
import numpy as np
from scipy import misc
import matplotlib.pyplot as plt
import scipy
import os.path as path
from PIL import Image
from skimage import img_as_ubyte

# add a line for pull test
class BaseModel(object):

    def __init__(self, sess, conf):
        self.sess = sess
        self.conf = conf
        self.is_training = True
        self.input_shape = [None, None, None, self.conf.in_channel]
        self.output_shape = [None, None, None, self.conf.out_channel]
        self.output_shape = [None, None, None, self.conf.out_channel]
        self.create_placeholders()

    def create_placeholders(self):
        with tf.name_scope('Input'):
            self.x = tf.placeholder(tf.float32, self.input_shape, name='input')
            self.y = tf.placeholder(tf.float32, self.output_shape, name='annotation')
            self.keep_prob = tf.placeholder(tf.float32)

    def loss_func(self):
        with tf.name_scope('Loss'):
            if self.conf.loss_type == 'MSE':
                with tf.name_scope('MSE'):
                    #norm_factor = tf.shape(self.y)[1] * tf.shape(self.y)[2]
                    #self.loss = tf.norm(self.y - self.y_pred)
                    self.loss=tf.losses.mean_squared_error(self.y , self.y_pred)#
            if self.conf.use_reg:
                with tf.name_scope('L2_loss'):
                    l2_loss = tf.reduce_sum(
                        self.conf.lmbda * tf.stack([tf.nn.l2_loss(v) for v in tf.get_collection('reg_weights')]))
                    self.loss += l2_loss
            self.mean_loss, self.mean_loss_op = tf.metrics.mean(self.loss)

    def configure_network(self):
        self.y_pred = tf.sigmoid(self.logits)
        self.loss_func()
        global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)
        learning_rate = tf.train.exponential_decay(self.conf.init_lr,
                                                   global_step,
                                                   decay_steps=500,
                                                   decay_rate=0.97,
                                                   staircase=True)
        self.learning_rate = tf.maximum(learning_rate, self.conf.lr_min)
        with tf.name_scope('Optimizer'):
            optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate)
            self.train_op = optimizer.minimize(self.loss, global_step=global_step)
        self.sess.run(tf.global_variables_initializer())
        trainable_vars = tf.trainable_variables()
        self.saver = tf.train.Saver(var_list=trainable_vars, max_to_keep=1000)
        self.train_writer = tf.summary.FileWriter(self.conf.logdir + self.conf.run_name + '/train/', self.sess.graph)
        self.valid_writer = tf.summary.FileWriter(self.conf.logdir + self.conf.run_name + '/valid/')
        self.configure_summary()
        print('*' * 50)
        print('Total number of trainable parameters: {}'.
              format(np.sum([np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()])))
        print('*' * 50)

    def configure_summary(self):
        summary_list = [tf.summary.scalar('learning_rate', self.learning_rate),
                        tf.summary.scalar('loss', self.mean_loss),
                        tf.summary.image('prediction_mask', self.y_pred, max_outputs=3),
                        tf.summary.image('original_mask', self.y, max_outputs=3)]
        self.merged_summary = tf.summary.merge(summary_list)

    def save_summary(self, summary, step):
        if self.is_training:
            self.train_writer.add_summary(summary, step)
        else:
            self.valid_writer.add_summary(summary, step)
        self.sess.run(tf.local_variables_initializer())

    def train(self):
        self.sess.run(tf.local_variables_initializer())
        if self.conf.reload_step > 0:
            self.reload(self.conf.reload_step)
            print('----> Continue Training from step #{}'.format(self.conf.reload_step))
        else:
            print('----> Start Training')
        self.best_validation_loss = 10000
        self.data_reader = DataLoader(self.conf)

        if self.idxs is not None:                   # if we have a mask for training, idxs is the list of pixels used for training. 1 in the mask
            self.data_reader.idxs= self.idxs

        # self.data_reader.load_crop_data(mode='valid')
        self.data_reader.get_data(mode='valid')
        self.num_val_batch = max (int(self.data_reader.y_valid.shape[0] / self.conf.batch_size),1)
        for train_step in range(1, self.conf.max_step + 1):
            self.is_training = True
            if train_step % self.conf.SUMMARY_FREQ == 0:
                x_batch, y_batch = self.data_reader.next_batch(mode='train')
                feed_dict = {self.x: x_batch, self.y: y_batch, self.keep_prob: self.conf.drop_out_rate}
                _, _, summary = self.sess.run([self.train_op,
                                               self.mean_loss_op,
                                               self.merged_summary],
                                              feed_dict=feed_dict)
                loss = self.sess.run(self.mean_loss)
                print('step: {0:<6}, train_loss= {1:.4f}'.format(train_step, loss))
                self.save_summary(summary, train_step + self.conf.reload_step)
            else:
                x_batch, y_batch = self.data_reader.next_batch(mode='train')
                feed_dict = {self.x: x_batch, self.y: y_batch, self.keep_prob: self.conf.drop_out_rate}
                self.sess.run([self.train_op, self.mean_loss_op], feed_dict=feed_dict)
            if train_step % self.conf.VAL_FREQ == 0:
                self.evaluate(train_step)

    def evaluate(self, train_step):
        self.is_training = False
        self.sess.run(tf.local_variables_initializer())
        # pred_mask = np.zeros_like(self.data_reader.y_valid)
        # for step in range(self.num_val_batch):
        #     start = step * self.conf.batch_size
        #     end = (step + 1) * self.conf.batch_size
        #     x_val, y_val = self.data_reader.next_batch(start, end, mode='valid')
        feed_dict = {self.x: self.data_reader.x_valid, self.y: self.data_reader.y_valid, self.keep_prob: 1}
        pred_mask,  _ = self.sess.run([self.y_pred, self.mean_loss_op], feed_dict=feed_dict)
        summary_valid = self.sess.run(self.merged_summary, feed_dict=feed_dict)
        valid_loss = self.sess.run(self.mean_loss)
        self.save_summary(summary_valid, train_step + self.conf.reload_step)
        self.save_fig(train_step,
                      pred_mask.reshape(-1, self.data_reader.val_img_size[0], self.data_reader.val_img_size[1], self.conf.out_channel),
                      self.data_reader.y_valid.reshape(-1, self.data_reader.val_img_size[0], self.data_reader.val_img_size[1], self.conf.out_channel),
                      valid_loss, mode='valid')
        if valid_loss < self.best_validation_loss:
            self.best_validation_loss = valid_loss
            improved_str = '(improved)'
            self.save(train_step + self.conf.reload_step)
        else:
            improved_str = ''
        print('-' * 25 + 'Validation' + '-' * 25)
        print('After {0} training step: val_loss= {1:.4f} {2}'
              .format(train_step, valid_loss, improved_str))
        print('-' * 60)

    def test(self, step_num):
        self.sess.run(tf.local_variables_initializer())
        self.reload(step_num)
        self.data_reader = DataLoader(self.conf)
        self.data_reader.get_data(mode='test')
        self.num_test_batch = max (int(self.data_reader.y_test.shape[0] / self.conf.batch_size) , 1)
        pred_mask = np.zeros_like(self.data_reader.y_test)
        self.is_training = False
        self.sess.run(tf.local_variables_initializer())
        for step in range(self.num_test_batch):
            start = step * self.conf.batch_size
            end = (step + 1) * self.conf.batch_size
            x_test, y_test = self.data_reader.next_batch(start, end, mode='test')
            feed_dict = {self.x: x_test, self.y: y_test, self.keep_prob: 1}
            pred_mask[start:end], _ = self.sess.run([self.y_pred, self.mean_loss_op], feed_dict=feed_dict)
        test_loss = self.sess.run(self.mean_loss)
        self.save_fig(step_num, pred_mask, self.data_reader.y_test, test_loss)
        print('-' * 18 + 'Test Completed' + '-' * 18)
        print('test_loss= {0:.4f}'.format(test_loss))
        print('-' * 50)

    def save(self, step):
        print('----> Saving the model at step #{0}'.format(step))
        checkpoint_path = os.path.join(self.conf.modeldir+self.conf.run_name, self.conf.model_name)
        self.saver.save(self.sess, checkpoint_path, global_step=step)

    def reload(self, step):
        checkpoint_path = os.path.join(self.conf.modeldir+self.conf.run_name, self.conf.model_name)
        model_path = checkpoint_path + '-' + str(step)
        if not os.path.exists(model_path + '.meta'):
            print('----> No such checkpoint found', model_path)
            return
        print('----> Restoring the model at step# {}'.format(step))
        self.saver.restore(self.sess, model_path)
        print('----> Model successfully restored')

    def save_fig(self, step, pred, y, loss, mode='test'):
        num_images = pred.shape[0]
        fig, axs = plt.subplots(nrows=num_images, ncols=3)
        axs = axs.reshape(-1, 3)
        for ii in range(num_images):
            pred_mask = pred[ii]
            true_mask = y[ii]
            ax = axs[ii, 0]
            ax.imshow(true_mask)
            ax.set_xticks([])
            ax.set_yticks([])
            ax.set_title('Groung Truth')
            ax = axs[ii, 1]
            ax.imshow(pred_mask)
            ax.set_xticks([])
            ax.set_yticks([])
            ax.set_title('Prediction')
            ax = axs[ii, 2]
            ax.imshow(np.abs(true_mask-pred_mask))
            ax.set_xticks([])
            ax.set_yticks([])
            ax.set_title('Difference)')
            ax.set_xlabel('loss='+str(loss))
            fig.set_size_inches(6, 2*num_images)
            fig.savefig(self.conf.modeldir+self.conf.run_name+'/'+mode+'_step_{0}_image{1}.png'.format(step, ii))
            misc.imsave(self.conf.modeldir+self.conf.run_name+'/'+mode+'_step_{0}_image{1}.bmp'.format(step, ii),pred_mask)
        plt.close('all')

    def load_mask(self):
        # get file names in masks directory sorted in alphabetical order
        mask_path = self.conf.data_dir +'mask/'
        file_list =[]
        mask_files=[]
        for f in os.listdir(mask_path):
            file_list.append(f)
        file_list.sort()
        #check if they are really file :)
        for f in file_list:
            if not f.startswith('.'):
                file_name = path.join(mask_path, f)
                if path.isfile(file_name) and not (file_name in mask_files):
                    mask_files.append(file_name)

        if len(mask_files) < 1:
            raise ValueError('Missing masks files in folder %s!' % mask_path)

        num_mask = len(mask_files)
        idx =[]
        for i in range(num_mask):
            mask = scipy.misc.imread(mask_files[i])
            idx_temp = np.transpose(np.nonzero(mask[:, :, 1])) # non_zero indices for each mask
            max_bottom_left_front_corner = (np.max(idx_temp[:, 0]) - self.conf.height - 1, np.max(idx_temp[:, 1]) - self.conf.width - 1)
            for j in idx_temp:                #check the nonzero points of masks and exclude border points
                if j[0] < max_bottom_left_front_corner[0] and j[1] < max_bottom_left_front_corner[1]:
                    idx.append(j)

        #mask_path = self.conf.data_dir + 'tr_mask.bmp'
        #mask = scipy.misc.imread(mask_path)
        #idx = np.transpose(np.nonzero(mask[:, :, 1]))
        #idxs = []  # indices close to borders which cannot be bottom_left corner are excluded from idx
        #max_bottom_left_front_corner = (np.max(idx[:,0]) - self.conf.height - 1, np.max(idx[:,1]) - self.conf.width - 1)
        #max_bottom_left_front_corner = (self.conf.img_size[0] - self.conf.height - 1, self.conf.img_size[1] - self.conf.width - 1)
        #for i in idx:
        #    if i[0] < max_bottom_left_front_corner[0] and i[1] < max_bottom_left_front_corner[1]:
        #         idxs.append(i)

        self.idxs = np.asarray(idx)


import logging
import boto3
from botocore.exceptions import ClientError

s3 = boto3.client('s3')
bucket_name = 'uploadingimageandgettingurl'
#for bucket in s3.buckets.all():
 #   print(bucket.name)

# with open('batmanRises.png', 'wb') as img:
#     s3.download_fileobj('uploadingimageandgettingurl', 'batmanRises.png' , img)

def upload_file(file_name,bucket,object_name=None):
    if object_name is None:
        object_name=file_name

    s3_client = boto3.client('s3')
    try:
        response = s3_client.upload_file(file_name,bucket,object_name)
    except ClientError as e:
        logging.error(e)
        return False
    return True

#with open('putatoe.jpg', 'rb') as img:
#     s3.upload_fileobj(img ,'uploadingimageandgettingurl', "putatoe_image")

s3.upload_file('batmanRises.png',bucket_name, 'batman_img' ,ExtraArgs = {'ACL':'public-read'})

key = "putatoe_image"
location = s3.get_bucket_location(Bucket = bucket_name)['LocationConstraint']
url = "https://s3-%s.amazonaws.com/%s/%s" % (location, bucket_name, key)
print(url)


from django.shortcuts import redirect, render
from .forms import userForm
from .models import userdata

# Create your views here.
def index(request):
    if request.method=='POST':
        user=userForm(request.POST)
        if user.is_valid(): #true
            user.save()
            print("Your data has been saved!")
        else:
            print(user.errors)
    return render(request,'index.html')

def showdata(request):
    data=userdata.objects.all()
    return render(request,'showdata.html',{'data':data})

def deletedata(request,stid):
   id=userdata.objects.get(id=stid)
   userdata.delete(id)
   return redirect('showdata')

def updatedata(request,stid):
    id=userdata.objects.get(id=stid)
    if request.method=='POST':
        updateuser=userForm(request.POST)
        if updateuser.is_valid():
            updateuser=userForm(request.POST,instance=id)
            updateuser.save()
            print("Your records has been updated!")
            return redirect('showdata')
        else:
            print(updateuser.errors)
    return render(request,'updatedata.html',{'stdata':userdata.objects.get(id=stid)})


import numpy as np
import matplotlib.pyplot as plt

from funbo.utils import *
from .utils import *
from .trials import plot_trials

try:
    import plotly.offline as ply
    import plotly.graph_objs as go

    if in_jupyter():
        print('plotly setup')
        ply.init_notebook_mode(connected=False)
except ImportError:
    ply = None

def plot_trial_area(optimiser, trial, true_best, quiet=False, ax=None):
    """
    """
    xmin, xmax = optimiser.domain_bounds[0]
    if not quiet:
        print('area between: {}'.format(multidimensional_integrate(lambda x: abs(true_best(x)-trial.f(x)), optimiser.domain_bounds)))
        print('R_g = {}'.format(trial.R_g))
        print('R_ls = {}'.format(format_list([r for x, r in trial.R_ls], 3)))
        print('f = {}'.format(trial.f))
    if ax is None:
        fig, ax = plt.subplots(figsize=(12, 6))
    xs = np.linspace(xmin, xmax, num=200)
    fs = [trial.f(x) for x in xs]

    ax.plot(xs, fs, color='salmon', label='f')
    ax.scatter(trial.f.control_xs, trial.f.control_ys, color='salmon', s=10, label='f control points')

    fits = [true_best(x) for x in xs]
    ax.plot(xs, fits, linestyle='--', color='k', label='to fit')
    ax.fill_between(xs, [min(a, b) for a,b in zip(fits, fs)], [max(a,b) for a,b in zip(fits, fs)], alpha=0.2)

    # sampled locations where local rewards were given
    if trial.R_ls:
        s_xs, s_rs = unzip(trial.R_ls)
        s_ys = [trial.f(x) for x in s_xs]
        ax.scatter(s_xs, s_ys, c='black', s=10, zorder=10, label='local rewards')

    ax.legend()

def plot_surrogate_with_trials(optimiser, trial_num, true_best=None, ylim=None, midpoint_fraction=None):
    if trial_num < 0:
        trial_num = len(optimiser.trials) + trial_num
    t = optimiser.trials[trial_num]
    fig = plot_surrogate(optimiser, t.surrogate, ylim, midpoint_fraction)
    plot_trials(optimiser, optimiser.trials[:trial_num+1], true_best, color_by_reward=False, ax=fig.axes[0])
    return fig

def plot_surrogate(optimiser, surrogate, ylim=None, midpoint_fraction=None, fig_ax=None):
    assert surrogate is not None, 'no surrogate'
    if fig_ax is None:
        fig, ax = plt.subplots(figsize=(20, 10))
    else:
        fig, ax = fig_ax
    # undo seaborn styling
    ax.grid(False)
    ax.set_facecolor((1, 1, 1))

    x_name = optimiser.domain_names[0]
    xmin, xmax = optimiser.domain_bounds[0]
    xs = np.linspace(xmin, xmax, num=100)

    if ylim is not None:
        ymin, ymax = ylim
        y_margin = 0
    else:
        ymin, ymax = optimiser.range_bounds
        y_margin = (ymax-ymin)*0.1

    ys = np.linspace(ymin-y_margin, ymax+y_margin, num=100)
    ax.margins(0, 0)

    grid = np.meshgrid(xs, ys)
    mus_points, var_points = surrogate.predict(grid_to_points(grid))
    mus_grid = vals_to_grid(mus_points, grid[0].shape)

    cmap = 'viridis' if optimiser.is_maximising() else 'viridis_r'
    if midpoint_fraction is None:
        norm = None
    else:
        norm = utils.MidpointNorm(np.min(mus_points), np.max(mus_points), midpoint=None, midpoint_fraction=midpoint_fraction)
    im = ax.pcolormesh(grid[0], grid[1], mus_grid, cmap=cmap, norm=norm)
    c = fig.colorbar(im, ax=ax)
    c.set_label(r'$R_l({},f({}))$'.format(x_name, x_name))

    ax.set_xlabel('${}$'.format(x_name))
    ax.set_ylabel(r'$f({})$'.format(x_name))

    return fig

def plot_surrogate_3D(optimiser, surrogate, show_var=True, flip_z=False):
    assert ply is not None, 'plotly not imported'
    assert surrogate is not None, 'no surrogate'

    x_name = optimiser.domain_names[0]
    xmin, xmax = optimiser.domain_bounds[0]
    xs = np.linspace(xmin, xmax, num=100)

    ymin, ymax = optimiser.range_bounds
    y_margin = (ymax-ymin)*0.1
    ys = np.linspace(ymin-y_margin, ymax+y_margin, num=100)

    grid = np.meshgrid(xs, ys)
    grid_shape = grid[0].shape
    mus_points, var_points = surrogate.predict(grid_to_points(grid))
    mus_grid = vals_to_grid(mus_points, grid_shape)
    if flip_z:
        mus_grid = -mus_grid

    data = [
        go.Surface(x=grid[0], y=grid[1], z=mus_grid, colorscale='Viridis', reversescale=optimiser.is_minimising() != flip_z),
    ]
    if show_var:
        sig_grid = vals_to_grid(np.sqrt(np.clip(var_points, 0, np.inf)), grid_shape)
        n_sig = 2
        color_row = ['#0000ff'] * grid[0].shape[1]
        colors = [color_row] * grid[0].shape[0]
        data.extend([
            go.Surface(x=grid[0], y=grid[1], z=mus_grid + n_sig*sig_grid, surfacecolor=colors, opacity=0.3, showscale=False),
            #go.Surface(x=grid[0], y=grid[1], z=mus_grid - n_sig*sig_grid, surfacecolor=colors, opacity=0.3, showscale=False),
        ])

    layout = go.Layout(
        title='3D surface',
        autosize=False,
        width=1000,
        height=750,
        margin=dict(l=0, r=0, b=0, t=0),
        scene=dict(
            xaxis=dict(title=x_name),
            yaxis=dict(title='f({})'.format(x_name)),
            zaxis=dict(title='{}R_l({},f({}))'.format('-' if flip_z else '', x_name, x_name)),
            aspectratio=dict(x=1, y=1, z=1)
        )
    )
    fig = go.Figure(data=data, layout=layout)
    ply.iplot(fig, show_link=False)

def surface_3D(x, y, z, tooltips=None, axes_names=('x','y','z')):
    '''plot a 3D surface using plotly

    Parameters should be of the form:
    ```
    X = np.arange(...)
    Y = np.arange(...)
    X, Y = np.meshgrid(X, Y)
    Z = f(X,Y)
    ```

    Args:
        tooltips: an array with the same length as the number of points,
            containing a string to display beside them
    '''
    data = [go.Surface(
        x=x, y=y, z=z,
        text=tooltips, colorscale='Viridis', opacity=1
    )]
    layout = go.Layout(
        title='3D surface',
        autosize=False,
        width=900,
        height=600,
        margin=dict(l=0, r=0, b=0, t=0),
        scene=dict(
            xaxis=dict(title=axes_names[0]),
            yaxis=dict(title=axes_names[1]),
            zaxis=dict(title=axes_names[2])
        )
    )
    fig = go.Figure(data=data, layout=layout)
    # show_link is a link to export to the 'plotly cloud'
    ply.iplot(fig, show_link=False)