import re
import datetime
import time
import os
import shutil
import codecs
import mmap
import json
import logging
import MySQL_interactions
import spider_toolkit
import zipfile
import spider_exceptions
import utilities

def unzip_archive(path, target, ignore_list, only_single_files=True):
    """Extract a zip archive from a supplied path to a supplied target, ignoring
    files in ignore_list. Extracts all nested archives, flattening structure.
    Defaults to only unpacking if there is only one non-ignored file in the archive.
    Removes archives once the contents have been extracted.
    Returns path to last unzipped file
    """
    final_path = ''
    with zipfile.ZipFile(path, 'r') as myzip:
        useful_files = [i for i in myzip.infolist() if i.filename not in ignore_list]
        if only_single_files and len(useful_files) > 1:
            logging.info("Too many files in zip archive - one required file expected")
            raise spider_exceptions.TooManyFilesInArchiveError
        for info in useful_files:
            myzip.extract(info, target)
            if info.filename.lower().endswith('.zip'):
                final_path = unzip_archive(os.path.join(target, info.filename), target, ignore_list)
            else:
                final_path = os.path.join(target, info.filename)
    os.remove(path)
    return final_path

def clean_singleton_unzip(path, target, ignore_list):
    """Extract a single file from a zip archive, doesn't unpack anything if there are
    multiple files in the zip. On error, cleans up any files or folders left over from intermediate
    unzip stages, leaving the originating directory as it was previously. NB: this is possible because
    the inner unzip calls itself recursively, thus doesn't remove any archives until the innermost
    one has been successfully extracted. Returns path to new file if successful, path to original file otherwise
    """
    parent_dir = os.path.dirname(path)
    start_list = os.listdir(parent_dir)
    try:
        return unzip_archive(path, target, ignore_list, True)
    except:
        for f in os.listdir(parent_dir):
            if f not in start_list:
                os.remove(os.path.join(parent_dir, f))
        return path

def clean_folder(sel_path):
    """Cleanly and completely clear out a folder with the path provided,
    removing all files and subdirectories
    """
    for root, dirs, files in os.walk(sel_path):
        for f in files:
            os.unlink(os.path.join(root, f))

        for d in dirs:
            shutil.rmtree(os.path.join(root, d))

def make_test_file_path(file_path):
    """Take a production file path and convert it to something analogous but local, for testing purposes"""
    paths_to_replace = [u'H:', u'W:', u'U:', u'\\nyccelerra.fitch.corp\sfna$', u'\\nyccelerra\sfna$', u'\\Frafp03',
                        u'nyccelerra.fitch.corp\sfna$', u'nyccelerra\sfna$']
    test_path_base = 'C:\\temp\\spiderTest\\Targets'
    for path in paths_to_replace:
        file_path = file_path.replace(path, test_path_base)
        if file_path.startswith('\\'):
            file_path = file_path[1:]
    return file_path

class FilePathDefiner():
    """Define all widely used file paths with reference to base paths loaded
    from the local config file
    """

    def __init__(self):
        config = utilities.load_local_config()
        self.local_folder_base = config['localstorage']
        self.local_download_folder_base = os.path.join(self.local_folder_base, 'DownloadStaging')
        self.tidied_deal_folder_base = os.path.join(self.local_folder_base, 'CompletedDealFolders')
        self.data_folder_base = config['datalocation']
        self.file_dump_base = os.path.join(self.data_folder_base, 'UnmappedDownloads')

class SpiderFileHandler(FilePathDefiner):

    def __init__(self, trusteeName):
        FilePathDefiner.__init__(self)
        self.trustee = trusteeName
        #initialised here but accessed through the property as may be modified during a spider run
        self.download_folder_increment = 1
        #check if there are multiple download folders from unfinished runs
        if os.path.exists(self.local_download_folder_base):
            folders = [x for x in os.listdir(self.local_download_folder_base) if x.startswith(trusteeName)]
            for f in folders:
                inc = f.split('_')
                self.download_folder_increment = int(inc[1]) if int(inc[1]) > self.download_folder_increment else self.download_folder_increment
        #create folder if it doesn't exist already
        if not os.path.exists(self.local_download_folder):
            os.makedirs(self.local_download_folder)
        #these are fine as attributes because they do not need to be modified at runtime
        self.download_log = os.path.join(self.data_folder_base, trusteeName + '_DownloadLog.txt')
        self.screenshot_folder = os.path.join(self.data_folder_base, 'Screenshots', trusteeName)
        if not os.path.exists(self.screenshot_folder):
            os.makedirs(self.screenshot_folder)
        self.completed_deal_download_folder = os.path.join(self.tidied_deal_folder_base, self.trustee)
        if not os.path.exists(self.completed_deal_download_folder):
            os.makedirs(self.completed_deal_download_folder)
        self.unmapped_file_dump = os.path.join(self.file_dump_base, self.trustee)
        if not os.path.exists(self.unmapped_file_dump):
            os.makedirs(self.unmapped_file_dump)

        self.orphaned_report_folder_base = os.path.join(self.file_dump_base, self.trustee)
        self.state_file_folder = os.path.join(self.data_folder_base, 'JSONStateSnapshots', self.trustee)
        if not os.path.exists(self.state_file_folder):
            os.makedirs(self.state_file_folder)
        self.state_file = self.state_file_folder + '\\currentstateinfo.txt'
        self.temp_dl_list = []

    #essentially a readonly property as only this class should handle runtime
    #modifications of the local download folder path
    @property
    def local_download_folder(self):
        """Return fully qualified local download folder created from base plus spider-specific extension"""
        trustee_folder = self.trustee + '_' + str(self.download_folder_increment)
        return os.path.join(self.local_download_folder_base, trustee_folder)

    def increment_download_folder(self):
        """In the case of a download failing, make a new download staging folder for future downloads.
        Also clear the temporary download list so further fileinfolists in bundles do not contain erroneous data.
        """
        self.download_folder_increment += 1
        os.mkdir(self.local_download_folder)
        self.temp_dl_list = []

    def clean_download_folder(self):
        """Attempts to completely clear out the local download folder in case of interrupted previous attempts.
           If unable to clear, creates new local download folder path for subsequent downloads"""
        try:
            clean_folder(self.local_download_folder)
        except:
            logging.warning("Possible partial download, unable to clear temp folder")
            self.increment_download_folder()

    def clean_alternate_download_folders(self):
        '''Attempts to remove any extra temp download folders created when partial downloads
           prevent move/delete during spider run'''
        tempDirs = os.listdir(self.local_download_folder_base)
        for item in tempDirs:
            try:
                #delete anything called '<Trustee>_<X>'
                if self.trustee in item:
                    os.rmdir(self.local_download_folder_base + '\\' + item)
                #reset the download folder increment
                self.download_folder_increment = 1
                os.mkdir(self.local_download_folder)
            except:
                logging.warning(r"Unable to delete extra download folder - please check C:\Spider\DownloadStaging and clear manually")
                #self.writeSpiderLog("","Warning","", r"Unable to delete extra download folder - please check C:\Temp and clear manually")

    """
    def make_download_target_list(self, dealMap, dealName, repDetails):
        #TODO: improve detail in 'failure to map' error logging so we can see ~why~ it failed
        #TODO: improve... this method -_- Or at least understand and comment the blighter so
        #future folks can understand it without screaming

        #horrible: ext check not too bad, but reportName one is checked again repeatedly in this routine
        if not repDetails.has_key("ext"):
            repDetails["ext"] = ""
        if not repDetails.has_key("reportName"):
            repDetails["reportName"] = "unnamed"

        #this can stay
        copy_list = [];

        for location in dealMap[dealName]['locations']:
            #ok, so here we are looking for a date to build a folder structure
            #only even looks for a date if the website has reported one
            if repDetails.has_key("date"):
                aMatch = re.search(location['dtRegex'] , repDetails["date"])
                #did we find a match? if we did, let's make it a proper date
                if aMatch:
                    c = time.strptime(aMatch.group(0),location['fmtFileIn'])
                    #we chould now do anything that definitely requires a date here, where we know we have a date for certain
                    monthDir = str(c.tm_mon)
                    yearDir = time.strftime('%Y',c)
                    if location['fmtFolderMonth'] == "Q%q": #quarters not supported by time lib
                        monthDir = "Q"+ str(int((c.tm_mon+2)/3))
                    else:
                        monthDir = time.strftime(location['fmtFolderMonth'], c)
                else:
                    #no match but report thinks it has a date! eek
                    monthDir = 'DateMatchFailed'
                    yearDir = 'DateMatchFailed'
            else:
                #report doesn't even have a clue about dates
                monthDir = 'NoDate'
                yearDir = 'NoDate'

            #so that gives us a folder with some sort of year/month structure if at all possible
            folder = os.path.join(os.pardir, 'Data', location['folder'], yearDir, monthDir)

            #now off to get a filename:
            foundType = ""
            typeInDB = ""
            #OZ TWEAK members of reportMappings are now dicts so can ref by named keys instead of indeces
            for mapping in location['reportMappings']:
                if repDetails.has_key("url"):
                    urlMatch = re.search(mapping['linkURLRegex'], repDetails["url"])
                else:
                    urlMatch = False
                if repDetails.has_key("reportName"):
                    textMatch = re.search(mapping['linkTextRegex'], repDetails["reportName"])
                else:
                    textMatch = False
                #DANGER: why is the next bit 'both' when they surely mean 'either'? The success message 'and/or' implies an 'or'
                #ah no, success message is just trying to cover all bases and db field should be called 'requiredmatch' or something
                #hang on, this is a dict. I can call the keys whatever I want. Do I have to nearly match the db, really?
                if ((mapping['requiredMatch'] == 'Both') and urlMatch and textMatch) or \
                    ((mapping['requiredMatch'] == 'URL') and urlMatch) or \
                    ((mapping['requiredMatch'] == 'Text') and textMatch): # urlMatch replaces not(urlMatch is None) to avoid empty strings being truthy
                    #what is going on in this next bit? seems to be 'if no ext from web OR no ext from DB OR we have both and they match'
                    #so... only if we have both and they don't match, it falls through?
                    #does 'not mapping['extension'] work if ext is null?
                    if (repDetails["ext"] == '') or (not mapping['extension']) or (repDetails["ext"].lower() == mapping['extension'].lower()):
                        if mapping['linkTextRegex'] and repDetails.has_key("reportName"): #dont sub "" and have a target
                            foundType = re.sub(mapping['linkTextRegex'], mapping['reportOutput'], repDetails["reportName"])
                        else:
                            foundType = repDetails["reportName"]
                        extension = repDetails["ext"]
                        typeInDB = mapping['reportOutput']
                        if mapping['extension']:
                            extension = mapping['extension']
                            #OZ NOTE: if there is no extension found at all, this still passes. Is that right?
                        # Only take first match - we shouldn't map to multiple names for the same
                        break
            if foundType <> "":
                #foundType = re.sub(r"[\\|\/|\:|\*|\?|\"|\<|\>|\|]", "", foundType) # windows reserved chars
                foundType = sanitised_string(foundType)
                #NB: mapping at this point references the last value it took before the break
                logging.info('Matched on URL: ' + mapping['linkURLRegex'] + ' and/or Text: ' + mapping['linkTextRegex'])
                local_name = location['fmtToken']
                local_name = local_name.replace('?a',dealMap[dealName]['abbr'])
                local_name = local_name.replace('?t',foundType)
                local_name = local_name.replace('?d',time.strftime(location['fmtFileOut'],c))
                local_name = local_name.replace('?e',extension)
                copy_list.append({"localName": local_name, "checkPath":folder, "reportType":typeInDB})
        if len(copy_list) == 0:
            downloadDetails = ""
            repType = "unmatched"
            #DODGY: this next will never fail but could just be 'unnamed'
            if repDetails.has_key("reportName"):
                repType = repDetails["reportName"]
                downloadDetails = downloadDetails +"(RPT: "+ repDetails["reportName"] +")"
            if repDetails.has_key("url"):
                if repType == "unmatched": #unreachable
                    repType = repDetails["url"]
                downloadDetails = downloadDetails +"(URL: "+ repDetails["url"] +")"
            #DODGY: again, this has been set above so key certainly present but could just be ""
            if repDetails.has_key("ext"):
                downloadDetails = downloadDetails +"(EXT: "+ repDetails["ext"] +")"
            #deal_id = dealMap[dealName]['dealId']
            #DateLogFMT = ""
            #if repDetails.has_key("date"):
                #DateLogFMT = repDetails["date"]
            #self.writeSpiderLog(self._dealData[deal]['busGrp'],"Warning", deal, "Could not map filename", source_location=downloadDetails, deal_id=deal_id)
            logging.info('Could not map filename')
            for opt in repDetails:
                logging.info(opt +'\t'+ repDetails[opt])
        return copy_list

    def old_make_download_target_list(self, dealMap, dealName, repDetails):
        #here for reference as I start to hack it apart OZ 20151026
        if not repDetails.has_key("ext"):
            repDetails["ext"] = ""
        if not repDetails.has_key("reportName"):
            repDetails["reportName"] = "unnamed"
        copy_list = [];
        for location in dealMap[dealName]['locations']:
            c = "DateDotParsed"
            if repDetails.has_key("date"):
                aMatch = re.search(location['dtRegex'] , repDetails["date"])
                if aMatch:
                    c = time.strptime(aMatch.group(0),location['fmtFileIn'])
            #DANGER: the rest of the date stuff depends on the previous 'if' executing or the whole thing fails
            monthDir = str(c.tm_mon)
            if location['fmtFolderMonth'] == "Q%q": #quarters not supported by time lib
                monthDir = "Q"+ str(int((c.tm_mon+2)/3))
            else:
                monthDir = time.strftime(location['fmtFolderMonth'],c)
            folder = os.path.join(os.pardir, 'Data',location['folder'], time.strftime('%Y',c), monthDir)
            foundType = ""
            typeInDB = ""
            #OZ TWEAK members of reportMappings are now dicts so can ref by readable keys instead of indeces
            for mapping in location['reportMappings']:
                if repDetails.has_key("url"):
                    urlMatch = re.search(mapping['linkURLRegex'], repDetails["url"])
                else:
                    urlMatch = False
                if repDetails.has_key("reportName"):
                    textMatch = re.search(mapping['linkTextRegex'], repDetails["reportName"])
                else:
                    textMatch = False
                #DANGER: why is the next bit 'both' when they surely mean 'either'? The success message 'and/or' implies an 'or'
                if ((mapping['useRegex'] == 'Both') and urlMatch and textMatch) or \
                    ((mapping['useRegex'] == 'URL') and urlMatch) or \
                    ((mapping['useRegex'] == 'Text') and textMatch): # urlMatch replaces not(urlMatch is None) to avoid empty strings being truthy
                    if (repDetails["ext"] == '') or (not mapping['extension']) or (repDetails["ext"].lower() == mapping['extension'].lower()):
                        if mapping['linkTextURL'] and repDetails.has_key("reportName"): #dont sub "" and have a target
                            foundType = re.sub(mapping['linkTextURL'], mapping['reportOutput'], repDetails["reportName"])
                        else:
                            foundType = repDetails["reportName"]
                        extension = repDetails["ext"]
                        typeInDB = mapping['reportOutput']
                        if mapping['extension']:
                            extension = mapping['extension']
                        # Only take first match - we shouldn't map to multiple names for the same
                        break
            if foundType <> "":
                foundType = re.sub(r"[\\|\/|\:|\*|\?|\"|\<|\>|\|]", "", foundType) # windows reserved chars
                #WTF: surely mapping var is out of scope now?! How is this reliable?!
                logging.info('Matched on URL: ' + mapping[1] + ' and/or Text: ' + mapping[2])
                local_name = location['fmtToken']
                local_name = local_name.replace('?a',dealMap[dealName]['abbr'])
                local_name = local_name.replace('?t',foundType)
                local_name = local_name.replace('?d',time.strftime(location['fmtFileOut'],c))
                local_name = local_name.replace('?e',extension)
                copy_list.append({"localName": local_name, "checkPath":folder, "reportType":typeInDB})
        if len(copy_list) == 0:
            downloadDetails = ""
            repType = "unmatched"
            #DODGY: this next will never fail but could just be 'unnamed'
            if repDetails.has_key("reportName"):
                repType = repDetails["reportName"]
                downloadDetails = downloadDetails +"(RPT: "+ repDetails["reportName"] +")"
            if repDetails.has_key("url"):
                if repType == "unmatched":
                    repType = repDetails["url"]
                downloadDetails = downloadDetails +"(URL: "+ repDetails["url"] +")"
            #DODGY: again, this has been set above so key certainly present but could just be ""
            if repDetails.has_key("ext"):
                downloadDetails = downloadDetails +"(EXT: "+ repDetails["ext"] +")"
            #deal_id = dealMap[dealName]['dealId']
            #DateLogFMT = ""
            #if repDetails.has_key("date"):
                #DateLogFMT = repDetails["date"]
            #self.writeSpiderLog(self._dealData[deal]['busGrp'],"Warning", deal, "Could not map filename", source_location=downloadDetails, deal_id=deal_id)
            logging.info('Could not map filename')
            for opt in repDetails:
                logging.info(opt +'\t'+ repDetails[opt])
        return copy_list
    """
    def report_in_download_log(self, textToCheck):
        """Compare supplied report text to saved download log, returning True if text present"""
        #very first thing - is there even a download log at all? I really hope so, but just in case:
        if os.path.isfile(self.download_log) == False:
            self.write_download_log("Created at local time " + str(datetime.datetime.now()))

        try:
            with codecs.open(self.download_log, 'r', 'utf-8') as log:
                checker = mmap.mmap(log.fileno(), 0, access=mmap.ACCESS_READ)
                #return checker.find(textToCheck) != -1

                #ok you forced me to make this case-insensitive for STUPID REASONS
                #leave the faster (more than 5x faster in fact) version here for when all the spiders are replaced
                #and I can do a one-time replace to nuke all the uppercase in the files
                cleaned = textToCheck.replace('(', r'\(').replace(')', r'\)').replace('?',r'\?').replace('|', r'\|')
                #non-regex won't need the sanitising crud either
                return re.search(cleaned, checker, re.IGNORECASE) is not None
        except IOError:
            logging.error("Problem reading download log")
            raise

    def write_download_log(self, loginfo):
        """Write supplied logging information (usually report identifier) to the text
        download log file so future runs can check if this file has already been
        downloaded
        """
        try:
            with codecs.open(self.download_log, 'a', 'utf-8') as log:
                log.write('\n' + loginfo.strip() + '\r\n')
        except IOError:
            logging.error('Problem writing to download log file')
            raise

    def note_latest_downloaded_file(self, file_dict):
        """Get name of most recent file in dl directory and add it to a supplied dict"""
        #don't bother with any clever order-files-by-timestamp stuff, it's too flakey
        logging.info('Attempting to get latest download name')
        #give it a few tries with an increasing wait just in case
        ignorelist = self.temp_dl_list + ['currentstateinfo.txt']
        for i in range(1,6):
            files = os.listdir(self.local_download_folder)
            for f in files:
                #check for a new, non-partial file
                if f not in ignorelist and f.lower().endswith('.part') is False:
                    self.temp_dl_list.append(f)
                    file_dict["DownloadedName"] = f
                    logging.info('Latest file: ' + f)
                    #found one - add it to the list and let calling func continue
                    return
                else:
                    #nothing new found, wait a while and have another go
                    time.sleep((i/2))
        else:
            self.temp_dl_list.append('No new file found')
            logging.warning('No new file detected')

    def stage_latest_download(self, filesList, logCheckName, targetList, deal_name, deal_map, db_interaction, rep_date=''):
        """Write download log and file target list for the latest downloaded file. Date is for DB log and if supplied
        should be a string in '%Y-%m-%d' format.
        """
        #build dict of file target info etc. to map to download order in ordered list for later file handling
        file_dict = {'NamePreDL':logCheckName, 'FileTargetList':targetList}
        self.note_latest_downloaded_file(file_dict)
        #20160215 - trying out a new version moving this to the bundling routine so incomplete deal bundles
        #get completely re-downloaded
        #self.write_download_log(logCheckName)
        filesList.append(file_dict)
        for target in targetList:
            #write database download log here - while not 'correct' in that the file may not actually be in the specified
            #location yet, it's how the old spiders did it and we have to match their behaviour to maintain compatibility
            #(at least for the moment)
            if target['localName'] == '' and file_dict.has_key("DownloadedName"):
                target['localName'] = file_dict["DownloadedName"]

            db_interaction.write_db_log(self.trustee, deal_map[deal_name]['busGrp'],
                                        'File downloaded', deal_name, filename=target['localName'],
                                        directory=target['checkPath'], dealID=deal_map[deal_name]['dealId'],
                                        repDate=rep_date, repType=target['reportType'], use_ntp=True)

    def package_deal_downloads(self, dealName, dealID, filesList, expectedCount, check_name_list):
        """Make a nice human-readable-named folder to put the downloaded files in, and move them there with the
        ordered list of useful info for a later process to use in order to rename and move to final resting places
        """
        sanitised_dealname = utilities.sanitised_string(dealName, True, True)
        dealFolder = os.path.join(self.completed_deal_download_folder, (str(dealID) + ' ' + sanitised_dealname + ' ' + time.strftime('%Y%m%d')))
        if not os.path.exists(dealFolder):
            os.makedirs(dealFolder)
        #check for .part files, empty files, and whether the count matches the number expected
        filenames = os.listdir(self.local_download_folder)
        for f in filenames:
            if f.lower().endswith('.part') or os.path.getsize(os.path.join(self.local_download_folder, f)) == 0:
                #we have a situation - don't move these ones, make a note and move on
                filesList.append({'Incomplete':f})
                continue
            elif os.path.isfile(os.path.join(dealFolder, f)):
                #who knows how this happened, but if the file is already there we can ignore and delete
                logging.warning('File already exists in staging destination!')
                os.remove(os.path.join(self.local_download_folder, f))
            else:
                os.rename(os.path.join(self.local_download_folder, f), os.path.join(dealFolder, f))
        #all done? write the check_names to the download log
        for check in check_name_list:
            self.write_download_log(check)
        #compare the counts
        successfiles = os.listdir(dealFolder)
        #try and make a note of the download order, which should correspond to the file details supplied
        filesList.append({'DownloadOrder':self.temp_dl_list})
        filesList.append({'DownloadsRequested':expectedCount, 'SuccessfulDownloads':len(successfiles)})
        #chuck the useful info in after the whole bang-shoot
        with open(dealFolder + '\\FileInfoList.txt', 'w') as t:
            json.dump(filesList, t)
        #any fragments left? Don't care. Do the cleanup/increment so everything is peachy for the next deal
        self.clean_download_folder()
        self.temp_dl_list = []

    def handle_deal_file_bundle(self, dealDownloadFolderPath, delete_original=True, local_test=False):
        """Take a folder of files downloaded for a deal and move them to folders specified
        in an attached info list
        """
        #NOTE: called from polling process in different thread to Spider instances
        #if this bit fails we are stuffed and would prefer to bomb out completely
        fullpath = os.path.join(dealDownloadFolderPath,'FileInfoList.txt')
        with open(fullpath, 'r') as fileinfo:
            fileinfolist = json.load(fileinfo)

        #slightly annoying to have to iterate to extract the download order list, but such is life
        for info in fileinfolist:
            if 'DownloadOrder' in info:
                filedownloadorder = info['DownloadOrder']
                break

        #go through non-fileinfolist files and move according to supplied targets
        dealfiles = os.listdir(dealDownloadFolderPath)
        for df in [x for x in dealfiles if x != 'FileInfoList.txt']:
            if df in filedownloadorder:
                downloadindex = filedownloadorder.index(df)
                fileinfodict = fileinfolist[downloadindex]
                self.handle_downloaded_file(os.path.join(dealDownloadFolderPath, df), fileinfodict, delete_original, local_test)
            else:
                continue

        #At the end, if there are any files left we can try to put them in a folder for manual handling
        remaining = os.listdir(dealDownloadFolderPath)
        if len(remaining)>1:
            #more than one file left? copy all that's left of the bundle to a dump folder for safety pls
            pathparts = dealDownloadFolderPath.split("\\")
            deal_dirname = pathparts[-1]
            trustee = pathparts[-2]
            target_dump = os.path.join(self.unmapped_file_dump, deal_dirname)
            shutil.copytree(dealDownloadFolderPath, target_dump)
            logging.warning("Download copy failure - moved to dump folder")
            #let's also shove a warning in the database if necessary
            if delete_original:
                dbint = MySQL_interactions.MySQLInteraction(True)
                dbint.write_db_log(trustee, "", "Warning", deal_dirname.split(' ',1)[1],
                                   directory="Trapdoor: could not move files, bundle moved to dump",
                                   dealID=deal_dirname.split(' ')[0], use_ntp=True)

        logging.info('Deal bundle file handling complete')
        #then attempt to remove the parent deal folder once it's copied if necessary
        if delete_original:
            clean_folder(dealDownloadFolderPath)
            os.rmdir(dealDownloadFolderPath)

    def handle_downloaded_file(self, fullDownloadFilePath, fileInfoDict, delete_orig=True, local_test=False):
        """Rename (if required) and move a downloaded file within a download bundle
        to its target folder according to supplied dict values
        """
        targetlist = fileInfoDict['FileTargetList']
        overrideable_delete_orig = delete_orig

        #quick check: if there's no targets, just throw it in the heap and move on
        if len(targetlist) == 0:
            self.handle_unmapped_file(fullDownloadFilePath, delete_orig)
            logging.warning("Download with no target detected: " + fullDownloadFilePath + " moved to dump folder")
            return

        for target in targetlist:
            #NEW 20160224 - targets may come in with blank localName if they both had no mapping AND missed the
            #latest-download checker in stage_latest_download. Not sure if this is even possible?
            #no actually, I'm pretty sure it isn't because if they miss latestdownload then they aren't in
            #the downloadlist that is being iterated on to call this routine. Thus if localname blank here,
            #we are in trouble and do need to bomb out pls

            #there's a routine in the existing system that compares the actual file extension to
            #what the db expected it to be, and alters the latter to match to former if necessary
            #I think this is worth doing - only a tiny bit better
            db_extension = target['localName'].split('.')[-1]
            actual_extension = fullDownloadFilePath.split('.')[-1]
            if actual_extension.lower() == 'zip' and db_extension.lower() != actual_extension.lower():
                #we have a zip and we weren't expecting it. Let's have a quick go at unzipping:
                fullDownloadFilePath = clean_singleton_unzip(fullDownloadFilePath, os.path.dirname(fullDownloadFilePath), ['MadDisclaimer.pdf', 'Mad_Disclaimer.pdf'])
                #unzip to same place, and ignore the USBank-specific disclaimer file
                #(add db-to-fileinfolist ignorelist injection if we ever require this for other trustees!)
                #so now we either have the single desired unzipped file or a multi-file zip
            #need to re-check extension matching now
            actual_extension = fullDownloadFilePath.split('.')[-1]
            if db_extension != actual_extension:
                target['localName'] = target['localName'].replace(db_extension, actual_extension)
            #are we in test mode? change the target to a local one pls
            if local_test == True:
                target['checkPath'] = make_test_file_path(target['checkPath'])
            #now we're into bits that can fail on broken permissions, network glitches or incorrect paths
            try:
                fullTargetPath = os.path.join(target['checkPath'], target['localName'])
                #see if the required directory exists and if not, make it
                if not os.path.exists(target['checkPath']):
                    os.makedirs(target['checkPath'])
                #if the file isn't already there, copy it across to the target path
                if not os.path.exists(fullTargetPath):
                    shutil.copy(fullDownloadFilePath, fullTargetPath)
                    logging.info(target['localName'] + ' File copied to ' + fullTargetPath)
                else:
                    logging.warning("File already exists in target location: " + fullTargetPath)
                    continue
            except:
                #I know, too general - but dammit it really could be anything
                overrideable_delete_orig = False
                logging.warning("Error copying file to: " + fullTargetPath)
                dbint = MySQL_interactions.MySQLInteraction()
                dbint.write_db_log(self.trustee, '',
                                        'Warning', 'File copy failed', filename=target['localName'],
                                        directory=target['checkPath'], use_ntp=True)

        #and if all successful, remove the old one.
        if overrideable_delete_orig:
            os.remove(fullDownloadFilePath)

    def handle_unmapped_file(self, download_file_path, delete_orig):
        """If a report has somehow been bundled without a matching report mapping, move or copy it
        into a special folder for manual handling. Returns the final path of the moved/copied file.
        """
        #get useful info from the full download path - reliable as this was built by package_deal_downloads above
        dl_path_parts = download_file_path.replace("\\", "/").split("/")
        trustee, deal_name, file_name = dl_path_parts[-3], dl_path_parts[-2], dl_path_parts[-1]
        orphan_target = os.path.join(self.orphaned_report_folder_base, deal_name)
        if not os.path.exists(orphan_target):
            os.makedirs(orphan_target)
        full_target = os.path.join(orphan_target, file_name)
        shutil.copy2(download_file_path, full_target)
        dbint = MySQL_interactions.MySQLInteraction(True)
        dbint.write_db_log(trustee, "", "Warning", deal_name.split(' ',1)[1],
                           directory="Trapdoor: could not map filename, file moved to dump",
                           dealID=deal_name.split(' ')[0], use_ntp=True)
        if delete_orig:
            os.remove(download_file_path)
        return full_target

    def initialise_logger(self):
        """Simple call to spider_toolkit logging initialiser, passing in the name
        of the trustee, the target data folder, and setting outputs to file and tkinter
        """
        spider_toolkit.init_logger(self.trustee, True, False, True, self.data_folder_base, 'w')
        #spider_toolkit.init_logger(self.trustee, True, False, True, 'C:\\temp\\spiderTest', 'w')

    def write_state_to_disk(self, statedict):
        """Write dict containing useful spider state info (e.g. current deal, last report clicked)
        to a human-readable file on disk
        """
        with open(self.state_file, 'w') as t:
            json.dump(statedict, t)

    def read_state_from_disk(self):
        """Read saved spider state file back into dict for spider to use
        Return empty dict if no state file found
        """
        statedict = {}
        if os.path.isfile(self.state_file):
            with open(self.state_file, 'r') as f:
                statedict = json.load(f)
        return statedict

    def clear_saved_state(self):
        if os.path.isfile(self.state_file):
            os.remove(self.state_file)

class DownloadTargetBuilder():
    """Method object building the list of target locations to which a downloaded file
    should be copied, as per the Locations/Report Mappings in the database
    """

    def __init__(self, dealMap, dealName, repDetails):
        self.deal_map = dealMap
        self.deal_name = dealName
        self.report_details = repDetails

        self.matched_date = 'unknown_date'
        self.month_dir = 'DateMatchFailed'
        self.year_dir = 'DateMatchFailed'
        self.rep_type_name = ''
        self.type_name_in_db = ''

        #set some safe vals here to make subsequent tests and substitutions easier
        self.safe_ext = 'unknown' if not self.report_details.has_key('ext') else self.report_details['ext']
        self.safe_name = 'unnamed' if not self.report_details.has_key('reportName') else self.report_details['reportName']

    def constructed_report_file_name(self, location, mapping):
        """Build target file name from db results and matched text"""
        repTypeName = utilities.sanitised_string(self.rep_type_name)
        #NB: mapping at this point references the last value it took before the break
        logging.info('Matched on URL: ' + mapping['linkURLRegex'] + ' and/or Text: ' + mapping['linkTextRegex'])
        constructed_name = location['fmtToken']
        constructed_name = constructed_name.replace('?a', self.deal_map[self.deal_name]['abbr'])
        constructed_name = constructed_name.replace('?t', repTypeName)
        date_string = self.matched_date if self.matched_date == 'unknown_date' else time.strftime(location['fmtFileOut'], self.matched_date)
        constructed_name = constructed_name.replace('?d', date_string)
        constructed_name = constructed_name.replace('?e', self.safe_ext)
        return constructed_name

    def report_date_match(self, location):
        """Get any date search matches using regex strings from db to match against values from website"""
        if self.report_details.has_key("date"): #with date regex from db, check for match first in repDetails['date']
            dateMatch = re.search(location['dtRegex'], self.report_details["date"])
        elif self.report_details.has_key("url"): #and failing that in repDetails['URL']
            dateMatch = re.search(location['dtRegex'], self.report_details["url"])
        else:
            logging.warning('Date match failed in make target folder name') #if they both fail or aren't present, log it and move on

        return dateMatch

    def set_date_folder_names(self, location):
        """Set names of year and month/quarter folders for target location"""
        self.year_dir = time.strftime('%Y', self.matched_date)
        if location['fmtFolderMonth'] == "Q%q": #quarters not supported by time lib
            self.month_dir = "Q" + str(int((self.matched_date.tm_mon + 2) / 3))
        else:
            self.month_dir = time.strftime(location['fmtFolderMonth'], self.matched_date)

    def mapping_match_found(self, mapping):
        """Check for match in report details to url or link text regex from db"""
        #use db regex strings to search URL and/or reportname from repDetails as appropriate
        urlMatch = re.search(mapping['linkURLRegex'], self.report_details["url"]) if self.report_details.has_key("url") else False
        textMatch = re.search(mapping['linkTextRegex'], self.report_details["reportName"]) if self.report_details.has_key("reportName") else False

        #lots of stuff to test here so sod efficient, I want readable - let's break out the conditions:
        bothMatched = (mapping['requiredMatch'] == 'Both') and urlMatch and textMatch
        urlMatched = (mapping['requiredMatch'] == 'URL') and urlMatch
        textMatched = (mapping['requiredMatch'] == 'Text') and textMatch

        match_found = bothMatched or urlMatched or textMatched
        return match_found

    def log_unmatched_file_details(self):
        """Make any notes possible about a downloaded file which doesn't match any db vals
        Save these notes to log file and database log
        """
        downloadDetails = ""
        repType = "unmatched"
        if self.report_details.has_key("reportName"):
            repType = self.report_details["reportName"]
            downloadDetails = downloadDetails + "(RPT: " + self.report_details["reportName"] + ")"
        if self.report_details.has_key("url"):
            if repType == "unmatched":
                repType = self.report_details["url"]
            downloadDetails = downloadDetails + "(URL: " + self.report_details["url"] + ")"
        if self.report_details.has_key("ext"):
            downloadDetails = downloadDetails + "(EXT: " + self.report_details["ext"] + ")"
        #deal_id = self.deal_map[self.deal_name]['dealId']
        #self.writeSpiderLog(self.trustee, self._dealData[self.deal_name]['busGrp'],"Warning", self.deal_name, "Could not map filename", source_location=downloadDetails, deal_id=deal_id)
        logging.info('Could not map filename')
        logging.info('Report details for unmatched download: ' + str(self.report_details))

    def make_report_type_name(self, mapping):
        """A name for the given report: from mapping text regex if present, supplied
        report details if not
        """
        if mapping['linkTextRegex']:
            repTypeName = re.sub(mapping['linkTextRegex'], mapping['reportOutput'], self.safe_name)
        else:
            if self.safe_name == "unnamed":
                #link text useless, url not parsed by calling spider page into a useful name
                repTypeName = mapping['reportOutput']
            else:
                repTypeName = self.safe_name
        return repTypeName

    def make_download_target_list(self):
        """Make list of target filenames and folders to which a downloaded file will be copied"""
        target_list = []

        #go through all target file locations as found in database
        for location in self.deal_map[self.deal_name]['locations']:
            #reset type name so we can use it to check that a mapping has been found for this location
            self.rep_type_name = ''
            dateMatch = self.report_date_match(location)
            if dateMatch:
                #we should now do anything that definitely requires a date here, where we know we have a date for certain
                self.matched_date = time.strptime(dateMatch.group(0),location['fmtFileIn'])
                self.set_date_folder_names(location)

            #construct target folder name with db location folder + year/month structure
            folder = os.path.join(os.pardir, 'Data', location['folder'], self.year_dir, self.month_dir)

            #filenames: iterate over the reportmappings for the current location
            for mapping in location['reportMappings']:
                if not (self.mapping_match_found(mapping)): #no match, try the next mapping
                    continue
                #got initial match, now check that either we only have one version of the file extension or that they match
                if not ((self.safe_ext == 'unknown') or (not mapping.has_key('extension')) or (self.safe_ext.lower() == mapping['extension'].lower())):
                    logging.warning('Inconsistent file extensions')
                    continue
                #so we have a match and either a consistent or unknown file extension - carry on
                self.rep_type_name = self.make_report_type_name(mapping)
                self.safe_ext = self.safe_ext if not mapping.has_key('extension') else mapping['extension']
                self.type_name_in_db = mapping['reportOutput']
                break #and dive out now we have a match - only grab first match (as per current production code)

            #if something matched, we have a report type name (even if it is 'unnamed')
            #build a constructed_name and add to copy_list with the folder name and a 'type' from the db
            if self.rep_type_name != '':
                constructed_name = self.constructed_report_file_name(location, mapping)
                target_list.append({"localName": constructed_name, "checkPath":folder, "reportType":self.type_name_in_db})

        #iff target_list is empty, log a load of stuff about it not matching anything
        if len(target_list) == 0:
            self.log_unmatched_file_details()
            #we now have to add a blank target for each location
            for location in self.deal_map[self.deal_name]['locations']:
                if location['folder'] != r'H:\03. ESF - CMBS\09. Workflow\Special Notices':
                    unmapped_folder = os.path.join(os.pardir, 'Data', location['folder'], self.year_dir, self.month_dir, 'Unmapped')
                    target_list.append({"localName": "", "checkPath":unmapped_folder, "reportType":""})

        return target_list

class file_download_started():

    def __init__(self, downloadHandler, prevCount):
        self.download_handler = downloadHandler
        self.prev_count = prevCount

    def __call__(self, driver):
        """Returns True if there is one more file than previously in the specified download location"""
        return len(os.listdir(self.download_handler.local_download_folder)) > self.prev_count

class file_download_finished():

    def __init__(self, downloadHandler):
        self.download_handler = downloadHandler

    def __call__(self, driver):
        """Returns False if there is a .PART or empty file in the specified download location"""
        for f in os.listdir(self.download_handler.local_download_folder):
            if f.lower().endswith('.part'):
                return False
            elif os.path.getsize(self.download_handler.local_download_folder + '\\' + f) == 0:
                return False
        return True

class ReportDefinition():
    """Data structure storing metadata about reports in cleaned formats.
    Provides the USBank-specific 'name-pre-download' which matches old records
    in the download log.
    """

    #TODO: not sure if this is too USBank specific, keep an eye on it
    def __init__(self, trusteeDealName, rawName = '', rawDate = '', fileExt = '', portfolioGroup = '', reportGroup = '', url = ''):
        """A collection of attributes used to create a target list for file downloads.
        'rawDate', if used, should contain date in format to match regex in database Report Mapping Set
        'url' can also contain the Report Mapping Set date regex and is used to search for URL regex match as per Report Mapping Data
        'rawName' is used to search for Text regex match as per Report Mapping Data
        """
        self.trustee_deal_name = trusteeDealName
        self.raw_name = rawName
        self.raw_date = rawDate
        self.file_extension = fileExt
        #set up dictionary of optional report params to use for download target list
        self.report_opts = {"namePreDL": self.name_pre_download.replace(u"\u2026","").decode('unicode_escape').encode('ascii','ignore')}
        if rawName != '':
            self.report_opts['reportName'] = rawName.replace(u"\u2026","").decode('unicode_escape').encode('ascii','ignore')
        if rawDate != '':
            self.report_opts['date'] = rawDate
        if fileExt != '':
            self.report_opts['ext'] = '.' + fileExt.replace(u"\u2026","").decode('unicode_escape').encode('ascii','ignore')
        if portfolioGroup != '':
            self.report_opts['tabGroup'] = portfolioGroup.replace(u"\u2026","").decode('unicode_escape').encode('ascii','ignore')
        if reportGroup != '':
            self.report_opts['reportTab'] = reportGroup.replace(u"\u2026","").decode('unicode_escape').encode('ascii','ignore')
        if url != '':
            self.report_opts['url'] = url.replace(u"\u2026","").decode('unicode_escape').encode('ascii','ignore')

    @property
    def name_pre_download(self):
        constructed_name = (self.trustee_deal_name.lower().replace(' ', '-').replace('.', '') +
                            "-" + self.raw_name.lower().replace(' ', '-').replace('.', '').replace('---', ' - ') +
                            "-" + self.raw_date.replace("/", '-') +"."+ self.file_extension.lower())
        return constructed_name


from time import time
import get_traj_data

import ann

import numpy as np
import matplotlib.pyplot as plt

import pickle

import subprocess
import os

all_data = get_traj_data.concatenate(0, 50)

P_min = np.min(all_data["P"])
P_max = np.max(all_data["P"])

T_min = np.min(all_data["T"])
T_max = np.max(all_data["T"])

def test_single_batch(Y, c, i, theta=None, tau=0.0001):
    t = time()

    d = 4
    K = 5
    h = 1
    it_max = 4000000
    tol = 1e-4

    (F, dF, Js, theta) = ann.train_ANN_and_make_model_function(
        Y, c, d, K, h,
        it_max, tol, tau=tau, padding_mode="zeros", log="log.txt", theta=theta,
        y_min=P_min, y_max=P_max, c_min=T_min, c_max=T_max)

    print(f"took {time() - t:.2f} seconds to train")

    plt.plot(Js)
    plt.yscale("log")
    plt.savefig(f"{i}.png")
    plt.clf()
    plt.cla()

    print(f"J = {Js[-1]}")

    return theta

foldername = "T_trained_4x5_new"
# foldername = "T_trained_4x5_bad"

def startup():
    subprocess.call(f"mkdir results/{foldername}", shell=True)
    batch_data = get_traj_data.generate_data(batch=0)
    os.chdir(f"results/{foldername}")
    Y = batch_data["P"][:, :1000]
    c = batch_data["T"][:1000]
    theta = test_single_batch(Y, c, 0, tau=0.0003)

    with open("theta.pickle", "wb") as file:
        pickle.dump(theta, file)

    os.chdir("../..")

def resume():
    os.chdir(f"results/{foldername}")
    with open("theta.pickle", "rb") as file:
        theta = pickle.load(file)

    for i in range(0, 25):
        os.chdir("../..")
        batch_data = get_traj_data.generate_data(batch=i)
        os.chdir(f"results/{foldername}")
        Y = batch_data["P"]
        c = batch_data["T"]
        theta = test_single_batch(Y, c, i + 1, theta=theta, tau=0.0001)

    with open("theta.pickle", "wb") as file:
        pickle.dump(theta, file)

    os.chdir("../..")

def resume_with_big_data(lo, hi, plot_index=0):
    os.chdir(f"results/{foldername}")
    with open("theta.pickle", "rb") as file:
        theta = pickle.load(file)

    os.chdir("../..")
    batch_data = get_traj_data.concatenate(lo, hi)
    os.chdir(f"results/{foldername}")

    Y = batch_data["P"]
    c = batch_data["T"]

    theta = test_single_batch(Y,
                              c,
                              plot_index,
                              theta=theta,
                              tau=0.00002 / (hi - lo))

    with open("theta.pickle", "wb") as file:
        pickle.dump(theta, file)

    os.chdir("../..")

if __name__ == "__main__":
    # startup()
    # resume()
    # for i in range(19, 20):
    #     resume_with_big_data(i, i + 2, i)
    # resume_with_big_data(0, 20, 21)
    # resume_with_big_data(10, 30, 22)
    # resume_with_big_data(20, 40, 23)
    resume_with_big_data(0, 40, 24)


# coding:utf-8

import cv2
import numpy as np

# 读取图片
img = cv2.imread("test.jpg")

# 通过NumPy模块，创建新的空数组，图像使用NumPy数组的属性来表示图像的尺寸和通道信息
emptyImage = np.zeros(img.shape, np.uint8)

# 复制图片
emptyImage2 = img.copy()

# 通过cvtColor获得图像副本
emptyImage3=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

# 将图片转为黑白图片
#emptyImage3[...]=0

# 显示图片
cv2.imshow("EmptyImage", emptyImage)
cv2.imshow("Image", img)
cv2.imshow("EmptyImage2", emptyImage2)
cv2.imshow("EmptyImage3", emptyImage3)

# 保存图像
"""
# 第一个参数是保存的路径及文件名，第二个是图像矩阵。其中，imwrite()有个可选的第三个参数，如下：
# 第三个参数针对特定的格式： 对于JPEG，其表示的是图像的质量，用0-100的整数表示，默认为95。 注意，cv2.IMWRITE_JPEG_QUALITY类型为Long，必须转换成int。
#                         对于PNG，第三个参数表示的是压缩级别。cv2.IMWRITE_PNG_COMPRESSION，从0到9,压缩级别越高，图像尺寸越小。默认级别为3：
"""
cv2.imwrite("test2.jpg", img, [int(cv2.IMWRITE_JPEG_QUALITY), 5])
cv2.imwrite("test3.jpg", img, [int(cv2.IMWRITE_JPEG_QUALITY), 100])
cv2.imwrite("test.png", img, [int(cv2.IMWRITE_PNG_COMPRESSION), 0])
cv2.imwrite("test2.png", img, [int(cv2.IMWRITE_PNG_COMPRESSION), 9])

cv2.waitKey(0)
cv2.destroyAllWindows()


import numpy as np
import gym

from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import Adam

from trickster.agent import PPO
from trickster.rollout import Rolling, Trajectory, RolloutConfig
from trickster.experience import Experience
from trickster.utility import visual

env = gym.make("CartPole-v1")
input_shape = env.observation_space.shape
num_actions = env.action_space.n

actor = Sequential([Dense(16, activation="relu", input_shape=input_shape),
                    Dense(16, activation="relu"),
                    Dense(num_actions, activation="softmax")])
actor.compile(loss="categorical_crossentropy", optimizer=Adam(1e-4))

critic = Sequential([Dense(16, activation="relu", input_shape=input_shape),
                     Dense(16, activation="relu"),
                     Dense(1, activation="linear")])
critic.compile(loss="mse", optimizer=Adam(1e-5))

agent = PPO(actor,
            critic,
            action_space=2,
            memory=Experience(max_length=10000),
            reward_discount_factor_gamma=0.99,
            entropy_penalty_coef=0.005)

rollout = Rolling(agent, env, config=RolloutConfig(max_steps=300))
test_rollout = Trajectory(agent, gym.make("CartPole-v1"))

rewards = []
actor_loss = []
actor_utility = []
actor_kld = []
actor_entropy = []
advantages = []
critic_loss = []

for episode in range(1, 2001):

    rollout.roll(steps=64, verbose=0, push_experience=True)
    agent_history = agent.fit(batch_size=32, verbose=0, reset_memory=True)

    actor_loss.append(np.mean(agent_history["actor_loss"]))
    actor_utility.append(np.mean(agent_history["actor_utility"]))
    actor_kld.append(np.mean(agent_history["actor_kld"]))
    actor_entropy.append(np.mean(agent_history["actor_entropy"]))
    advantages.append(np.mean(agent_history["advantage"]))
    critic_loss.append(np.mean(agent_history["critic_loss"]))

    test_history = test_rollout.rollout(verbose=0, push_experience=False, render=False)
    rewards.append(test_history["reward_sum"])

    print("\rEpisode {:>4} RWD {:>5.2f} A {:>5.2f} ACTR {:>7.4f} UTIL {:>7.4f} KKL {:>7.4f} ENTR {:>7.4f} CRIT {:>7.4f}"
          .format(
            episode,
            np.mean(rewards[-10:]),
            np.mean(advantages[-10:]),
            np.mean(actor_loss[-10:]),
            np.mean(actor_utility[-10:]),
            np.mean(actor_kld[-10:])*1000,
            np.mean(actor_entropy[-10:]),
            np.mean(critic_loss[-10:])), end="")
    if episode % 10 == 0:
        print()

visual.plot_vectors([rewards, actor_loss, actor_utility, actor_kld, actor_entropy, critic_loss],
                    ["Reward", "Actor Loss", "Actor Utility", "Actor KLD", "Actor Entropy", "Critic Loss"],
                    smoothing_window_size=10)


import numpy as np
import keras as ks
from Balerion import balerion
from keras.models import Sequential
from keras.layers import Dense
import matplotlib.pyplot as plt
#from scipy.ndimage.filters import gaussian_filter1d
from scipy.interpolate import spline
from docplex.mp.model import Model
from time import time

def createInstances( jobs, machines):
    sequences = []
    length = np.random.choice(machines, replace=True, size=jobs)
    n = 0
    for i in range(len(length)):
        if length[i] <= machines/2:
            length[i] = machines
        n = n + length[i]
    for i in range(jobs):
        array = np.random.choice(machines, replace=False, size=length[i])
        comp = np.random.randint(low = 1, high = jobs*10, size = machines)
        sequence = []
        for s, t in zip(array, comp):
            add = t
            if add < 1:
                add = 1
            sequence.append((s, add))
        sequences.append(sequence)
    print("JOBS: ", jobs)
    print("MACHINES: ", machines)
    for i in range(jobs):
        print("Job N°", i, " : ", sequences[i])
    return sequences, n

def createModel( layers, instance_dimension ):
    model = Sequential()

    tup = layers[0]
    model.add(Dense(tup[0], input_dim=instance_dimension, activation=tup[2],  kernel_initializer=tup[1]))
    for i in range(1, len(layers)):
        tup = layers[i]
        model.add(Dense( tup[0], activation=tup[2],  kernel_initializer=tup[1]))
    #optimizer =  ks.optimizers.SGD(lr=0.1, decay=1e-6, momentum=0.1, nesterov=True)
    optimizer = ks.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)
    # w = model.get_weights()
    # w[0][0,1] = 10
    # model.set_weights(w)
    # outputTensor = model.output
    # listOfVariableTensors = model.trainable_weights
    # gradients = k.gradients(outputTensor, listOfVariableTensors)
    # print("gradients", gradients)
    # trainingExample = np.random.random((1, 8))
    # sess = tf.InteractiveSession()
    # sess.run(tf.initialize_all_variables())
    #

    #print("INPUT", model.input)

    #evaluated_gradients = sess.run(gradients, feed_dict={model.input: })
    #print("EVALUATED GRADIENTS",evaluated_gradients)
    #sess.close()

    #print("weigth",model.get_weights())
    return model, optimizer

def cplexmodel(jobs, machines, sequences):
    cplexmodel = Model(name = 'simple_job_shop')

    t = []

    for i in range(jobs):
        name = "t" + str(i)+"_"
        tj = cplexmodel.continuous_var_list(
            range(machines), lb = 0, name = name)
        t.append(tj)

    z = []
    for j in range(jobs):
        zj = []
        for k in range(jobs):
            name = "z"+str(j)+"_"+str(k)
            zjk = cplexmodel.binary_var_list(
                range(machines), name = name
            )
            zj.append(zjk)
        z.append(zj)
    m = cplexmodel.continuous_var(name = 'makespan')

    bigM = 0
    for seq in sequences:
        for s in seq:
            bigM = bigM + s[1]
    bigM = bigM*2
    #####CONSTRAINTS####
    for j in range(jobs):
        tj = t[j]
        seq = sequences[j]
        for i in range(1, len(seq)):
            ohj = seq[i][0]
            ohm1j = seq[i-1][0]
            pohm1j = seq[i-1][1]
            xohj = tj[ohj]
            xohm1j = tj[ohm1j]
            con = cplexmodel.add_constraint( xohj >= xohm1j + pohm1j)
            #print("CIAO",cplexmodel.pprint_as_string(con))
            if i == len(seq) -1:
                pohj = seq[i][1]
                cplexmodel.add_constraint( m >= xohj + pohj)

    for j in range(jobs):
        seqj = sequences[j]
        for k in range(jobs):
            if j != k:
                seqk = sequences[k]
                mach = machmerge(seqj, seqk)
                for machine in mach:
                    xij = t[j][machine]
                    #print(xij)
                    xik = t[k][machine]
                    #print(xik)
                    pij = 0
                    pik = 0
                    for s in seqj:
                        if s[0] == machine:
                            pij = s[1]
                            break
                    for s in seqk:
                        if s[0] == machine:
                            pik = s[1]
                            break

                    cplexmodel.add_constraint(xij >= xik + pik - bigM * z[j][k][machine])
                    cplexmodel.add_constraint(xik >= xij + pij - bigM + bigM * z[j][k][machine])

    #####Objective
    cplexmodel.minimize( m )
    #####Solution
    start = time()
    #print(cplexmodel.export_as_lp_string())

    solution = cplexmodel.solve(log_output = True)

    return (time() - start), solution, cplexmodel.objective_value

def machmerge( seqj, seqk):
    lista = []
    listb = []
    ##lista.append( seqj[i][0] for i in range(len(seqj)))
    for s in seqj:
        lista.append(s[0])
    lista = np.array(lista)
    #listb.append( seqk[i][0] for i in range(len(seqk)))
    for s in seqk:
        listb.append(s[0])
    listb = np.array(listb)
    intersection = np.intersect1d(lista, listb)
    #print("lista", lista,"\nlistb",listb,"\ninter", intersection)
    return intersection

def mainbalerion():
    seed = 300#70
    np.random.seed(seed)

    ############################################create instances###
    jobs = 2
    machines = 2
    sequences, n = createInstances(jobs = jobs, machines = machines)

    state_dimension = n + machines
    instance_dimension = n + machines + n + machines + machines
                     #jobs + machines + nextjobs + nextmachines + action
    print("State Dimension: ", state_dimension, "  Instance dimension: ", instance_dimension)

    ###############################################create model###
    innerlayers = [
        (instance_dimension*10+machines, 'uniform', 'relu'),
        #(2*state_dimension+machines, 'uniform', 'relu'),
        #(instance_dimension, 'uniform', 'relu'),
        (machines, 'uniform', 'sigmoid')
    ]

    outerlayers = [
        (instance_dimension+machines, 'uniform', 'relu'),
        (2*state_dimension, 'uniform', 'relu'),
        # (instance_dimension, 'uniform', 'relu'),
        (1, 'uniform', 'linear')
    ]
    #model, optimizer = createModel(  layers, instance_dimension ) #fi you need to check particular settings
                                                                 # for the optimizer please go deeper into the method

    #####################################Reinforcement Learning###

    min_experience = 1
    max_experience =  2
    min_gamma = 4
    max_gamma = 5

    M = 30

    record = []

    for experience in range(min_experience, max_experience):
        for gamma in range(min_gamma, max_gamma):

            seed = gamma*experience

            outermodel, outeroptimizer = createModel(outerlayers, instance_dimension+machines)  # fi you need to check particular settings

            innermodel, inneroptimizer = createModel(innerlayers, state_dimension+machines)  # fi you need to check particular settings

            # for the optimizer please go deeper into the method
            gamma = gamma/10
            trainer = balerion(
                innermodel,
                inneroptimizer,
                outermodel,
                outeroptimizer,
                jobs,
                machines,
                sequences,
                path=None,
                M=M,
                T=jobs * machines,
                peps=0.9,
                delta=0.75,
                gamma=gamma,
                experience=experience,
                size=10000000000,
                randomized=False,
                finalreward=200*2,#1e1,
                batch_size=5,
                verbose=True,
                npseed=seed)
            episode, reward, rewards, solution, elapsedtime, timings = trainer.solve()
            makespan = 0
            for timing in timings:
                makespan = makespan + timing
            print("Experience", experience," Gamma",gamma, " Best reward", reward," MAKESPAN", makespan, " Elapsed time", elapsedtime)

            record.append( ( experience, gamma, episode, reward, rewards, solution, elapsedtime, timings ) )
                            #   0          1       2       3        4         5          6          7

    ###show resuts###

    fig = plt.figure(1, figsize = (10,4.8))
    chart = fig.add_subplot(111)
    for i in record:
        name = str(i[0]) + "-"+ str(i[1])
        #y = gaussian_filter1d(i[4], sigma=1)
        x = np.array(range(len(i[4])))
        x_smooth = np.linspace(x.min(), x.max(), M)
        y_smooth = spline(x, np.array(i[4]), x_smooth)
        chart.plot( x_smooth, y_smooth, label = name)
    #chart.legend()
    plt.show()

    # episode, reward, rewards, solution, elapsedtime, timings = trainer.play()
    # makespan = 0
    # for timing in timings:
    #     makespan = makespan + timing
    # print("PLAY Experience", experience, " Gamma", gamma, " Best reward", reward, " MAKESPAN", makespan,
    #       " Elapsed time",
    #       elapsedtime)

    elaptime, sol, obj =cplexmodel(jobs, machines, sequences)

    print("CPLEX_SOL  Elapsed time", elaptime, " MAKESPAN", obj)

    return

mainbalerion()


import os
import torch.utils.data
import numpy as np
from PIL import Image
import json

from ssd.structures.container import Container

def get_valid_person(json_path):
    """Get the valid person object, which needs to satisfy the following conditions.
    *: The label of the object is pedestrian.
    *: The height of pedestrian is more than 50 pixel.
    *: The ridio of pedestrain is less than 0.35 (not sure)
    Args:
        json_path: the path of json file that will be decoded

    Retuens:
        valid_person: a dictionary and the keys of this dict is 'valid' and 'bbox'
        the value of 'valid' is a bool that valid == True means we get a valid person data
        the value of 'bbox_list' is a list that contains y_min, x_min, y_max, x_max
    """
    valid_person = dict()
    bbox_list = list()
    label_list = list()
    person_label = {'pedestrian': 1, 'rider': 2}
    with open(json_path, 'r') as label_json:
        annot = json.load(label_json)
    valid_index = 0
    for obj in annot['objects']:
        if obj['label'] == 'pedestrian':
            x, y, w, h = obj['bbox']
            if h > 60:
                x_vis, y_vis, w_vis, h_vis = obj['bboxVis']
                ratio = 1 - (w_vis * h_vis) / (w * h)
                if ratio < 0.35:
                    valid_index += 1
                    bbox = [y-1, x-1, y-1+h, x-1+w]
                    bbox_list.append(bbox)
                    label_list.append(person_label[obj['label']])
    if valid_index > 0:
        valid = True
    else:
        valid = False
    valid_person['valid'] = valid
    valid_person['bbox_list'] = bbox_list
    valid_person['label_list'] = label_list
    return valid_person

def get_data_list(img_path, json_path, split='train'):
    """Get the images and bbox labels lists, which contain people.
    Args:
        img_path: the path of images that contain people, actually the path is '/cityscape/leftImg8bit'
        label_path: the path of bbox labels that contain people, actually the path is '/cityscape/gtBboxCityPersons'

    Returns:
        valid image list and label list
    """

    img_list, json_list = list(), list()
    if split not in ['train', 'val']:
        return print("You should check the parameter of split!")
    else:
        i_dir = os.path.join(img_path, split)
        j_dir = os.path.join(json_path, split)
        city_list = sorted(os.listdir(j_dir))
        for city in city_list:
            i_city_dir = os.path.join(i_dir, city)
            j_city_dir = os.path.join(j_dir, city)
            js_list = sorted(os.listdir(j_city_dir))
            for js in js_list:
                label_json_path = os.path.join(j_city_dir, js)
                if get_valid_person(label_json_path)['valid'] == True:
                    json_name = label_json_path
                    img_name = os.path.join(i_city_dir, js).replace('gtBboxCityPersons.json', 'leftImg8bit.png')
                    img_list.append(img_name)
                    json_list.append(json_name)

        return img_list, json_list

class CitypersonDataset(torch.utils.data.Dataset):
    class_names = ('__background__',
                   'pedestrian')
    data_dir = '/home/szy/data/cityscape'
    #img_path = '/home/szy/data/cityscape/leftImg8bit'
    #json_path = '/home/szy/data/cityscape/gtBboxCityPersons'

    def __init__(self, data_dir, split, transform=None, target_transform=None, keep_difficult=False):
        self.data_dir = data_dir
        self.split = split
        self.transform = transform
        self.target_transform = target_transform
        self.keep_difficult = keep_difficult
        self.img_path = os.path.join(data_dir, 'leftImg8bit')
        self.json_path = os.path.join(data_dir, 'gtBboxCityPersons')
        self.img_list, self.json_list = get_data_list(self.img_path, self.json_path, split=split)

        self.class_dict = {class_name: i for i, class_name in enumerate(self.class_names)}

    def __getitem__(self, index):
        boxes, labels, is_difficult = self._get_annotation(index)
        if not self.keep_difficult:
            boxes = boxes[is_difficult == 0]
            labels = labels[is_difficult == 0]
        image = self._read_image(index)
        if self.transform:
            image, boxes, labels = self.transform(image, boxes, labels)
        if self.target_transform:
            boxes, labels = self.target_transform(boxes, labels)
        target = Container(
            boxes=boxes,
            labels=labels,
        )
        return image, target, index

    def __len__(self):
        return len(self.img_list)

    def _get_annotation(self, image_id):
        annotation_file = self.json_list[image_id]
        bbox_list = get_valid_person(annotation_file)['bbox_list']
        label_list = get_valid_person(annotation_file)['label_list']
        boxes = np.array(bbox_list, dtype=np.float32)
        labels = np.array(label_list, dtype=np.int64)
        is_difficult = np.zeros(labels.size, dtype=np.int8)
        return boxes, labels, is_difficult

    def _read_image(self, image_id):
        image_file = self.img_list[image_id]
        image = Image.open(image_file).convert("RGB")
        image = np.array(image)
        return image

if __name__ == '__main__':
    img_path = '/home/szy/data/cityscape/leftImg8bit'
    json_path = '/home/szy/data/cityscape/gtBboxCityPersons'
    img_list, json_list = get_data_list(img_path, json_path)
    print(img_list)
    data_dir = '/home/szy/data/cityscape'
    cityperson = CitypersonDataset(data_dir, split='train')
    image, target, index = cityperson.__getitem__(0)
    print('done')


import tensorflow as tf
import numpy as np
import pandas as pd
import pymysql
#import os
import sys

from sys import argv

script, usuario_id = argv

usuario_id = int(usuario_id)
#usuario_id = 1
#print (usuario_id)

#mysql_connection = pymysql.connect(host='localhost',
 #                   user='root',
  #                  password='',
   #                 db='libros',
    #                charset='utf8',
     #               cursorclass= pymysql sudo apt-get install python3-pymysql.cursors.DictCursor)

mysql_connection = pymysql.connect(host='localhost',
                    user='root',
                    password='',
                    db='libros',
                    charset='utf8')

cursor = mysql_connection.cursor()

sql1 = "SELECT * FROM `bibliografia`"
sql2 = "SELECT * FROM `user_libros`"
libros_df = pd.read_sql(sql1, mysql_connection)
ratings_df = pd.read_sql(sql2, mysql_connection)

#print (libros_df.head(5))

#ratings_df = pd.read_csv(r'data/busquedas.csv', sep=',', header=None, engine='python')
#print (ratings_df.head(5))

libros_df.columns = ['bibliografiaID', 'Title', 'Autor1','Autor2','Volumen']
#ratings_df.columns = ['UserID', 'bibliografiaID', 'Rating', 'Timestamp']
ratings_df.columns = ['ID','UserID','bibliografiaID','busquedaID', 'Rating', 'Creado','Modificado']
#ratings_df = ratings_df[np.isfinite(ratings_df['bibliografiaID'])]
#ratings_df['bibliografiaID'] = ratings_df.bibliografiaID.astype(int)
#Dropping unecessary columns
ratings_df = ratings_df.drop(['ID','busquedaID','Creado','Modificado'], axis=1)
#print (ratings_df.head(5))

user_rating_df = ratings_df.pivot(index='UserID', columns='bibliografiaID', values='Rating')
#print (user_rating_df.head())

norm_user_rating_df = user_rating_df.fillna(0) / 5.0
trX = norm_user_rating_df.values

#print (trX[0:5])

hiddenUnits = 20
visibleUnits =  len(user_rating_df.columns)
vb = tf.placeholder("float", [visibleUnits]) #Cantidad de libros unicos
hb = tf.placeholder("float", [hiddenUnits]) #Numero de caracteristicas que va aprender
W = tf.placeholder("float", [visibleUnits, hiddenUnits])

#Fase 1: Input Processing, Forward Pass

v0 = tf.placeholder("float", [None, visibleUnits])
_h0 = tf.nn.sigmoid(tf.matmul(v0, W) + hb)
h0 = tf.nn.relu(tf.sign(_h0 - tf.random_uniform(tf.shape(_h0))))

#Fase 2: Reconstruction

_v1 = tf.nn.sigmoid(tf.matmul(h0, tf.transpose(W)) + vb)
v1 = tf.nn.relu(tf.sign(_v1 - tf.random_uniform(tf.shape(_v1))))
h1 = tf.nn.sigmoid(tf.matmul(v1, W) + hb)

#Tasa de aprendizaje
alpha = 1.0
#Crear gradientes
w_pos_grad = tf.matmul(tf.transpose(v0), h0)
w_neg_grad = tf.matmul(tf.transpose(v1), h1)

#Calcula la Divergencia Contrastiva para maximizar
CD = (w_pos_grad - w_neg_grad) / tf.to_float(tf.shape(v0)[0])

#Crear métodos para actualizar los pesos y bias

update_w = W + alpha * CD
update_vb = vb + alpha * tf.reduce_mean(v0 - v1, 0)
update_hb = hb + alpha * tf.reduce_mean(h0 - h1, 0)

err = v0 - v1
err_sum = tf.reduce_mean(err * err)

#Current weight
cur_w = np.zeros([visibleUnits, hiddenUnits], np.float32)
#Current visible unit biases
cur_vb = np.zeros([visibleUnits], np.float32)
#Current hidden unit biases
cur_hb = np.zeros([hiddenUnits], np.float32)
#Previous weight
prv_w = np.zeros([visibleUnits, hiddenUnits], np.float32)
#Previous visible unit biases
prv_vb = np.zeros([visibleUnits], np.float32)
#Previous hidden unit biases
prv_hb = np.zeros([hiddenUnits], np.float32)
sess = tf.Session()
sess.run(tf.global_variables_initializer())

epochs = 20
batchsize = 100
errors = []
for i in range(epochs):
    for start, end in zip( range(0, len(trX), batchsize), range(batchsize, len(trX), batchsize)):
        batch = trX[start:end]
        cur_w = sess.run(update_w, feed_dict={v0: batch, W: prv_w, vb: prv_vb, hb: prv_hb})
        cur_vb = sess.run(update_vb, feed_dict={v0: batch, W: prv_w, vb: prv_vb, hb: prv_hb})
        cur_nb = sess.run(update_hb, feed_dict={v0: batch, W: prv_w, vb: prv_vb, hb: prv_hb})
        prv_w = cur_w
        prv_vb = cur_vb
        prv_hb = cur_hb
    errors.append(sess.run(err_sum, feed_dict={v0: trX, W: cur_w, vb: cur_vb, hb: cur_hb}))

   # print (errors[-1])

#Seleccionando el usuario de entrada
mock_user_id = usuario_id

inputUser = trX[mock_user_id-1].reshape(1, -1)
#print (inputUser[0:5])

#Alimentar el usuario de entrada y reconstruir la entrada

hh0 = tf.nn.sigmoid(tf.matmul(v0, W) + hb)
vv1 = tf.nn.sigmoid(tf.matmul(hh0, tf.transpose(W)) + vb)
feed = sess.run(hh0, feed_dict={ v0: inputUser, W: prv_w, hb: prv_hb})
rec = sess.run(vv1, feed_dict={ hh0: feed, W: prv_w, vb: prv_vb})

#print(rec)

#Se recomiendan 10 libros a nuestro usuario de prueba clasificándolos por sus puntajes proporcionados por nuestro modelo.
scored_libros_df_mock = libros_df[libros_df['bibliografiaID'].isin(user_rating_df.columns)]
scored_libros_df_mock = scored_libros_df_mock.assign(RecommendationScore = rec[0])
#print (scored_libros_df_mock.sort_values(["RecommendationScore"], ascending=False).head(20))

libros_df_mock = ratings_df[ratings_df['UserID'] == mock_user_id]

#combinar libros_df con ratings_df por bibliografiaID
merged_df_mock = scored_libros_df_mock.merge(libros_df_mock, on='bibliografiaID', how='outer')

#borrar columnas innecesarias
merged_df_mock = merged_df_mock.drop('Title', axis=1).drop(['UserID','Autor1','Autor2','Volumen',], axis=1)

rec = merged_df_mock[merged_df_mock["Rating"].isnull()].sort_values(["RecommendationScore"], ascending=False).head(10)
rec = rec.drop('Rating',axis=1)
print (rec)
#print (merged_df_mock.sort_values(["RecommendationScore"], ascending=False).head(20))


#! /usr/bin/python
# -*- coding: utf-8 -*-

import sys
import os

sys.path.append( os.path.join(sys.path[0], 'system/networks') )
sys.path.append( os.path.join(sys.path[0], 'system/devices') )

from deviceConstants import *
import deviceObject

from networkConstants import *

from modbusConstants import *
# import modbusConstants

import copy
from collections import OrderedDict

import logsystem
log = logsystem.getLogger()

deviceType = "LAE AT1-5BS2U-BG"
deviceTypeName = "LAE AT1-5BS2U-BG"
executionType = deviceNetworkExecution
executionTypeName = deviceNetworkExecutionText

valueDescriptions = OrderedDict()

# OUTPUTS - Dedicated Register value
valueDescriptions["T1 TEMP"] = {"displayName": "T1-Air Probe", "dataType": dataTypeFloat, "unitType": unitTypeTemperature, "valueType": valueTypeOutput, "significantDigits": 1, "defaultLog":"true"}
valueDescriptions["T2 TEMP"] = {"displayName": "T2-Evap Probe", "dataType": dataTypeFloat, "unitType": unitTypeTemperature, "valueType": valueTypeOutput, "significantDigits": 1, "defaultLog":"true"}

# OUTPUT - Address 100 Status Flag bits
valueDescriptions["ALARM"] = {"displayName": "General Alarm", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "On", "false": "Off"} }
valueDescriptions["MUTE"] = {"displayName": "Alarm Mute", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "On", "false": "Off"} }
valueDescriptions["DEFROST"] = {"displayName": "Defrost Status", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "On", "false": "Off"}, "defaultLog":"true" }
valueDescriptions["DOOR"] = {"displayName": "Door Status", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "Open", "false": "Closed"}, "defaultLog":"true" }

# OUTPUT - Address 101 Alarm Flag bits
valueDescriptions["T1FAILUREALARM"] = {"displayName": "T1 Probe FAILUREALARM", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "On", "false": "Off"} }
valueDescriptions["T2FAILUREALARM"] = {"displayName": "T2 Probe FAILUREALARM", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "On", "false": "Off"} }
valueDescriptions["HIGHTEMPALARM"] = {"displayName": "High Temp Alarm", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "On", "false": "Off"} }
valueDescriptions["LOWTEMPALARM"] = {"displayName": "Low Temp Alarm", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "On", "false": "Off"} }
valueDescriptions["CONDCLEANALARM"] = {"displayName": "Dirty Condenser", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "On", "false": "Off"} }
valueDescriptions["DOOROPENALARM"] = {"displayName": "Door Open Alarm", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "On", "false": "Off"} }

# OUTPUT - Address 102 Output Flag bits
valueDescriptions["COMP RLY"] = {"displayName": "Compressor", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "On", "false": "Off"}, "defaultLog":"true"}
valueDescriptions["AUX RLY"] = {"displayName": "AUX", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "On", "false": "Off"}, "defaultLog":"true"}

# Virtual (function to load value)
# valueDescriptions["MDEF"] = {"dataType":dataTypeBool,"dataList":{"true":"On", "false": "Off"},"valueType":valueTypeConfig,"displayName":"MDEF","description":"Turn On Manual Defrost"}
valueDescriptions["STANDBY"] = {"displayName": "Standby Status", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeOutput, "dataList": {"true": "Standby", "false": "No"} }

# CONFIG
valueDescriptions["SCL"] = {"displayName": "SCL", "description": "Readout scale", "dataType": dataTypeList, "valueType": valueTypeConfig, "dataList": {"0": "1°C", "1": "2°C", "2": "°F"},"default":"0"}
valueDescriptions["SPL"] = {"displayName": "SPL", "description": "Minimum setpoint", "dataType": dataTypeFloat, "valueType": valueTypeConfig, "unitType": unitTypeTemperature, "significantDigits": 1,"default":"-25.0"}
valueDescriptions["SPH"] = {"displayName": "SPH", "description": "Maximum setpoint", "dataType": dataTypeFloat, "valueType": valueTypeConfig, "unitType": unitTypeTemperature, "significantDigits": 1,"default":"25.0"}
valueDescriptions["SP"] = {"displayName": "SP", "description": "Setpoint", "dataType": dataTypeFloat, "valueType": valueTypeConfig, "unitType": unitTypeTemperature, "significantDigits": 1,"default":"0.0"}
valueDescriptions["CHHEATCOOL"] = {"displayName": "C-H", "description": "Refrigerating or heating mode", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeConfig, "dataList": {"true": "Heating", "false": "Cooling"},"default":"true"}
valueDescriptions["HYS"] = {"displayName": "HYS", "description": "Hysteresis", "dataType": dataTypeFloat, "valueType": valueTypeConfig, "unitType": unitTypeDeltaTemperature, "significantDigits": 1,"default":"3.0"}
valueDescriptions["CRT"] = {"displayName": "CRT", "description": "Compressor rest time", "dataType": dataTypeInt, "valueType": valueTypeConfig, "unitText": "minutes","default":"1"}
valueDescriptions["CT1"] = {"displayName": "CT1", "description": "Compressor run with sensor T1 failure", "dataType": dataTypeInt, "valueType": valueTypeConfig, "unitText": "minutes","default":"3"}
valueDescriptions["CT2"] = {"displayName": "CT2", "description": "Compressor stop with T1 failure", "dataType": dataTypeInt, "valueType": valueTypeConfig, "unitText": "minutes","default":"6"}
valueDescriptions["CSD"] = {"displayName": "CSD", "description": "Compressor stop delay after door open", "dataType": dataTypeInt, "valueType": valueTypeConfig, "unitText": "minutes","default":"1"}
valueDescriptions["DFR"] = {"displayName": "DFR", "description": "Defrost frequency", "dataType": dataTypeInt, "valueType": valueTypeConfig, "unitText": "2.5 minute increments","default":"0"}
valueDescriptions["DLI"] = {"displayName": "DLI", "description": "Defrost end temperature", "dataType": dataTypeFloat, "valueType": valueTypeConfig, "unitType": unitTypeTemperature, "significantDigits": 1,"default":"10.0"}
valueDescriptions["DTO"] = {"displayName": "DTO", "description": "Maximum defrost duration", "dataType": dataTypeInt, "valueType": valueTypeConfig, "unitText": "minutes","default":"30"}
valueDescriptions["DTY"] = {"displayName": "DTY", "description": "Defrost type", "dataType": dataTypeList, "valueType": valueTypeConfig, "dataList": {"14": "OFF", "15": "ELE", "16": "GAS"},"default":"14"}
valueDescriptions["DDY"] = {"displayName": "DDY", "description": "Display controls during defrost", "dataType": dataTypeInt, "valueType": valueTypeConfig, "unitText": "minutes","default":"0"}
valueDescriptions["ATM"] = {"displayName": "ATM", "description": "Alarm threshold management", "dataType": dataTypeList, "valueType": valueTypeConfig, "dataList": {"11": "NON", "12": "ABS", "13": "REL"},"default":"11"}
valueDescriptions["ALA"] = {"displayName": "ALA", "description": "Low temperature alarm", "dataType": dataTypeFloat, "valueType": valueTypeConfig, "unitType": unitTypeTemperature, "significantDigits": 1,"default":"-50.0"}
valueDescriptions["AHA"] = {"displayName": "AHA", "description": "High temperature alarm", "dataType": dataTypeFloat, "valueType": valueTypeConfig, "unitType": unitTypeTemperature, "significantDigits": 1,"default":"120.0"}
valueDescriptions["ALR"] = {"displayName": "ALR", "description": "Low alarm differential", "dataType": dataTypeFloat, "valueType": valueTypeConfig, "unitType": unitTypeDeltaTemperature, "significantDigits": 1,"default":"0.0"}
valueDescriptions["AHR"] = {"displayName": "AHR", "description": "High alarm differential", "dataType": dataTypeFloat, "valueType": valueTypeConfig, "unitType": unitTypeDeltaTemperature, "significantDigits": 1,"default":"0.0"}
valueDescriptions["ATD"] = {"displayName": "ATD", "description": "Temperature alarm delay", "dataType": dataTypeInt, "valueType": valueTypeConfig, "unitText": "minutes","default":"60"}
valueDescriptions["ADO"] = {"displayName": "ADO", "description": "Door alarm delay", "dataType": dataTypeInt, "valueType": valueTypeConfig, "unitText": "minutes","default":"5"}
valueDescriptions["ACC"] = {"displayName": "ACC", "description": "Condenser cleaning", "dataType": dataTypeInt, "valueType": valueTypeConfig, "unitText": "weeks","default":"0"}
valueDescriptions["STANDBYBUTTONENABLE"] = {"displayName": "SB","description":"Standby button enable", "dataType": dataTypeBool, "valueType": valueTypeConfig, "dataList": {"true": "Yes", "false": "No"},"default":"true"}
valueDescriptions["DSDOORENABLE"] = {"displayName": "DS", "description": "Door Enable", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeConfig, "dataList": {"true": "Yes", "false": "No"},"default":"false"}
valueDescriptions["OAU"] = {"displayName": "OAU", "description": "Auxiliary output control mode", "dataType": dataTypeList, "valueType": valueTypeConfig, "dataList": {"17": "NON", "18": "O-I", "19": "DEF", "20": "LGT", "21": "ALR"},"default":"17"}
valueDescriptions["INP"] = {"displayName": "INP", "description": "Probe type selection", "dataType": dataTypeList, "valueType": valueTypeConfig, "dataList": {"3": "SN4", "4": "ST1"},"default":"3"}
valueDescriptions["OS1"] = {"displayName": "OS1", "description": "Probe T1 offset", "dataType": dataTypeFloat, "valueType": valueTypeConfig, "unitType": unitTypeDeltaTemperature, "significantDigits":1,"default":"0.0"}
valueDescriptions["ENABLET2"] = {"displayName": "T2", "description": "Probe T2 Enable", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeConfig, "dataList": {"true": "Yes", "false": "No"},"default":"false"}
valueDescriptions["OS2"] = {"displayName": "OS2", "description": "Probe T2 offset", "dataType": dataTypeFloat, "valueType": valueTypeConfig, "unitType": unitTypeDeltaTemperature, "significantDigits": 1,"default":"0.0"}
valueDescriptions["TLD"] = {"displayName": "TLD", "description": "Delay for min/max temperature storage", "dataType": dataTypeInt, "valueType": valueTypeConfig, "unitText": "minutes","default":"5"}
valueDescriptions["SIM"] = {"displayName": "SIM", "description": "Display slowdown", "dataType": dataTypeInt, "valueType": valueTypeConfig,"default":"3"}

# Cfg_1 Address 229
valueDescriptions["LOCKKEYBOARD"] = {"displayName": "LOC", "description": "Lock Keyboard", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeConfig, "dataList": {"true": "Yes", "false": "No"}}
valueDescriptions["STBY"] = {"displayName": "STBY", "description": "Standby", "dataType": dataTypeBool,"unitType":"OnOff", "valueType": valueTypeConfig, "dataList": {"true": "Standby", "false": "No"}}

# ALARMS
alarmDescriptions = OrderedDict()
alarmDescriptions["T1FAILUREALARM"] = {"description": "T1 Probe Failure" }
alarmDescriptions["T2FAILUREALARM"] = {"description": "T2 Probe Failure" }
alarmDescriptions["HIGHTEMPALARM"] = {"description": "High Temp" }
alarmDescriptions["LOWTEMPALARM"] = {"description": "Low Temp" }
alarmDescriptions["CONDCLEANALARM"] = {"description": "Dirty Condenser" }
alarmDescriptions["DOOROPENALARM"] = {"description": "Door Open Alarm" }

class Device(deviceObject.NetworkDeviceObject):
  def __init__(self, deviceManager, name, description, network, networkAddress, image, method = ""):
    deviceObject.NetworkDeviceObject.__init__(self, deviceManager, name, description, network, networkAddress, deviceType, deviceTypeName, image)

    self._valueDescriptions = valueDescriptions
    self._alarmDescriptions = alarmDescriptions

    self.loadValuesFromDatabase()
    self.loadAdvisoriesFromDatabase()

    self.configModbusList = [
                      ["1", "SCL", 200],
                      ["1", "SPL", 201],
                      ["1", "SPH", 202],
                      ["2", "SP",  203],
                      ["2", "HYS", 204],
                      ["2", "CRT", 205],
                      ["3", "CT1", 206],
                      ["3", "CT2", 207],
                      ["3", "CSD", 208],
                      ["4", "DFR", 209],
                      ["4", "DLI", 210],
                      ["4", "DTO", 211],
                      ["5", "DTY", 212],
                      ["5", "DDY", 213],
                      ["5", "ATM", 214],
                      ["6", "ALA", 215],
                      ["6", "AHA", 216],
                      ["6", "ALR", 217],
                      ["7", "AHR", 218],
                      ["7", "ATD", 219],
                      ["7", "ADO", 220],
                      ["8", "ACC", 221],
                      ["8", "OAU", 222],
                      ["8", "INP", 223],
                      ["9", "OS1", 224],
                      ["9", "OS2", 225],
                      ["9", "TLD", 226],
                      ["10", "SIM", 227],
                      ["11", "CFG_1", 229]
               #       ["12","CMD_1",100]
    ]

  def _prepareSetDeviceConfigurationTransactions(self):
    if self._newDeviceConfigurationValues is None:
      return None

    retval = []
    networkTransTag = ""
    networkTrans = None

    for configItem in self.configModbusList:
      # [0] = MessageGroup/Tag; [1] = actual parameter, [2] = register address
      if networkTransTag != configItem[0]:
        networkTransTag = configItem[0]
        networkTrans = NetworkTransaction("WriteConfig" + networkTransTag)
        retval.append(networkTrans)

      key = configItem[1]

   #   if len(self._newDeviceConfigurationValues) == 1:
   #     value = 0
   #     value = value | 0x04 if self._newDeviceConfigurationValues["MDEF"] else value
   #     networkTrans.transactions.append(NetworkMessage(writeHoldingRegister(100, 0x04), "MDEF"))
   #     break

   #   if key == "CMD_1":
   #     value = 0
   #     if "MDEF" in self._newDeviceConfigurationValues:
   #       value = value | 0x04 if self._newDeviceConfigurationValues["MDEF"] else value

      if key == "CFG_1":
        value = 0
        value = value | 0x01 if self._newDeviceConfigurationValues["DSDOORENABLE"] else value
        value = value | 0x02 if self._newDeviceConfigurationValues["CHHEATCOOL"] else value
        value = value | 0x04 if self._newDeviceConfigurationValues["ENABLET2"] else value
        value = value | 0x20 if self._newDeviceConfigurationValues["LOCKKEYBOARD"] else value
        value = value | 0x40 if self._newDeviceConfigurationValues["STBY"] else value
        value = value | 0x80 if self._newDeviceConfigurationValues["STANDBYBUTTONENABLE"] else value

      else:
        newConfigValue = self._newDeviceConfigurationValues[key]

     # LEO sending data to Device
        if self._valueDescriptions[key]["dataType"] == dataTypeFloat:
          if (self._newDeviceConfigurationValues["SCL"]) == "2" or (self._newDeviceConfigurationValues["SCL"]) == 2:
            if self._valueDescriptions[key]["unitType"] == unitTypeTemperature:
              newConfigValue = self._convertC2F(newConfigValue)
            elif self._valueDescriptions[key]["unitType"] == unitTypeDeltaTemperature:
              newConfigValue = self._convertDeltaC2F(newConfigValue)
          elif (self._newDeviceConfigurationValues["SCL"]) == "0" or (self._newDeviceConfigurationValues["SCL"]) == 0:
            newConfigValue = newConfigValue * 10.0
          value = self._convertFromFloatValue(newConfigValue)
        elif self._valueDescriptions[key]["dataType"] == dataTypeBool:
          value = 0 if newConfigValue == False else 1
        else:
          value = self._convertFromIntValue(newConfigValue)
      networkTrans.transactions.append(NetworkMessage(writeHoldingRegister(configItem[2], value), configItem[1]))
    return retval

  def _prepareUpdateDeviceConfigurationTransactions(self):
    retval = []
    networkTransTag = ""
    networkTrans = None

    for configItem in self.configModbusList:
      if networkTransTag != configItem[0]:
        networkTransTag = configItem[0]
        networkTrans = NetworkTransaction("ReadConfig" + networkTransTag)
        retval.append(networkTrans)
      networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(configItem[2], 1), configItem[1]))
    return retval

  def _prepareLoggingTransactions(self, valueToLog):
    if len(valueToLog) == 0:
      return None

    networkTrans = NetworkTransaction("Logging")

    if "T1 TEMP" in valueToLog:
      networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(0, 1), "T1 TEMP"))

    if "T2 TEMP" in valueToLog:
      networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(1, 1), "T2 TEMP"))

    setValueToLog = set(valueToLog)
    setStatusFlag = set(["ALARM", "MUTE", "DEFROST", "DOOR"])
    if len(setValueToLog & setStatusFlag) > 0:
      networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(100, 1), "StatusFlag"))

    # Alarm flag
    setAlarmFlag = set(["T1FAILUREALARM", "T2FAILUREALARM", "HIGHTEMPALARM", "LOWTEMPALARM", "CONDCLEANALARM", "DOOROPENALARM"])
    if len(setValueToLog & setAlarmFlag) > 0:
      networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(101, 1), "AlarmFlag"))

    # Output flag
    setOutputFlag = set(["COMP RLY", "AUX RLY"])
    if len(setValueToLog & setOutputFlag) > 0:
      networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(102, 1), "OutputFlag"))

    # Standby flag
    setStandbyFlag = set(["STANDBY"])
    if len(setValueToLog & setStandbyFlag) > 0:
      networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(229, 1), "StandbyFlag"))

    return [ networkTrans ]

  def _prepareUpdateStatusTransactions(self):
    networkTrans = NetworkTransaction("Status1")
    networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(0, 1), "T1 TEMP"))
    networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(1, 1), "T2 TEMP"))
    networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(203, 1), "SP"))
    networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(100, 1), "StatusFlag"))
    networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(102, 1), "OutputFlag"))
    networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(229, 1), "StandbyFlag"))
    return [ networkTrans ] + self._prepareUpdateAlarmsTransactions()

  def _prepareUpdateAlarmsTransactions(self):
    networkTrans = NetworkTransaction("Alarms")
    networkTrans.transactions.append(NetworkMessage(readHoldingRegisters(101, 1), "AlarmFlag")) # function 3
    return [ networkTrans ]

  def _executeTransaction(self, networkTrans):
    if not networkTrans.online:
      self._nullOutputValues()
    else:

      for transaction in networkTrans.transactions:
        with self.lock:
          if isinstance(transaction.response, readHoldingRegistersResponse):
            value = transaction.response.registers[0]

            if transaction.tag == "StatusFlag":
              self._values["ALARM"] = ((value & 0x01) > 0)
              self._alarm = self._values["ALARM"]
              self._values["MUTE"] = ((value & 0x02) > 0)
              self._values["DEFROST"] = ((value & 0x04) > 0)
              self._values["DOOR"] = ((value & 0x08) > 0)

            elif transaction.tag == "AlarmFlag":
              self._values["T1FAILUREALARM"] = ((value & 0x01) > 0)
              self.checkBooleanAdvisory("T1FAILUREALARM", self._values["T1FAILUREALARM"])
              self._values["T2FAILUREALARM"] = ((value & 0x02) > 0)
              self.checkBooleanAdvisory("T2FAILUREALARM", self._values["T2FAILUREALARM"])
              self._values["HIGHTEMPALARM"] = ((value & 0x04) > 0)
              self.checkBooleanAdvisory("HIGHTEMPALARM", self._values["HIGHTEMPALARM"])
              self._values["LOWTEMPALARM"] = ((value & 0x08) > 0)
              self.checkBooleanAdvisory("LOWTEMPALARM", self._values["LOWTEMPALARM"])
              self._values["CONDCLEANALARM"] = ((value & 0x10) > 0)
              self.checkBooleanAdvisory("CONDCLEANALARM", self._values["CONDCLEANALARM"])
              self._values["DOOROPENALARM"] = ((value & 0x20) > 0)
              self.checkBooleanAdvisory("DOOROPENALARM", self._values["DOOROPENALARM"])

            elif transaction.tag == "OutputFlag":
              self._values["COMP RLY"] = ((value & 0x01) > 0)
              self._values["AUX RLY"] = ((value & 0x02) > 0)

            elif transaction.tag == "StandbyFlag":
              self._values["STANDBY"] = ((value & 0x40) > 0)

            elif transaction.tag == "CFG_1":
              self._values["DSDOORENABLE"] = ((value & 0x01) > 0)
              self._values["CHHEATCOOL"] = ((value & 0x02) > 0)
              self._values["ENABLET2"] = ((value & 0x04) > 0)
              self._values["LOCKKEYBOARD"] = ((value & 0x20) > 0)
              self._values["STBY"] = ((value & 0x40) > 0)
              self._values["STANDBYBUTTONENABLE"] = ((value & 0x80) > 0)

        #    elif transaction.tag == "CMD_1":
        #      self._values["MDEF"] = ((value & 0x4) > 0)

            else:    # LEO reading up from Device
              key = transaction.tag
              if self._valueDescriptions[key]["dataType"] == dataTypeFloat:
                self._values[key] = self._convertToFloatValue(value)
                if self._values["SCL"] == 2:
                  if self._valueDescriptions[key]["unitType"] == unitTypeTemperature:
                    self._values[key] = self._convertF2C(self._values[key])
                  elif self._valueDescriptions[key]["unitType"] == unitTypeDeltaTemperature:
                    self._values[key] = self._convertDeltaF2C(self._values[key])
                elif self._values["SCL"] == 0:
                  self._values[key] = self._values[key] / 10.0
              elif self._valueDescriptions[key]["dataType"] == dataTypeBool:
                self._values[key] = False if value == 0 else True
              else:
                self._values[key] = self._convertToIntValue(value)

      if "ReadConfig" in networkTrans.tag:
        self.saveValuesToDatabase()
      elif "WriteConfig" in networkTrans.tag:
        self.updateDeviceConfiguration()

  def _convertF2C(self, value):
    return (value - 32) / 1.8

  def _convertC2F(self, value):
    return (value * 1.8) + 32

  def _convertDeltaF2C(self, value):
    return value / 1.8

  def _convertDeltaC2F(self, value):
    return value * 1.8

  def _convertToFloatValue(self, value):
    if value > 0x7fff:
      return float(value - 0x10000)
    return float(value)

  def _convertFromFloatValue(self, value):
    if value < 0:
      return int(round(value + 0x10000))
    return int(round(value))

  def _convertToIntValue(self, value):
    if value > 0x7fff:
      return int(value - 0x10000)
    return int(value)

  def _convertFromIntValue(self, value):
    if value < 0:
      return int(value + 0x10000)
    return int(value)