#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# coding: utf-8

"""
Author: Arnaud Ferré
Mail: arnaud.ferre.pro@gmail.com
Description: Training module for C-Norm method

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

#######################################################################################################
# Import modules & set up logging
#######################################################################################################

from optparse import OptionParser
import json
import gzip
from os.path import dirname, exists
from os import makedirs

import numpy
import gensim
from tensorflow.keras import layers, models, Model, Input, regularizers, optimizers, metrics, losses, initializers, backend

from word2term import getSizeOfVST
from onto import loadOnto, ontoToVec

#######################################################################################################
# Utils
#######################################################################################################

# Normalization of token embeddings:
def normalizeEmbedding(vst_onlyTokens):
    for token in vst_onlyTokens.keys():
        vst_onlyTokens[token] = vst_onlyTokens[token] / numpy.linalg.norm(vst_onlyTokens[token])
    return vst_onlyTokens

# CHoose with getMatricForCNN...?
def prepare2D_data(vst_onlyTokens, dl_terms, dl_associations, vso, phraseMaxSize):

    #ToDo: Keep a constant size of the input matrix between train and prediction
    nbTerms = len(dl_terms.keys())
    sizeVST = getSizeOfVST(vst_onlyTokens)
    sizeVSO = getSizeOfVST(vso)

    X_train = numpy.zeros((nbTerms, phraseMaxSize, sizeVST))
    Y_train = numpy.zeros((nbTerms, 1, sizeVSO))

    l_unkownTokens = list()
    l_uncompleteExpressions = list()

    for i, id_term in enumerate(dl_associations.keys()):
        # stderr.write('id_term = %s\n' % str(id_term))
        # stderr.write('len(dl_associations[id_term]) = %d\n' % len(dl_associations[id_term]))

        for id_concept in dl_associations[id_term]:
            Y_train[i][0] = vso[id_concept]
            for j, token in enumerate(dl_terms[id_term]):
                if j < phraseMaxSize:
                    if token in vst_onlyTokens.keys():
                        X_train[i][j] = vst_onlyTokens[token]
                    else:
                        l_unkownTokens.append(token)
                else:
                    l_uncompleteExpressions.append(id_term)
            break # Because it' easier to keep only one concept per mention (mainly to calculate size of matrix).
            # ToDo: switch to object to include directly size with these structures.

    return X_train, Y_train, l_unkownTokens, l_uncompleteExpressions

#
def loadJSON(filename):
    if filename.endswith('.gz'):
        f = gzip.open(filename)
    else:
        # f = open(filename, encoding='utf-8')
        f = open(filename, "r", encoding="utf-8")
    result = json.load(f)
    f.close()
    return result;

#######################################################################################################
# Concept-Normalization (C-Norm)
#######################################################################################################

def CNorm(vst_onlyTokens, dl_terms, dl_associations, vso,
          nbEpochs=30, batchSize=64,
          l_numberOfFilters=[4000], l_filterSizes=[1],
          phraseMaxSize=15):

    # Preparing data for SLFNN and S-CNN components:
    dataSCNN, labels, l_unkownTokens, l_uncompleteExpressions = prepare2D_data(vst_onlyTokens, dl_terms, dl_associations, vso, phraseMaxSize)
    dataSLFNN = numpy.zeros((dataSCNN.shape[0], dataSCNN.shape[2]))
    for i in range( dataSCNN.shape[0]):
        numberOfToken = 0
        for embedding in dataSCNN[i]:
            if not numpy.any(embedding):
                pass
            else:
                numberOfToken += 1
                dataSLFNN[i] += embedding

        if numberOfToken > 0:
            dataSLFNN[i] = dataSLFNN[i] / numberOfToken

    # Input layers:
    inputLP = Input(shape=dataSLFNN.shape[1])
    inputCNN = Input(shape=[dataSCNN.shape[1],dataSCNN.shape[2]])

    # SLFNN component:
    ontoSpaceSize = labels.shape[2]
    denseLP = layers.Dense(units=ontoSpaceSize, use_bias=True, kernel_initializer=initializers.GlorotUniform())(inputLP)
    modelLP = Model(inputs=inputLP, outputs=denseLP)

    # Shallow-CNN component:
    l_subLayers = list()
    for i, filterSize in enumerate(l_filterSizes):

        convLayer = (layers.Conv1D(l_numberOfFilters[i], filterSize, strides=1, kernel_initializer=initializers.GlorotUniform()))(inputCNN)

        outputSize = phraseMaxSize - filterSize + 1
        pool = (layers.MaxPool1D(pool_size=outputSize))(convLayer)

        activationLayer = (layers.LeakyReLU(alpha=0.3))(pool)

        l_subLayers.append(activationLayer)

    if len(l_filterSizes) > 1:
        concatenateLayer = (layers.Concatenate(axis=-1))(l_subLayers)  # axis=-1 // concatenating on the last dimension
    else:
        concatenateLayer = l_subLayers[0]

    denseLayer = layers.Dense(ontoSpaceSize, kernel_initializer=initializers.GlorotUniform())(concatenateLayer)
    modelCNN = Model(inputs=inputCNN, outputs=denseLayer)

    convModel = Model(inputs=inputCNN, outputs=concatenateLayer)
    fullmodel = models.Sequential()
    fullmodel.add(convModel)

    # Combination of the two components:
    combinedLayer = layers.average([modelLP.output, modelCNN.output])
    fullModel = Model(inputs=[inputLP, inputCNN], outputs=combinedLayer)
    fullModel.summary()

    # Compile and train:
    fullModel.compile(optimizer=optimizers.Nadam(), loss=losses.LogCosh(), metrics=[metrics.CosineSimilarity(), metrics.MeanSquaredError()])
    fullModel.fit([dataSLFNN, dataSCNN], labels, epochs=nbEpochs, batch_size=batchSize)

    return fullModel, vso, l_unkownTokens

#######################################################################################################
# Run class:
#######################################################################################################
class Train(OptionParser):

    def __init__(self):

        OptionParser.__init__(self, usage='usage: %prog [options]')

        self.add_option('--word-vectors', action='store', type='string', dest='word_vectors',
                        help='path to word vectors JSON file as produced by word2vec')
        self.add_option('--word-vectors-bin', action='store', type='string', dest='word_vectors_bin',
                        help='path to word vectors binary file as produced by word2vec')
        self.add_option('--terms', action='store', type='string', dest='terms',
                        help='path to terms file in JSON format (map: id -> array of tokens)')
        self.add_option('--attributions', action='store', type='string', dest='attributions',
                        help='path to attributions file in JSON format (map: id -> array of concept ids)')
        self.add_option('--ontology', action='store', type='string', dest='ontology',
                        help='path to ontology file in OBO format')

        self.add_option('--outputModel', action='store', type='string', dest='model', help='path to save the NN model directory')

        # Methods hyperparameters:
        self.add_option('--factor', action='store', type='float', dest='factors', default=0.65, help='parent concept weight factor (default=0.6).')
        self.add_option('--epochs', action='store', type='int', dest='epochs', default=150, help='number of epochs (default=150).')
        self.add_option('--batch', action='store', type='int', dest='batch', default=64, help='number of samples in batch (default=64).')
        self.add_option('--filtersSize', action='append', type='int', dest='filtersSize', help='list of the different size of filters (default=1)')
        self.add_option('--filtersNb', action='append', type='int', dest='filtersNb', help='list of the number of filters from filtersSize (default=100)')
        self.add_option('--phraseMaxSize', action='store', type='int', dest='phrase_max_size', default=15, help='max considered size of phrases in inputs (default=15).')
        self.add_option('--normalizedInputs', action='store', type='string', dest='normalizedInputs', default="True", help='unit normalize embeddings if "True" (default: True).')

    def run(self):

        options, args = self.parse_args()
        if len(args) > 0:
            raise Exception('stray arguments: ' + ' '.join(args))

        if options.word_vectors is None and options.word_vectors_bin is None:
            raise Exception('missing either --word-vectors or --word-vectors-bin')
        if options.word_vectors is not None and options.word_vectors_bin is not None:
            raise Exception('incompatible --word-vectors or --word-vectors-bin')
        if options.ontology is None:
            raise Exception('missing --ontology')
        if not options.terms:
            raise Exception('missing --terms')
        if not options.attributions:
            raise Exception('missing --attributions')
        if not options.model:
            raise Exception('missing --outputModel')

        if options.filtersSize is None:
            options.filtersSize = [1]
        if options.filtersNb is None:
            options.filtersNb = [100]
        if options.filtersSize is not None and options.filtersNb is not None:
            if len(options.filtersSize) != len(options.filtersNb):
                raise Exception('ERROR: number of elements in --filtersSize different from number of elements in --filtersNb')

        # Selected hyperparameters (can have an important influence...):
        print("\nRuning C-Norm with next hyperparameters:")
        print("factor=", options.factors)
        print("epochs=", options.epochs)
        print("batch=", options.batch)
        print("filtersSize=", options.filtersSize)
        print("filtersNb=", options.filtersNb)
        print("phraseMaxSize=", options.phrase_max_size)
        print("normalizedInputs=", options.normalizedInputs)

        # Loading ontology:
        print("\nloading ontology:", options.ontology)
        ontology = loadOnto(options.ontology)

        # Loading word embeddings:
        if options.word_vectors is not None:
            print("loading word embeddings:", options.word_vectors)
            word_vectors = loadJSON(options.word_vectors)
        elif options.word_vectors_bin is not None:
            print("loading word embeddings:", options.word_vectors_bin)
            EmbModel = gensim.models.Word2Vec.load(options.word_vectors_bin)
            word_vectors = dict((k, list(numpy.float_(npf32) for npf32 in EmbModel.wv[k])) for k in EmbModel.wv.vocab.keys())

        # Loading all mentions:
        print("Loading terms:", options.terms)
        dl_terms = loadJSON(options.terms)

        # Loading training examples:
        print("Loading attributions:", options.attributions)
        attributions = loadJSON(options.attributions)

        # Scaling of all embeddings:
        if options.normalizedInputs is not None:
            if options.normalizedInputs == "True":
                print("\nScaling: Unit normalization of input embeddings (recommended)...")
                word_vectors = normalizeEmbedding(word_vectors)
                print("Scaling done.\n")
        else:
            print("No scaling of input embeddings (not recommended, see normalizedInputs option).")

        # Building ontological space
        print("Building ontological space (with factor applied)...")
        vso = ontoToVec(ontology, options.factors)
        print("Ontological space built.\n")

        print("C-Norm training...")
        model, ontology_vector, _ = CNorm(word_vectors, dl_terms, attributions, vso,
                                nbEpochs=options.epochs, batchSize=options.batch,
                                l_numberOfFilters=options.filtersNb, l_filterSizes=options.filtersSize,
                                phraseMaxSize=options.phrase_max_size)
        print("C-Norm training done.\n")

        # Saving model:
        if options.model is not None:
            print("Saving trained Tensorflow model...")
            d = dirname(options.model)
            if not exists(d) and d != '':
                makedirs(d)
            model.save(options.model)
            print("Model saved.")

#######################################################################################################
# Test section
#######################################################################################################

if __name__ == '__main__':

    Train().run()


# -*- coding: utf-8 -*-
# After execution of the main pipeline, files are organized in a variety of
# locations, and some information is spread across multiple files. This
# gathers various charts and creates tables using provided data, storing it
# all in folders organized by paper section.
import utils
import graphUtils
import numpy as np
import scipy.stats as stats
import os
import re
import subprocess

import mainParams as mp
from groupWords import getWordGroupsRangeTest

# ==============================================================================
# ==============================================================================

# Color constants for heat-map tables
COLOR_BLUE = (66, 134, 244)
COLOR_ORANGE = (244, 179, 66)
COLOR_GRAY = (239, 239, 239)

# given a start color, an end color, and a current value within a min-max
# range, find a linear interpolation between start and end color
def colorConvert(min, max, current, startColor, endColor):
    sr, sg, sb = startColor
    er, eg, eb = endColor
    scale = (current - min) / (max - min)
    red = sr + scale*(er-sr)
    green = sg + scale*(eg-sg)
    blue = sb + scale*(eb-sb)
    return (red/255, green/255, blue/255)

# create tables with information evaluating given metrics
def makeMetricEvalTables(suffix, topStr, comparableTopStr, topNum, poetryNum, comparableNum, simMetrics, baseFolder):
    baseScoreInfo = [
        ("Cosine", 0),
        ("Burrows' Delta", 0),
    ]

    bestMetricName = "Jensen-Shannon (250)" #Jensen-Shannon+p
    bestMetricSigWork = []
    bestMetricSigAuthor = []

    evalTableOutput = []
    evalTableOutput.append("""\\begin{table}[!bt]
  \\centering
  \\def\\arraystretch{1}
  \\begin{tabular}{| l | r | r |}
\\hline
 & \\multicolumn{2}{c|}{\\textbf{Percentage of segments most similar to a segment...}} \\\\

\\textbf{Metric}& \\textbf{from the same work} & \\textbf{by the same author} \\\\\\hline
""")

    sameWorkTableOutput = []
    sameAuthorTableOutput = []
    temp = """\\begin{table}[!bt]
  \\centering
  \\def\\arraystretch{1}
  \\begin{tabular}{| l | c | c | c |}
\\hline
    """
    sameWorkTableOutput.append(temp)
    sameAuthorTableOutput.append(temp)

    temp = "& & \\textbf{Top %d +} & \\\\" % (topNum)
    sameWorkTableOutput.append(temp)
    sameAuthorTableOutput.append(temp)

    temp = "\\textbf{Metric}& \\textbf{Top %d} & \\textbf{Top %d in Poetry} & \\textbf{Top %d} \\\\\\hline" % (topNum, poetryNum, comparableNum)
    sameWorkTableOutput.append(temp)
    sameAuthorTableOutput.append(temp)

    workSigReport = []
    authorSigReport = []

    # & \\textbf{Sim to another work} & \\textbf{Closest to diff author} & \\textbf{Median}

    # Get the list of authors and works the metric got correct
    scoreLists = {}
    for simMetric in simMetrics:
        dir, metricName = simMetric
        scoreLists[metricName] = {}
        for i, params in enumerate([(False, False), (True, False), (False, True), ]):
            name = metricName
            addP, comparable = params
            metricTopStr = topStr
            if addP:
                metricTopStr += "+p"
                name += "+p"
            # look at comparable number of non-poetry words
            elif comparable:
                metricTopStr = comparableTopStr
                name += " (%d)" % comparableNum
            else:
                name += " (%d)" % topNum

            fname = "output/greek/no_split/%s/%s/metric/Books/scores.json" % (metricTopStr, dir)
            scores = utils.getContent(fname, True)
            scoreLists[metricName][i] = scores
            scoreLists[metricName][i]["name"] = name

    baseScores = []
    for bsi in baseScoreInfo:
        baseScoreMetric, baseScoreIndex = bsi
        baseScores.append(scoreLists[baseScoreMetric][baseScoreIndex])

    # Create a table of the information using the provided scores
    for metricName in scoreLists:
        cell2 = "\\textbf{%s}" % (metricName)
        cell3 = "\\textbf{%s}" % (metricName)
        for i in scoreLists[metricName]:
            currentScores = scoreLists[metricName][i]
            authorScores = currentScores["author"]
            workScores = currentScores["work"]
            name = currentScores["name"]
            sameWork = "%.2f%%" % (100*np.mean(workScores))
            sameAuth = "%.2f%%" % (100*np.mean(authorScores))
            # sameWork = "%.2f%%, (%d/%d)" % (100*np.mean(workScores), np.sum(workScores), len(workScores))
            # sameAuth = "%.2f%%, (%d/%d)" % (100*np.mean(authorScores), np.sum(authorScores), len(authorScores))

            # cell = "%s & %s & %s & %s & %s & %s" % (name, sameAuth, sameWork, otherWork, diffAuthClosest, median)
            cell = "%s & %s & %s" % (name, sameWork, sameAuth)
            cell = cell.replace("%", "\\%")
            evalTableOutput.append("%s\\\\\\hline" % cell)

            cell2 += " & %s" % (sameWork) # work_p
            cell3 += " & %s" % (sameAuth) # , author_p)

            for j, baseScore in enumerate(baseScores):
                a = baseScore["work"]
                b = currentScores["work"]
                work_t, work_p = stats.ttest_rel(a, b)
                workSigReport.append(name)
                # Degrees of freedom
                df = len(b) - 1
                workSig = "  (M=%.3f, SD=%.3f) t(%d)=%.3f, p=%.3e" % (np.mean(b), np.std(b), df, work_t, work_p)
                workSigReport.append(workSig)

                a = baseScore["author"]
                b = currentScores["author"]
                author_t, author_p = stats.ttest_rel(a, b)
                authorSigReport.append(name)
                # Degrees of freedom
                df = len(b) - 1
                authorSig = "  (M=%.3f, SD=%.3f) t(%d)=%.3f, p=%.3e" % (np.mean(b), np.std(b), df, author_t, author_p)
                authorSigReport.append(authorSig)

                if (name == bestMetricName or name == baseScore["name"]):
                    bestMetricSigWork.append("%s vs %s" % (name, baseScore["name"]))
                    bestMetricSigWork.append(workSig)

                    bestMetricSigAuthor.append("%s vs %s" % (name, baseScore["name"]))
                    bestMetricSigAuthor.append(authorSig)

                #print("  Author: t-statistic = %6.3f pvalue = %f" %  stats.ttest_rel(a, b))

                # Significance notes
                if (j == 0):
                    if (work_p < 0.01):
                        cell2 += "\\textbf{†}"
                    elif (work_p < 0.05):
                        cell2 += "\\textbf{*}"
                    if (author_p < 0.01):
                        cell3 += "\\textbf{†}"
                    elif (author_p < 0.05):
                        cell3 += "\\textbf{*}"
                else:
                    if (work_p < 0.01):
                        cell2 += "\\textbf{‡}"
                    if (author_p < 0.01):
                        cell3 += "\\textbf{‡}"

        cell2 = cell2.replace("%", "\\%")
        sameWorkTableOutput.append("%s\\\\\\hline" % cell2)

        cell3 = cell3.replace("%", "\\%")
        sameAuthorTableOutput.append("%s\\\\\\hline" % cell3)

    evalTableOutput.append("""
      \\end{tabular}
      \\caption{How well similarity metrics identify whether two segments come from the same work or the same author.}
      \\label{table:metric_eval}
    \\end{table}
    """)

    utils.safeWrite("%smetric/extraInfo/metricEvalTable%s.tex" % (baseFolder, suffix), "\n".join(evalTableOutput))

    sameWorkTableOutput.append("\\end{tabular}")
    sameWorkTableOutput.append("\\caption[How well similarity metrics based on a given set of words identify whether two segments come from the same work.]{")
    sameWorkTableOutput.append("How well similarity metrics based on a given set of words identify whether two segments come from the same work. \\newline")
    sameWorkTableOutput.append("†: Results very significant (p < 0.01) when compared to %s. \\newline" % baseScores[0]["name"])
    sameWorkTableOutput.append("*: Results significant (p < 0.05) when compared to %s. \\newline" % baseScores[0]["name"])
    sameWorkTableOutput.append("‡: Results very significant (p < 0.01) when compared to %s. " % baseScores[1]["name"])
    sameWorkTableOutput.append("}")
    sameWorkTableOutput.append("\\label{table:metric_eval_work}")
    sameWorkTableOutput.append("\\end{table}")

    utils.safeWrite("%smetric/sameWorkEvalTable%s.tex" % (baseFolder, suffix), "\n".join(sameWorkTableOutput))

    sameAuthorTableOutput.append("\\end{tabular}")
    sameAuthorTableOutput.append("\\caption[How well similarity metrics based on a given set of words identify whether two segments come from the same author.]{")
    sameAuthorTableOutput.append("How well similarity metrics based on a given set of words identify whether two segments come from the same author. \\newline")
    sameAuthorTableOutput.append("†: Results very significant (p < 0.01) when compared to %s. \\newline" % baseScores[0]["name"])
    sameAuthorTableOutput.append("*: Results significant (p < 0.05) when compared to %s. \\newline" % baseScores[0]["name"])
    sameAuthorTableOutput.append("‡: Results very significant (p < 0.01) when compared to %s. " % baseScores[1]["name"])
    sameAuthorTableOutput.append("}")
    sameAuthorTableOutput.append("\\label{table:metric_eval_author}")
    sameAuthorTableOutput.append("\\end{table}")

    utils.safeWrite("%smetric/sameAuthorEvalTable%s.tex" % (baseFolder, suffix), "\n".join(sameAuthorTableOutput))

    sigReport = "Work:\n" + ("\n".join(bestMetricSigWork)) + "\n\n-------------\n\nAuthor:\n" + ("\n".join(bestMetricSigAuthor))
    utils.safeWrite("%smetric/bestMetricSignificance%s.txt" % (baseFolder, suffix), sigReport)
    # utils.safeWrite("%smetric/bestMetricSignificanceWork%s.txt" % (baseFolder, suffix), "\n".join(bestMetricSigWork))
    # utils.safeWrite("%smetric/bestMetricSignificanceAuthor%s.txt" % (baseFolder, suffix), "\n".join(bestMetricSigAuthor))

    utils.safeWrite("%smetric/extraInfo/metricSignificanceReportWork%s.txt" % (baseFolder, suffix), "\n".join(workSigReport))
    utils.safeWrite("%smetric/extraInfo/metricSignificanceReportAuthor%s.txt" % (baseFolder, suffix), "\n".join(authorSigReport))

# create tables with information evaluating performance of each metric with/without
# smoothing and remainder words
def makeMetricInternalTables(suffix, topStr, simMetrics, baseFolder):
    metricInternalTables = []
    for simMetric in simMetrics:
        dir, metricName = simMetric

        # skip Jensen-Shannon
        if metricName == "Jensen-Shannon":
            continue

        tableOutput = []
        temp = """
\\begin{table}[!bt]
  \\centering
  \\def\\arraystretch{1}
  \\begin{tabular}{| l | c | c | c |}
\\hline
        """
        tableOutput.append(temp)

        temp = "\\textbf{Metric Options} & \\textbf{Author} & \\textbf{Work} & \\textbf{Total} \\\\\\hline"
        tableOutput.append(temp)

        workSigReport = []
        authorSigReport = []
        totalSigReport = []

        # & \\textbf{Sim to another work} & \\textbf{Closest to diff author} & \\textbf{Median}

        metricOptions = [
            ("Baseline", "-remainder-smoothed"),
            ("+1 Smoothing", "-remainder+smoothed"),
            ("Remainder", "+remainder-smoothed"),
            ("Both", "+remainder+smoothed")
        ]

        # Get the list of authors and works the metric got correct
        scoreLists = {}
        for _, opt in metricOptions:
            scoreLists[opt] = {}
            name = opt
            # Use Poetry Words
            metricTopStr = topStr

            fname = "output/greek/no_split/%s/%s/metric%s/Books/scores.json" % (metricTopStr, dir, opt)
            scores = utils.getContent(fname, True)
            scoreLists[opt] = scores
            scoreLists[opt]["name"] = name

        baseScore = scoreLists["-remainder-smoothed"]
        # baseScores = []
        # for bsi in baseScoreInfo:
        #     baseScoreMetric, baseScoreIndex = bsi
        #     baseScores.append(scoreLists[baseScoreMetric][baseScoreIndex])

        # Create a table of the information using the provided scores
        for optName, opt in metricOptions:
            cell = "\\textbf{%s}" % (optName)

            currentScores = scoreLists[opt]
            authorScores = currentScores["author"]
            workScores = currentScores["work"]
            name = currentScores["name"]
            sameWork = "%.2f%%, (%d/%d)" % (100*np.mean(workScores), np.sum(workScores), len(workScores))
            sameAuth = "%.2f%%, (%d/%d)" % (100*np.mean(authorScores), np.sum(authorScores), len(authorScores))
            all = np.concatenate((workScores, authorScores))
            total = "%.2f%%, (%d/%d)" % (100*np.mean(all), np.sum(all), len(all))

            wrk = " & %s" % (sameWork)
            auth = " & %s" % (sameAuth)
            tot = " & %s" % (total)

            # Calculate significance
            a = baseScore["work"]
            b = currentScores["work"]
            work_t, work_p = stats.ttest_rel(a, b)
            workSigReport.append(name)
            # Degrees of freedom
            df = len(b) - 1
            workSig = "  (M=%.3f, SD=%.3f) t(%d)=%.3f, p=%.3e" % (np.mean(b), np.std(b), df, work_t, work_p)
            workSigReport.append(workSig)

            a = baseScore["author"]
            b = currentScores["author"]
            author_t, author_p = stats.ttest_rel(a, b)
            authorSigReport.append(name)
            # Degrees of freedom
            df = len(b) - 1
            authorSig = "  (M=%.3f, SD=%.3f) t(%d)=%.3f, p=%.3e" % (np.mean(b), np.std(b), df, author_t, author_p)
            authorSigReport.append(authorSig)

            a = np.concatenate((baseScore["work"], baseScore["author"]))
            b = np.concatenate((currentScores["work"], currentScores["author"]))
            all_t, all_p = stats.ttest_rel(a, b)
            totalSigReport.append(name)
            # Degrees of freedom
            df = len(b) - 1
            totalSig = "  (M=%.3f, SD=%.3f) t(%d)=%.3f, p=%.3e" % (np.mean(b), np.std(b), df, all_t, all_p)
            totalSigReport.append(totalSig)

            # if (name == bestMetricName or name == baseScore["name"]):
            #     bestMetricSigWork.append("%s vs %s" % (name, baseScore["name"]))
            #     bestMetricSigWork.append(workSig)
            #
            #     bestMetricSigAuthor.append("%s vs %s" % (name, baseScore["name"]))
            #     bestMetricSigAuthor.append(authorSig)

            #print("  Author: t-statistic = %6.3f pvalue = %f" %  stats.ttest_rel(a, b))

            # Significance notes
            if (work_p < 0.01):
                wrk += "\\textbf{†}"
            elif (work_p < 0.05):
                wrk += "\\textbf{*}"
            if (author_p < 0.01):
                auth += "\\textbf{†}"
            elif (author_p < 0.05):
                auth += "\\textbf{*}"
            if (all_p < 0.01):
                tot += "\\textbf{†}"
            elif (all_p < 0.05):
                tot += "\\textbf{*}"

            # wrk += " %.4f" % work_p
            # auth += " %.4f" % author_p
            # tot += " %.4f" % all_p

            cell += "%s%s%s" % (wrk, auth, tot)

            cell = cell.replace("%", "\\%")
            tableOutput.append("%s\\\\\\hline" % cell)

        tableOutput.append("\\end{tabular}")
        tableOutput.append("\\caption{")
        tableOutput.append("How well %s performs with the remainder words and smoothing included. " % metricName)
        tableOutput.append("†: Results very significant (p < 0.01) when compared to baseline. ")
        tableOutput.append("*: Results significant (p < 0.05) when compared to baseline. ")
        tableOutput.append("}")
        tableOutput.append("\\label{table:metric_options_eval_%s}" % dir)
        tableOutput.append("\\end{table}")

        tableOutput.append("")
        tableOutput.append("")
        metricInternalTables.append("\n".join(tableOutput))
        utils.safeWrite("%smetric/%s_optionsEvalTable%s.tex" % (baseFolder, metricName, suffix), "\n".join(tableOutput))

        # sigReport = "Work:\n" + ("\n".join(bestMetricSigWork)) + "\n\n-------------\n\nAuthor:\n" + ("\n".join(bestMetricSigAuthor))
        # utils.safeWrite("%smetric/bestMetricSignificance%s_2.txt" % (baseFolder, suffix), sigReport)

        # utils.safeWrite("%smetric/extraInfo/metricSignificanceReportWork%s_2.txt" % (baseFolder, suffix), "\n".join(workSigReport))
        # utils.safeWrite("%smetric/extraInfo/metricSignificanceReportAuthor%s_2.txt" % (baseFolder, suffix), "\n".join(authorSigReport))
    utils.safeWrite("%smetric/extraInfo/optionsEvalTables%s.tex" % (baseFolder, suffix), "\n".join(metricInternalTables))

# Get author pairs for authors 4 centuries apart with high similarity.
def fourCenturiesTables(topStr, simMetrics, baseFolder):
    comparisonOutput = []
    topSimsToExamine = 100

    # Grab this from the best metric
    authorSims = utils.getContent("output/greek/no_split/%s/jensen-shannon/metric/Authors/sims.txt" % (topStr), False).split("\n")
    topDistantSims = []
    topDistantAuthors = {}
    for i, sim in enumerate(authorSims):
        centuries_apart = int(sim.split("(")[-1].split(" ")[0])
        if (centuries_apart >= 4 and i < topSimsToExamine):
            topDistantSims.append(sim)
            topDistantAuthors[sim[11:]] = {}

        authors = " (".join(sim.split(" - ")[1].split(" (")[:-1])
        if authors == "Isocrates, Lysias" or authors == "Plato, Xenophon" or authors == "AratusSolensis, Callimachus" or authors == "Herodotus, Thucydides":
            comparisonOutput.append("Rank %d: %s" % (i+1, sim))

    fourCenturiesApartOutput = []
    fourCenturiesApartOutput.append("%d of the top %d are at least 4 centuries apart." % (len(topDistantSims), topSimsToExamine))
    fourCenturiesApartOutput.append("---")
    fourCenturiesApartOutput.extend(topDistantSims)

    utils.safeWrite("%swordUse/fourCenturiesApart.txt" % baseFolder, "\n".join(fourCenturiesApartOutput))

    # Comparison to English and Icelandic
    numGreek = len(authorSims)
    fracGreek = topSimsToExamine/numGreek
    numDistantGreek = len(topDistantSims)

    englishSims = utils.getContent("output/english/no_split/%s/jensen-shannon/metric/Authors/sims.txt" % (topStr), False).split("\n")
    numEnglish = len(englishSims)
    topSimsEnglish = int(np.ceil(numEnglish*fracGreek))
    fracEnglish = topSimsEnglish/numEnglish
    numDistantEnglish = 0
    num2English = 0
    for sim in englishSims[:topSimsEnglish]:
        centuries_apart = int(sim.split("(")[-1].split(" ")[0])
        if (centuries_apart >= 2):
            num2English += 1
        if (centuries_apart >= 4):
            numDistantEnglish += 1

    iceSims = utils.getContent("output/icelandic/no_split/%s/jensen-shannon/metric/Authors/sims.txt" % (topStr), False).split("\n")
    numIcelandic = len(iceSims)
    topSimsIcelandic = int(np.ceil(numIcelandic*fracGreek))
    fracIcelandic = topSimsIcelandic/numIcelandic
    numDistantIcelandic = 0
    for sim in iceSims[:topSimsIcelandic]:
        centuries_apart = int(sim.split("(")[-1].split(" ")[0])
        if (centuries_apart >= 4):
            numDistantIcelandic += 1

    comparisonOutput.append("\n=========\n")
    comparisonOutput.append("Top similar pairs")
    comparisonOutput.append("Greek:")
    comparisonOutput.append("  examining top %d of %d pairs (%.2f%%)" % (topSimsToExamine, numGreek, 100*fracGreek))
    comparisonOutput.append("  %d (%.2f%%) are at least 4 centuries apart" % (numDistantGreek, 100*numDistantGreek/topSimsToExamine))
    comparisonOutput.append("English:")
    comparisonOutput.append("  examining top %d of %d pairs (%.2f%%)" % (topSimsEnglish, numEnglish, 100*fracEnglish))
    comparisonOutput.append("  %d (%.2f%%) are at least 4 centuries apart" % (numDistantEnglish, 100*numDistantEnglish/topSimsEnglish))
    comparisonOutput.append("  %d (%.2f%%) are at least 2 centuries apart" % (num2English, 100*num2English/topSimsEnglish))
    comparisonOutput.append("Icelandic:")
    comparisonOutput.append("  examining top %d of %d pairs (%.2f%%)" % (topSimsIcelandic, numIcelandic, 100*fracIcelandic))
    comparisonOutput.append("  %d (%.2f%%) are at least 4 centuries apart" % (numDistantIcelandic, 100*numDistantIcelandic/topSimsIcelandic))

    utils.safeWrite("%swordUse/fourApartComparisonInfo.txt" % baseFolder, "\n".join(comparisonOutput))

    # Table
    for simMetric in simMetrics:
        dir, name = simMetric
        # "" or "+p" depending on which is better
        metricSims = utils.getContent("output/greek/no_split/%s/%s/metric/Authors/sims.txt" % (topStr, dir), False).split("\n")
        for i, sim in enumerate(metricSims):
            pairName = sim[11:]
            if pairName in topDistantAuthors:
                topDistantAuthors[pairName][dir] = i + 1

    # prepare values for coloring table cells
    maxVal = 0
    minVal = 1000000

    for authorPair in topDistantAuthors:
        for simDir, _ in simMetrics:
            val = topDistantAuthors[authorPair][simDir]
            minVal = min(minVal, val)
            maxVal = max(maxVal, val)

    pairRankOutput = []
    pairRankOutputSimple = []
    pairRankOutput.append("""
    \\begin{table}[!bt]
      \\centering
      \\def\\arraystretch{1}
      \\begin{tabular}{| l | c | c | c | c | c | c |}
    \\hline
    & \\multicolumn{5}{c|}{\\textbf{Rank according to}} \\\\
    & \\textbf{Jensen-} & \\textbf{Burrows'} & & & & \\\\
    \\textbf{Authors} & \\textbf{Shannon} & \\textbf{Delta} & \\textbf{Min-Max} & \\textbf{Manhattan} & \\textbf{Canberra} & \\textbf{Cosine} \\\\\\hline
    """)
    pairRankOutputSimple.append("%s,%s,%s,%s,%s,%s,%s" % ("Authors", "Jensen-Shannon", "Burrow's Delta", "Min-Max", "Manhattan", "Canberra", "Cosine"))
    authorConvert = {
        "ApolloniusRhodius": "Apollonius",
        "DionysiusOfHalicarnassus": "Dionysius",
        "EusebiusOfCaesarea": "Eusebius",
        "ClementOfAlexandria": "Clement",
        "BasilBishopOfCaesarea": "Basil",
        "Anonymous(Hymns_Aphrodite)": "Hymns Aphrodite",
        "Anonymous(Hymns_Apollo)": "Hymns Apollo",
        "Anonymous(Hymns_Demeter)": "Hymns Demeter",
        "Anonymous(Hymns_Hermes)": "Hymns Hermes",
        "Anonymous(Hymns_Rest)": "Hymns Rest",
    }
    for authorPair in topDistantAuthors:
        pair = "(".join(authorPair.split(" (")[:-1])
        pairSplit = pair.split(", ")
        author1 = pairSplit[0]
        author2 = pairSplit[1]

        if author1 in authorConvert:
            author1 = authorConvert[author1]
        if author2 in authorConvert:
            author2 = authorConvert[author2]

        pairName = author1 + ", " + author2
        cell = "%s &" % pairName
        cellSimple = "%s," % re.sub(", ", "/", pairName)
        firstVal = None
        for simDir, _ in simMetrics:
            val = topDistantAuthors[authorPair][simDir]

            cutoff = 100
            if (val < cutoff):
                r, g, b = colorConvert(minVal, cutoff, val, COLOR_ORANGE, COLOR_GRAY)
            else:
                r, g, b = colorConvert(cutoff, maxVal, val, COLOR_GRAY, COLOR_BLUE)
            cell += "\\cellcolor[rgb]{%.3f,%.3f,%.3f} " % (r, g, b)

            if (firstVal == None):
                firstVal = val
                cell += "%d & " % (val)
                cellSimple += "%d," % (val)
            else:
                cell += "%d (%+d) & " % (val, firstVal - val)
                rel = "(%d)" % (firstVal - val)
                cellSimple += "%d %s," % (val, rel)
        cell = cell[:-2]
        pairRankOutput.append("%s\\\\\\hline" % cell)
        pairRankOutputSimple.append(cellSimple)
    pairRankOutput.append("""
      \\end{tabular}
      \\caption{Rank of these pair's similarity by different metrics.}
      \\label{table:pair_rank}
    \\end{table}
    """)

    utils.safeWrite("%swordUse/pairRankTable.tex" % baseFolder, "\n".join(pairRankOutput))
    utils.safeWrite("%swordUse/pairRankTableSimple.csv" % baseFolder, "\n".join(pairRankOutputSimple))

# Get info on number of words used and create the table of top words
def getWordUseInfo(topStr, baseFolder):
    # total +p words
    tops = utils.getContent("output/greek/no_split/%s/wordInfo_%s.txt" % (topStr, topStr), False).split("\n")[1:]
    poetrys = utils.getContent("output/greek/no_split/top_p/wordInfo_top_p.txt", False).split("\n")[1:]
    # Top plus poetry
    totals = utils.getContent("output/greek/no_split/%s+p/wordInfo_%s+p.txt" % (topStr, topStr), False).split("\n")[1:]

    numWordsOutput = []
    numWordsOutput.append("Number of Top Words: %d" % len(tops))
    numWordsOutput.append("Number of Poetry Words: %d" % len(poetrys))
    numWordsOutput.append("Total Number of Words: %d" % len(totals))
    utils.safeWrite("%s/wordUse/totalWords.txt" % baseFolder, "\n".join(numWordsOutput))

    # Create Table of words
    topRanks = {}
    poetryRanks = {}

    for i, line in enumerate(tops):
        w = line.split(":")[0]
        topRanks[w] = i + 1

    for i, line in enumerate(poetrys):
        w = line.split(":")[0]
        poetryRanks[w] = i + 1

    rankInfo = []
    for line in totals:
        w = line.split(":")[0]
        topRank = ""
        if w in topRanks:
            topRank = "%d" % topRanks[w]
        poetryRank = ""
        if w in poetryRanks:
            poetryRank = "%d" % poetryRanks[w]

        rankInfo.append((w, topRank, poetryRank))

    rankTableOutput = []
    rankTableOutput.append("""
    \\begin{table}[!hbt]
      \\centering
      \\def\\arraystretch{1}
      \\begin{tabular}{| l | l | l ||| l | l | l ||| l | l | l ||| l | l | l |}
    \\hline

    \\textbf{Token} & \\textbf{A} & \\textbf{P} & \\textbf{Token} & \\textbf{A} & \\textbf{P} & \\textbf{Token} & \\textbf{A} & \\textbf{P} & \\textbf{Token} & \\textbf{A} & \\textbf{P}\\\\\\hline
    """)

    columnHeight = 43;
    for i in range(columnHeight):
        cells =  []
        for j in range(4):
            index = i + j*columnHeight
            cell = ""
            if (index < len(rankInfo)):
                cell = "%s & %s & %s" % rankInfo[index]

            cells.append(cell)
        rankTableOutput.append("%s \\\\\\hline" % (" & ".join(cells)))

    rankTableOutput.append("""
      \\end{tabular}
      \\caption{List of tokens used, along with their rank in the top 150 tokens found in all texts (\\textbf{A}) and rank in the top 100 tokens found in poetry texts (\\textbf{P}).}
      \\label{table:top_words}
    \\end{table}
    """)

    utils.safeWrite("%swordUse/topWordsTable.tex" % baseFolder, "\n".join(rankTableOutput))

# Get author and book counts for our languages for the data section
def getAuthorBookCounts(baseFolder):
    ab_counts_output = []
    splitter = "\n------\n"

    ab_counts_output.append("Greek:\n")
    ab_counts_output.append(utils.getContent("output/greek/numberOfAuthors_Books.txt", False))
    ab_counts_output.append(utils.getContent("output/greek/numberOfTypes_Tokens.txt", False))
    ab_counts_output.append(splitter)
    ab_counts_output.append("English:\n")
    ab_counts_output.append(utils.getContent("output/english/numberOfAuthors_Books.txt", False))
    ab_counts_output.append(utils.getContent("output/english/numberOfTypes_Tokens.txt", False))
    ab_counts_output.append(splitter)
    ab_counts_output.append("Icelandic:\n")
    ab_counts_output.append(utils.getContent("output/icelandic/numberOfAuthors_Books.txt", False))
    ab_counts_output.append(utils.getContent("output/icelandic/numberOfTypes_Tokens.txt", False))
    ab_counts_output.append(splitter)

    utils.safeWrite("%s/AuthorBookNumbers.txt" % baseFolder, "\n".join(ab_counts_output))

# Get word overlap info
def getOverlapInfo(baseFolder):
    output = []
    splitter = "\n------\n"

    output.append("Greek:\n")
    output.append(utils.getContent("output/greek/topWordOverlapOverTime.txt", False))
    output.append(splitter)
    output.append("English:\n")
    output.append(utils.getContent("output/english/topWordOverlapOverTime.txt", False))
    output.append(splitter)
    output.append("Icelandic:\n")
    output.append(utils.getContent("output/icelandic/topWordOverlapOverTime.txt", False))
    output.append(splitter)

    utils.safeWrite("%s/topWordOverlapOverTime.txt" % baseFolder, "\n".join(output))

# Get information about words that were skipped
def getSkippedWordInfo(baseFolder):
    output = []
    splitter = "\n------\n"

    output.append("Greek:\n")
    output.append(utils.getContent("output/greek/no_split/top250/chosenWordInfo.txt", False))
    output.append("\nPoetry:")
    output.append(utils.getContent("output/greek/no_split/top250+p/chosenWordInfoPoetry.txt", False))
    output.append(splitter)
    output.append("English:\n")
    output.append(utils.getContent("output/english/no_split/top250/chosenWordInfo.txt", False))
    output.append("\nPoetry:")
    output.append(utils.getContent("output/english/no_split/top250+p/chosenWordInfoPoetry.txt", False))
    output.append(splitter)
    output.append("Icelandic:\n")
    output.append(utils.getContent("output/icelandic/no_split/top250/chosenWordInfo.txt", False))
    output.append(splitter)

    utils.safeWrite("%s/skippedWords.txt" % baseFolder, "\n".join(output))

def getDataInfo(topStr, baseFolder):
    baseFolder = baseFolder + "/data"

    getAuthorBookCounts(baseFolder)

    getOverlapInfo(baseFolder)

    getSkippedWordInfo(baseFolder)

    subprocess.run("cp output/greek/no_split/%s/wordsTable.csv %s/wordsTableGreek_%s.csv" % (topStr, baseFolder, topStr), shell=True)
    subprocess.run("cp output/english/no_split/%s/wordsTable.csv %s/wordsTableEnglish_%s.csv" % (topStr, baseFolder, topStr), shell=True)
    subprocess.run("cp output/icelandic/no_split/%s/wordsTable.csv %s/wordsTableIcelandic_%s.csv" % (topStr, baseFolder, topStr), shell=True)

# Get information on the top authors
def makeTopAuthorTable(topStr, baseFolder):
    # Grab this from the best metric
    fname = "output/greek/no_split/%s/jensen-shannon/metric/Authors/sims.txt" % (topStr)
    allAuthorSims = utils.getContent(fname, False).split("\n")

    topAuthorPairs = []

    topAuthorPairs.append("""\\begin{table}[!bt]
  \\centering
  \\def\\arraystretch{1.2}
  \\begin{tabular}{| r | l | l | l | l |} \\hline
  & \\textbf{Author 1} & \\textbf{Author 2} & \\textbf{Score} & \\textbf{Notes}  \\\\\\hline
""")

    for i, pair in enumerate(allAuthorSims[:10]):
        splt1 = pair.split(" - ")
        sim = splt1[0]
        auths = splt1[1].split(" (")[0].split(", ")
        topAuthorPairs.append("  %.2d & %s & %s & %s & TODO \\\\\\hline" % (i+1, auths[0], auths[1], sim))

    topAuthorPairs.append("""
  \\end{tabular}
  \\caption{Top author pairs by similarity score according to Jensen-Shannon Similarity.}
  \\label{table:top_author_pairs}
\\end{table}
    """)

    utils.safeWrite("%smetric/topAuthorPairs.tex" % baseFolder, "\n".join(topAuthorPairs))

# create tables showing performance of light machine learning algorithms on predicting various categories
def makeMLTable(source, norm, filename):
    output = []

    output.append("""\\begin{table}[!bt]
  \\centering
  \\def\\arraystretch{1.2}
""")

    # No naive bayes if normed due to negative data
    if norm:
        output.append("  \\begin{tabular}{| r | l | l |} \\hline")
        output.append("  \\textbf{Prediction Task} & \\textbf{Majority Class} & \\textbf{KNN}  \\\\\\hline")
    else:
        output.append("  \\begin{tabular}{| r | l | l | l |} \\hline")
        output.append("  \\textbf{Prediction Task} & \\textbf{Majority Class} & \\textbf{KNN} & \\textbf{Naive Bayes}  \\\\\\hline")

    for t in ["Authors", "Books", "Books_2"]:
        cats = ["genre", "dialect", "timeframe"]
        if (t == "Books"):
            cats.append("author")
        if (t == "Books_2"):
            cats = ["work", "genre", "dialect", "timeframe", "author"]

        for cat in cats:
            fname = source + "res_%s_%s.txt" % (cat, t)
            lines = utils.getContent(fname, False).split("\n")
            maj_class = lines[1].split(" - ")[0].strip()
            knn = lines[2].split(" - ")[0].strip()
            naive_bayes = lines[3].split(" - ")[0].strip()

            t_name = t
            if t_name == "Books":
                t_name = "Segments"
            if t_name == "Books_2":
                t_name = "Segments*"
            if norm:
                output.append(" %s of %s & %s & %s \\\\\\hline" % (cat, t_name, maj_class, knn))
            else:
                output.append(" %s of %s & %s & %s & %s \\\\\\hline" % (cat, t_name, maj_class, knn, naive_bayes))

    output.append("""
  \\end{tabular}
  \\caption{Results of running simple machine learning on the frequency data.}
  \\label{table:ml+p}
\\end{table}
    """)

    utils.safeWrite(filename, "\n".join(output))

# =============================================================================
# =============================================================================

# Get information on genre
def getGenreInfo(topStr, baseFolder):
    # 2-dimensional tSNE projection
    subprocess.run("cp output/greek/no_split/%s/authors/tSNE/genre_tSNE_2D_no_labels.pdf %sgenre/tSNE_topWords_nolabels.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s/authors/tSNE/genre_tSNE_2D_labels.pdf %sgenre/tSNE_topWords_labels.pdf" % (topStr, baseFolder), shell=True)

# Get information on the metrics
def getMetricInfo(topStr, comparableTopStr, topNum, poetryNum, comparableNum, simMetrics, baseFolder):
    # Copy full eval files for jensen-shannon
    subprocess.run("cp output/greek/no_split/%s/jensen-shannon/metric/Books/comparisonInfo.txt %smetric/extraInfo/metricEvaluation_tops.txt" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s+p/jensen-shannon/metric/Books/comparisonInfo.txt %smetric/extraInfo/metricEvaluation_+p.txt" % (topStr, baseFolder), shell=True)

    # Grab median distance
    fname = "output/greek/no_split/%s/jensen-shannon/metric/Books/comparisonInfo.txt" % (topStr)
    metricEvalInfo = utils.getContent(fname, False).split("=========")[-2].split("\n")[2:-1]
    sameAuthorRanks = []
    for i, line in enumerate(metricEvalInfo):
        sameAuthorRank = line.split("with same author: ")[1].split(".")[0]
        sameAuthorRanks.append(int(sameAuthorRank))

    median = np.median(sameAuthorRanks)

    utils.safeWrite("%smetric/extraInfo/medianForDifferentAuthor.txt" % (baseFolder), "Median distance for closest author: %f" % median)

    # get info on the indica
    subprocess.run("cp output/greek/no_split/%s/jensen-shannon/metric/Books/sims/Arrian.Indica.1.txt %smetric/extraInfo/arrianIndica.txt" % (topStr, baseFolder), shell=True)

    # Info on book distance
    # Grab this from the best metric
    fname = "output/greek/no_split/%s/jensen-shannon/metric/Books/sims.txt" % (topStr)
    allBookSims = utils.getContent(fname, False).split("\n")

    utils.safeWrite("%smetric/lowestSimilarity.txt" % (baseFolder), "Lowest similarity between segments: %s" % allBookSims[-1])

    # Info on top similar authors
    makeTopAuthorTable(topStr, baseFolder)

    # ===============================

    makeMetricEvalTables("", topStr, comparableTopStr, topNum, poetryNum, comparableNum, simMetrics, baseFolder)

    # baseScoreMetric = "Burrows' Delta"
    # baseScoreIndex = 1
    # makeMetricEvalTables("_cmp_burrows", topStr, comparableTopStr, topNum, poetryNum, comparableNum, simMetrics, baseScoreMetric, baseScoreIndex)

# Get information comparing various texts across centuries
def getCenturyInfo(topStr, baseFolder):
    subprocess.run("cp output/greek/no_split/%s/jensen-shannon/metric/Authors/century_sims_overall_no_labels.pdf %scentury/centuriesGreek.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s/jensen-shannon/metric/Authors/century_sims_overall_labels.pdf %scentury/extraInfo/Greek_CenturyOverall_Label.pdf" % (topStr, baseFolder), shell=True)

    subprocess.run("cp output/greek/no_split/%s/jensen-shannon/metric/Authors/simRange.txt %scentury/extraInfo/Greek_SimRange.txt" % (topStr, baseFolder), shell=True)

    # -------------------------
    # Century similarity data
    subprocess.run("cp output/greek/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_no_labels.pdf %scentury/extraInfo/Greek_Century_No_Label.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_labels.pdf %scentury/extraInfo/Greek_Century_Label.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s+p/jensen-shannon/metric/Authors/century_sims_genre_no_labels.pdf %scentury/extraInfo/Greek+p_Century_No_Label.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s+p/jensen-shannon/metric/Authors/century_sims_genre_labels.pdf %scentury/extraInfo/Greek+p_Century_Label.pdf" % (topStr, baseFolder), shell=True)

    subprocess.run("cp output/greek/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_under_9_no_labels.pdf %scentury/extraInfo/Greek_Century_Cutoff_No_Label.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_under_9_labels.pdf %scentury/extraInfo/Greek_Century_Cutoff_Label.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s+p/jensen-shannon/metric/Authors/century_sims_genre_under_9_no_labels.pdf %scentury/extraInfo/Greek+p_Century_Cutoff_No_Label.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s+p/jensen-shannon/metric/Authors/century_sims_genre_under_9_labels.pdf %scentury/extraInfo/Greek+p_Century_Cutoff_Label.pdf" % (topStr, baseFolder), shell=True)

    subprocess.run("cp output/greek/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_under_9_no_labels.pdf %scentury/centuriesGreek2.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_under_9_no_labels_violin.pdf %scentury/centuriesGreekViolin.pdf" % (topStr, baseFolder), shell=True)

    subprocess.run("cp output/english/no_split/%s/jensen-shannon/metric/Authors/simRange.txt %scentury/extraInfo/English_SimRange.txt" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/english/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_no_labels.pdf %scentury/centuriesEnglish.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/english/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_no_labels_violin.pdf %scentury/centuriesEnglishViolin.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/english/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_labels.pdf %scentury/extraInfo/English_Century_Label.pdf" % (topStr, baseFolder), shell=True)

    subprocess.run("cp output/icelandic/no_split/%s/jensen-shannon/metric/Authors/simRange.txt %scentury/extraInfo/Icelandic_SimRange.txt" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/icelandic/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_no_labels.pdf %scentury/centuriesIcelandic.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/icelandic/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_no_labels_violin.pdf %scentury/centuriesIcelandicViolin.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/icelandic/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_labels.pdf %scentury/extraInfo/Icelandic_Century_Label.pdf" % (topStr, baseFolder), shell=True)

    # Get pvalue + other regression information for charts
    greekPval = utils.getContent("output/greek/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_under_9_pslope.txt" % (topStr), False)
    englishPval = utils.getContent("output/english/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_pslope.txt" % (topStr), False)
    icelandicPval = utils.getContent("output/icelandic/no_split/%s/jensen-shannon/metric/Authors/century_sims_genre_pslope.txt" % (topStr), False)

    pvalOutput = []
    pvalOutput.append("Greek:")
    pvalOutput.append(greekPval)
    pvalOutput.append("English:")
    pvalOutput.append(englishPval)
    pvalOutput.append("Icelandic:")
    pvalOutput.append(icelandicPval)

    utils.safeWrite("%scentury/century_pvals.txt" % baseFolder, "\n".join(pvalOutput))

# Get charts on word usage across authors and by specific author pairs
def getWordUsageInfo(topStr, baseFolder):
    # Word usage charts
    # Grab these from the best metric
    subprocess.run("cp output/greek/no_split/%s/wordImportance/Jensen-shannon-sorted/ignoreBestWords.pdf %swordUse/ignoreBestWords.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s/wordImportance/Jensen-shannon-sorted/all-diffs-cumul-cloud.pdf %swordUse/extraInfo/broadWordUsage.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s/wordImportance/Jensen-shannon-sorted/all-diffs-cumul.pdf %swordUse/extraInfo/all-diffs-cumul.pdf" % (topStr, baseFolder), shell=True)

    # Word organization charts
    subprocess.run("cp output/greek/no_split/%s/textsOnlyTopWords/9_group/tSNE/Word_Groupings_tSNE_2D_labels.pdf %swordUse/wordGroups.pdf" % (topStr, baseFolder), shell=True)

    subprocess.run("cp output/greek/no_split/%s/textsOnlyTopWords/9_group/images/dhct.pdf %swordUse/eightUp_9Group.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s/textsOnlyTopWords/pos_group/images/dhct.pdf %swordUse/eightUp_posGroup.pdf" % (topStr, baseFolder), shell=True)

# create each of the folders in a list if they do not exist yet
def createFolders(folders, baseFolder):
    for folder in folders:
        if not(os.path.exists(baseFolder + folder)):
            subprocess.run("mkdir " + baseFolder + folder, shell=True)

# =============================================================================
# =============================================================================
# =============================================================================

# List of similarity metrics
SIM_METRICS = [
    ("jensen-shannon", "Jensen-Shannon"),
    ("burrowsdelta", "Burrows' Delta"),
    ("minmax", "Min-Max"),
    ("cityblock", "Manhattan"),
    ("canberra", "Canberra"),
    ("cosine", "Cosine"),
]

# Gather full set of files
def gatherFilesFull(topStr, topNum, comparableTopStr, comparableNum, poetryNum):
    baseFolder = "output/full/"

    folders = [
        "",
        "data",
        "genre",
        "metric",
        "metric/extraInfo",
        "century",
        "century/extraInfo",
        "wordUse",
        "wordUse/extraInfo",
        "wordUse/grouping",
    ]
    createFolders(folders, baseFolder)

    # Get info for the data section
    getDataInfo(topStr, baseFolder)

    # Get info for approach section
    getWordUseInfo(topStr, baseFolder)

    # Get genre info
    getGenreInfo(topStr, baseFolder)
    # Gather 4up tsne charts for standard data and data normalized by genre
    # Grab this from the best metric
    subprocess.run("cp output/greek/no_split/%s/Authors/tSNE/info_no_labels_4Up.pdf %sgenre/groupings.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s/Books/tSNE/outliers4up.pdf %sgenre/bookOutliers.pdf" % (topStr, baseFolder), shell=True)

    # Get book tsne charts
    # Grab this from the best metric
    subprocess.run("cp output/greek/no_split/%s/Books/tSNE/tSNE_2D_no_labels.pdf %sgenre/books_tSNE_no_labels.pdf" % (topStr, baseFolder), shell=True)
    subprocess.run("cp output/greek/no_split/%s/Books/tSNE/tSNE_2D_labels.pdf %sgenre/books_tSNE_labels.pdf" % (topStr, baseFolder), shell=True)
    # To get a look at these, run python3 visualizeBooks

    # Get info for standard and normalized by poetry
    makeMLTable("output/greek/no_split/%s/dataPreds/" % (topStr), False, "%sgenre/ml_table.tex" % baseFolder)
    # makeMLTable("output/greek/no_split/%s+p/dataPreds/" % (topStr), False, "%sgenre/ml_table+p.tex" % baseFolder)

    # =========================

    # Get info for results section

    # -----------
    # Metric
    getMetricInfo(topStr, comparableTopStr, topNum, poetryNum, comparableNum, SIM_METRICS, baseFolder)

    makeMetricInternalTables("", topStr, SIM_METRICS, baseFolder)
    makeMetricInternalTables("", topStr + "+p", SIM_METRICS, baseFolder)

    # -----------
    # Century
    # Get information on century comparison
    getCenturyInfo(topStr, baseFolder)
    # Get pvalue + other regression information for charts that are + p
    greekPval = utils.getContent("output/greek/no_split/%s+p/jensen-shannon/metric/Authors/century_sims_genre_under_9_pslope.txt" % (topStr), False)
    englishPval = utils.getContent("output/english/no_split/%s+p/jensen-shannon/metric/Authors/century_sims_genre_pslope.txt" % (topStr), False)

    pvalOutput = []
    pvalOutput.append("Greek:")
    pvalOutput.append(greekPval)
    pvalOutput.append("English:")
    pvalOutput.append(englishPval)

    utils.safeWrite("%scentury/century_pvals+p.txt" % baseFolder, "\n".join(pvalOutput))

    # -------------------------
    # Grab this from the best metric
    subprocess.run("cp output/greek/no_split/%s/jensen-shannon/metric/Authors/sims.txt %swordUse/authorSims.txt" % (topStr, baseFolder), shell=True)

    fourCenturiesTables(topStr, SIM_METRICS, baseFolder)

    # get word usage charts and info
    getWordUsageInfo(topStr, baseFolder)

    # We didn't end up using grouping charts
    # groups = getWordGroupsRangeTest(topNum)
    # for g in groups:
    #     subprocess.run("cp output/greek/no_split/%s+p/textsOnlyTopWords/%d_group/images/groupingCompare.png %swordUse/grouping/%.2d.png" % (topStr, g, baseFolder, g), shell=True)

if __name__ == "__main__":
    #gatherFiles(mp.topStr, mp.topNum, mp.comparableTopStr, mp.comparableTopNum, mp.poetryNum)
    gatherFilesFull(mp.topStr, mp.topNum, mp.comparableTopStr, mp.comparableTopNum, mp.poetryNum)


from scipy.interpolate import interp1d
import os
from astropy.io import fits
import astropy.units as u
import h5py
import numpy as np
from astropy.constants import h, c

__all__ = ['Spectrum1D', 'ObservationArchive', 'nirspec_pixel_wavelengths',
           'Simulation']

bg_path = os.path.join(os.path.dirname(__file__), os.pardir, 'data', 'etc',
                       'image_detector.fits')

wl_path = os.path.join(os.path.dirname(__file__), os.pardir, 'data', 'etc',
                       'lineplot_wave_pix.fits')

outputs_dir_path = os.path.join(os.path.dirname(__file__), os.pardir, 'data',
                                'outputs')

JWST_aperture_area = 25 * u.m**2

def nirspec_pixel_wavelengths():
    return fits.getdata(wl_path)['WAVELENGTH'] * u.um

class Spectrum1D(object):
    def __init__(self, wavelength, flux, error=None, header=None, t_eff=None):
        self.wavelength = wavelength
        self.flux = flux
        self.error = error
        self.header = header
        self.t_eff = t_eff

    def plot(self, ax=None, **kwargs):
        import matplotlib.pyplot as plt
        if ax is None:
            ax = plt.gca()

        ax.plot(self.wavelength, self.flux, **kwargs)

        return ax

    @u.quantity_input(new_wavelengths=u.m)
    def interp_flux(self, new_wavelengths):
        f = interp1d(self.wavelength.value, self.flux, kind='linear',
                     bounds_error=False, fill_value=0)
        interped_fluxes = f(new_wavelengths)

        if hasattr(self.flux, 'unit') and self.flux.unit is not None:
            return interped_fluxes * self.flux.unit
        return interped_fluxes

    def __add__(self, other_spectrum):

        if not hasattr(other_spectrum, 'wavelength'):
            raise NotImplementedError()

        interp_flux = (np.interp(other_spectrum.wavelength.value,
                                 self.wavelength.value, self.flux.value) *
                       self.flux.unit)

        return Spectrum1D(other_spectrum.wavelength,
                          interp_flux + other_spectrum.flux,
                          header=self.header)

    def __rmul__(self, multiplier):
        if not np.isscalar(multiplier):
            raise NotImplementedError()

        return Spectrum1D(self.wavelength, multiplier * self.flux,
                          header=self.header)

    def n_photons(self, wavelengths, exp_time, J):
        """
        Estimate the number of photons received from a target with J magnitude
        ``J`` over exposure time ``exp_time``.

        Parameters
        ----------
        wavelengths : `~astropy.units.Quantity`
            Wavelengths to test
        exp_time : `~astropy.units.Quantity`
            Exposure time
        J : float
            J-band magnitude of the target

        Returns
        -------
        fluxes : `~numpy.ndarray`
            Counts that reach the telescope at each wavelength
        """
        if not hasattr(self.flux, 'unit'):
            raise NotImplementedError("Flux must have units")

        interped_fluxes = self.interp_flux(wavelengths)

        delta_lambda = np.nanmedian(np.diff(wavelengths))
        n_photons_template = (interped_fluxes * wavelengths / h / c *
                              JWST_aperture_area * delta_lambda *
                              exp_time).decompose().value

        relative_target_flux = 10**(0.4 * (float(self.header['J']) - J))

        return relative_target_flux * n_photons_template

class ObservationArchive(object):
    def __init__(self, fname, mode='r', outputs_dir=None):

        if outputs_dir is None:
            outputs_dir = outputs_dir_path

        self.path = os.path.join(outputs_dir, fname + '.hdf5')
        self.target_name = fname
        self.archive = None
        self.mode = mode

    def __enter__(self):
        self.archive = h5py.File(self.path, self.mode)

        planets = [i for i in list('bcdefgh') if i in self.archive]

        for planet in planets:
            simulations = []
            for iteration in self.archive[planet]:
                attrs = dict(self.archive[planet][iteration].attrs)
                simulations.append(Simulation(self.archive[planet][iteration],
                                              attrs=attrs,
                                              path="/{0}/{1}/".format(planet,
                                                                      iteration)))
            setattr(self, planet, simulations)

        return self

    def __exit__(self, *args):
        self.archive.close()

class Simulation(object):
    def __init__(self, observation, attrs=None, path=None):
        self.observation = observation
        self.attrs = attrs
        self.path = path

    @property
    def times(self):
        return self.observation['times'][:]

    @property
    def areas(self):
        return self.observation['spotted_area'][:]

    @property
    def spitzer_var(self):
        return self.observation['spitzer_var'][:]

    @property
    def flares(self):
        return self.observation['flares'][:]

    @property
    def fluxes(self):
        return self.observation['fluxes'][:]

    @property
    def spectra(self):
        return self.observation['spectra'][:]

    @property
    def samples_depth(self):
        return self.observation['samples/depth'][:]

    @property
    def samples_t0(self):
        return self.observation['samples/t0'][:]

    @property
    def samples_amp(self):
        return self.observation['samples/amp'][:]

    @property
    def samples_log_S0(self):
        return self.observation['samples/log_S0'][:]

    @property
    def samples_log_omega0(self):
        return self.observation['samples/log_omega0'][:]

    @property
    def samples_median(self):
        samples = (self.samples_log_S0, self.samples_log_omega0,
                   self.samples_amp, self.samples_depth, self.samples_t0)
        return np.array([np.median(s) for s in samples])

    def plot(self):
        wl = nirspec_pixel_wavelengths()

        fig, ax = plt.subplots(2, 5, figsize=(14, 6))
        ax[0, 0].plot(self.times, self.areas)
        ax[0, 0].set(xlabel='Time', ylabel='Spotted area')

        ax[0, 1].plot(self.times, self.fluxes)
        ax[0, 1].set(xlabel='Time', ylabel='Stellar flux')

        monochromatic_flares = np.sum(self.flares, axis=1)
        monochromatic_flares /= np.median(monochromatic_flares)
        ax[0, 2].plot(self.times, monochromatic_flares)
        ax[0, 2].set(xlabel='Time', ylabel='Flare flux')

        ax[0, 3].plot(self.times, self.spitzer_var)
        ax[0, 3].set(xlabel='Time', ylabel='Spitzer var.')

        ax[0, 4].plot(self.times, self.transit)
        ax[0, 4].set(xlabel='Time', ylabel='Transit')

        ax[1, 0].imshow(self.spectra, extent=[0.6, 5.3, 0, self.times.ptp()])
        ax[1, 0].set(title='Spectrophotometry', aspect=3/(self.times.ptp()),
                     xlabel='Wavelength [$\mu$m]', ylabel='Time [d]')

        short_bin = np.sum(self.spectra[:, :100], axis=1)
        mid_bin = np.sum(self.spectra[:, 100:200], axis=1)
        long_bin = np.sum(self.spectra[:, 200:], axis=1)
        ax[1, 1].plot(self.times, long_bin/long_bin.max(), ',', color='C0',
                      label=r'{0:.2f}-{1:.2f} $\mu$m'
                      .format(wl[0].value, wl[100].value))
        ax[1, 2].plot(self.times, mid_bin/mid_bin.max(), ',', color='C2',
                      label=r'{0:.2f}-{1:.2f} $\mu$m'
                      .format(wl[100].value, wl[200].value))
        ax[1, 3].plot(self.times, short_bin/short_bin.max(), ',', color='r',
                      label=r'{0:.2f}-{1:.2f} $\mu$m'
                      .format(wl[200].value, wl[-1].value))

        ax[1, 4].plot(self.times, np.sum(self.spectra, axis=1), ',')
        ax[1, 4].set(xlabel='Time', ylabel='NIRSpec counts',
                     title='Band-integrated')

        for axis in [ax[1, 1], ax[1, 2], ax[1, 3]]:
            axis.get_shared_y_axes().join(axis, ax[1, 1])
            axis.legend()
            axis.set_xlabel('Time')
            axis.set_ylabel('Flux')

        fig.tight_layout()

        return fig, ax

import numpy,from scipy,import astropy,from astropy


# -*- coding: utf-8 -*-
"""
Adobe Wide Gamut RGB Colourspace
================================

Defines the *Adobe Wide Gamut RGB* colourspace:

-   :attr:`colour.models.RGB_COLOURSPACE_ADOBE_WIDE_GAMUT_RGB`.

References
----------
-   :cite:`Wikipedia2004c` : Wikipedia. (2004). Wide-gamut RGB color space.
    Retrieved April 13, 2014, from
    http://en.wikipedia.org/wiki/Wide-gamut_RGB_color_space
"""

import numpy as np
from functools import partial

from colour.colorimetry import CCS_ILLUMINANTS
from colour.models.rgb import (
    RGB_Colourspace,
    gamma_function,
    normalised_primary_matrix,
)

__author__ = 'Colour Developers'
__copyright__ = 'Copyright (C) 2013-2021 - Colour Developers'
__license__ = 'New BSD License - https://opensource.org/licenses/BSD-3-Clause'
__maintainer__ = 'Colour Developers'
__email__ = 'colour-developers@colour-science.org'
__status__ = 'Production'

__all__ = [
    'PRIMARIES_ADOBE_WIDE_GAMUT_RGB',
    'WHITEPOINT_NAME_ADOBE_WIDE_GAMUT_RGB',
    'CCS_WHITEPOINT_ADOBE_WIDE_GAMUT_RGB',
    'MATRIX_ADOBE_WIDE_GAMUT_RGB_TO_XYZ',
    'MATRIX_XYZ_TO_ADOBE_WIDE_GAMUT_RGB',
    'RGB_COLOURSPACE_ADOBE_WIDE_GAMUT_RGB',
]

PRIMARIES_ADOBE_WIDE_GAMUT_RGB = np.array([
    [0.7347, 0.2653],
    [0.1152, 0.8264],
    [0.1566, 0.0177],
])
"""
*Adobe Wide Gamut RGB* colourspace primaries.

PRIMARIES_ADOBE_WIDE_GAMUT_RGB : ndarray, (3, 2)
"""

WHITEPOINT_NAME_ADOBE_WIDE_GAMUT_RGB = 'D50'
"""
*Adobe Wide Gamut RGB* colourspace whitepoint name.

WHITEPOINT_NAME_ADOBE_WIDE_GAMUT_RGB : str
"""

CCS_WHITEPOINT_ADOBE_WIDE_GAMUT_RGB = (
    CCS_ILLUMINANTS['CIE 1931 2 Degree Standard Observer'][
        WHITEPOINT_NAME_ADOBE_WIDE_GAMUT_RGB])
"""
*Adobe Wide Gamut RGB* colourspace whitepoint chromaticity coordinates.

CCS_WHITEPOINT_ADOBE_WIDE_GAMUT_RGB : ndarray
"""

MATRIX_ADOBE_WIDE_GAMUT_RGB_TO_XYZ = normalised_primary_matrix(
    PRIMARIES_ADOBE_WIDE_GAMUT_RGB, CCS_WHITEPOINT_ADOBE_WIDE_GAMUT_RGB)
"""
*Adobe Wide Gamut RGB* colourspace to *CIE XYZ* tristimulus values matrix.

MATRIX_ADOBE_WIDE_GAMUT_RGB_TO_XYZ : array_like, (3, 3)
"""

MATRIX_XYZ_TO_ADOBE_WIDE_GAMUT_RGB = np.linalg.inv(
    MATRIX_ADOBE_WIDE_GAMUT_RGB_TO_XYZ)
"""
*CIE XYZ* tristimulus values to *Adobe Wide Gamut RGB* colourspace matrix.

MATRIX_XYZ_TO_ADOBE_WIDE_GAMUT_RGB : array_like, (3, 3)
"""

RGB_COLOURSPACE_ADOBE_WIDE_GAMUT_RGB = RGB_Colourspace(
    'Adobe Wide Gamut RGB',
    PRIMARIES_ADOBE_WIDE_GAMUT_RGB,
    CCS_WHITEPOINT_ADOBE_WIDE_GAMUT_RGB,
    WHITEPOINT_NAME_ADOBE_WIDE_GAMUT_RGB,
    MATRIX_ADOBE_WIDE_GAMUT_RGB_TO_XYZ,
    MATRIX_XYZ_TO_ADOBE_WIDE_GAMUT_RGB,
    partial(gamma_function, exponent=1 / (563 / 256)),
    partial(gamma_function, exponent=563 / 256),
)
RGB_COLOURSPACE_ADOBE_WIDE_GAMUT_RGB.__doc__ = """
*Adobe Wide Gamut RGB* colourspace.

References
----------
:cite:`Wikipedia2004c`

RGB_COLOURSPACE_ADOBE_WIDE_GAMUT_RGB : RGB_Colourspace
"""


# -*- coding: utf-8 -*-

from __future__ import division, print_function

__all__ = ["Vetter"]

import os
import h5py
import transit
import numpy as np
from scipy.optimize import minimize
from scipy.signal import lombscargle

from .pipeline import Pipeline

def _nll_transit(p, system, lcs):
    system.set_vector(p)

    ll = 0.0
    for lc in lcs:
        mod = 1e3*(system.light_curve(lc.time, texp=lc.texp)-1.0)
        r = mod - lc.flux
        ll += lc.lnlike_eval(r)
    return -ll

def _nll_and_grad_transit(p, system, lcs):
    system.set_vector(p)

    ll = 0.0
    ll_grad = np.zeros_like(p)
    for lc in lcs:
        mod, grad = system.light_curve_gradient(lc.time, texp=lc.texp)
        r = 1e3 * (mod - 1.0) - lc.flux
        if np.any(~np.isfinite(r)):
            assert 0
        grad *= 1e3
        a, b = lc.grad_lnlike_eval(r, grad)
        ll += a
        ll_grad += b
    return -ll, ll_grad

def _ln_evidence_basic(lcs):
    ll = sum(lc.lnlike_eval(lc.flux) for lc in lcs)
    return ll, ll

def _ln_evidence_outlier(lcs, period, duration, t0):
    hp = 0.5 * period
    hd = 0.5 * duration

    lnlike = 0.0
    norm = 0.0
    depths = np.empty(len(lcs))
    ivars = np.empty(len(lcs))
    for i, lc in enumerate(lcs):
        r = lc.flux - lc.predict()
        m = np.fabs((lc.time - t0 + hp) % period - hp) < hd
        if not np.any(m):
            lnlike += lc.ll0
            continue
        tloc = lc.time[m][np.argmin(r[m])]

        def model(t):
            mod = np.zeros_like(t)
            mod[t == tloc] = -1.0
            return mod

        l0, depths[i], ivars[i] = lc.lnlike(model)
        lnlike += lc.ll0
        if ivars[i] > 0.0:
            lnlike += l0
            norm += 0.5 * (np.log(2*np.pi) - np.log(ivars[i]))

    return lnlike, lnlike + norm

def _ln_evidence_box0(lcs, period, duration, t0):
    def model(t, hp=0.5*period, hd=0.5*duration):
        mod = np.zeros_like(t)
        mod[np.fabs((t - t0 + hp) % period - hp) < hd] = -1.0
        return mod

    # Compute the evidence.
    lnlike = 0.0
    depths = np.empty(len(lcs))
    ivars = np.empty(len(lcs))
    for i, lc in enumerate(lcs):
        l0, depths[i], ivars[i] = lc.lnlike(model)
        lnlike += lc.ll0
        if ivars[i] > 0.0:
            lnlike += l0

    m = ivars > 0.0
    depths = depths[m]
    ivars = ivars[m]

    depth = np.sum(ivars * depths) / np.sum(ivars)
    ivar = np.sum(ivars)

    lnlike -= 0.5 * np.sum(depths**2 * ivars)
    lnlike += 0.5 * depth**2 * ivar
    lnlike += 0.5 * np.sum(np.log(ivars)) - 0.5*len(depths)*np.log(2*np.pi)

    return lnlike, lnlike - 0.5 * np.log(ivar) + 0.5 * np.log(2*np.pi)

def _ln_evidence_box(lcs, period, duration, t0, eps=1.123e-2):
    lnlike, f0 = _ln_evidence_box0(lcs, period, duration, t0)
    lnd2fdx2 = 0.0

    _, fp = _ln_evidence_box0(lcs, period + eps, duration, t0)
    _, fm = _ln_evidence_box0(lcs, period - eps, duration, t0)
    lnd2fdx2 += np.log(2*f0 - fp - fm) - 2 * np.log(eps) - np.log(2*np.pi)

    _, fp = _ln_evidence_box0(lcs, period, duration + eps, t0)
    _, fm = _ln_evidence_box0(lcs, period, duration - eps, t0)
    lnd2fdx2 += np.log(2*f0 - fp - fm) - 2 * np.log(eps) - np.log(2*np.pi)

    _, fp = _ln_evidence_box0(lcs, period, duration, t0 + eps)
    _, fm = _ln_evidence_box0(lcs, period, duration, t0 - eps)
    lnd2fdx2 += np.log(2*f0 - fp - fm) - 2 * np.log(eps) - np.log(2*np.pi)

    if not np.isfinite(lnd2fdx2):
        return lnlike, f0

    return lnlike, f0 - 0.5 * lnd2fdx2

def _ln_evidence_vee0(lcs, period, duration, t0):
    def model(t, hp=0.5*period, hd=0.5*duration):
        mod = np.zeros_like(t)
        dt = (t - t0 + hp) % period - hp
        m = np.fabs(dt) < hd
        mod[m] = np.abs(dt[m]) / hd - 1.0
        return mod

    # Compute the evidence.
    lnlike = 0.0
    depths = np.empty(len(lcs))
    ivars = np.empty(len(lcs))
    for i, lc in enumerate(lcs):
        l0, depths[i], ivars[i] = lc.lnlike(model)
        lnlike += lc.ll0
        if ivars[i] > 0.0:
            lnlike += l0

    m = ivars > 0.0
    depths = depths[m]
    ivars = ivars[m]

    depth = np.sum(ivars * depths) / np.sum(ivars)
    ivar = np.sum(ivars)

    lnlike -= 0.5 * np.sum(depths**2 * ivars)
    lnlike += 0.5 * depth**2 * ivar
    lnlike += 0.5 * np.sum(np.log(ivars)) - 0.5*len(depths)*np.log(2*np.pi)

    return lnlike, lnlike - 0.5 * np.log(ivar) + 0.5 * np.log(2*np.pi)

def _ln_evidence_vee(lcs, period, duration, t0, eps=1.123e-2):
    lnlike, f0 = _ln_evidence_vee0(lcs, period, duration, t0)
    lnd2fdx2 = 0.0

    _, fp = _ln_evidence_vee0(lcs, period + eps, duration, t0)
    _, fm = _ln_evidence_vee0(lcs, period - eps, duration, t0)
    lnd2fdx2 += np.log(2*f0 - fp - fm) - 2 * np.log(eps) - np.log(2*np.pi)

    _, fp = _ln_evidence_vee0(lcs, period, duration + eps, t0)
    _, fm = _ln_evidence_vee0(lcs, period, duration - eps, t0)
    lnd2fdx2 += np.log(2*f0 - fp - fm) - 2 * np.log(eps) - np.log(2*np.pi)

    _, fp = _ln_evidence_vee0(lcs, period, duration, t0 + eps)
    _, fm = _ln_evidence_vee0(lcs, period, duration, t0 - eps)
    lnd2fdx2 += np.log(2*f0 - fp - fm) - 2 * np.log(eps) - np.log(2*np.pi)

    if not np.isfinite(lnd2fdx2):
        return lnlike, f0

    return lnlike, f0 - 0.5 * lnd2fdx2

def _ln_evidence_transit(p, *args):
    h = 1.1234e-3
    x0 = np.array(p)
    lnd2fdx2 = np.empty_like(x0)
    f0 = _nll_transit(x0, *args)
    for i in range(len(x0)):
        x0[i] += h
        fp = _nll_transit(x0, *args)
        x0[i] -= 2*h
        fm = _nll_transit(x0, *args)
        x0[i] += h
        d = fp - 2*f0 + fm
        if d <= 0.0:
            lnd2fdx2[i] = np.inf
        else:
            lnd2fdx2[i] = np.log(fp - 2*f0 + fm)
    lnd2fdx2 -= 2*np.log(h)
    lnZ = -f0 - 0.5 * np.sum(lnd2fdx2) + 0.5 * len(x0) * np.log(2*np.pi)
    return -f0, lnZ

def _ln_evidence_period(lcs):
    if not hasattr(lcs[0], "detrend_flux"):
        return -np.inf, -np.inf
    x = np.concatenate([lc.time for lc in lcs])
    y = np.concatenate([lc.detrend_flux for lc in lcs])
    ivar = np.concatenate([1./lc.detrend_ferr**2 for lc in lcs])
    delta = np.max(x) - np.min(x)
    f = 2*np.pi*np.arange(max(2./24., 10./delta), 1.0 / 0.025, 0.1 / delta)
    p = lombscargle(x, y, f)
    omega = f[np.argmax(p)]
    AT = np.vstack([np.sin(omega*x), np.cos(omega*x), np.ones(len(x))])
    A = AT.T
    cov = np.dot(AT, A * ivar[:, None])
    w = np.linalg.solve(cov, np.dot(AT, y * ivar))

    def model(t):
        A = np.vstack([np.sin(omega*t), np.cos(omega*t)]).T
        return np.dot(A, w[:2])

    # Compute the evidence.
    lnlike = 0.0
    depths = np.empty(len(lcs))
    ivars = np.empty(len(lcs))
    for i, lc in enumerate(lcs):
        l0, depths[i], ivars[i] = lc.lnlike(model)
        lnlike += lc.ll0
        if ivars[i] > 0.0:
            lnlike += l0

    m = ivars > 0.0
    depths = depths[m]
    ivars = ivars[m]

    depth = np.sum(ivars * depths) / np.sum(ivars)
    ivar = np.sum(ivars)

    lnlike -= 0.5 * np.sum(depths**2 * ivars)
    lnlike += 0.5 * depth**2 * ivar
    lnlike += 0.5 * np.sum(np.log(ivars)) - 0.5*len(depths)*np.log(2*np.pi)

    return (lnlike, lnlike - 0.5 * np.log(ivar) + 0.5 * np.log(2*np.pi),
            2*np.pi/omega)

class Vetter(Pipeline):

    cache_ext = ".h5"
    query_parameters = dict(
        t0_rng=(0.2, False),
        period_rng=(0.1, False),
    )

    def get_result(self, query, parent_response):
        # Get the results from the pipeline so far.
        peaks = parent_response.peaks
        lcs = parent_response.model_light_curves

        # Save the initial flux values.
        flux0 = [np.array(lc.flux) for lc in lcs]

        # Loop over the peaks and compute the evidence for each one.
        for peak in peaks:
            # Set up the Keplerian fit.
            system = transit.SimpleSystem(
                period=peak["period"], t0=peak["t0"],
                ror=np.sqrt(1e-3*peak["depth"]),
                duration=peak["duration"],
                impact=0.5,
            )

            # Fit the transit model.
            p0 = system.get_vector()
            ln_period_rng = np.log((
                peak["period"] - query["period_rng"],
                peak["period"] + query["period_rng"],
            ))
            t0_rng = (
                peak["t0"] - query["t0_rng"],
                peak["t0"] + query["t0_rng"],
            )
            bounds = [(None, None), ln_period_rng, t0_rng,
                      (None, None), (None, None),
                      (-10.0, 10.0), (-10.0, 10.0)]
            result = minimize(_nll_and_grad_transit, p0, method="L-BFGS-B",
                              jac=True,
                              args=(system, lcs),
                              bounds=bounds)
            system.set_vector(result.x)

            # Compute the transit evidence.
            x = result.x
            peak["transit_q1"] = system.q1
            peak["transit_q2"] = system.q2
            peak["transit_period"] = system.period
            peak["transit_ror"] = system.ror
            peak["transit_duration"] = system.duration
            peak["transit_t0"] = system.t0
            peak["transit_b"] = system.impact
            peak["lnlike_transit"], peak["lnZ_transit"] = \
                _ln_evidence_transit(x, system, lcs)

            # Compute the evidence for the competing models.
            peak["lnlike_none"], peak["lnZ_none"] = _ln_evidence_basic(lcs)
            peak["lnlike_box"], peak["lnZ_box"] = _ln_evidence_box(
                lcs, peak["transit_period"], peak["transit_duration"],
                peak["transit_t0"])
            peak["lnlike_vee"], peak["lnZ_vee"] = _ln_evidence_vee(
                lcs, peak["transit_period"], peak["transit_duration"],
                peak["transit_t0"])
            peak["lnlike_outlier"], peak["lnZ_outlier"] = _ln_evidence_outlier(
                lcs, peak["transit_period"], peak["transit_duration"],
                peak["transit_t0"])

            peak["lnlike_period"], peak["lnZ_period"], peak["osc_period"] = \
                _ln_evidence_period(lcs)

            # Subtract the best fit transit model.
            for lc in lcs:
                mod = 1e3*(system.light_curve(lc.time, texp=lc.texp)-1.0)
                lc.flux -= mod

        # Return the fluxes to their original values.
        for lc, f in zip(lcs, flux0):
            lc.flux[:] = f[:]

        return dict(
            peaks=peaks,
        )

    def save_to_cache(self, fn, response):
        try:
            os.makedirs(os.path.dirname(fn))
        except os.error:
            pass

        # Parse the peaks into a structured array.
        peaks = response["peaks"]
        if len(peaks):
            dtype = [(k, np.float64) for k in sorted(peaks[0].keys())]
            peaks = [tuple(peak[k] for k, _ in dtype) for peak in peaks]
            peaks = np.array(peaks, dtype=dtype)

        with h5py.File(fn, "w") as f:
            f.create_dataset("peaks", data=peaks, compression="gzip")

    def load_from_cache(self, fn):
        if os.path.exists(fn):
            with h5py.File(fn, "r") as f:
                try:
                    peaks = [dict((k, peak[k]) for k in peak.dtype.names)
                             for peak in f["peaks"]]
                    return dict(
                        peaks=peaks,
                    )
                except KeyError:
                    pass
        return None


"""A 1D simple velocity gradient model to calculate lines with the LVG method

Original IDL model by Kees Dullemond, Python translation by Attila Juhasz
"""
from __future__ import absolute_import
from __future__ import print_function
import traceback
try:
    import numpy as np
except ImportError:
    np = None
    print(' Numpy cannot be imported ')
    print(' To use the python module of RADMC-3D you need to install Numpy')
    print(traceback.format_exc())

from .. natconst import *

def getModelDesc():
    """Provides a brief description of the model
    """

    return "Example model: A 1D simple velocity gradient model to calculate lines with the LVG method"

def getDefaultParams():
    """Provides default parameter values

    Returns a list whose elements are also lists with three elements:
    1) parameter name, 2) parameter value, 3) parameter description
    All three elements should be strings. The string of the parameter
    value will be directly written out to the parameter file if requested,
    and the value of the string expression will be evaluated and be put
    to radmc3dData.ppar. The third element contains the description of the
    parameter which will be written in the comment field of the line when
    a parameter file is written.
    """

    defpar = [['mstar', '1.0*ms', 'Mass of the star(s)'],
              ['pstar', '[0., 0., 0.]', 'Position of the star(s) (cartesian coordinates)'],
              ['rstar', '1.0*rs', 'Radius of the star(s)'],
              ['tstar', '1.0*ts', 'Effective temperature of the star(s)'],
              ['crd_sys', "'car'", 'Coordinate system used (car/sph)'],
              ['nx', '10', 'Number of grid points in the first dimension'],
              ['ny', '1', 'Number of grid points in the second dimension'],
              ['nz', '1', 'Number of grid points in the third dimension'],
              ['xbound', '[-1000.0*au, 1000.0*au]', 'Boundaries for the x-grid'],
              ['ybound', '[-1000.0*au/nx, 1000.0*au/nx]', 'Boundaries for the y-grid'],
              ['zbound', '[-1000.0*au/nx, 1000.0*au/nx]', 'Boundaries for the z-grid'],
              ['nw', '[20,100,30]', 'Number of points in the wavelength grid'],
              ['wbound', '[0.1, 7., 25., 1e4]', 'Boundaries for the wavelength grid'],
              ['dustkappa_ext', "['silicate']", 'Dust opacity file name extension'],
              ['nphot', '1000000', 'Number of photons in the thermal Monte Carlo simulation'],
              ['lines_mode', '3', ''],
              ['scattering_mode_max', '1', '0 - no scattering, 1 - isotropic scattering, 2 - anizotropic scattering'],
              ['gasspec_mol_name', "['co']", ''],
              ['gasspec_mol_abun', '[1e-4]', ''],
              ['gasspec_mol_dbase_type', "['leiden']", ''],
              ['gasspec_colpart_name', "['h2']", ''],
              ['gasspec_colpart_abun', '[1e0]', ''],
              ['gasspec_vturb', '1e5', 'Microturbulent linewidth'],
              ['abun_h2', '0.5', ''],
              ['abun_he', '0.1', ''],
              ['nh2', '1e5', ''],
              ['temp0', '30.', ''],
              ['tdust0', '30.', ''],
              ['dusttogas', '1e-2', ''],
              ['dvdau', '1e-2*1e5', '']]

    return defpar

def getGasTemperature(grid=None, ppar=None):
    """Calculates the gas temperature

    Parameters
    ----------
    grid : radmc3dGrid
            An instance of the radmc3dGrid class containing the spatial and wavelength grid

    ppar : dictionary
            Dictionary containing all parameters of the model

    Returns
    -------
    Returns the gas temperature in K
    """

    tgas = np.zeros([grid.nx, grid.ny, grid.nz], dtype=np.float64) + ppar['temp0']
    return tgas

def getDustTemperature(grid=None, ppar=None):
    """Calculates/sets the dust temperature

    Parameters
    ----------
    grid : radmc3dGrid
            An instance of the radmc3dGrid class containing the spatial and wavelength grid

    ppar : dictionary
            Dictionary containing all parameters of the model

    Returns
    -------
    Returns the dust temperature in K

    """

    tdust = np.zeros([grid.nx, grid.ny, grid.nz, 1], dtype=np.float64) + ppar['tdust0']
    return tdust

def getGasAbundance(grid=None, ppar=None, ispec=''):
    """Calculates/sets the molecular abundance of species ispec
    The number density of a molecule is rhogas * abun

    Parameters
    ----------
    grid  : radmc3dGrid
            An instance of the radmc3dGrid class containing the spatial and wavelength grid

    ppar  : dictionary
            Dictionary containing all parameters of the model

    ispec : str
            The name of the gas species whose abundance should be calculated

    Returns
    -------
    Returns the abundance as an ndarray
    """
    # Mass of gas per H2-molecule
    # mgas    = mp*(2.0*ppar['abun_h2']+4*ppar['abun_he'])/ppar['abun_h2']

    if ispec in ppar['gasspec_mol_name']:
        gasabun = np.zeros([grid.nx, grid.ny, grid.nz], dtype=np.float64)
        ind = ppar['gasspec_mol_name'].index(ispec)
        gasabun[:, :, :] = ppar['gasspec_mol_abun'][ind]  # /mgas

    elif ispec in ppar['gasspec_colpart_name']:
        gasabun = np.zeros([grid.nx, grid.ny, grid.nz], dtype=np.float64)
        ind = ppar['gasspec_colpart_name'].index(ispec)
        gasabun[:, :, :] = ppar['gasspec_colpart_abun'][ind]  # /mgas
    else:
        raise ValueError(' The abundance of "'+ispec+'" is not specified in the parameter file')

    return gasabun

def getGasDensity(grid=None, ppar=None):
    """Calculates the total gas density distribution

    Parameters
    ----------
    grid : radmc3dGrid
            An instance of the radmc3dGrid class containing the spatial and wavelength grid

    ppar : dictionary
            Dictionary containing all parameters of the model

    Returns
    -------
    Returns the gas volume density in g/cm^3
    """
    # Mass of gas per H2-molecule
    mgas = mp*(2.0*ppar['abun_h2']+4*ppar['abun_he'])/ppar['abun_h2']
    rhogas = np.zeros([grid.nx, grid.ny, grid.nz], dtype=np.float64) + ppar['nh2'] * mgas
    return rhogas

def getDustDensity(grid=None, ppar=None):
    """Calculates the dust density distribution

    Parameters
    ----------
    grid : radmc3dGrid
            An instance of the radmc3dGrid class containing the spatial and wavelength grid

    ppar : dictionary
            Dictionary containing all parameters of the model

    Returns
    -------
    Returns the dust volume density in g/cm^3
    """
    rhogas = getGasDensity(grid=grid, ppar=ppar)
    rhodust = np.zeros([grid.nx, grid.ny, grid.nz, 1], dtype=np.float64)
    rhodust[:, :, :, 0] = rhogas * ppar['dusttogas']
    return rhodust

def getVTurb(grid=None, ppar=None):
    """Calculates/sets the turbulent velocity field

    Parameters
    ----------
    grid : radmc3dGrid
            An instance of the radmc3dGrid class containing the spatial and wavelength grid

    ppar : dictionary
            Dictionary containing all parameters of the model

    Returns
    -------
    Returns the turbulent velocity in cm/s
    """
    vturb = np.zeros([grid.nx, grid.ny, grid.nz], dtype=np.float64) + ppar['gasspec_vturb']
    return vturb

def getVelocity(grid=None, ppar=None):
    """Calculates/sets the gas velocity field

    Parameters
    ----------
    grid : radmc3dGrid
            An instance of the radmc3dGrid class containing the spatial and wavelength grid

    ppar : dictionary
            Dictionary containing all parameters of the model

    Returns
    -------
    Returns the turbulent velocity in cm/s
    """
    vel = np.zeros([grid.nx, grid.ny, grid.nz, 3], dtype=np.float64)
    for ix in range(grid.nx):
        vel[ix, :, :, 0] = ppar['dvdau']*grid.x[ix]/au

    return vel


#! /usr/bin/env python

import numpy as np

def framewiseEval(resTracks, groundTruthTracks):
    GTTracks = groundTruthTracks
    # aligning the tracks:
    Tref = GTTracks.shape[0]
    refTimes = GTTracks[:,0]
    estTimes = resTracks[:,0]
    reference = np.zeros([Tref, GTTracks.shape[1]-1])
    reference[:,:] = GTTracks[:,1:]
    print reference.shape

    nTracks = resTracks.shape[1] - 1
    newResTracks = np.zeros([Tref, nTracks])
    for i in range(Tref):
        indexMin = np.argmin(np.abs(refTimes[i] - estTimes))
        newResTracks[i,:] = resTracks[indexMin,1:]

    resTracks0 = np.copy(newResTracks)
    GTTracks0 = np.copy(reference)

    # Consider negative frequency values as 0:
    newResTracks[newResTracks < 0] = 0
    # Setting same number of tracks:
    ## nTracks = np.maximum(nTracks, reference.shape[1])

    print newResTracks.shape

    # setting the boundaries for tolerance of error:
    reference_low = 0.97 * reference
    reference_hig = 1.03 * reference

    # values for score computation
    TP = np.zeros(Tref) # True Positives
    FN = np.zeros(Tref) # False Negatives
    TN = np.zeros(Tref) # True Negatives

    for n in range(Tref):
        for track in range(nTracks):
            for track2 in range(reference.shape[1]):
                if reference_low[n, track2] < newResTracks[n, track] and \
                       newResTracks[n, track] < reference_hig[n, track2] and \
                       reference[n, track2] > 0:
                    TP[n] += 1
                    # inhibiting the TP match
                    newResTracks[n, track] = -1
                    reference_low[n, track2] = -1
                    reference_hig[n, track2] = -1
                    reference[n, track2] = -1

    FP = np.sum(newResTracks>0) # True Negatives
    # True Negatives - not sure if this quantity is right
    TN = np.minimum(np.sum(reference<=0,
                           axis=1),
                    np.sum(np.array(newResTracks==0) + \
                           np.array(newResTracks<-1),
                           axis=1))
    Ncorr = np.copy(TP) # number of correctly estimated pitches for each frames (MIREX 2007 criteria)
    TP = np.sum(TP) # total TP
    FN = np.maximum(0, np.sum(np.array(newResTracks==0) + \
                              np.array(newResTracks<-1),
                              axis=1) -\
                    np.sum(reference<=0,
                           axis=1)) # False Negatives, per frame
    TN = np.sum(TN) # total TN
    FN = np.sum(FN) # total FN

    # dictionary containing all of the desired values
    resStruct = {}
    resStruct['TP'] = TP
    resStruct['TN'] = TN
    resStruct['FP'] = FP
    resStruct['FN'] = FN
    resStruct['Precision'] = 100.0 * TP / np.double(TP+FP)
    resStruct['Recall'] = 100.0 * TP / np.double(np.sum(GTTracks0>0))# np.sum(reference!=0, dtype=np.double)
    resStruct['FMeasure'] = 2.0 * resStruct['Precision'] * resStruct['Recall'] / (resStruct['Precision'] + resStruct['Recall'])
    resStruct['Accuracy'] = 100.0 * (TP + TN) / np.double(TP + TN + FN) # recall, including silence in the classes to retrieve.

    # Additional MIREX 2007 evaluation criteria:
    Nref = np.sum(GTTracks0>0, axis=1, dtype=np.double)
    Nsys = np.sum(resTracks0>0, axis=1, dtype=np.double)

    resStruct['AccuracyMirex07'] = 100.0 * TP / np.double(TP+FN+FP)

    NrefTotal = np.sum(Nref, dtype=np.double)
    resStruct['Mirex07Etot'] = 100.0 * \
                               np.sum(np.maximum(Nref,Nsys) - \
                                      Ncorr) / \
                                      NrefTotal
    resStruct['Mirex07Esub'] = 100.0 * \
                               np.sum(np.minimum(Nref,Nsys) - \
                                      Ncorr) / \
                                      NrefTotal
    resStruct['Mirex07Emis'] = 100.0 * \
                               np.sum(np.maximum(0, Nref-Nsys)) / \
                                      NrefTotal
    resStruct['Mirex07Efal'] = 100.0 * \
                               np.sum(np.maximum(0, Nsys-Nref)) / \
                                      NrefTotal

    return resStruct, GTTracks0, resTracks0

def framewiseMono(resTracks, groundTruthTracks):
    GTTracks = groundTruthTracks
    # aligning the tracks:
    Tref = GTTracks.shape[0]
    refTimes = GTTracks[:,0]
    estTimes = resTracks[:,0]
    reference = GTTracks[:,1]

    newResTracks = np.zeros(Tref)
    for i in range(Tref):
        indexMin = np.argmin(np.abs(refTimes[i] - estTimes))
        newResTracks[i] = resTracks[indexMin,1]

    resTracks0 = np.copy(newResTracks)
    GTTracks0 = np.copy(reference)

    # Consider negative frequency values as 0:
    newResTracks[newResTracks < 0] = 0

    errorInSemitone = np.zeros(Tref)
    for n in range(Tref):
        if reference[n] > 0 and newResTracks[n] != 0:
            errorInSemitone[n] = 12.0 * np.log2(np.abs(newResTracks[n])/np.double(reference[n]))

    return errorInSemitone

def compareFilesByName(fileGT, fileRes):
    GTTracks = np.loadtxt(fileGT)
    resTracks = np.loadtxt(fileRes)

    errorInSemitone = framewiseMono(resTracks, GTTracks)
    resStruct, GTTracks0, resTracks0 = framewiseEval(resTracks, GTTracks)

    return errorInSemitone, resStruct, GTTracks0, resTracks0