#Caroline Parsons
#parsonsc
#Diego Hagans
#2019-4-18
#Final Project

"""
Space Invaders Final Project Code.
"""
import arcade
import random
import os

#Set the screen size and title of the screen
screen_width = 700
screen_height = 600
screen_title = "Space Invaders"
column_spacing = 60
row_spacing = 50
left_margin = 70
bottom_margin = 250
explosion_duration = 13

#Class directly adapted from example code in Arcade Academy:
#http://arcade.academy/examples/sprite_explosion.html#sprite-explosion
#Small adaptations and edits were made to fit the design elements of the project
class Explosion(arcade.Sprite):
    """
    Creates an explosion animation when an enemy is killed by the user.
    """

    def __init__(self, texture_list):
        super().__init__("explosions.png")

        # Start at the first frame
        self.current = 0
        self.textures = texture_list

    def update(self):

        # Update to the next frame of the animation.
        self.current = self.current + 1
        if self.current < len(self.textures):
            self.set_texture(self.current)
        else:
            self.kill()

#Class directly adapted from example code in Arcade Academy:
#http://arcade.academy/examples/gui_text_button.html#gui-text-button
#Small adaptations and edits were made to fit the design elements of the project
#Docstrings were added to follow the dtyle guide
class TextButton:
    """
    Text-based button class for GUI.
    """
    #Initialization of variables
    def __init__(self, center_x, center_y, width, height, text, font_size=18,
                 font_face="Arial", face_color=arcade.color.LIGHT_GRAY,
                 highlight_color=arcade.color.WHITE,
                 shadow_color=arcade.color.GRAY,
                 button_height=2):
        """
        Initializes class variables.
        """
        self.center_x = center_x
        self.center_y = center_y
        self.width = width
        self.height = height
        self.text = text
        self.font_size = font_size
        self.font_face = font_face
        self.pressed = False
        self.face_color = face_color
        self.highlight_color = highlight_color
        self.shadow_color = shadow_color
        self.button_height = button_height

    def draw(self):
        """
        Draw the button.
        """
        arcade.draw_rectangle_filled(self.center_x, self.center_y, self.width,
                                     self.height, self.face_color)
        if not self.pressed:
            color = self.shadow_color
        else:
            color = self.highlight_color
        #Bottom horizontal
        arcade.draw_line(self.center_x - self.width / 2,
                        self.center_y - self.height / 2,
                        self.center_x + self.width / 2,
                        self.center_y - self.height / 2,
                         color, self.button_height)
        #Right vertical
        arcade.draw_line(self.center_x + self.width / 2,
                        self.center_y - self.height / 2,
                        self.center_x + self.width / 2,
                        self.center_y + self.height / 2,
                         color, self.button_height)
        if not self.pressed:
            color = self.highlight_color
        else:
            color = self.shadow_color
        #Top horizontal
        arcade.draw_line(self.center_x - self.width / 2,
                        self.center_y + self.height / 2,
                        self.center_x + self.width / 2,
                        self.center_y + self.height / 2,
                         color, self.button_height)
        #Left vertical
        arcade.draw_line(self.center_x - self.width / 2,
                        self.center_y - self.height / 2,
                        self.center_x - self.width / 2,
                        self.center_y + self.height / 2,
                         color, self.button_height)
        x = self.center_x
        y = self.center_y
        if not self.pressed:
            x -= self.button_height
            y += self.button_height
        arcade.draw_text(self.text, x, y,
                         arcade.color.BLACK, font_size=self.font_size,
                         width=self.width, align="center",
                         anchor_x="center", anchor_y="center")

    def on_press(self):
        """
        Check to see if button is pressed.
        """
        self.pressed = True

    def on_release(self):
        """
        Check to see if button is released.
        """
        self.pressed = False

def check_mouse_press_for_buttons(x, y, button_list):
    """
    Given an x, y, see if we need to register any button clicks.

    Args:
        x(float): X position of mouse.
        y(float): Y position of mouse.
        button_list(list): List of buttons.
    """
    for button in button_list:
        if x > button.center_x + button.width / 2:
            continue
        if x < button.center_x - button.width / 2:
            continue
        if y > button.center_y + button.height / 2:
            continue
        if y < button.center_y - button.height / 2:
            continue
        button.on_press()

def check_mouse_release_for_buttons(x, y, button_list):
    """
    If a mouse button has been released, see if we need to process
    any release events.
    """
    for button in button_list:
        if button.pressed:
            button.on_release()

class StartTextButton(TextButton):
    """
    Text-based button for GUI, resumes game from parent button class.
    """
    #Initialization of variables
    def __init__(self, center_x, center_y, action_function):
        super().__init__(center_x, center_y, 100, 40, "RESUME", 14, "Arial")
        """
        Initializes class variables.
        """
        self.action_function = action_function

    def on_release(self):
        super().on_release()
        self.action_function()

class StopTextButton(TextButton):
    """
    Text-based button for GUI, pauses game from parent button class.
    """
    #Initialization of variables
    def __init__(self, center_x, center_y, action_function):
        super().__init__(center_x, center_y, 100, 40, "PAUSE", 14, "Arial")
        """
        Initializes class variables.
        """
        self.action_function = action_function

    def on_release(self):
        super().on_release()
        self.action_function()
#END OF CODE DIRECTLY USED FROM ARCADE ACADEMY FOR A GUI INTERFACE
###############################################################################

###############################################################################
#Start of code by Caroline Parsons
###############################################################################
class Sprite(arcade.Sprite):
    """
    Sprite class for Space Invaders"
    """
    def __init__(self):
        self.player_list = None
        self.enemy_list = None
        self.enemy_list2 = None
        self.bullet_list = None
        self.player = None
        self.death_list = None

class Game(arcade.Window):
    """
    Main game class for Space Invaders.
    """
    #Initialization of variables
    def __init__(self, width, height, title):
        """
        Initializes class variables.

        Args:
            width(int): Width of screen.
            height(int): Height of screen.
            title(string): Title of game.
        """
        super().__init__(width, height, title)
        #If neccessary in the future make the mouse not visible on the screen
        #self.set_mouse_visible(False)
        self.pause = False
        self.background = None
        self.frame_count = 0
        self.score = 0
        self.explosion_list = []

        for x in range(explosion_duration):
            texture = "explosions.png"
            self.explosion_list.append(arcade.load_texture(texture))

    #Setup function
    def setup(self):
        """
        Setup the game to render.
        """
        #Set up the background of the screen
        self.background = arcade.load_texture("backgroundGScreen.png")
        #Technique from:
        #http://arcade.academy/arcade.html#module-arcade.sprite_list
        Sprite.player_list = arcade.SpriteList()
        Sprite.enemy_list = arcade.SpriteList()
        self.enemy_list2 = arcade.SpriteList()
        Sprite.bullet_list = arcade.SpriteList()
        Sprite.enemy_bullet_list = arcade.SpriteList()
        self.explosions_list = arcade.SpriteList()
        #Set the score to 0
        self.score = 0
        self.counts = 0
        #Add user ship
        Sprite.player = arcade.Sprite("ship.png", 0.05)
        Sprite.player_list.append(Sprite.player)
         # Create our on-screen GUI buttons
        self.button_list = []

        play_button = StartTextButton(60, 570, self.resume_program)
        self.button_list.append(play_button)

        quit_button = StopTextButton(60, 515, self.pause_program)
        self.button_list.append(quit_button)
        #Add enemy aliens with a for loop
        #Technique from:
        #http://arcade.academy/examples/nested_loops_box.html#nested-loops-box
        #For loop over the x
        for row in range(3):
            #For loop over the y
            for column in range(10):
                #Calculate the location
                x = column * column_spacing + left_margin
                y = row * row_spacing + bottom_margin
                #make the alien sprite
                enemy = arcade.Sprite("alien.png", 0.20)
                enemy.center_x = x
                enemy.center_y = y
                Sprite.enemy_list.append(enemy)

        for row in range(1):
            #For loop over the y
            for column in range(10):
                #Calculate the location
                x = column * column_spacing + left_margin
                y = row * row_spacing + bottom_margin
                #make the alien sprite
                enemy = arcade.Sprite("ye.png", 0.125)
                enemy.center_x = x
                enemy.center_y = 400
                Sprite.enemy_list.append(enemy)

        #Play background song
        #arcade.play_sound(arcade.sound.load_sound("mo_bamba.wav"))
        #arcade.play_sound(arcade.sound.load_sound("hbfs.wav"))
        arcade.play_sound(arcade.sound.load_sound("digi.wav"))

    #on_draw is already estblished in the acrade package
    #In the arcade.Window class
    #Technique from:
    #http://arcade.academy/arcade.html#module-arcade.draw_commands
    def on_draw(self):
        """
        Render the screen before drawing.
        """
        #Gets ready to start the render
        #Must be called before drawing on the screen
        #Technique from:
        #http://arcade.academy/arcade.html#module-arcade.window_commands
        arcade.start_render()
        #Create the background for the game
        arcade.draw_texture_rectangle(screen_width // 2, screen_height // 2,
                            screen_width, screen_height, self.background)
        # Draw the buttons
        for button in self.button_list:
            button.draw()
        #Create the Sprites
        Sprite.enemy_list.draw()
        self.enemy_list2.draw()
        Sprite.bullet_list.draw()
        Sprite.player_list.draw()
        Sprite.enemy_bullet_list.draw()
        self.explosions_list.draw()

        #Set the score on the screen
        score_txt = f"SCORE: {self.score}"
        #Set the number of shots on the screen
        number_shots = f"SHOTS: {self.counts}"
        arcade.draw_text(score_txt, 300, 550, arcade.color.WHITE, 20)
        arcade.draw_text(number_shots, 550, 550, arcade.color.WHITE, 20)

    def on_mouse_motion(self, x, y, delta_x, delta_y):
        """
        Called whenever the mouse moves.

        Args:
            x(float): X position of mouse.
            y(float): Y position of mouse.
            delta_x(float): Change in x.
            delta_y(float): Change in y.
        """
        Sprite.player.center_x = x
        Sprite.player.center_y = 20

    def on_mouse_press(self, x, y, button, modifiers):
        """
        Shoots bullet on press of the mouse.

        Args:
            x(foat): X position of the mouse
            y(float): Y position of the mouse
            button(int): Mouse hit
            modifiers(int): Shift/click, ctrl/click, etc.
        """
        check_mouse_press_for_buttons(x, y, self.button_list)
        #Make the shots fired noise when pressed
        arcade.sound.play_sound(arcade.sound.load_sound("shoot.wav"))
        ship_bullet = arcade.Sprite("green_bullet.png", 0.03)
        ship_bullet.change_y = 8
        #Make the center of the bullet equal to the center of the ship
        ship_bullet.center_x = Sprite.player.center_x
        ship_bullet.bottom = Sprite.player.top
        Sprite.bullet_list.append(ship_bullet)
        #keep count of each bullet
        self.counts += 1

    def on_mouse_release(self, x, y, button, key_modifiers):
        """
        Called when a user releases a mouse button.
        """
        check_mouse_release_for_buttons(x, y, self.button_list)

    def pause_program(self):
        """
        Pauses game.
        """
        self.pause = True

    def resume_program(self):
        """
        Resumes game.
        """
        self.pause = False

    def update(self, delta_time):
        """
        Update/play the game once it is setup and rendered.

        Args:
            #http://arcade.academy/arcade.html
            delta_time(float): Moves everything, collision checks
        """
        if self.pause:
            return
        #Loop through the enemies
        for enemy in Sprite.enemy_list:
            #http://arcade.academy/arcade.html#module-arcade.sprite_list
            # Have a random 1 in 2000 change of shooting each frame
            if random.randrange(2000) == 0:
                #Set the bullet sprite to a picture and designate the size
                enemy_bullet = arcade.Sprite("bullet.png", 0.07)
                #Shoot the bullet from the center of the enemy
                enemy_bullet.center_x = enemy.center_x
                enemy_bullet.top = enemy.bottom
                enemy_bullet.change_y = -8
                Sprite.enemy_bullet_list.append(enemy_bullet)
        #Remove bullet when it goes off the screen
        for bullet in Sprite.enemy_bullet_list:
            if bullet.top <0:
                bullet.kill()
        #Update the bullet lists
        Sprite.enemy_bullet_list.update()
        self.explosions_list.update()
        Sprite.bullet_list.update()
        #Loop through each bullet
        for bullet in Sprite.bullet_list:
            #Check each bullet for a collision with an enemy
            hits_list = arcade.check_for_collision_with_list(bullet,
                        Sprite.enemy_list)
            #If there is a hit
            if len(hits_list) > 0:
            	#Have the explosion png appear in the location of the enemy
                explosion = Explosion(self.explosion_list)
                explosion.center_x = hits_list[0].center_x
                explosion.center_y = hits_list[0].center_y
                self.explosions_list.append(explosion)
                #Get the bullet of the screen
                bullet.kill()
            # If the bullet flies off-screen, remove it.
            if bullet.bottom > screen_height:
                #Get the bullet of the screen
                bullet.kill()
            #For every enemy in the hit list
            for enemy in hits_list:
                #Kill the enemy
                enemy.kill()
                #Increase the score
                self.score += 10
                #Hit Sound
                arcade.sound.play_sound(arcade.sound.load_sound("inKilled.wav"))

        for bullet in Sprite.bullet_list:
            #Check each bullet for a collision with an enemy
            hits_list = arcade.check_for_collision_with_list(bullet,
                        self.enemy_list2)

            #If there is a hit
            if len(hits_list) > 0:
                #Get the bullet of the screen
                bullet.kill()
            #For every enemy we hit

            for enemy2 in hits_list:
                #Kill the enemy
                enemy2.kill()
                #Increase the score
                self.score += 10

                #Hit Sound
                arcade.sound.play_sound(arcade.sound.load_sound("inKilled.wav"))
            # If the bullet flies off-screen, remove it.
            if bullet.bottom > screen_height:
                #Get the bullet of the screen
                bullet.kill()

#Main fuction
def main():
    """
    Main method to run game.
    """
    window = Game(screen_width, screen_height, screen_title)
    window.setup()
    arcade.run()

if __name__ == "__main__":
    main()


'''
Contains functions required by sed_main.ipynb.

Created for the purpose of sound event detection.
by Reynaldo Vazquez (rexvaz.com)
version June 5, 2019

Licensed under CC BY 4.0 (http://creativecommons.org/licenses/by/4.0/)
'''

import os, sys, io, h5py, IPython
import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
import soundfile, librosa
import librosa.display
from scipy.io import wavfile
from tabulate import tabulate
from io import BytesIO
from zipfile import ZipFile
from pydub import AudioSegment
AudioSegment.converter =  '/anaconda3/ffmpeg'

from __main__ import *

mpl.rcParams.update({'text.color' : "#4d4d4d", 'axes.labelcolor' : "#4d4d4d",
                     'font.size': 13, 'xtick.color':'#4d4d4d','ytick.color':'#4d4d4d',
                     'axes.spines.top': False, 'axes.spines.right': False,
                     'axes.edgecolor': '#a6a6a6', 'axes.linewidth':1.0, 'figure.figsize':[8, 4]})

def get_wav_info_from_zip(path_in_archive, archive, target_fs = 44100):
    '''
    Extracts data from audio file in .wav format contained in a zip
    archive. Standardizes to 1 channel and to a sample rate of 44.1KHz.
    '''
    audio_zip = archive.read(path_in_archive)
    bytes_io = BytesIO(audio_zip)
    rate, data = wavfile.read(bytes_io)
    if data.ndim > 1:
        data = np.mean(data, axis=1)
    if target_fs is not None and rate != target_fs:
        data = librosa.resample(data, orig_sr=rate, target_sr=target_fs)
        rate = target_fs
    return rate, data

def log_mel_features(audio_data, sr, mel_power):
    '''
    Extracts the log mel features from audio data.
    '''
    S = librosa.feature.melspectrogram(y=audio_data, sr=sr)
    lmf = librosa.power_to_db(S**mel_power, ref=np.max)
    return lmf

def plot_log_mel(log_mel, mel_id=None, sr = 44100):
    '''
    Plots the log mel spectrogram
    '''
    plt.box(False)
    librosa.display.specshow(log_mel, cmap='Blues_r',
                             x_axis='time', sr = sr)
    cb = plt.colorbar(format='%+2.0f dB')
    cb.outline.set_visible(False)
    plt.title('log mel spectrogram')
    plt.locator_params(axis='x', nbins=10)
    plt.xlim(0,10)
    return

def show_audio_info(archive, track_num, meta_segments, prefix):
    '''
    Displays the meta information, sound waves, mel spectrogram and play widget,
    of an audio file  contained in a zip archive.
    '''
    print('Audio clip id: ', str(track_num))
    path_in_archive = prefix+str(track_num)+'.wav'
    rate, data = get_wav_info_from_zip(path_in_archive, archive)
    logmel = log_mel_features(data, rate, mel_power=0.5)
    audio_zip = archive.read(path_in_archive)
    plt.figure(figsize=(17, 3))
    plt.subplot(1,2,2)
    plot_log_mel(logmel, track_num, rate)
    plt.subplot(1,2,1)
    librosa.display.waveplot(data, sr=44100)
    plt.title('sound waves')
    plt.locator_params(axis='x', nbins=10)
    plt.show()

    events = meta_segments[meta_segments['trackID'] == track_num].sort_values('event_start')
    if len(events) == 0:
        print('No audio events in this track, only background')
    else:
        table_keys=['event type', 'start time (sec)', 'end time (sec)']
        event_df = pd.concat([events['event'], round(events['event_start']/1000,2),
                 round(events['event_end']/1000,2)], axis=1, keys=table_keys)
        print(tabulate(event_df, headers='keys', showindex=False))

    bytes_io = BytesIO(audio_zip)
    old = AudioSegment.from_wav(bytes_io)
    old.export('media/temp_mp3.mp3', format='mp3')
    return IPython.display.Audio('media/temp_mp3.mp3')

def dataset_totals(meta_track, dataset_type):
    '''
    Calculates prints audio distribution counts using meta data
    '''
    number_of_tracks = meta_track.shape[0]
    num_positive_tracks = sum(meta_track['class_dummy'] == 1)
    num_non_positive_tracks = sum(meta_track['class_dummy'] == 0)
    print('Totals for ', dataset_type, ' dataset' )
    print('number of tracks:', number_of_tracks)
    print('number of tracks with positive event occurrence:', num_positive_tracks)
    print('number of tracks without positive event occurrence:', num_non_positive_tracks, '\n')
    return number_of_tracks, num_positive_tracks, num_non_positive_tracks

def make_bar_plot(x_names, y_values, color, width = 0.8):
    '''
    Make a bar plot
    '''
    xpos = np.arange(len(x_names))
    plt.bar(xpos, y_values, align='center', alpha=0.7, color = color, width=width)
    plt.xticks(xpos, x_names)

def tracks_content_info(meta_track, meta_segments, dataset_type, track_dur, event_type):
    '''
    Calculates the audio content on the dataset using meta data
    '''
    total_audio_time_sec = meta_track.shape[0]*track_dur
    total_audio_time_min = round(total_audio_time_sec/60, 2)

    meta_segments['event_dur'] = meta_segments['event_end'] - meta_segments['event_start']
    total_time_event_ms = sum(meta_segments['event_dur'][meta_segments['event'] == event_type])
    total_time_event_min = round(total_time_event_ms/(60*1000), 2)

    total_time_neg_ms = sum(meta_segments['event_dur'][meta_segments['event'] != event_type])
    total_time_neg_min = round(total_time_neg_ms/(60*1000), 2)

    total_time_bgd_only_min = round(total_audio_time_min - total_time_event_min - total_time_neg_min,2)
    print('Audio contents of', dataset_type, ' dataset' )
    print('Total audio minutes:', total_audio_time_min)
    print('Total glassbreak minutes:', total_time_event_min)
    print('Total gunshot minutes:', total_time_neg_min)
    print('Total background only minutes:', total_time_bgd_only_min, '\n')
    return total_audio_time_min, total_time_bgd_only_min, total_time_event_min, total_time_neg_min

def distribution_plot(counts, x_names, y_lim, lab_offset1, lab_offset2, y_label = None, color = '#0059b3'):
    '''
    Creates audio distribution plots
    '''
    plt.subplots_adjust(top=.75)
    for i, name in enumerate(counts):
        plt.subplot(1,2,i+1)
        y_values = [counts[name][i] for i in range(1,len(counts[name]))]
        y_pct = [val/sum(y_values) for val in y_values]
        bar_lab = ['   '+str(format(y_values[j],'.0f'))+
                   '\n'+'('+str(format(y_pct[j]*100,'.1f'))+'%)' for j in range(len(y_values))]
        make_bar_plot(x_names, y_values, width = 0.55, color = color)
        plt.title(name + ' audioset (total '+ str(counts[name][0]) +')\n')
        plt.ylim(0,y_lim)
        plt.ylabel(y_label)
        for j, v in enumerate(y_values):
            plt.text(j-lab_offset1, v+lab_offset2, bar_lab[j])
    plt.show()
    return

# Part 2 - additions for ETL
def get_log_mel_features_from_zip(archive, meta, Tx, n_freq, target_fs, mel_power, prefix):
    '''
    Extracts log mel features from audio file in .wav format contained in a zip
    archive.
    '''
    mel_feats = None
    tracks_n = meta.shape[0]
    mel_feats = np.zeros((tracks_n, Tx, n_freq))
    for i, trackID in enumerate(meta.trackID):
        path = prefix+str(trackID)+'.wav'
        rate, data = get_wav_info_from_zip(path, archive, target_fs)
        xi = log_mel_features(data, rate, mel_power)
        xi = np.transpose(xi)
        mel_feats[i] = xi
    return mel_feats

def create_or_load_mel_features(zip_train, meta_train_track, zip_eval, meta_eval_track,
                                Tx, n_freq, sample_rate, mel_power, prefix_train,
                                prefix_eval):
    '''
    Extracts and saves mel features for all audio files contained in a zip
    archive. Loads them if the featues alerady exist.
    '''
    file_names = ['X_train_mel', 'X_eval_mel']
    if not os.path.exists('output_data'):
        os.makedirs('output_data')
        X_train_mel = get_log_mel_features_from_zip(zip_train, meta_train_track, Tx,
                                        n_freq, sample_rate, mel_power, prefix_train)
        X_eval_mel = get_log_mel_features_from_zip(zip_eval, meta_eval_track, Tx,
                                        n_freq, sample_rate, mel_power, prefix_eval)
        datasets_to_save = [X_train_mel, X_eval_mel]
        for i, dataset in enumerate(datasets_to_save):
            h5f = h5py.File('output_data/'+file_names[i]+'.h5', 'w')
            h5f.create_dataset('dataset', data=dataset)
            h5f.close()
    else:
        X_sets = []
        for i, name in enumerate(file_names):
            file_name = 'output_data/'+name+'.h5'
            h5f = h5py.File(file_name,'r')
            X_sets.append(h5f['dataset'][:])
            h5f.close()
        [X_train_mel, X_eval_mel] = X_sets
    return X_train_mel, X_eval_mel

# Part 3 - additions for Feature Engineering
def insert_ones(y, segment_start_ms, segment_end_ms, Ty, track_dur_ms, during, lag):
    """
    Update the label vector y at specified times.

    Arguments:
        y -- numpy array of shape (1, Ty), the labels of the training example
        segment_start_ms -- the start time of the segment in ms
        segment_end_ms -- the end time of the segment in ms
        during -- whether the labels are placed during the event (True), or after (False)
        lag -- number of lagged time frames

    Returns: y -- updated labels
    """
    segment_end_y = int(segment_end_ms * Ty / track_dur_ms)
    if during == False:
        for i in range(segment_end_y + 1, segment_end_y + lag + 1):
            if i < Ty:
                y[0, i] = 1
    else:
        segment_start_y = int(segment_start_ms * Ty / track_dur_ms)
        for i in range(segment_start_y + lag, segment_end_y + lag):
            if i < Ty:
                y[0, i] = 1
    return y

def create_labels(meta_track, meta_segments, Ty, track_dur_ms, during, lag, event_label):
    '''
    Creates a vector of labels for an audio track.
    '''
    labels = None
    tracks_n = meta_track.shape[0]
    labels = np.zeros((tracks_n, Ty, 1))

    for i, trackID in enumerate(meta_track.trackID):
        yi = np.zeros((1,Ty))
        if meta_track.class_dummy[i] == 1:
            sub = meta_segments.loc[meta_segments['trackID'] == trackID]
            for e, event in enumerate(sub['event']):
                if event == event_label:
                    segment_start_ms = sub['event_start'].iloc[e]
                    segment_end_ms = sub['event_end'].iloc[e]
                    yi = insert_ones(yi, segment_start_ms, segment_end_ms, Ty, track_dur_ms, during, lag)
        yi = np.transpose(yi)
        labels[i] = yi
    return labels

# Part 4 - additions for Model definition.
# Part 5 - additions for Model training
# Part 6 - additions for Model Evaluation
from sklearn.metrics import accuracy_score, classification_report, f1_score, precision_score, recall_score

# Functions to build the ground truth event dictionaries for train and eval sets
def make_instance_dictionary(file_name, onset, offset, scene_label, event_label):
    '''
    Makes a dictionary containing meta data for each event.
    '''
    d = {}
    d['event_label'] = event_label
    d['event_onset'] = onset
    d['event_offset'] = offset
    d['file'] = file_name
    d['scene_label'] = scene_label
    return d

def get_segments_ref(meta_track, meta_segments, event_label):
    '''
    Builds the ground truth event dictionaries for dev and eval sets
    '''
    segments_dicts_ref = []
    for i, trackID in enumerate(meta_track.trackID):
        if meta_track.class_dummy[i]:
            file_name = str(trackID) + '.wav'
            sub = meta_segments.loc[meta_segments['trackID'] == trackID]
            for e, event in enumerate(sub['event']):
                if event == event_label:
                    onset = sub['event_start'].iloc[e]
                    offset = sub['event_end'].iloc[e]
                    segment_dict = make_instance_dictionary(file_name, onset/1000,
                                            offset/1000, scene_label, event_label)
                    segments_dicts_ref.append(segment_dict)
    return segments_dicts_ref

# Functions to build the dictionaries of events predicted by the model.
def predicted_event_segments(pred_index, Y_pred):
    '''
    Extracts the indexes in which a positive prediction has been made
    '''
    pos_index = np.where(Y_pred[pred_index,:,0] == 1)[0]
    starts = [pos_index[0]]
    ends = []
    for t in range(len(pos_index)-1):
        if pos_index[t+1] - pos_index[t] > 1:
            ends.append(pos_index[t])
            starts.append(pos_index[t+1])
    ends.append(pos_index[len(pos_index)-1])
    return starts, ends
# For event based statistics (True Positives have no use or meaning)
def segments_dicts_est(predictions_matrix, meta_track, event_label = 'glassbreak', track_dur = 10.0, Ty = 212):
    '''
    Builds the dictionaries of all events predicted by the model
    predictions_matrix must be already turned into 0s and 1s by threshold,
    not raw predicitons
    '''
    lists_of_predicted_segments = []
    for t in range(predictions_matrix.shape[0]):
        if 1 in predictions_matrix[t,:,0]:
            starts, ends = predicted_event_segments(t, predictions_matrix)
            lists_of_predicted_segments.append([starts, ends])
        else:
            lists_of_predicted_segments.append(None)

    segments_dicts_est = []
    for p in range(len(predictions_matrix)):
        segments = lists_of_predicted_segments[p]
        if segments != None:
            file_name = str(meta_track.trackID[p]) + '.wav'
            for s in range(len(segments[0])):
                onset = (segments[0][s]-lag)*track_dur/Ty
                offset = (segments[1][s]-lag)*track_dur/Ty
                segment_dict = make_instance_dictionary(file_name, onset, offset,
                                                scene_label, event_label)
                segments_dicts_est.append(segment_dict)
    return segments_dicts_est

# Function to calculate Segment and Event based metrics
def event_based_metrics(segments_dicts_ref, segments_dicts_est,
                                t_col = 0.5, pct_len = 0.5, eval_offset = False):
    '''
    Calculates and displays event based metrics
    '''
    reference_event_list = dcase_util.containers.MetaDataContainer(segments_dicts_ref)
    estimated_event_list = dcase_util.containers.MetaDataContainer(segments_dicts_est)
    event_based_metrics = sed_eval.sound_event.EventBasedMetrics(
        event_label_list=reference_event_list.unique_event_labels,
        t_collar=t_col, percentage_of_length = pct_len, evaluate_offset=eval_offset,
        evaluate_onset=True, event_matching_type='optimal')
    for filename in reference_event_list.unique_files:
        reference_event_list_for_current_file = reference_event_list.filter(
            filename=filename)
        estimated_event_list_for_current_file = estimated_event_list.filter(
            filename=filename)
        event_based_metrics.evaluate(
            reference_event_list=reference_event_list_for_current_file,
            estimated_event_list=estimated_event_list_for_current_file)
    event_metrics = event_based_metrics.results_overall_metrics()
    error_rate_metrics_dict = event_metrics['error_rate']
    f_metrics_dict = event_metrics['f_measure']
    error_rate_metrics_df = pd.DataFrame.from_dict(error_rate_metrics_dict, orient='index')
    f_metrics_df = pd.DataFrame.from_dict(f_metrics_dict, orient='index')
    error_rate_metrics_df.index = error_rate_metrics_df.index.map(lambda x: x.replace('_', ' '))
    f_metrics_df.rename(index={'f_measure':'F1'},inplace=True)
    print('F-measure')
    print(tabulate(f_metrics_df, floatfmt='.2f', tablefmt='rst'), '\n')
    print('Error Rate')
    print(tabulate(error_rate_metrics_df, floatfmt='.2f', tablefmt='rst'))
    return event_metrics

# Function to return event based metrics tables
def ebm_tables(segments_dicts_ref, segments_dicts_est,
                                t_col = 0.5, pct_len = 0.5, eval_offset = False):
    '''
    Creates event based metrics tables
    '''
    reference_event_list = dcase_util.containers.MetaDataContainer(segments_dicts_ref)
    estimated_event_list = dcase_util.containers.MetaDataContainer(segments_dicts_est)
    event_based_metrics = sed_eval.sound_event.EventBasedMetrics(
        event_label_list=reference_event_list.unique_event_labels,
        t_collar=t_col, percentage_of_length = pct_len, evaluate_offset=eval_offset,
        evaluate_onset=True, event_matching_type='optimal')
    for filename in reference_event_list.unique_files:
        reference_event_list_for_current_file = reference_event_list.filter(
            filename=filename)
        estimated_event_list_for_current_file = estimated_event_list.filter(
            filename=filename)
        event_based_metrics.evaluate(
            reference_event_list=reference_event_list_for_current_file,
            estimated_event_list=estimated_event_list_for_current_file)
    event_metrics = event_based_metrics.results_overall_metrics()
    error_rate_metrics_dict = event_metrics['error_rate']
    f_metrics_dict = event_metrics['f_measure']
    error_rate_metrics_df = pd.DataFrame.from_dict(error_rate_metrics_dict, orient='index')
    f_metrics_df = pd.DataFrame.from_dict(f_metrics_dict, orient='index')
    error_rate_metrics_df.index = error_rate_metrics_df.index.map(lambda x: x.replace('_', ' '))
    f_metrics_df.rename(index={'f_measure':'f1-score'},inplace=True)
    print('F-measure')
    print(tabulate(f_metrics_df, floatfmt='.2f', tablefmt='rst'), '\n')
    print('Error Rate')
    print(tabulate(error_rate_metrics_df, floatfmt='.2f', tablefmt='rst'))
    return event_metrics, f_metrics_df, error_rate_metrics_df

x_axis_vals = np.arange(0,Ty)*track_dur/Ty

def plot_pred_vs_true(meta_track, actual_labels, raw_predictions, dataset,
                        zip_audioset, prefix, track_index, threshold = 0.50):
    '''
    Plots model predictions versus ground truth, displays audio play button.
    '''
    track_id = meta_track.trackID.iloc[track_index]
    y_test = actual_labels[track_index,:,0]
    y_pred = raw_predictions[track_index,:,0]
    title = dataset + ' clip id: '+str(track_id)
    fig, ax = plt.subplots(figsize=(12, 4))

    l2,=ax.plot(x_axis_vals, y_pred, color='#6699ff', alpha=0.4)
    ax.fill_between(x_axis_vals, 0, y_pred, color='#6699ff', alpha=0.4)
    l1,=ax.plot(x_axis_vals, y_test, color='#003399', linewidth=2.3)

    y_test_gs = actual_labels[track_index,:,1]
    y_pred_gs = raw_predictions[track_index,:,1]

    l4,=ax.plot(x_axis_vals, y_pred_gs, color='#ff4dd2', alpha=0.4)
    ax.fill_between(x_axis_vals, 0, y_pred_gs, color='#ff4dd2', alpha=0.4)
    l3,=ax.plot(x_axis_vals, y_test_gs, color='#800060', linewidth=2.3)

    ax.set_title(title, {'fontsize': 15}).set_color('#4d4d4d')
    ax.set_xlabel('time (s)', {'fontsize': 14}).set_color('#4d4d4d')
    ax.set_ylabel('probability', {'fontsize': 14}).set_color('#4d4d4d')
    ax.spines["top"].set_visible(False)
    ax.spines["right"].set_visible(False)
    ax.tick_params(length=2, width=0.5, colors='0.4',labelsize = 12)
    leg = plt.legend([l1, l2, l3, l4],["glassbreak truth", "glassbreak prediction", "gunshot truth", "gunshot prediction"],
                     prop={'size': 12}, frameon=False, loc=(1.05,0.4))
    for text in leg.get_texts(): plt.setp(text, color = '#4d4d4d')
    for legthickness in leg.legendHandles: legthickness.set_linewidth(2.5)

    plt.ylim(0,1.05)
    plt.xlim(0,10)
    plt.locator_params(axis='x', nbins=10)
    fig.tight_layout()
    plt.show()
    audio_zip = zip_audioset.read(prefix+str(track_id)+'.wav')
    bytes_io = BytesIO(audio_zip)
    old = AudioSegment.from_wav(bytes_io)
    old.export('media/temp_mp3.mp3', format='mp3')
    IPython.display.display(IPython.display.Audio('media/temp_mp3.mp3'))
    return


import os
import numpy as np
import cv2
import json

class DataProvider():
    "this class creates machine-written text for a word list. TODO: change getNext() to return your samples."

    def __init__(self):
        words = os.listdir("HKR/ann")
        self.wordList = []
        for i in words:
            with open(f"/Users/alex/Documents/pythonProject/Terly/hw_recognition/HKR/ann/{i}") as f:
                if all(map(lambda x: not x.isalpha() or 1040 <= ord(x) <= 1103, json.loads(f.read())["description"])):
                    self.wordList.append(i)
        self.idx = 0

    def hasNext(self):
        "are there still samples to process?"
        return self.idx < len(self.wordList)

    def getNext(self):
        "TODO: return a sample from your data as a tuple containing the text and the image"
        word = self.wordList[self.idx]
        img = cv2.imread(f"/Users/alex/Documents/pythonProject/Terly/hw_recognition/HKR/img/{word.split('.')[0]}.jpg",
                         cv2.IMREAD_GRAYSCALE)
        with open(f"/Users/alex/Documents/pythonProject/Terly/hw_recognition/HKR/ann/{word}") as f:
            word = json.loads(f.read())["description"]
        self.idx += 1
        return (word, img)

def createIAMCompatibleDataset(dataProvider):
    "this function converts the passed dataset to an IAM compatible dataset"

    # create files and directories
    f = open('words.txt', 'w+')
    if not os.path.exists('sub'):
        os.makedirs('sub')
    if not os.path.exists('sub/sub-sub'):
        os.makedirs('sub/sub-sub')

    # go through data and convert it to IAM format
    ctr = 0
    while dataProvider.hasNext():
        sample = dataProvider.getNext()

        # write img
        cv2.imwrite('sub/sub-sub/sub-sub-%d.png' % ctr, sample[1])

        # write filename, dummy-values and text
        line = 'sub-sub-%d' % ctr + ' X X X X X X X ' + sample[0] + '\n'
        f.write(line)

        ctr += 1

if __name__ == '__main__':
    dataProvider = DataProvider()
    createIAMCompatibleDataset(dataProvider)


import os
from supabase import create_client, Client

url: str = os.environ.get("SUPABASE_URL")
key: str = os.environ.get("SUPABASE_KEY")
supabase: Client = create_client(url, key)

response = supabase.table('Users').select("*").execute()
print(response)
data, count = supabase.table('Users').insert({"id": 1, "email": "t@t.com"}).execute()
print(data)

#func = supabase.functions()
#@asyncio.coroutine
#async def test_func(loop):
#  resp = await func.invoke("hello-world",invoke_options={'body':{}})
#  return resp

#loop = asyncio.get_event_loop()
#resp = loop.run_until_complete(test_func(loop))
#loop.close()


'''
	A simple Flask app for making Stripe payments
	Author: Muneeb Ali | http://muneeb.org
	Modified from examples at http://stripe.com/docs/
	License: MIT (see LICENSE)
'''

from flask import Flask, render_template, request
import stripe

from config import stripe_keys
from config import DEFAULT_CHARGE_AMOUNT, DEFAULT_CHARGE_DESCRIPTION
from config import DB_NAME

stripe.api_key = stripe_keys['secret_key']

app = Flask(__name__)

#-----------------------------------
# MongoDB for stripe customers
from pymongo import MongoClient
c = MongoClient()
db = c[DB_NAME]
posts = db.posts

#-----------------------------------
@app.route('/')
def index():
	return render_template('index.html',
					key=stripe_keys['publishable_key'],
					amount_usd=get_amount_usd()
					)

#-----------------------------------
def get_amount_usd(amount=DEFAULT_CHARGE_AMOUNT):
		return float(amount) / 100

#-----------------------------------
def get_stripe_customer(customer_email):

	customer_info = posts.find_one({"customer_email":customer_email})

	if(customer_info == None):
		return False, customer_info
	else:
		return True, customer_info

#-----------------------------------
def save_stripe_customer(customer_email, customer_id):

	# If already have a customer by that email, delete old entry
	reply, entry = get_stripe_customer(customer_email)
	if(reply):
		posts.remove(entry)
	else:
		post = {"customer_email":customer_email,"customer_id":customer_id}
		posts.insert(post)

#-----------------------------------
def stripe_charge(customer_id,
			charge_amount=DEFAULT_CHARGE_AMOUNT,
			charge_descripton=DEFAULT_CHARGE_DESCRIPTION):

	try:
		charge = stripe.Charge.create(
			customer=customer_id,
			amount=charge_amount,
			currency='usd',
			description=charge_descripton
			)

	except stripe.CardError as e:

		# The card has been declined (or other error)
		return False, e

	return True, "successfully charged"

#-----------------------------------
@app.route('/charge', methods=['POST'])
def charge():

	customer_email = request.form['email']

	# Check if already have this customer
	customer_valid, customer_info = get_stripe_customer(customer_email)

	if(customer_valid):
		customer_id = customer_info['customer_id']
	else:

		customer = stripe.Customer.create(
					email=customer_email,
					card=request.form['stripeToken']
					)

		customer_id = customer.id

		# save the customer ID in DB for later use
		save_stripe_customer(customer_email, customer_id)

	# charge the customer (new or already present in DB)
	charge_valid, charge_msg = stripe_charge(customer_id)

	return render_template('charge.html',
					amount_usd=get_amount_usd(),
					customer_email=customer_email,
					charge_valid=charge_valid,
					charge_msg=charge_msg)

#-----------------------------------
@app.route('/recurring', methods=['POST'])
def recurring():

	customer_email = request.form['email']

	# Get customer_id from Mongo
	customer_valid, customer_info = get_stripe_customer(customer_email)

	if(customer_valid):
		customer_id = customer_info['customer_id']

		# Charge the customer (new or already present in DB)
		charge_valid, charge_msg = stripe_charge(customer_id)

	else:
		charge_valid = False
		charge_msg = "recurring customer doesn't exist"

	return render_template('charge.html',
					amount_usd=get_amount_usd(),
					customer_email=customer_email,
					charge_valid=charge_valid,
					charge_msg=charge_msg)

#-----------------------
if __name__ == '__main__':

	app.run(debug=True)


# -*- coding: utf-8 -*-
#!/usr/bin/python2
from lxml import etree
import sys
import os

from cobra.io import read_sbml_model, write_sbml_model
from cobra.io.sbml import create_cobra_model_from_sbml_file
from cobra import *

def without_metabo(file_draft,metabo,outFile) :

	parser = etree.XMLParser(remove_blank_text=True)

	with open(file_draft,"r") as file :
		line0 = file.readlines()[0]

	draft = etree.parse(file_draft,parser)
	root = "}".join(draft.getroot().tag.split("}")[:-1])+"}"

	listR = draft.getroot().find(root+'model/'+root+"listOfReactions")
	for reaction in listR :
		remove_reac = False
		reversible = reaction.attrib["reversible"]
		listOfReactant = reaction.find(root+"listOfReactants")
		try :
			for specie in listOfReactant :
				if specie.attrib['species'] == metabo :
					listR.remove(reaction)
					remove_reac = True
					break
		except :
			pass
		if remove_reac==False and reversible == "true" :
			listOfProduct = reaction.find(root+"listOfProducts")
			try :
				for specie in listOfProduct :
					if specie.attrib['species'] == metabo :
						listR.remove(reaction)
						break
			except :
				pass

	new_file = "/".join(file_draft.split("/")[:-1])+"/"+outFile
	with open(new_file,"wb") as file :
		file.write(line0.encode('utf-8'))
		draft.write(file, pretty_print=True)

	return new_file

if __name__ == '__main__':
	file_draft = sys.argv[1]
	metabo = sys.argv[2]
	try :
			outFile = sys.argv[3]
	except IndexError :
			outFile = "new_draft.sbml"

	new_file = without_metabo(file_draft,metabo,outFile)
	print(new_file)


#! /Library/Frameworks/Python.framework/Versions/3.7/bin/python3
# By Nicholas Coletta, in the year 2019.
# This is part of my tour on codewars.com. The code below is a solution to a problem.

# INSTRUCTIONS: Complete the function scramble(str1, str2) that returns true if a portion of str1 characters can be rearranged to match str2, otherwise returns false.
# EXAMPLE:
# scramble('rkqodlw', 'world') ==> True
# scramble('cedewaraaossoqqyt', 'codewars') ==> True
# scramble('katas', 'steak') ==> False
from collections import Counter
def scramble(s1, s2):
    sone = Counter(s1)
    stwo = Counter(s2)
    diff = stwo - sone
    if len(diff) == 0:
        return True
    else:
        return False
# Both sets of code work. The commented is slower by about 2000ms. Which this challenge relied on performance.
''' compList = []
    for letter in s1:
        if letter in s2:
            if s2.count(letter) > compList.count(letter):
                compList.append(letter)
    print(compList)
    if len(compList) == len(s2):
        return True
    else:
        return False'''

print(scramble('rkqodlw', 'world'))


from setuptools import find_packages, setup

install_requires = [
    "tqdm>=4.15.0,<5",
    "wagtail>=2.15,<4.2",
]

testing_extras = [
    "coverage>=3.7.0",
    "mock>=1.0.0",
]

setup(
    name="wagtail-inventory",
    url="https://github.com/cfpb/wagtail-inventory",
    author="CFPB",
    author_email="tech@cfpb.gov",
    description="Lookup Wagtail pages by block content",
    long_description=open("README.rst", "r", encoding="utf-8").read(),
    license="CCO",
    version="1.6",
    include_package_data=True,
    packages=find_packages(),
    python_requires=">=3.8",
    install_requires=install_requires,
    extras_require={"testing": testing_extras},
    classifiers=[
        "Framework :: Django",
        "Framework :: Django :: 3.2",
        "Framework :: Django :: 4.0",
        "Framework :: Wagtail",
        "Framework :: Wagtail :: 2",
        "Framework :: Wagtail :: 3",
        "Framework :: Wagtail :: 4",
        "License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication",
        "License :: Public Domain",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
    ],
)


import numpy as np
from sklearn.utils import shuffle
from .utils import encode

class ADALINEClassifier:
    def __init__(self):
        self.avg_error = []
        self.W = None

    def fit(self, X, y, alpha=0.001, epoches=10):
        Y = encode(y) # Encode labels into binary matrix
        Y = 2*Y - 1 # Transform 0 to -1

        # Init weight matrix
        self.W = 0.1*np.random.randn(Y.shape[1], X.shape[1])
        for _ in range(epoches):
            error_lis = []
            X, Y = shuffle(X, Y, random_state=100)
            for i in range(X.shape[0]):
                x = X[i, :]
                y_pred = self.W.dot(x)
                error = Y[i, :] - y_pred.T

                # Normalize x vector
                x_norm = x/np.linalg.norm(X[0, :])

                # Update the weights
                delta = np.outer(error, x_norm)
                self.W = self.W + alpha*delta
                error_lis.append(sum(error**2))
            self.avg_error.append(sum(error_lis)/len(X))

    def predict(self, X):
        Y_score = self.W.dot(X.T)
        y_pred = np.argmax(Y_score, 0)
        return y_pred


import inspect
import copy
from aorist import *
from aorist_recipes import programs
from common import endpoints

"""
Defining schema
"""
attributes = [
    Attribute(KeyStringIdentifier("id")),
    Attribute(StringIdentifier("author")),
    Attribute(StringIdentifier("subreddit")),
    Attribute(POSIXTimestamp("created_utc")),
    Attribute(FreeText("title")),
    Attribute(FreeText("selftext", nullable=True)),
]
"""
A row in our table is a struct.
"""
subreddit_datum = RowStruct(
    name="subreddit",
    attributes=attributes,
)
tmp_dir = "tmp/subreddits"
"""
Data will be replicated to Hive
"""
local = HiveTableStorage(
    location=HiveLocation(MinioLocation(name='reddit')),
    encoding=Encoding(NewlineDelimitedJSONEncoding()),
    layout=TabularLayout(StaticTabularLayout()),
)
"""
Declaring where our subreddits live, i.e. in PushShift
"""
subreddits = ['france']#, 'newzealand']
tabular_schema = default_tabular_schema(
    subreddit_datum, subreddit_datum.name, attributes
)
assets = {x: StaticDataTable(
    name=x,
    schema=DataSchema(tabular_schema),
    setup=StorageSetup(RemoteStorageSetup(
        remote=Storage(RemoteStorage(
            location=RemoteLocation(
                PushshiftAPILocation(
                    subreddit=x
                )
            ),
            layout=APIOrFileLayout(
                APILayout(
                    PushshiftSubredditPostsAPILayout()
                ),
            ),
            encoding=Encoding(
                NewlineDelimitedJSONEncoding()
            ),
        )),
    )),
    tag=x,
    ) for x in subreddits}

embedding = FasttextEmbedding(
    name="embedding",
    comment="Fasttext embedding of size 128",
    schema=DataSchema(FasttextEmbeddingSchema(
        dim=128,
        source_schema=tabular_schema,
        text_attribute_name="selftext",
    )),
    setup=StorageSetup(LocalStorageSetup(
        Storage(local),
        '/tmp/fasttext_embedding',
    )),
    source_assets=[Asset(v) for v in assets.values()],
)
assets = {
    k: Asset(v) for (k, v) in assets.items()
}
assets['embedding'] = Asset(embedding)
"""
Creating the dataset
"""
subreddits = DataSet(
    name="subreddits",
    description="""
    A selection of small region-based Subreddits to demonstrate
    collecting Reddit data via [Pushshift](https://pushshift.io/).
    """,
    source_path=__file__,
    datum_templates=[DatumTemplate(subreddit_datum)],
    assets=assets,
    access_policies=[],
)
"""
Dataset will be replicated.
"""
subreddits = subreddits.replicate_to_local(
    Storage(local), tmp_dir, Encoding(CSVEncoding())
)
universe = Universe(
    name="my_cluster",
    datasets=[subreddits],
    endpoints=endpoints,
)
universe.compute_uuids()

result = dag(
    universe,
    ["UploadFasttextToMinio"],
    "airflow",
    programs,
    dialect_preferences=[
        R(),
        Python([]),
        Bash(),
        Presto(),
    ],
)
print(result)


import cv2 as cv

# img = cv.imread("prototype.png")

# cv.imshow("image", img)

# cv.waitKey(0)

capture = cv.VideoCapture(r"C:\Users\ACER\Downloads\ariandel.mp4")

while True:
    isTrue, frame = capture.read()
    cv.imshow('Video', frame)

    if cv.waitKey(20) & 0xFF==ord('d'):
        break
capture.release()
cv.destroyAllWindows()


import time
import multiprocessing

def fatorial(n):
    fat = n
    for i in range(n - 1, 1, -1):
        fat = fat * i
    return fat

def fatorial_lista(lista, result_parcial, id):
    lista_retorno = []
    for item in lista:
        resultado = fatorial(item)
        lista_retorno.append(resultado)
    result_parcial[id] = lista_retorno

if __name__ == "__main__":
    quantidade_numeros = 5000
    hora_inicio = time.time()

    lista = []
    for i in range(quantidade_numeros):
        lista.append(i)

    num_processos = 4
    hora_inicio = time.time()

    with multiprocessing.Manager() as manager:
        # Cria lista na memória do SP
        lista = manager.list(lista)
        # Vetor para salvar a soma parcial de cada thread
        result_parcial = manager.list(num_processos * [0])

        lista_processos = []
        for i in range(num_processos):
            ini = i * int(quantidade_numeros / num_processos)
            fim = (i + 1) * int(quantidade_numeros / num_processos)
            p = multiprocessing.Process(target=fatorial_lista, args=(lista[ini:fim],result_parcial, i))
            p.start()
            lista_processos.append(p)

        for p in lista_processos:
            p.join()

        resultado = [x for l in result_parcial for x in l]

        hora_final = time.time()
        tempo_decorrido = hora_final - hora_inicio

        print("Primeiros resultados:", resultado[:5])
        print(f"Tempo multiprocess: {tempo_decorrido}")

        arq = open("08C.txt", "w")
        for item in resultado:
            arq.write(f"{item}\n")
        arq.close()


from settings import LMD_MATCHED_FOLDER, RWC_DATASET_PATH
import os
import pretty_midi
from data_preprocess import prepare_quantization, get_piano_roll, get_quantized_melody
import numpy as np
import matplotlib.pyplot as plt
import sklearn.metrics
from crf import CRFDecoder
from metrical_crf import get_ternary_transition
import torch

def hierarchical_viterbi_decode(novelty_score):
    log_observations = torch.cumsum(torch.log(torch.tensor(novelty_score)), dim=1)
    log_observations = torch.cat([torch.zeros(log_observations.shape[0], 1), log_observations], dim=1) * 4
    # log_transitions = get_log_transitions(4)
    log_transitions, indices = get_ternary_transition(np.array([-2.6, -2.0, -1.4, -0.8]), np.array([-5.0, -4.2, -3.4, -2.6]))
    log_observations = log_observations[:, indices]
    crf = CRFDecoder(torch.tensor(log_transitions))
    result = crf.viterbi_decode(log_observations[None]).squeeze(0).numpy()
    return indices[result]

def joint_analyze(file_name, subbeat_count=4, save_type='midi', load_type='none', data_aug=False, drums=1, melody=1, others=1, tracks=None):
    try:
        midi = pretty_midi.PrettyMIDI(file_name)
    except:
        print('Midi load failed: %s' % file_name)
        return None
    n_subbeat, downbeat_bins, boundaries, subbeat_time = prepare_quantization(midi, subbeat_count)
    output = pretty_midi.Instrument(program=0, is_drum=True, name='Layers')
    piano_rolls = [get_piano_roll(ins, boundaries, False, ignore_drums=False) for ins in midi.instruments]
    context_length = subbeat_count * 4
    novelty_scores = []
    n_layers = 4
    for k in range(n_layers):
        way_mean_ssm = []
        for way in [0, 1]:
            ssms = []
            for j, ins in enumerate(midi.instruments):
                if (ins.is_drum):
                    if (drums == 0 or (tracks is not None and j not in tracks)):
                        continue
                elif ('mel' in ins.name.lower() or 'vocal' in ins.name.lower()):
                    if (melody == 0 or (tracks is not None and j not in tracks)):
                        continue
                else:
                    if (others == 0 or (tracks is not None and j not in tracks)):
                        continue
                features = []
                for i, downbeat_bin in enumerate(downbeat_bins):
                    feature = piano_rolls[j][downbeat_bin:downbeat_bin + context_length] if way == 1 else \
                        piano_rolls[j][downbeat_bin - context_length:downbeat_bin]
                    if (feature.shape[0] < context_length):
                        pad = (0, context_length - feature.shape[0]) if way == 1 else \
                            (context_length - feature.shape[0], 0)
                        feature = np.pad(feature, (pad, (0, 0)))
                    feature = feature.reshape(-1)
                    if (np.abs(feature.sum()) > 1e-6):
                        feature = feature / np.linalg.norm(feature)
                    features.append(feature)
                features = np.stack(features, axis=0)
                ssm = np.matmul(features, features.T)
                if (ssm.any()):
                    ssms.append(ssm)
            mean_ssm = np.mean(np.stack(ssms, axis=0), axis=0)
            mean_ssm = (mean_ssm - mean_ssm.mean()) / np.std(mean_ssm)
            way_mean_ssm.append(mean_ssm)
        novelty_score = np.abs(way_mean_ssm[1] - way_mean_ssm[0]).mean(axis=0)
        novelty_scores.append(novelty_score)
        context_length *= 2
        # plt.plot(np.arange(len(novelty_score)), novelty_score)
        # plt.show()
    novelty_scores = np.stack(novelty_scores, axis=1)
    result = hierarchical_viterbi_decode(novelty_scores)
    gt_file_path = 'annotation/%s_gt.mid' % os.path.basename(file_name)
    if (load_type == 'midi'):
        # assert (save_type == 'numpy')
        midi_gt = pretty_midi.PrettyMIDI(gt_file_path)
        n_subbeat_gt, downbeat_bins_gt, boundaries_gt, subbeat_time_gt = prepare_quantization(midi_gt, subbeat_count)
        notes = [[np.searchsorted(boundaries_gt, note.start), note.pitch] for note in midi_gt.instruments[0].notes]
        note_dict = np.zeros(max(n_subbeat, n_subbeat_gt), dtype=np.int16)
        print('before:', result)
        for note in notes:
            if (note[1] >= 40):
                note_dict[note[0]] = max(note_dict[note[0]], note[1] - 39)
        for i in range(len(result)):
            result[i] = note_dict[downbeat_bins[i]]
        print('after:', result)

    if (save_type == 'midi'):
        for i, pred in enumerate(result):
            for k in range(pred):
                onset_time = subbeat_time[downbeat_bins[i]]
                output.notes.append(pretty_midi.Note(velocity=100, pitch=40 + k, start=onset_time, end=onset_time + 0.5))

        midi.instruments.append(output)
        midi.write('output/%s_hierarchical_dp.mid' % os.path.basename(file_name))
    elif (save_type == 'f1'):
        if (os.path.exists(gt_file_path)):
            eval_result = evaluate_result(result, gt_file_path, downbeat_bins, subbeat_count, n_layers)
            print(('%s:\t' % file_name) + '\t'.join(
                str(x) for x in eval_result))
            return eval_result
    elif (save_type == 'numpy'):
        output_context_length = 64
        context_length = output_context_length
        results = []
        onset_rolls = [get_piano_roll(ins, boundaries, True, ignore_drums=False) for ins in midi.instruments]
        beat_bins = np.searchsorted(boundaries, midi.get_beats())
        k_min, k_max = 0, 0
        if (data_aug):
            k_min, k_max = -1, 1
        for k in range(k_min, k_max + 1):
            for beat_bin in beat_bins:
                context_left = beat_bin - context_length
                context_right = beat_bin + context_length
                js = np.argwhere(downbeat_bins == beat_bin)
                if (context_left >= 0 and context_right < n_subbeat):
                    for i, ins in enumerate(midi.instruments):
                        if (np.any(piano_rolls[i][context_left: context_right])):
                            if (ins.is_drum):
                                roll = onset_rolls[i][context_left: context_right] * 2
                            else:
                                roll = piano_rolls[i][context_left: context_right] + onset_rolls[i][context_left: context_right]
                            target = result[js[0]] + 1 if len(js) > 0 else 0.0
                            target = np.clip(target - k, 0, n_layers + 1)
                            roll = roll.reshape(output_context_length * 2, -1, roll.shape[-1]).max(axis=1)
                            results.append(np.concatenate([
                                np.array([target, ins.is_drum, ins.program], dtype=np.uint8),
                                roll.reshape(-1).astype(np.uint8)
                            ]))
            context_length *= 2
        results = np.stack(results, axis=0)
        return results
    else:
        raise NotImplementedError()

def generate_labels(start_id, end_id, name, load_type='none'):
    train_data = []
    test_data = []
    for i in range(start_id, end_id):
        result = joint_analyze(os.path.join(LMD_MATCHED_FOLDER,
                                            R'E:\Dataset\RWC\AIST.RWC-MDB-P-2001.SMF_SYNC\RM-P%03d.SMF_SYNC.MID' % (
                                                        i + 1)),
                               save_type='numpy', load_type=load_type, data_aug=True, subbeat_count=8)
        if (result is not None):
            if (i % 10 == 0):
                test_data.append(result)
            else:
                train_data.append(result)
    np.savez_compressed('data/%s.npz' % name, np.concatenate(train_data, axis=0))

def eval_beat_result(predicted_result, gt_result):
    if (len(predicted_result) >= len(gt_result)):
        predicted_result = predicted_result[:len(gt_result)]
    else:
        predicted_result = np.pad(predicted_result, (0, len(gt_result) - len(predicted_result)))
    return sklearn.metrics.f1_score(gt_result > 0, predicted_result > 0)

def evaluate_result(predicted_result, gt_midi_path, downbeat_bins, subbeat_count, n_layers):
    midi_gt = pretty_midi.PrettyMIDI(gt_midi_path)
    n_subbeat_gt, downbeat_bins_gt, boundaries_gt, subbeat_time_gt = prepare_quantization(midi_gt, subbeat_count)
    notes = [[np.searchsorted(boundaries_gt, note.start), note.pitch] for note in midi_gt.instruments[0].notes]
    note_dict = np.zeros(n_subbeat_gt, dtype=np.int16)
    gt_result = np.zeros(len(downbeat_bins_gt), dtype=np.int16)
    if (len(predicted_result) >= len(gt_result)):
        predicted_result = predicted_result[:len(gt_result)]
    else:
        predicted_result = np.pad(predicted_result, (0, len(gt_result) - len(predicted_result)))
    for note in notes:
        if (note[1] >= 40):
            note_dict[note[0]] = max(note_dict[note[0]], note[1] - 39)
    for i in range(len(gt_result)):
        if (downbeat_bins[i] < n_subbeat_gt):
            gt_result[i] = note_dict[downbeat_bins[i]]
    return [sklearn.metrics.f1_score(gt_result >= i, predicted_result >= i) for i in range(1, n_layers + 1)]

def get_split(data_file, split):
    f = open('./data/%s.split.txt.names' % data_file, 'r')
    tokens = [line.strip().split(',') for line in f.readlines() if line.strip() != '']
    f.close()
    if (split == 'train'):
        return tokens[0]
    elif (split == 'val'):
        return tokens[1]
    elif (split == 'test'):
        return tokens[2]
    else:
        raise Exception('No such split')

def oracle_analyze(file_name, subbeat_count=4):
    midi = pretty_midi.PrettyMIDI(file_name)
    n_subbeat, downbeat_bins, boundaries, subbeat_time = prepare_quantization(midi, subbeat_count)
    template = np.array([4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0])
    results = []
    for i in range(16):
        template = np.roll(template, 1)
        pred = template[np.arange(len(downbeat_bins)) % 16]
        gt_file_path = 'annotation/%s_gt.mid' % os.path.basename(file_name)
        results.append(evaluate_result(pred, gt_file_path, downbeat_bins, subbeat_count, 4))
    # print(results)
    return np.max(results, axis=0)

def evaluations_to_latex(set_name, evaluations):
    mean_evaluations = np.mean(evaluations, axis=0)
    std_evaluations = np.std(evaluations, axis=0)
    if ('\n' in set_name):
        set_name = R'\begin{tabular}[c]{@{}l@{}}%s\end{tabular}' % (set_name.replace('\n', '\\\\ '))
    return(set_name + '\t' + '\t'.join(R'& \begin{tabular}[c]{@{}r@{}}%.4f \\ $\pm$%.4f\end{tabular}' % (mean_evaluation, std_evaluation)
                     for (mean_evaluation, std_evaluation) in zip(mean_evaluations, std_evaluations)) + R' \\ \midrule')

if __name__ == '__main__':
    # generate_labels(23, 40, 'hierarchical_dp_analysis_rwc_test_v3', 'none')
    # generate_labels(0, 33, 'hierarchical_gt_rwc_train_v6', 'midi')

    split_files = get_split('rwc_multitrack_hierarchy_v6_supervised', 'test')
    print(evaluations_to_latex('Oracle',
        [oracle_analyze(os.path.join(RWC_DATASET_PATH, 'AIST.RWC-MDB-P-2001.SMF_SYNC', file)) for file in split_files]))
    print(evaluations_to_latex('Rule',
        [joint_analyze(os.path.join(RWC_DATASET_PATH, 'AIST.RWC-MDB-P-2001.SMF_SYNC', file), save_type='f1',
                    drums=1, melody=1, others=1) for file in split_files]))
    print(evaluations_to_latex('Rule\n (mel. only)',
        [joint_analyze(os.path.join(RWC_DATASET_PATH, 'AIST.RWC-MDB-P-2001.SMF_SYNC', file), save_type='f1',
                    drums=0, melody=1, others=0) for file in split_files]))
    print(evaluations_to_latex('Rule\n (no drums)',
        [joint_analyze(os.path.join(RWC_DATASET_PATH, 'AIST.RWC-MDB-P-2001.SMF_SYNC', file), save_type='f1',
                    drums=0, melody=1, others=1) for file in split_files]))

    print('\n\nPOP909\n\n')
    print(evaluations_to_latex('Oracle', [oracle_analyze(R'input/POP909-%d.mid' % (i + 1)) for i in range(5)]))
    print(evaluations_to_latex('Rule', [joint_analyze(R'input/POP909-%d.mid' % (i + 1), save_type='f1') for i in range(5)]))
    print(evaluations_to_latex('Rule\n(mel. only)', [joint_analyze(R'input/POP909-%d.mid' % (i + 1), tracks=[0], save_type='f1') for i in range(5)]))
    joint_analyze(R'E:\Dataset\RWC\AIST.RWC-MDB-P-2001.SMF_SYNC\RM-P051.SMF_SYNC.MID', save_type='f1')
    exit(0)


import json

from django.conf import settings
from django.http import HttpResponse

from tags.models import Tag

from django.apps import apps
get_model = apps.get_model

# Including these declarations here because it would be nice to not rely on the taggit autosuggest app
MAX_SUGGESTIONS = getattr(settings, 'TAGGIT_AUTOSUGGEST_MAX_SUGGESTIONS', 20)
TAG_MODELS = getattr(settings, 'TAGGIT_AUTOSUGGEST_MODELS', {'default': ('taggit', 'Tag')})
if not type(TAG_MODELS) == dict:
    TAG_MODELS = {'default': TAG_MODELS}

def list_all_tags(request):
    """Returns all the tags in the database"""
    all_tags = Tag.objects.all().values_list('name', flat=True)
    return HttpResponse(json.dumps(list(all_tags)), content_type='application/json')

def list_tags(request, tagmodel=None):
    """
    Returns a list of JSON objects with a `name` and a `value` property that
    all start like your query string `q` (not case sensitive).
    """
    if not tagmodel or tagmodel not in TAG_MODELS:
        TAG_MODEL = get_model(*TAG_MODELS['default'])
    else:
        TAG_MODEL = get_model(*TAG_MODELS[tagmodel])

    query = request.GET.get('q', '')
    limit = request.GET.get('limit', MAX_SUGGESTIONS)
    try:
        request.GET.get('limit', MAX_SUGGESTIONS)
        limit = min(int(limit), MAX_SUGGESTIONS)  # max or less
    except ValueError:
        limit = MAX_SUGGESTIONS
    except TypeError:
        pass

    tag_name_qs = TAG_MODEL.objects.filter(name__icontains=query).order_by('name', 'id'). \
        values('pk', 'name')

    if callable(getattr(TAG_MODEL, 'request_filter', None)):
        tag_name_qs = tag_name_qs.filter(TAG_MODEL.request_filter(request)).distinct()

    if MAX_SUGGESTIONS:
        data = [{'id': n['pk'], 'name': n['name'], 'value': n['name']} for n in tag_name_qs[:limit]]
    else:
        data = [{'id': n['pk'], 'name': n['name'], 'value': n['name']} for n in tag_name_qs]

    return HttpResponse(json.dumps(data), content_type='application/json')


from selenium.webdriver.chrome.options import Options
from selenium import webdriver
from playsound import playsound
import time
import pyttsx3
import sys
import os
import ctypes
import winsound

# Returns true if file path exists
def is_valid_path(path):
    return os.path.exists(path)

# Check file path and exit with message if invalid
def invalid_path_exit(path):
    if not is_valid_path(path):
        print(f"'{path}' is invalid, ending script")
        sys.exit()

# Function for generating text-to-speech
def text_to_speech(text):
    # Initialize the text-to-speech engine
    engine = pyttsx3.init()

    # Convert text to speech
    engine.say(text)
    engine.runAndWait()

# Function for generating a windows popup
def show_popup(text):
    # Get the file name of the script
    script_file = os.path.basename(__file__)
    # Generate the popup
    ctypes.windll.user32.MessageBoxW(0, text, script_file, 0)

# Checks a target url for a string at a given interval and runs specified alert behaviour
def open_and_refresh(url, headless_mode, success_type, refresh_interval, check_string, alert_type, mp3_path=''):
    # Prep webdriver
    if headless_mode:  # Don't show the browser window
        chrome_options = Options()
        chrome_options.add_argument("--headless")  # Make the browser run in headless mode
        driver = webdriver.Chrome(options=chrome_options)  # Pass the options when creating the webdriver
    else:  # Do show the browser window
        driver = webdriver.Chrome()

    driver.get(url)

    # Core loop
    while True:
        # Wait and print message per second to console based on refresh interval
        for i in range(refresh_interval, 0, -1):
            print(f"Refreshing in {i} seconds...")
            time.sleep(1)
        print("Refreshing now!")
        driver.refresh()

        # Get the page source as a string
        html = driver.page_source

        # Function to run conditional alert behaviour
        def alert(append_string):
            # Function to make string for messages
            def message_string(message_type):
                output_string = f"Target URL: '{url}'\n" \
                                f"Check string: '{check_string}'\n" \
                                f"Success Type: {success_type}\n" \
                                f"Alert Type: '{message_type}'\n" \
                                f"Result / Action: '{append_string}"
                return output_string

            # Generate that string
            alert_message_string = message_string(alert_type)

            # For text to speech
            if alert_type == 'tts':
                print(alert_message_string)
                text_to_speech(alert_message_string)
            # For mp3
            elif alert_type == 'mp3':
                print(alert_message_string)
                invalid_path_exit(mp3_path)
                playsound(mp3_path)
            elif alert_type == 'popup':
                print(alert_message_string)
                winsound.PlaySound("SystemAsterisk", winsound.SND_ASYNC)
                show_popup(alert_message_string)
            # If no valid option was selected
            else:
                print("Choose 'tts' or 'mps' for var alert_type")
                sys.exit()

        # Actions to take if success_type is 'not in'. i.e. script ends if string is NOT found
        if success_type == 'string not found':
            if check_string in html:
                print(f"'{check_string}' string still found - repeating loop")
            else:
                alert('SUCCESS: string not found - breaking loop')
                break

        # Actions to take if success_type is 'in'. i.e. script ends if string IS found
        if success_type == 'string found':
            if check_string in html:
                alert('SUCCESS: string found - breaking loop')
                break
            else:
                print(f"'{check_string}' string not found - repeating loop")

# Path definition
target_url = "https://example.com.au/collections/product-in-stock"
alert_sound_audio = r"C:\Users\kylec\Audio\alert_sound.mp3"

# Running the function
open_and_refresh(
    url=target_url,  # URL to check
    headless_mode=True,  # True or False - False if you want to see the browser window
    success_type='string not found',  # 'string not found' or 'string found'
    refresh_interval=30,  # How often to recheck the page in seconds
    check_string='Stock is empty',  # The string to check the html for
    alert_type='popup',  # 'tts', 'mp3' or 'popup'
    mp3_path=alert_sound_audio)  # path of mp3 audio file to play - OPTIONAL


import os, sys, re
from PIL import Image
from os import listdir
from os.path import isfile, join

fscreenshots=open('../../public_html/screenshots/screenshots_.html','w')
f800x660=open('../../public_html/screenshots/800x660.html','w')
f800x600=open('../../public_html/screenshots/800x600.html','w')
f640x512=open('../../public_html/screenshots/640x512.html','w')
felse=open('../../public_html/screenshots/else.html','w')
v800x660=[]
v800x600=[]
v640x512=[]
velse=[]

def replaceStringInFile(filePath):
	'''
	output = open(tempName,'w')
	output.write(outtext)
	output.close()
	'''
	i1=Image.open(filePath)
	if (i1.size[0]==800) and (i1.size[1]==660):
		v800x660.append(filePath)
		ln='%s: (%d, %d)\n' % (i, i1.size[0], i1.size[1])
		ln='''
%s
\n''' % (filePath, filePath)
		ln='''

\n''' % (filePath, filePath)
		ln='''

 %s
''' % (filePath, filePath, filePath)
		ln=ln.replace('../../public_html/','')
		ln=ln.replace('screenshots/','')
		ln=ln.replace('dx9_','../dx9_')
		f800x660.write(ln);
	elif (i1.size[0]==800) and (i1.size[1]==600):
		v800x600.append(filePath)
		ln='%s: (%d, %d)\n' % (i, i1.size[0], i1.size[1])
		ln='''
%s
\n''' % (filePath, filePath)
		ln='''

\n''' % (filePath, filePath)
		ln='''

 %s
''' % (filePath, filePath, filePath)
		ln=ln.replace('../../public_html/','')
		ln=ln.replace('screenshots/','')
		ln=ln.replace('dx9_','../dx9_')
		f800x600.write(ln);
	elif (i1.size[0]==640) and (i1.size[1]==512):
		v640x512.append(filePath)
		ln='%s: (%d, %d)\n' % (i, i1.size[0], i1.size[1])
		ln='''
%s
\n''' % (filePath, filePath)
		ln='''

\n''' % (filePath, filePath)
		ln='''

 %s
''' % (filePath, filePath, filePath)
		ln=ln.replace('../../public_html/','')
		ln=ln.replace('screenshots/','')
		ln=ln.replace('dx9_','../dx9_')
		f640x512.write(ln);
	else:
		#print filePath, i1.size[0], i1.size[1]
		velse.append(filePath)
		ln='%s: (%d, %d)\n' % (i, i1.size[0], i1.size[1])
		ln='''
%s
\n''' % (filePath, filePath)
		ln='''

\n''' % (filePath, filePath)
		ln='''

 %s
''' % (filePath, filePath, filePath)
		ln=ln.replace('../../public_html/','')
		ln=ln.replace('screenshots/','')
		ln=ln.replace('dx9_','../dx9_')
		felse.write(ln);

def myfun(dummy, dirr, filess):
	#print '#3'
	#print dummy, dirr, filess
	#onlyfiles = [f for f in listdir(dirr) if isfile(join(dirr, f))]
	#print onlyfiles
	filess = [f for f in listdir(dirr) if isfile(join(dirr, f))]
	#print filess
	for child in filess:
	    #print child
	    if '.jpg' == os.path.splitext(child)[1] and os.path.isfile(dirr+'/'+child):
		    replaceStringInFile(dirr+'/'+child)
		    #print child

ln='''<TITLE>Screenshots</TITLE>
Screenshots

\n'

Screenshots (800, 660)

Screenshots (800, 600)

Screenshots (640, 512)

Screenshots

'''

ln1='<TITLE>Screenshots</TITLE>
Screenshots

\n'
ln2='

\n'

fscreenshots.write(ln)
f800x660.write(ln1)
f800x600.write(ln1)
f640x512.write(ln1)
felse.write(ln1)
v=['../../public_html']
for i in v:
	os.path.walk(i, myfun, 13)

f800x660.write(ln2)
f800x600.write(ln2)
f640x512.write(ln2)
felse.write(ln2)
f800x660.close()
f800x600.close()
f640x512.close()
felse.close()
fscreenshots.close()


import os
import json
import itertools
import numpy as np
from scipy.linalg import lstsq
from types import SimpleNamespace

from phasefinder import jackknife
from phasefinder.optimization import cvxopt_solve_qp
from phasefinder.datasets import Ising
from phasefinder.utils import build_path

###process magnetization, order parameter curves, and U_4 Binder cumulant curves

def gather_magnetizations(data_dir, results_dir, J, L, N_test=None):
	temperatures = []
	measurements = []
	L_dir = os.path.join(data_dir, J, "L{:d}".format(L))
	for temperature_dir in sorted(os.listdir(L_dir)):
		if temperature_dir[0] != "T":
			continue
		I = Ising()
		Ms = I.magnetization(os.path.join(L_dir, temperature_dir), per_spin=True, staggered=(J=="antiferromagnetic"))[-2::-2]
		temperatures.append(I.T)
		measurements.append(Ms)
	temperatures = np.array(temperatures)
	measurements = np.stack(measurements, 0)
	if N_test is not None:
		measurements = measurements[:,:N_test]
	output_dir = "{}/{}/magnetization/L{:d}".format(results_dir, J, L)
	os.makedirs(output_dir, exist_ok=True)
	np.savez(os.path.join(output_dir, "measurements.npz"), temperatures=temperatures, measurements=measurements)

def calculate_stats(results_dir, J, observable_name, L, N_test, N=None, fold=None, seed=None, L_test=None, bins=50):
	dir = build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, L_test=L_test)
	measurements = np.load(os.path.join(dir, "measurements.npz"))
	temperatures = measurements["temperatures"]
	measurements = measurements["measurements"].T[:N_test]
	order_means, order_stds = jackknife.calculate_mean_std( jackknife.calculate_samples(np.abs(measurements)) )
	u4_means, u4_stds = jackknife.calculate_mean_std( \
		1 - jackknife.calculate_samples(measurements**4)/( 3*jackknife.calculate_samples(measurements**2)**2 ) )
	distribution_range = (measurements.min(), measurements.max())
	distributions = []
	for slice in measurements.T:
		hist, _ = np.histogram(slice, bins=bins, range=distribution_range, density=False)
		hist = hist/len(slice)
		distributions.append(hist)
	distributions = np.stack(distributions, 0)
	distribution_range = np.array(list(distribution_range))
	output_dir = build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, L_test=L_test, subdir="processed")
	os.makedirs(output_dir, exist_ok=True)
	np.savez(os.path.join(output_dir, "stats.npz"), temperatures=temperatures, distributions=distributions, distribution_range=distribution_range, order_means=order_means, order_stds=order_stds, u4_means=u4_means, u4_stds=u4_stds)

### process critical temperature estimates

def U4_samples(measurements):
	samples_2 = jackknife.calculate_samples(measurements**2)
	samples_4 = jackknife.calculate_samples(measurements**4)
	samples = 1 - samples_4/(3*samples_2**2)
	return samples

def critical_temperature_samples(temperatures, u4samples):
	n_samples, n_temperatures = u4samples.shape
	nums = np.arange(n_temperatures+1)[None,:]
	nums[0,0] = 1
	cumsums = np.concatenate([np.zeros((n_samples, 1)), np.cumsum(u4samples, 1)], 1)
	means_forward = cumsums/nums
	means_backward = (cumsums[:,-1,None]-cumsums)/nums[:,::-1]
	tc_samples = []
	for i in range(n_samples):
		means = np.tril(np.repeat(means_forward[i,:,None], n_temperatures, axis=1), k=-1) \
			+ np.triu(np.repeat(means_backward[i,:,None], n_temperatures, axis=1), k=0)
		j = np.argmin( np.mean((u4samples[None,i,:]-means)**2, 1) )
		tc_samples.append( (temperatures[j]+temperatures[j-1])/2 )
	tc_samples = np.array(tc_samples)
	return tc_samples

def minimize_std(temperatures, samples):
	upper_bound_indices = np.less(temperatures[:,None], samples[None,:]).sum(0)
	upper_bounds = temperatures[upper_bound_indices]
	lower_bounds = temperatures[upper_bound_indices-1]
	n = len(samples)-1
	P = np.eye(n+1)
	P[:,0] = -1
	P[0,:] = -1
	P[0,0] = n
	P = P.astype(np.double)
	q = np.zeros((n+1)).astype(np.double)
	G = np.concatenate([np.eye(n+1), -np.eye(n+1)], 0).astype(np.double)
	h = np.concatenate([upper_bounds, -lower_bounds], 0).astype(np.double)
	new_samples = cvxopt_solve_qp(P, q, G=G, h=h)
	new_samples = new_samples + ( np.min(upper_bounds-new_samples) - np.min(new_samples-lower_bounds) )/2
	return new_samples

def minimize_std_without_bias(temperatures, samples):
	upper_bound_indices = np.less(temperatures[:,None], samples[None,:]).sum(0)
	upper_bounds = temperatures[upper_bound_indices]
	lower_bounds = temperatures[upper_bound_indices-1]
	upper_bounds = upper_bounds[1:]
	lower_bounds = lower_bounds[1:]
	n = len(samples)-1
	P = np.eye(n+1)
	P[:,0] = -1
	P[0,:] = -1
	P[0,0] = n
	P = P.astype(np.double)
	q = np.zeros((n+1)).astype(np.double)
	G = np.concatenate([np.eye(n+1)[1:], -np.eye(n+1)[1:]], 0).astype(np.double)
	h = np.concatenate([upper_bounds, -lower_bounds], 0).astype(np.double)
	A = np.full((1, n+1), -1, dtype=np.double)
	A[0,0] = n
	b = np.zeros((1), dtype=np.double)
	new_samples = cvxopt_solve_qp(P, q, G=G, h=h, A=A, b=b)
	new_samples = new_samples + ( np.min(upper_bounds-new_samples[1:]) - np.min(new_samples[1:]-lower_bounds) )/2
	new_samples[0] = new_samples[1:].mean()
	return new_samples

def lstsq_samples(x, y_samples, weights=None):
	ones = np.ones((x.shape[0], 1))
	if weights is None:
		weights = ones
	if weights.ndim == 1:
		weights = weights[:,None]
	if x.ndim == 1:
		x = x[:,None]
	x = np.concatenate((ones, x), 1)
	fit_samples = lstsq(x*weights, y_samples*weights)[0]
	yhat_samples = x.dot(fit_samples)
	y_samples_centered = y_samples - y_samples.mean(0, keepdims=True)
	yhat_samples_centered = yhat_samples - yhat_samples.mean(0, keepdims=True)
	r2_samples = (y_samples_centered*yhat_samples_centered).sum(0)**2/( np.sum(y_samples_centered**2, 0)*np.sum(yhat_samples_centered**2, 0) )
	return fit_samples, r2_samples

def calculate_critical_temperatures(results_dir, J, observable_name, L, N=None, fold=None, seed=None, L_test=None, optimize_std=False, optimize_reduce_bias=True):
	dir = build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, L_test=L_test)
	measurements = np.load(os.path.join(dir, "measurements.npz"))
	temperatures = measurements["temperatures"]
	measurements = measurements["measurements"].T
	samples = critical_temperature_samples(temperatures, U4_samples(measurements))
	if optimize_std:
		if optimize_reduce_bias:
			samples = minimize_std(temperatures, samples)
		else:
			samples = minimize_std_without_bias(temperatures, samples)
	mean, std, mean_bias, std_bias = jackknife.calculate_mean_std(samples, reduce_bias=False, return_bias=True)
	output_dir = build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, L_test=L_test, subdir="processed")
	os.makedirs(output_dir, exist_ok=True)
	np.savez(os.path.join(output_dir, "tc.npz"), mean=mean, std=std, mean_bias=mean_bias, std_bias=std_bias, samples=samples)

def calculate_critical_temperature_extrapolates(results_dir, J, observable_name, Ls, N=None, fold=None, seed=None, jitter=1e-6):
	load_tc = lambda L: np.load(os.path.join(build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, subdir="processed"), "tc.npz"))
	x = 1/np.array(Ls)
	y_samples = np.stack([load_tc(L)["samples"] for L in Ls], 0)
	weights = 1/(np.array([float(load_tc(L)["std"]) for L in Ls]) + jitter)
	fit_samples, r2_samples = lstsq_samples(x, y_samples, weights=weights)
	samples = np.concatenate([fit_samples, r2_samples[None,:]], 0).T
	mean, std, mean_bias, std_bias = jackknife.calculate_mean_std(samples, reduce_bias=False, return_bias=True)
	stats = ["mean", "std", "mean_bias", "std_bias"]
	values = [mean, std, mean_bias, std_bias]
	kwds = ["yintercept", "slope", "r2"]
	output_dict = {stat: value[0] for (stat, value) in zip(stats, values)}
	for ((i, kwd), (stat, value)) in itertools.product(enumerate(kwds), zip(stats, values)):
		output_dict["{}_{}".format(kwd, stat)] = value[i]
	output_dir = build_path(results_dir, J, observable_name, None, N, fold, seed, subdir="processed")
	os.makedirs(output_dir, exist_ok=True)
	np.savez(os.path.join(output_dir, "tc.npz"), **output_dict)

### process execution times

def calculate_times(results_dir, J, observable_name, L, N_test, N=None, fold=None, seed=None, L_test=None, n_Ls=1):
	L_dir = os.path.join("data", J, "L{:d}".format(L))
	T_dirs = sorted(os.listdir(L_dir))
	T_dirs = [os.path.join(L_dir, dir) for dir in T_dirs if dir[0] == "T"]
	with open(os.path.join(T_dirs[0], "args.json"), "r") as fp:
		args = json.load(fp)
	N_max = args["nmcs"]//args["nmeas"]
	generation_time = 0
	for dir in T_dirs:
		with open(os.path.join(dir, "time.txt"), "r") as fp:
			generation_time += float(fp.read())
	generation_time_ieq = generation_time*args["ieq"]/(args["ieq"] + args["nmcs"])
	generation_time_mc = generation_time*args["nmcs"]/(args["ieq"] + args["nmcs"])
	if observable_name == "magnetization":
		N = 0
		training_time = 0
		preprocessing_time = 0
	else:
		dir = build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, L_test=None)
		with open(os.path.join(dir, "results.json"), "r") as fp:
			results = json.load(fp)
		training_time = results["time"]
		with open(os.path.join(L_dir, "aggregate", "times.json"), "r") as fp:
			key = "states_symmetric" if results["args"]["symmetric"] else "states"
			preprocessing_time = json.load(fp)[key]
	generation_time_mc *= (N_test + N/n_Ls)/N_max
	preprocessing_time *= (N_test + N/n_Ls)/N_max
	output_dir = build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, L_test=L_test, subdir="processed")
	os.makedirs(output_dir, exist_ok=True)
	np.savez(os.path.join(output_dir, "time.npz"), generation=generation_time_ieq+generation_time_mc, preprocessing=preprocessing_time, training=training_time)

def calculate_time_extrapolates(results_dir, J, observable_name, Ls, N=None, fold=None, seed=None):
	dicts = [dict(np.load(os.path.join(build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, subdir="processed"), "time.npz"))) for L in Ls]
	output_dict = {key: sum(D[key] for D in dicts) for key in dicts[0]}
	output_dir = build_path(results_dir, J, observable_name, None, N=N, fold=fold, seed=seed, subdir="processed")
	os.makedirs(output_dir, exist_ok=True)
	np.savez(os.path.join(output_dir, "time.npz"), **output_dict)

### calculate functional correlations

def calculate_functional_cors(results_dir, J, observable_name, L, N=None, fold=None, seed=None, L_test=None):
	dir = build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, L_test=L_test)
	measurements = np.load(os.path.join(dir, "measurements.npz"))
	temperatures = measurements["temperatures"]
	measurements = measurements["measurements"].T
	magnetization = np.load(os.path.join(build_path(results_dir, J, "magnetization", L), "measurements.npz"))["measurements"].T
	cor = np.sum(measurements*magnetization)/np.sqrt(np.sum(measurements**2)*np.sum(magnetization**2))
	cor = np.maximum(0, 1-cor**2)
	onsager = np.where(temperatures<2/np.log(1+np.sqrt(2)), np.clip(1-1/np.sinh(2/temperatures)**4, 0, None)**(1/8), np.zeros_like(temperatures))
	samples = jackknife.calculate_samples(np.abs(measurements))
	samples = samples.dot(onsager)/np.sqrt(np.sum(samples**2, 1)*np.sum(onsager**2))
	samples = np.maximum(0, 1-samples**2)
	mean, std, mean_bias, std_bias = jackknife.calculate_mean_std(samples, reduce_bias=False, return_bias=True)
	output_dir = build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, L_test=L_test, subdir="processed")
	os.makedirs(output_dir, exist_ok=True)
	np.savez(os.path.join(output_dir, "cor.npz"), magnetization=cor, onsager_mean=mean, onsager_std=std, onsager_mean_bias=mean_bias, onsager_std_bias=std_bias, onsager_samples=samples)

def calculate_functional_cor_extrapolates(results_dir, J, observable_name, Ls, N=None, fold=None, seed=None, jitter=1e-6):
	load_tc = lambda L: np.load(os.path.join(build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, subdir="processed"), "cor.npz"))
	x = 1/np.array(Ls)
	y_samples = np.stack([load_tc(L)["onsager_samples"] for L in Ls], 0)
	weights = 1/(np.array([float(load_tc(L)["onsager_std"]+load_tc(L)["onsager_std_bias"]) for L in Ls]) + jitter)
	fit_samples, r2_samples = lstsq_samples(x, y_samples, weights=weights)
	samples = np.concatenate([fit_samples, r2_samples[None,:]], 0).T
	mean, std, mean_bias, std_bias = jackknife.calculate_mean_std(samples, reduce_bias=False, return_bias=True)
	stats = ["mean", "std", "mean_bias", "std_bias"]
	values = [mean, std, mean_bias, std_bias]
	kwds = ["yintercept", "slope", "r2"]
	output_dict = {stat: value[0] for (stat, value) in zip(stats, values)}
	for ((i, kwd), (stat, value)) in itertools.product(enumerate(kwds), zip(stats, values)):
		output_dict["{}_{}".format(kwd, stat)] = value[i]
	output_dict = {"onsager_"+key: value for (key, value) in output_dict.items()}
	output_dict["magnetization"] = 0
	output_dir = build_path(results_dir, J, observable_name, None, N, fold, seed, subdir="processed")
	os.makedirs(output_dir, exist_ok=True)
	np.savez(os.path.join(output_dir, "cor.npz"), **output_dict)

### gather critical temperatures, times, and correlations into CSV

def aggregate_tc(results_dir, J, observable_name, L, N=None, folds=None, seeds=None, reduce_bias=False, jackknife_std=True, r2=False):
	stats = ["mean", "std", "mean_bias", "std_bias"]
	key = SimpleNamespace(**{stat: stat for stat in stats})
	if r2:
		for stat in stats:
			setattr(key, stat, "r2_"+stat)
	if observable_name == "magnetization":
		with np.load(os.path.join(build_path(results_dir, J, observable_name, L, subdir="processed"), "tc.npz")) as D:
			mean, std = D[key.mean]+D[key.mean_bias]*int(reduce_bias), np.sqrt(D[key.std]**2+D[key.std_bias]**2*int(reduce_bias))*int(jackknife_std)
		return mean, std
	else:
		means, stds = [], []
		for (fold, seed) in itertools.product(folds, seeds):
			with np.load(os.path.join(build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, subdir="processed"), "tc.npz")) as D:
				means.append(D[key.mean]+D[key.mean_bias]*int(reduce_bias))
				stds.append(np.sqrt(D[key.std]**2+D[key.std_bias]**2*int(reduce_bias))*int(jackknife_std))
		means = np.array(means)
		stds = np.array(stds)
		mean = means.mean()
		std = np.sqrt( (stds**2).mean() + means.std()**2 )
		return mean, std

def aggregate_time(results_dir, J, observable_name, L, N=None, folds=None, seeds=None, key="training", reduce=np.mean):
	if observable_name == "magnetization":
		with np.load(os.path.join(build_path(results_dir, J, observable_name, L, subdir="processed"), "time.npz")) as D:
			t = D[key]
		return t
	else:
		ts = []
		for (fold, seed) in itertools.product(folds, seeds):
			with np.load(os.path.join(build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, subdir="processed"), "time.npz")) as D:
				ts.append(D[key])
		ts = np.array(ts)
		t = reduce(ts)
		return t

def aggregate_cor(results_dir, J, observable_name, L, N=None, folds=None, seeds=None, key="magnetization", func=np.abs, reduce_bias=False, jackknife_std=True):
	assert key in ["magnetization", "onsager"], "Keyword argument key must be either magnetization or onsager; got {} instead.".format(reduce)
	if observable_name == "magnetization":
		with np.load(os.path.join(build_path(results_dir, J, observable_name, L, subdir="processed"), "cor.npz")) as D:
			mean = func(D["magnetization"]) if key=="magnetization" else D["onsager_mean"]+D["onsager_mean_bias"]*int(reduce_bias)
			std = 0 if key=="magnetization" else np.sqrt(D["onsager_std"]**2+D["onsager_std_bias"]**2*int(reduce_bias))*int(jackknife_std)
		return mean, std
	else:
		means, stds = [], []
		for (fold, seed) in itertools.product(folds, seeds):
			with np.load(os.path.join(build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, subdir="processed"), "cor.npz")) as D:
				mean = func(D["magnetization"]) if key=="magnetization" else D["onsager_mean"]+D["onsager_mean_bias"]*int(reduce_bias)
				std = 0 if key=="magnetization" else np.sqrt(D["onsager_std"]**2+D["onsager_std_bias"]**2*int(reduce_bias))*int(jackknife_std)
				means.append(mean)
				stds.append(std)
		means = np.array(means)
		stds = np.array(stds)
		mean = means.mean()
		std = np.sqrt( (stds**2).mean() + means.std()**2 )
		return mean, std

def aggregate_slope_yintercept(results_dir, J, observable_name, L, N=None, folds=None, seeds=None, reduce_bias=False, prefix="tc"):
	assert prefix in ["tc", "onsager"], "Keyword prefix must be either tc or onsager."
	if L is not None:
		return 0, 0
	kwds = {"slope": "slope_mean", "slope_bias": "slope_mean_bias", "yintercept": "yintercept_mean", "yintercept_bias": "yintercept_mean_bias"}
	if prefix == "onsager":
		kwds = {key: prefix+"_"+value for (key, value) in kwds.items()}
	key = SimpleNamespace(**kwds)
	filename = "tc.npz" if prefix == "tc" else "cor.npz"
	if observable_name == "magnetization":
		with np.load(os.path.join(build_path(results_dir, J, observable_name, L, subdir="processed"), filename)) as D:
			slope, yintercept = D[key.slope]+D[key.slope_bias]*int(reduce_bias), D[key.yintercept]+D[key.yintercept_bias]*int(reduce_bias)
		return slope, yintercept
	else:
		slopes, yintercepts = [], []
		for (fold, seed) in itertools.product(folds, seeds):
			with np.load(os.path.join(build_path(results_dir, J, observable_name, L, N=N, fold=fold, seed=seed, subdir="processed"), filename)) as D:
				slopes.append(D[key.slope]+D[key.slope_bias]*int(reduce_bias))
				yintercepts.append(D[key.yintercept]+D[key.yintercept_bias]*int(reduce_bias))
		slope = np.array(slopes).mean()
		yintercept = np.array(yintercepts).mean()
		return slope, yintercept

def gather(results_dir, Js, observable_names, Ls, Ns, folds, seeds, tc_reduce_bias=False, cor_reduce_bias=True, jackknife_std=True):
	tc = 2/np.log(1+np.sqrt(2))
	output_dir = 	os.path.join(results_dir, "processed")
	os.makedirs(output_dir, exist_ok=True)
	with open(os.path.join(output_dir, "gathered.csv"), "w") as fp:
		fp.write("J,L,observable,N,tc_mean,tc_std,tc_slope,tc_yintercept,generation_time,preprocessing_time,training_time,cor_magnetization_mean,cor_magnetization_std,cor_magnetization_sign,cor_onsager_mean,cor_onsager_std,cor_onsager_slope,cor_onsager_yintercept\n")
		for (J, L, observable_name, N) in itertools.product(Js, Ls, observable_names, Ns):
			tc_mean, tc_std = aggregate_tc(results_dir, J, observable_name, L, N=N, folds=folds, seeds=seeds, reduce_bias=tc_reduce_bias, jackknife_std=jackknife_std)
			tc_slope, tc_yintercept = aggregate_slope_yintercept(results_dir, J, observable_name, L, N=N, folds=folds, seeds=seeds, reduce_bias=tc_reduce_bias, prefix="tc")
			tc_mean = 100*(tc_mean/tc - 1)
			tc_std = 100*tc_std/tc
			tc_slope = 100*tc_slope/tc
			tc_yintercept = 100*(tc_yintercept/tc - 1)
			generation_time = aggregate_time(results_dir, J, observable_name, L, N=N, folds=folds, seeds=seeds, key="generation", reduce=np.mean)/60
			preprocessing_time = aggregate_time(results_dir, J, observable_name, L, N=N, folds=folds, seeds=seeds, key="preprocessing", reduce=np.mean)/60
			training_time = aggregate_time(results_dir, J, observable_name, L, N=N, folds=folds, seeds=seeds, key="training", reduce=np.sum)/60
			cor_magnetization_mean, cor_magnetization_std = aggregate_cor(results_dir, J, observable_name, L, N=N, folds=folds, seeds=seeds, key="magnetization", reduce_bias=cor_reduce_bias, jackknife_std=jackknife_std)
			cor_onsager_mean, cor_onsager_std = aggregate_cor(results_dir, J, observable_name, L, N=N, folds=folds, seeds=seeds, key="onsager", reduce_bias=cor_reduce_bias, jackknife_std=jackknife_std)
			cor_onsager_slope, cor_onsager_yintercept = aggregate_slope_yintercept(results_dir, J, observable_name, L, N=N, folds=folds, seeds=seeds, reduce_bias=cor_reduce_bias, prefix="onsager")
			cor_magnetization_sign, _ = aggregate_cor(results_dir, J, observable_name, L, N=N, folds=folds, seeds=seeds, key="magnetization", func=np.sign, reduce_bias=cor_reduce_bias, jackknife_std=jackknife_std)
			L_str = "{:d}".format(L) if L is not None else "inf"
			fp.write("{},{},{},{:d},{:.3f},{:.3f},{:.3f},{:.3f},{:.3f},{:.3f},{:.3f},{:.3f},{:.3f},{:.3f},{:.3f},{:.3f},{:.3f},{:.3f}\n".format(J, L, observable_name, N, tc_mean, tc_std, tc_slope, tc_yintercept, generation_time, preprocessing_time, training_time, 100*cor_magnetization_mean, 100*cor_magnetization_std, 100*cor_magnetization_sign, 100*cor_onsager_mean, 100*cor_onsager_std, 100*cor_onsager_slope, 100*cor_onsager_yintercept))

### process symmetry generators

def calculate_generators(results_dir, Js, encoder_names, Ls, Ns, folds, seeds):
	generator_types = ["spatial", "internal"]
	output_dict = {J: {name: {gen_type: {} for gen_type in generator_types} for name in encoder_names} for J in Js}
	for (J, name, gen_type) in itertools.product(Js, encoder_names, generator_types):
		gens = []
		for (L, N, fold, seed) in itertools.product(Ls, Ns, folds, seeds):
			with open(os.path.join(build_path(results_dir, J, name, L, N=N, fold=fold, seed=seed), "generator_reps.json"), "r") as fp:
				gens.append( json.load(fp)[gen_type] )
		gens = np.array(gens)
		output_dict[J][name][gen_type]["mean"] = float(gens.mean())
		output_dict[J][name][gen_type]["std"] = float(gens.std())
	output_dir = os.path.join(results_dir, "processed")
	os.makedirs(output_dir, exist_ok=True)
	with open(os.path.join(output_dir, "generators.json"), "w") as fp:
		json.dump(output_dict, fp, indent=2)

if __name__ == "__main__":
	results_dir = "results"
	Js = ["ferromagnetic", "antiferromagnetic"]
	Ls = [16, 32, 64, 128]
	Ns = [8, 16, 32, 64, 128, 256]
	N_test = 2048

	observable_names = ["magnetization", "latent", "latent_equivariant"]
	encoder_names = ["latent", "latent_equivariant", "latent_multiscale_4"]
	n_Lss = [1, 1, 4]

	folds = list(range(8))
	seeds = list(range(3))

	print("Gathering magnetizations . . . ")
	for (J, L) in itertools.product(Js, Ls):
		gather_magnetizations("data", results_dir, J, L, N_test=N_test)

	print("Calculating stats . . . ")
	for (J, name) in itertools.product(Js, observable_names):
		calculate_stats(results_dir, J, name, Ls[-1], N_test, N=Ns[-1], fold=0, seed=0)

	print("Calculating critical temperatures . . . ")
	for J in Js:
		print("J:", J)
		print("magnetization")
		for L in Ls:
			calculate_critical_temperatures(results_dir, J, "magnetization", L, optimize_std=True, optimize_reduce_bias=True)
		for encoder_name in encoder_names:
			print(encoder_name)
			for (L, N, fold, seed) in itertools.product(Ls, Ns, folds, seeds):
				calculate_critical_temperatures(results_dir, J, encoder_name, L, N=N, fold=fold, seed=seed, optimize_std=True, optimize_reduce_bias=True)

	print("Calculating critical temperature extrapolates . . . ")
	for J in Js:
		print("J:", J)
		print("magnetization")
		calculate_critical_temperature_extrapolates(results_dir, J, "magnetization", Ls)
		for encoder_name in encoder_names:
			print(encoder_name)
			for (N, fold, seed) in itertools.product(Ns, folds, seeds):
				calculate_critical_temperature_extrapolates(results_dir, J, encoder_name, Ls, N=N, fold=fold, seed=seed)

	print("Calculating execution times . . . ")
	for J in Js:
		print("J:", J)
		print("magnetization")
		for L in Ls:
			calculate_times(results_dir, J, "magnetization", L, N_test)
		for (encoder_name, n_Ls) in zip(encoder_names, n_Lss):
			print(encoder_name)
			for (L, N, fold, seed) in itertools.product(Ls, Ns, folds, seeds):
				calculate_times(results_dir, J, encoder_name, L, N_test, N=N, fold=fold, seed=seed, n_Ls=n_Ls)

	print("Calculating execution time extrapolates . . . ")
	for J in Js:
		print("J:", J)
		print("magnetization")
		for L in Ls:
			calculate_time_extrapolates(results_dir, J, "magnetization", Ls)
		for encoder_name in encoder_names:
			print(encoder_name)
			for (N, fold, seed) in itertools.product(Ns, folds, seeds):
				calculate_time_extrapolates(results_dir, J, encoder_name, Ls, N=N, fold=fold, seed=seed)

	print("Calculating functional correlations . . . ")
	for J in Js:
		print("J:", J)
		print("magnetization")
		for L in Ls:
			calculate_functional_cors(results_dir, J, "magnetization", L)
		for encoder_name in encoder_names:
			print(encoder_name)
			for (L, N, fold, seed) in itertools.product(Ls, Ns, folds, seeds):
				calculate_functional_cors(results_dir, J, encoder_name, L, N=N, fold=fold, seed=seed)

	print("Calculating functional correlation extrapolates . . . ")
	for J in Js:
		print("J:", J)
		print("magnetization")
		calculate_functional_cor_extrapolates(results_dir, J, "magnetization", Ls)
		for encoder_name in encoder_names:
			print(encoder_name)
			for (N, fold, seed) in itertools.product(Ns, folds, seeds):
				calculate_functional_cor_extrapolates(results_dir, J, encoder_name, Ls, N=N, fold=fold, seed=seed)

	print("Gathering results into CSV . . . ")
	gather(results_dir, Js, ["magnetization"]+encoder_names, Ls+[None], Ns, folds, seeds, tc_reduce_bias=False, cor_reduce_bias=True, jackknife_std=True)

	print("Calculating symmetry generators . . . ")
	calculate_generators(results_dir, Js, ["latent_equivariant"], Ls, Ns, folds, seeds)

	print("Done!")