import copy
import time
from torchvision.utils import make_grid
import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.nn as nn
import torchvision
from torchnet import meter
from torch.autograd import Variable
from tqdm import tqdm
import os

from OXFORD_IIIT.src.data_loader import dset_classes, dset_loaders, dset_sizes, dsets
from OXFORD_IIIT.utils.config import LR, LR_DECAY_EPOCH, NUM_EPOCHS, NUM_IMAGES, MOMENTUM, BATCH_SIZE
from OXFORD_IIIT.utils.logger import Logger
from OXFORD_IIIT.utils.custom_plot import plot_acc_loss
from OXFORD_IIIT.src.densenet_DIY import densenet_DIY
from OXFORD_IIIT.vis_sample import visualize_sample

def record(filepath, mode, results):
    # if not os.path.exists(filepath):
    #     open(filepath,'w')
    with open(filepath, mode) as f:
        f.write(results)

def to_np(x):
    return x.data.cpu().numpy()

def exp_lr_scheduler(optimizer, epoch, init_lr=LR, lr_decay_epoch=LR_DECAY_EPOCH):
    lr = init_lr * (0.1**(epoch // lr_decay_epoch))

    if epoch % lr_decay_epoch == 0:
        print('Learning Rate: {}'.format(lr))

    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

    return optimizer

def train_model(model, criterion, optimizer, lr_scheduler, num_epochs=NUM_EPOCHS):
    since = time.time()

    best_model = model
    best_acc = 0.0
    loss_list, acc_list, val_loss_list, val_acc_list = [], [], [], []

    record_file = 'Results/txt/model_breeds_Epoch_%s_LR_%s_batchSize_%s.txt'% (str(num_epochs),str(LR),str(BATCH_SIZE))

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch + 1, num_epochs))
        print('-' * 50)
        record(record_file, 'a',  ('Epoch {}/{}\n'.format(epoch + 1, num_epochs)) + ('--' * 50) + '\n')

        for phase in ['train', 'val']:
            if phase == 'train':
                optimizer = lr_scheduler(optimizer, epoch)
                model.train(True)  # Set model to training mode
            else:
                model.train(False)  # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            for inputs, labels in tqdm(dset_loaders[phase]):
                '''
                # inputs torch.Size([batch_size, 3, 224, 224])
                # labels torch.Size([batch_size])
                '''
                if torch.cuda.is_available():
                    inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())
                else:
                    inputs, labels = Variable(inputs), Variable(labels)

                optimizer.zero_grad()

                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs.data, 1)
                    loss = criterion(outputs, labels)

                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                running_loss += loss.data.item() * inputs.size(0)  # inputs.size(0) 即 batch_size
                running_corrects += (preds == labels).sum().item()

            epoch_loss = running_loss / dset_sizes[phase]
            epoch_acc = running_corrects / dset_sizes[phase]

            print('{} Loss: {:.8f} Acc: {:.8f}'.format(phase, epoch_loss, epoch_acc))
            record(record_file, 'a', ('{} Loss: {:.8f} Acc: {:.8f}\n'.format(phase, epoch_loss, epoch_acc)) + '\n')

            if phase == 'train':
                loss_list.append(epoch_loss)
                acc_list.append(epoch_acc)
            else:
                val_loss_list.append(epoch_loss)
                val_acc_list.append(epoch_acc)

            if phase == 'val' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model = copy.deepcopy(model)

                # time_log = time.strftime('%Y-%m-%d-%H-%M', time.localtime(time.time()))
                save_model_path = 'Results/model/model_breeds_Epoch_%s_LR_%s_batch_size_%s_.pkl ' % (
                str(num_epochs), str(LR), str(BATCH_SIZE))
                torch.save(best_model.state_dict(), save_model_path)
                print("save_model ... ")

            if phase == 'val':
                # ============ TensorBoard logging ============#
                # (1) Log the scalar values
                info = {'loss': epoch_loss, 'accuracy': epoch_acc}
                for tag, value in info.items():
                    logger.scalar_summary(tag, value, epoch + 1)

                # (2) Log values and gradients of the parameters (histogram)
                for tag, value in model.named_parameters():
                    tag = tag.replace('.', '/')
                    logger.histo_summary(tag, to_np(value), epoch + 1)
                    logger.histo_summary(tag + '/grad', to_np(value.grad), epoch + 1)

                # (3) Log the images
                info = {'images': to_np(inputs.view(-1, 3, 224, 224)[:24])}  # view()创建了一个新的对象，与原对象共享相同的数据 #  R G B
                for tag, inputs in info.items():  # inputs
                    logger.image_summary(tag, inputs, epoch + 1)

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:8f}\n'.format(best_acc))
    results = ('\nTraining complete in {:.0f}m {:.0f}s\n'.format(time_elapsed // 60, time_elapsed % 60)) + ('Best val Acc: {:8f}\n'.format(best_acc))
    record(record_file, 'a', results)

    plot_path = 'Results/plot/'
    plot_acc_loss(loss_list, val_loss_list, acc_list, val_acc_list, plot_path)

    return best_model

def imshow(inp, title=None):
    """Imshow for Tensor."""
    inp = inp.numpy().transpose((1, 2, 0))
    mean = np.array([0.485, 0.456, 0.406])  # the same as data_loader config
    std = np.array([0.229, 0.224, 0.225])
    inp = std * inp + mean
    plt.imshow(inp)
    if title is not None:
        plt.title(title)
    # plt.pause(1)

def visualize_model(model, num_images=NUM_IMAGES):
    images_so_far = 0
    plt.figure()

    for _, data in enumerate(dset_loaders['val']):
        inputs, labels = data

        if torch.cuda.is_available():
            inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())
        else:
            inputs, labels = Variable(inputs), Variable(labels)

        outputs = model(inputs)
        _, preds = torch.max(outputs.data, 1)

        for j in range(inputs.size()[0]):
            images_so_far += 1
            ax = plt.subplot(num_images//2, 2, images_so_far)
            ax.axis('off')
            ax.set_title('predicted: {}'.format(dset_classes[preds[j]]))
            imshow(inputs.cpu().data[j])

            if images_so_far == num_images:
                plt.savefig(os.path.join(record_path, "val_sampling_preds.png"))
                return

def visualize_prediction(model, env_name, num_images = 1000):

    from visdom import Visdom
    viz = Visdom(env=env_name)
    images_so_far = 0

    for _, data in enumerate(dset_loaders['val']):

        if images_so_far == num_images:
            break

        inputs, labels = data

        if torch.cuda.is_available():
            inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())
        else:
            inputs, labels = Variable(inputs), Variable(labels)  # 若内存不够,就用cpu

        outputs = model(inputs)
        _, preds = torch.max(outputs.data, 1)

        for j in range(inputs.size()[0]):

            viz.image(inputs.cpu().data[j],  # Val_dataset with No transforms.Normalize
                      opts=dict(title='predicted: {}'.format(dset_classes[preds[j]])))

            images_so_far += 1

class CNNModel(nn.Module):

    def __init__(self):
        super(CNNModel, self).__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True),
            nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),

            nn.Conv2d(64, 128, kernel_size=5, padding=2),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.Conv2d(128, 192, kernel_size=5, padding=2),
            nn.BatchNorm2d(192),
            nn.ReLU(inplace=True),
            nn.Conv2d(192, 192, kernel_size=3, padding=1),
            nn.BatchNorm2d(192),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),

            nn.Conv2d(192, 384, kernel_size=3, padding=1),
            nn.BatchNorm2d(384),
            nn.ReLU(inplace=True),
            nn.Conv2d(384, 256, kernel_size=3, padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, kernel_size=3, padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 256, kernel_size=3, padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(inplace=True),
            nn.Conv2d(256, 512, kernel_size=3, padding=1),
            nn.BatchNorm2d(512),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
        )
        self.classifier = nn.Sequential(
            nn.Linear(512 * 6 * 6, 2048),
            nn.BatchNorm1d(2048),
            nn.ReLU(inplace=True),
            nn.Dropout(0.5),
            nn.Linear(2048, 2048),
            nn.BatchNorm1d(2048),
            nn.ReLU(inplace=True),
            nn.Dropout(0.5),
            nn.Linear(2048, 37),
            nn.LogSoftmax()
        )

    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), -1)
        x = self.classifier(x)
        return x

if __name__ == '__main__':

    print('\nProcessing Model Breeds...\n')

    logger = Logger('Results/logs')
    record_path = 'Results/txt'
    classes_breeds = dsets['train'].classes

    model = CNNModel()

    # densenet
    model = torchvision.models.densenet161(pretrained=True)
    num_ftrs = model.classifier.in_features
    model.classifier = nn.Linear(num_ftrs, 37)

    # resnet
    model = torchvision.models.resnet50(pretrained=True)
    num_ftrs = model.fc.in_features
    model.classifier = nn.Linear(num_ftrs, 37)

    # densenet-DIY
    model = densenet_DIY()

    # load the pre-saved model
    try:
        saved_model = 'Results/model/DenseNet/densenet161/'
        last_saved_model = sorted(os.listdir(saved_model))[-1]
        load_model_path = saved_model + last_saved_model
        if 'pkl' in last_saved_model:
            model.load_state_dict(torch.load(load_model_path))
            print('load the saved %s successfully ~' % load_model_path)
    except Exception as e:
        print(e)
        pass

    if torch.cuda.is_available():
        model.cuda()

    criterion = nn.CrossEntropyLoss().cuda()

    optimizer = torch.optim.SGD(model.parameters(), lr=LR, momentum=MOMENTUM)

    model = train_model(model, criterion, optimizer, exp_lr_scheduler, num_epochs=NUM_EPOCHS)

    visualize_prediction(model,'visualize-prediction')
    visualize_model(model)

    plt.ioff()
    plt.show()


#!/usr/bin/python
# -*- codding: utf-8 -*-
import os
import sys
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
from common.execute_command import write_two_parameter

# url : https://awscli.amazonaws.com/v2/documentation/api/latest/reference/dms/test-connection.html
if __name__ == '__main__':
    """
	delete-connection : https://awscli.amazonaws.com/v2/documentation/api/latest/reference/dms/delete-connection.html
	describe-connections : https://awscli.amazonaws.com/v2/documentation/api/latest/reference/dms/describe-connections.html
    """

    parameter_display_string = """
    # replication-instance-arn : The Amazon Resource Name (ARN) of the replication instance.
    # endpoint-arn : The Amazon Resource Name (ARN) string that uniquely identifies the endpoint.
    """
    add_option_dict = {}
    add_option_dict["parameter_display_string"] = parameter_display_string
    # ex: add_option_dict["no_value_parameter_list"] = "--single-parameter"
    write_two_parameter("dms", "test-connection", "replication-instance-arn", "endpoint-arn", add_option_dict)


import os
import time
from unittest import TestCase

import allure
from allure_commons.types import AttachmentType
from selenium import webdriver
from selenium.webdriver import DesiredCapabilities
from selenium.webdriver.support.abstract_event_listener import AbstractEventListener
from selenium.webdriver.support.event_firing_webdriver import EventFiringWebDriver

from libs.api.FirmwareServices.firmware_api import FirmwareApi
from libs.api.GroupManagementServices.group_management_api import GroupManagementApi
from libs.utils.RestFrontEnd import RestFrontEnd
from libs.utils.aruba_automation_config import ArubaAutomationConfig
from libs.utils.customer_logger import CustomLogger

class BaseUniversalTest(TestCase):

    @classmethod
    def setUpClass(cls):
        # super().setUpClass()
        # os.environ[
        #     "ARUBA_AUTOMATION_TESTCASE_PATH"] = "/Users/sibasishmohanta/Documents/Development/ArubaAutomation/tests"
        cls.log = CustomLogger()
        cls.log.setup_logger()
        cls.aruba_automation_config = ArubaAutomationConfig(dump_flag=True, quiet=False)
        # Setting up Driver instance

    def get_central_sign_in_url(self):
        return self.aruba_automation_config.get_property("central", "login_url")

    def central_site_setup(self):
        driver = self.setup_browser()
        time.sleep(2)
        driver.get(self.get_central_sign_in_url())
        return driver

    def get_driver(self, selenium_env, capabilities, command_executor, chrome_path=None):
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('--start-maximized')
        # fixing issue - WebDriverException: Message: unknown error: DevToolsActivePort file doesn't exist
        chrome_options.add_argument('--disable-dev-shm-usage')
        BROWSER_MAPPING = {
            "firefox": webdriver.Firefox,
            "chrome": webdriver.Chrome,
            "safari": webdriver.Safari,
            "remote": webdriver.Remote
        }
        browser = capabilities.get("browserName")
        if selenium_env == 'remote':
            self.start = BROWSER_MAPPING[selenium_env](command_executor=command_executor,
                                                       desired_capabilities=capabilities,
                                                       options=chrome_options)
            return EventFiringWebDriver(self.start, ScreenshotListener())
        else:
            browser = BROWSER_MAPPING[browser](capabilities=capabilities) \
                if not chrome_path else BROWSER_MAPPING[browser](chrome_path)
            return browser

    def setup_browser(self):
        selenium_env = self.aruba_automation_config.get_property("BrowserSettings", "selenium_env")
        select_browser = self.aruba_automation_config.get_property("BrowserSettings", "browser_name")
        implicit_wait = self.aruba_automation_config.get_property("BrowserSettings", "implicit_wait")
        command_executor = self.aruba_automation_config.get_property("BrowserSettings", "selenium_hub")
        if select_browser == "firefox":
            firefox_capabilities = DesiredCapabilities.FIREFOX
            firefox_capabilities['marionette'] = True
            self.driver = self.get_driver(selenium_env, firefox_capabilities, command_executor)
        elif select_browser == "chrome":
            chrome_capabilities = DesiredCapabilities.CHROME
            chrome_path = '{}/libs/browser/chromedriver'.format(
                os.path.dirname(os.path.dirname((os.path.abspath(__file__)))))
            self.driver = self.get_driver(selenium_env, chrome_capabilities, command_executor, chrome_path=chrome_path)
        else:
            raise Exception("Invalid browser selected - Supported browsers: Firefox, Chrome")
        self.driver.fullscreen_window()
        self.driver.implicitly_wait(implicit_wait)
        return self.driver

    @classmethod
    def tearDownClass(cls):
        cls.log.printLog("Inside Universal tearDownClass....")

    def tearDown(self):
        try:
            for method, err in self._outcome.errors:
                if err:
                    self.log.printLog("Test ended with an error. Taking screenshot")
                    allure.attach('test_error_screenshot', self.driver.get_screenshot_as_png(), type=AttachmentType.PNG)
            self.log.printLog('Test done - shutting down the browser')
            self.driver.close()
            self.driver.quit()
        except:
            self.log.printLog("Driver is already closed")

class ScreenshotListener(AbstractEventListener):
    """Listener for taking screenshot."""

    def on_exception(cls, exception, driver):
        """Take screenshot when the exception doesn't relate to stale element."""
        msg = "The element reference is stale. Either the element is no longer attached to the DOM or the page has" + \
              " been refreshed."
        if msg in exception.msg:
            print("Passing StaleElementReferenceException")
        else:
            allure.attach('exception_screenshot', driver.get_screenshot_as_png(), type=AttachmentType.PNG)
            driver.quit()


from aoiintegration import tools
import rasterio
import numpy as np
import os

repo = '/Users/rachelwegener/repos/floodwatch/processes/aoiintegration/tests/data/'
# save_location = os.path.join(repo, 'relative_windows_miny_offset_res.tif')
# east, south, west, north = -7, -21, 2, -6
# res = 3
# height, width = int((north - south) / res), int((west - east) / res)
# a = rasterio.transform.from_bounds(east, south, west, north, width, height)
# blank = np.ones([1, height, width], dtype=rasterio.ubyte)
# with rasterio.open(save_location, 'w', driver='GTiff', height=height, width=width, count=1, dtype=blank.dtype,
#                    crs='epsg:4326', transform=a) as grids_reader:
#     grids_reader.write(blank)
#
# with rasterio.open(save_location, 'r')as src:
#     print(src.transform)
#     print(src.width, src.height)

path_relative_windows_reference = os.path.join(repo, 'relative_windows_reference.tif')
path_relative_windows_contains = os.path.join(repo, 'relative_windows_miny_offset_res.tif')
with rasterio.open(path_relative_windows_reference, 'r') as rasterA_reader, \
     rasterio.open(path_relative_windows_contains, 'r') as rasterB_reader:
    print(rasterA_reader.transform, rasterA_reader.width, rasterA_reader.height)
    print()
    print(rasterB_reader.transform, rasterB_reader.width, rasterB_reader.height)
    window_overlapinA, window_overlapinB = tools.relative_windows(rasterA_reader, rasterB_reader)
    print(window_overlapinA)
    print(window_overlapinB)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('inference2', '0030_version_active'),
    ]

    operations = [
        migrations.AlterField(
            model_name='version',
            name='title',
            field=models.CharField(max_length=50, unique=True),
        ),
    ]


import flask
from telebot import types
from mongoengine import connect

from .. import config
from .bot_handlers import BotHandlers

connect(config.PROJECT_NAME, alias='default')

app = flask.Flask(__name__)
bot_handler = BotHandlers(config.BOT_TOKEN)

@app.route(config.WEBHOOK_URL_PATH, methods=['POST', 'GET'])
def web_hook():
    if flask.request.headers.get('content-type') == 'application/json':
        json_string = flask.request.get_data().decode('utf-8')
        update = types.Update.de_json(json_string)
        bot_handler.bot.process_new_updates([update])
        return ''
    else:
        flask.abort(403)


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-x86_64/egg/helpcenter/migrations/0006_auto_20160715_1917.py
# Compiled at: 2016-09-30 00:56:02
from __future__ import unicode_literals
from django.db import migrations

def add_edit_time(apps, schema_editor):
    """ Add edit_time to all existing Article instances """
    Article = apps.get_model(b'helpcenter', b'Article')
    for article in Article.objects.all():
        if not article.time_edited:
            article.time_edited = article.time_published
        article.save()

class Migration(migrations.Migration):
    dependencies = [
     ('helpcenter', '0005_article_time_edited')]
    operations = [
     migrations.RunPython(add_edit_time)]


# -*- coding: utf-8 -*-
"""
Created on Sat Dec  7 17:33:35 2019

@author: genev
"""

def heatmapdraw(jobs, search_job):
    '''
   This function is used to visualize the data of salary and job location in format of choropleth map
    Input arguments: cleaned data of job information collected from Glassdoor.
    This fuction plots 2 Heat Maps.
    THe first one is a heatmap for job number distribution.
    The second one is a heatmap for average salary distribution.

    Parameters
    ----------
    jobs : TYPE
        DESCRIPTION.
    search_job : TYPE
        DESCRIPTION.

    Returns
    -------
    list
        DESCRIPTION.

    '''
    import pandas as pd
    from collections import Counter
    z = jobs['State']
    a = Counter(z)
    df = pd.DataFrame.from_dict(a, orient='index').reset_index()
    df = df[df['index'].str.len() <= 2]
    import plotly.graph_objects as go

    # Load data frame and tidy it.

    figstate = go.Figure(data=go.Choropleth(
        locations=df['index'], # Spatial coordinates
        z = df[0].astype(float), # Data to be color-coded
        locationmode = 'USA-states', # set of locations match entries in `locations`
        colorscale = 'Blues',
        colorbar_title = "# of jobs",
    ))

    figstate.update_layout(
        title_text = '# of '+search_job+' jobs in USA by State',
        geo_scope='usa', # limite map scope to USA
    )

    figstate.show()

    salstate = jobs[['Average Sal','State']]

    sumavesal = pd.DataFrame()
    sumavesal['Sum'] = salstate.groupby(['State']).sum()['Average Sal']
    sumavesal['Count'] = salstate.groupby(['State']).count()['Average Sal']
    sumavesal['Result'] = sumavesal['Sum']/sumavesal['Count']

    figsal = go.Figure(data=go.Choropleth(
        locations=sumavesal.index, # Spatial coordinates
        z = sumavesal['Result'].astype(float), # Data to be color-coded
        locationmode = 'USA-states', # set of locations match entries in `locations`
        colorscale = 'Blues',
        colorbar_title = "K dollars",
    ))

    figsal.update_layout(
        title_text = 'Average salaries of '+search_job+' jobs in USA by State',
        geo_scope='usa', # limit map scope to USA
    )

    figsal.show()
    return [figstate, figsal]

'''
Function:This funtion generates 3 plots.
The first is a histogram of salary distribution in the nationwide.
The second is a barchart showing the top 5 job positions salary distribution
The third is a horizonal barchart presenting top 10 job
Input: cleaned job data from Glassdoor in pandas dataframe
Output: One histrogram and two bar charts
'''
def barchartdata(jobs):

    import matplotlib.pyplot as plt
    import pandas as pd
    import seaborn as sns
    from io import BytesIO
    import base64
    import numpy as np

    salDf = jobs
    df = salDf[pd.notnull(salDf['Average Sal'])]   # remove NaN

    # histogram salary distribution

    # summarize mean,median, 75% and 25% quantiles
    m = df['Average Sal'].mean()   # mean salary for all positions
    me = df['Average Sal'].median()
    q1 = df['Average Sal'].quantile(0.25)
    q2 = df['Average Sal'].quantile(0.75)
    sns.set_style("dark")
    salDist = sns.distplot(df['Average Sal'])
    salDist.axvline(me, lw=1.5, ls='dashed', color='black')
    salDist.axvline(m, lw=1.5, ls='dashed', color='red')
    salDist.axvline(q1, lw=1.5, ls='dashed', color='green')
    salDist.axvline(q2, lw=1.5, ls='dashed', color='green')
    plt.xlabel('Average Salary')
    plt.title('Salary($k) Distribution with Median (black), Mean (red) and Quantiles (green)')

    # save the plot
    buffer = BytesIO()
    plt.savefig(buffer)
    plot_data = buffer.getvalue()

    imb = base64.b64encode(plot_data)
    ims = imb.decode()
    imd = "data:image/png;base64,"+ims
    salbar_im = """""" % imd

    # plot the average salary for top 5 positions
    df['jobTitle'] = df['jobTitle'].str.replace('Sr.','Senior')
    df['jobTitle'] = df['jobTitle'].str.replace('Sr','Senior')
    jobCnt = df['jobTitle'].value_counts()
    top5Cnt = jobCnt[0:5].to_dict()

    # average the salaries by job title
    top5Sal = {}
    for job in list(top5Cnt.keys()):
        row = df[df['jobTitle'].str.match(job)]
        top5Sal[job] = row['Average Sal'].mean()

    job = list(top5Sal.keys())
    sal = list(top5Sal.values())

    plt.figure(figsize=(10, 6))
    plt.bar(job, sal)
    plt.xlabel('Job Title')
    plt.ylabel('Average Salary ($k)')
    plt.axhline(y = me, ls='dashed', color="black")
    plt.axhline(y = m, ls='dashed', color="red")

    plt.title('Average Salary for Top 5 Job Positions with Median (black) and Mean (red)')
    plt.tight_layout()  # show conplete labels in the saved image
    buffer = BytesIO()
    plt.savefig(buffer)
    plot_data = buffer.getvalue()

    # save the plot
    imb = base64.b64encode(plot_data)
    ims = imb.decode()
    imd = "data:image/png;base64,"+ims
    salpobar_im = """""" % imd

    # plot the counts of top 10 job titles

    data = jobs
    title_count=data['jobTitle'].value_counts()

    top=10
    title_count1=title_count[:top]
    titles=list(title_count1.index)
    counts=list(title_count1.values)

    fig1, ax1 = plt.subplots(dpi = 1200)
    y_pos = np.arange(len(titles))

    ax1.barh(y_pos,counts)
    ax1.set_yticks(y_pos)
    ax1.set_yticklabels(titles)
    ax1.invert_yaxis()  # labels read top-to-bottom
    title_String='Number ('+str(len(data['jobTitle']))+' Job Posts in Total)'
    ax1.set_xlabel(title_String)
    for i, v in enumerate(counts):
        ax1.text(v, i, str(v),fontsize=9)
    ax1.set_title('Top 10 Job Titles',fontsize=18)

    # save the plot
    buffer = BytesIO()
    plt.savefig(buffer)
    plot_data = buffer.getvalue()

    imb = base64.b64encode(plot_data)
    ims = imb.decode()
    imd = "data:image/png;base64,"+ims
    pobar_im = """""" % imd
    fig1.savefig('pobar_im.png', bbox_inches="tight")

    return [salbar_im,salpobar_im,pobar_im]

'''
Function: calculate the words frequency in job description and generate a wordCloud
Input: a list of job description
Output: the visualization of wordCloud
'''
def wordclouddata(indeed_jobs):
    import matplotlib.pyplot as plt
    from io import BytesIO
    import base64
    from wordcloud import WordCloud
    import nltk
    from nltk.corpus import stopwords
    from sklearn.feature_extraction.text import TfidfVectorizer
    plt.clf()
    data = indeed_jobs.drop_duplicates().reset_index(drop=True)

    # Change the dataframe to a string list
    jobDescription = []
    for i in range(len(data)):
        jobDescription.append(data['job summary'][i])

    # Use nltk to get the english stop words and add some new words to it
    nltk.download('stopwords')
    stop_words = stopwords.words('english')
    extra_stop_words = ["experience", "position", "work", "please", "click", "must", "may", "required", "preferred",
                        "type", "including", "strong", "ability", "needs", "apply", "skills", "requirements", "company",
                        "knowledge", "job", "responsibilities", "develop", "solution", "support", "build", "new",
                        "development", "years", "solutions", "etc", "using", "management", "environment", "time",
                        "related", "help", "working", "analysis", "tools", "technologies", "design", "scientist",
                        "research", "information", "techniques", "degree", "models", "status", "us", "people",
                        "opportunity", "role", "understanding", "advanced", "use", "well", "insights", "provide",
                        "products", "complex", "based", "across", "plus", "problems", "teams", "employees",
                        "employers", "one", "employer", "employee", "within", "improve", "industry", "reporting",
                        "field", "excellent", "best", "quality", "e", "impact", "g", "key", "building", "methods",
                        "looking", "deep", "developing", "create", "high", "multiple", "make", "benefits", "care",
                        'statistical', 'deliver', 'non', 'focus', 'maintenance', 'sets', 'manage', 'various']
    stop_words += extra_stop_words
    print("Generating Word Cloud...")

    # Build the parameter for calculating word frequency
    tfidf_para = {
        "stop_words": stop_words,
        "analyzer": 'word',   # analyzer in 'word' or 'character'
        "token_pattern": r'\w{1,}',    # match any word with 1 and unlimited length
        "sublinear_tf": False,  # False for smaller data size  #Apply sublinear tf scaling, to reduce the range of tf with 1 + log(tf)
        "dtype": int,   # return data type
        "norm": 'l2',     # apply l2 normalization
        "smooth_idf":False,   # no need to one to document frequencies to avoid zero divisions
        "ngram_range" : (1, 2),   # the min and max size of tokenized terms
        "max_features": 60    # the top 60 weighted features
    }
    tfidf_vect = TfidfVectorizer(**tfidf_para)
    transformed_job_desc = tfidf_vect.fit_transform(jobDescription)
    freqs_dict = dict([(word, transformed_job_desc.getcol(idx).sum()) for word, idx in tfidf_vect.vocabulary_.items()])

    # Use the word frequency to generate a wordCloud object with top 60 words
    wordcloud = WordCloud(
        background_color='white',
        width=1500,
        height=960,
        margin=10,
        mode='RGBA',
        max_words=60).fit_words(freqs_dict)

    # Use matplotlib to show and save the image
    plt.imshow(wordcloud)
    plt.axis("off")
    wordcloud.to_file('my_test1.png')

    buffer = BytesIO()
    plt.savefig(buffer)
    plot_data = buffer.getvalue()
    imb = base64.b64encode(plot_data)
    ims = imb.decode()
    imd = "data:image/png;base64,"+ims
    text_im = """""" % imd
    return text_im


import numpy as np
import pickle
import argparse
import os.path as ospath

from utils.ImgUtils import loadstack
from multiprocessing import Pool
from os import listdir, mkdir

def sumStackIntensity(stackDir):
    stack = loadstack(stackDir)
    return [np.sum(img) for img in stack]

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="A script to process images stacks and output measurements into a pickled structure.")
    parser.add_argument('WorkDir', help="The directory containing all the image stacks (only ends with .tiff). Should be generated from Pipeline_ProbStack.py")
    parser.add_argument('-o', "--Output",  default='./', help="The directory for pickled output")
    parser.add_argument('-t', "--MultiThread", type=int, default=1, help="Number of thread to use, default = 1")
    parser.add_argument('--keyword', type=str, help="Keyword used to filter the file names, only filenames that contains keywords will be processed")

    args = parser.parse_args()

    wDir = args.WorkDir

    oDir = ospath.normpath(args.Output)
    if not ospath.exists(oDir):
        mkdir(oDir)

    print("Output to " + oDir)

    mtNum = args.MultiThread
    if mtNum == 1:
        multiP = False
    elif mtNum < 1:
        print("Not valid thread number, now working in single thread mode")
        multiP = False
    else:
        multiP = True

    keyW = args.keyword
    if not keyW:
        keyW = ''

    # Making the list of stacks need to be processed
    stackNames = [name for name in listdir(wDir) if name.endswith('.tiff') and (keyW in name)]
    stackDirs = ['{0}/{1}'.format(wDir, name) for name in stackNames]

    # Processing the images
    if multiP:
        print("Starting {0} threads......".format(mtNum))
        with Pool(mtNum) as p:
            intensitySums = p.map(sumStackIntensity, stackDirs)
        resultDic = dict(zip(stackNames, intensitySums))
    else:
        print("Single thread mode")
        resultDic = {}
        for idx, stackDir in enumerate(stackDirs):
            resultDic[stackNames[idx]] = sumStackIntensity(stackDir)

    saveFilePath = '{0}/IntensitySum.p'.format(oDir)
    print('Finished processing, saving at {0}'.format(saveFilePath))
    with open(saveFilePath, 'wb') as f:
        pickle.dump(resultDic, f)
    print('Scripts finished successfully')


##
 #######################################################################################################################
 #
 #  Copyright (c) 2022-2023 Advanced Micro Devices, Inc. All Rights Reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal
 #  in the Software without restriction, including without limitation the rights
 #  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 #  copies of the Software, and to permit persons to whom the Software is
 #  furnished to do so, subject to the following conditions:
 #
 #  The above copyright notice and this permission notice shall be included in all
 #  copies or substantial portions of the Software.
 #
 #  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 #  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 #  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 #  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 #  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 #  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 #  SOFTWARE.
 #
 #######################################################################################################################

import argparse
import json
import sys
from typing import List
from ruamel.yaml import YAML

def extract_build_types(parent: dict) -> List:
    build_types = parent.get("BuildTypes", None)
    or_build_types = parent.get("OrBuildTypes", None)

    assert (build_types is None) or (or_build_types is None), f"parent: {parent}"

    if build_types is not None:
        return build_types
    elif or_build_types is not None:
        return or_build_types
    else:
        return []

def convert_to_settings2_top_level_tags(tags: List) -> List:
    """Convert the top level 'Tags'."""
    new_tags = []
    for tag in tags:
        if isinstance(tag, dict):
            build_types = extract_build_types(tag)
            if len(build_types) > 0:
                new_tag = {"name": tag["Name"], "buildtypes": build_types}
            else:
                new_tag = tag["Name"]
            new_tags.append(new_tag)
        else:
            new_tags.append(tag)
    return new_tags

def convert_to_settings2_enum_helper(valid_values: dict) -> dict:
    name = valid_values["Name"]
    description = valid_values.get("Description", name)
    enum = {"name": name, "description": description}

    values = []
    for valid_val in valid_values["Values"]:
        value = dict({"name": valid_val["Name"], "value": valid_val["Value"]})

        desc = valid_val.get("Description", None)
        if desc is None:
            name = valid_val["Name"]
            valid_values_name = valid_values["Name"]
            print(
                f'[WARN] Enum field "{name}" of "{valid_values_name}" has no description'
            )
        else:
            value["description"] = desc

        build_types = extract_build_types(valid_val)
        if len(build_types) > 0:
            value["buildtypes"] = build_types

        values.append(value)

    assert len(values) > 0
    enum["values"] = values

    build_types = extract_build_types(valid_values)
    if len(build_types) > 0:
        enum["buildtypes"] = build_types

    return enum

def convert_to_settings2_type(type: str) -> str:
    type_lower = type.strip().lower()

    if type_lower == "struct":
        raise AssertionError()
    if type_lower in ["size_t", "gpusize"]:
        return "uint64"
    elif type_lower == "enum":
        return "uint32"
    elif type_lower == "string":
        return "str"
    else:
        return type_lower

def convert_to_settings2_defaults(defaults: dict, type: str) -> dict:
    def fix(default, type: str):
        fixed_default = default

        if type == "float" and isinstance(default, str):
            fixed_default = float(default.replace("f", ""))  # e.g. "1.25f" -> 1.25"
        elif type.startswith("uint") or type.startswith("int"):
            if isinstance(default, str):
                if default.isdigit():
                    fixed_default = int(default, 0)
                elif default.startswith("0x"):
                    fixed_default = int(default, 16)
                else:
                    # default is an enum name, we will convert it to its
                    # corresponding enum value later
                    pass
        elif isinstance(default, str):
            default_lowercase = default.lower()
            if default_lowercase == "true":
                fixed_default = True
            elif default_lowercase == "false":
                fixed_default = False
            else:
                fixed_default = default.replace("\\", "\\\\")

        return fixed_default

    new_type = convert_to_settings2_type(type)

    new_defaults = {
        "default": fix(defaults["Default"], new_type),
        "type": new_type,
    }

    default_win = defaults.get("WinDefault", None)
    if default_win is not None:
        new_defaults["windows"] = fix(default_win, new_type)

    default_linux = defaults.get("LnxDefault", None)
    if default_linux is not None:
        new_defaults["linux"] = fix(default_linux, new_type)

    return new_defaults

def validvalues_missing_name_fix(setting_name: str) -> str:
    if setting_name == "CmdBufferLoggerAnnotations":
        return "CmdBufferLoggerAnnotations"
    elif setting_name in ["BasePreset", "ElevatedPreset"]:
        return "PresetLogFlags"
    elif setting_name == "FileSettingsFlags":
        return "FileSettingsFlagsBitmask"
    elif setting_name == "FileAccessFlags":
        return "FileAccessFlagsBitmask"
    elif setting_name == "OrigTypeMask":
        return "OrigTypeMask"
    else:
        print(f'[WARN] ValidValues of {setting_name} has no "Name" field')

def validvalue_name_fix(valid_values: dict):
    """This fixes the missing name of individual values of a valid_values object."""
    if valid_values["Name"] == "GpuProfilerCaptureTriggerKey":
        for value in valid_values["Values"]:
            name = value["Name"]
            value["Name"] = name.replace("GpuProfilerCaptureTriggerKey::", "")
    else:
        for value in valid_values["Values"]:
            if "Name" not in value:
                if valid_values["Name"] == "PresetLogFlags":
                    description = value["Description"]
                    value["Name"] = description[: description.index(":")]
                else:
                    value["Name"] = value["Description"]

                print(
                    "[WARN] Missing value name in ValidValues {}. Set it to {}".format(
                        valid_values["Name"], value["Name"]
                    )
                )

def convert_to_settings2_enum(setting: dict, enums: List[dict]):
    flags = setting.get("Flags", None)
    is_bitmask = False
    valid_values = setting.get("ValidValues")
    if flags:
        if flags.get("IsBitmask", False):
            is_bitmask = True
        elif flags.get("IsHex", False):
            if valid_values:
                # fix for bad json
                is_bitmask = valid_values.get("IsEnum", False)

    if setting["Type"] == "enum" or (is_bitmask and valid_values):
        valid_values = setting["ValidValues"]
        if "Values" in valid_values:
            if "Name" not in valid_values:
                valid_values["Name"] = validvalues_missing_name_fix(setting["Name"])

            validvalue_name_fix(valid_values)

            enum = convert_to_settings2_enum_helper(valid_values)

            if "buildtypes" not in enum:
                # check buildtypes associated with this setting
                build_types = list(extract_build_types(setting))
                if len(build_types) > 0:
                    enum["buildtypes"] = build_types

            if not any(map(lambda x: x["name"] == enum["name"], enums)):
                enums.append(enum)
        else:
            # if "Values" is a not field, then its "Name" field references an
            # already defined ValidValues
            pass

def fix_setting_name(name: str) -> str:
    assert name[0].isalpha(), "Setting name must start with an alphabet letter."

    name = name[0].upper() + name[1:]
    return name

def convert_to_settings2_setting(setting: dict, group_name: str) -> dict:
    setting_name = setting["Name"]
    new_setting = {
        "name": fix_setting_name(setting_name),
        "description": setting["Description"],
    }

    if group_name:
        new_setting["group"] = group_name

    tags = setting.get("Tags", None)
    if tags:
        new_setting["tags"] = tags

    is_bitmask = False
    flags = setting.get("Flags", None)
    if flags:
        new_flags = {}
        if flags.get("IsHex", False):
            new_flags["isHex"] = True
        if flags.get("IsPath", False):
            new_flags["isDir"] = True
        if flags.get("IsFile", False):
            new_flags["isFile"] = True
        if flags.get("RereadSetting", False):
            new_flags["ReadAfterOverride"] = True
        if flags.get("IsBitmask"):
            is_bitmask = True

        if len(new_flags) != 0:
            new_setting["flags"] = new_flags

    new_setting["defaults"] = convert_to_settings2_defaults(
        setting["Defaults"], setting["Type"]
    )

    registry_scope = setting.get("Scope", "PrivatePalKey")
    new_setting["scope"] = registry_scope

    if setting["Type"] == "enum" and not is_bitmask:
        new_setting["enum"] = setting["ValidValues"]["Name"]
    elif is_bitmask:
        valid_values = setting.get("ValidValues", None)
        if valid_values:
            name = valid_values.get("Name", None)
            if name is None:
                new_setting["enum"] = validvalues_missing_name_fix(setting_name)
            else:
                new_setting["enum"] = name
        else:
            print("[WARN] {} is a bitmask but has no ValidValues.".format(setting_name))

    build_types = extract_build_types(setting)
    if len(build_types) > 0:
        new_setting["buildtypes"] = build_types

    return new_setting

def convert_to_settings2_setting_or_struct(setting: dict, enums: List[dict]) -> dict:
    converted_settings = []

    setting_struct = setting.get("Structure", None)
    struct_build_types = extract_build_types(setting)
    struct_tags = setting.get("Tags", [])
    if setting_struct:
        for subsetting in setting_struct:
            # Extract build types from struct and attach them to subsettings.
            build_types = extract_build_types(subsetting)
            if len(build_types) == 0 and len(struct_build_types) > 0:
                subsetting["BuildTypes"] = list(struct_build_types)
            elif len(build_types) > 0:
                for buildtype in struct_build_types:
                    if buildtype not in build_types:
                        build_types.append(buildtype)

            # Extract tag from struct and attach them to subsettings.
            subsetting_tags = subsetting.get("Tags", [])
            if len(subsetting_tags) == 0 and len(struct_tags) > 0:
                subsetting["Tags"] = list(struct_tags)
            elif len(subsetting_tags) > 0:
                for tag in struct_tags:
                    if tag not in subsetting_tags:
                        subsetting_tags.append(tag)

            convert_to_settings2_enum(subsetting, enums)

            group_name = setting["Name"]
            new_setting = convert_to_settings2_setting(subsetting, group_name)
            converted_settings.append(new_setting)
    else:
        convert_to_settings2_enum(setting, enums)
        new_setting = convert_to_settings2_setting(setting, None)
        converted_settings.append(new_setting)

    return converted_settings

def convert_to_settings2(old_settings: dict) -> dict:
    settings2 = dict()

    settings2["version"] = 2
    settings2["component"] = old_settings["ComponentName"]

    if settings2["component"] == "PalCore":
        settings2["component"] = "Pal"

    tags = convert_to_settings2_top_level_tags(old_settings["Tags"])
    if len(tags) > 0:
        settings2["tags"] = tags

    enums = []
    if "Enums" in old_settings:
        enums = [convert_to_settings2_enum_helper(v) for v in old_settings["Enums"]]
    settings2["enums"] = enums

    new_settings_list = []
    settings2["settings"] = new_settings_list

    for setting in old_settings["Settings"]:
        # Populate top-level Enums list
        convert_to_settings2_enum(setting, enums)

        # Populate Settings
        converted_settings = convert_to_settings2_setting_or_struct(setting, enums)
        new_settings_list.extend(converted_settings)

    return settings2

def convert_settings2_enum_value_to_integer(settings2: dict):
    """Convert default values of enum settings to integers.

    If they are strings of enum variants, use the corresponding values of those
    variants.
    """

    def variant_name_to_int(variant_name: str, enum: dict) -> int:
        result = 0
        if variant_name.startswith("("):
            variant_name = variant_name[1 : variant_name.index(")")]

        variant_name_list = variant_name.split("|")

        for name in variant_name_list:
            val = next(v["value"] for v in enum["values"] if v["name"] == name.strip())
            if isinstance(val, str):
                val = int(val, 0)
            result |= int(val)
        return result

    enums = settings2["enums"]

    for setting in settings2["settings"]:
        enum_name = setting.get("enum", None)
        if enum_name:
            enum = next(e for e in enums if e["name"] == enum_name)
            assert enum is not None, "Setting {} references an unknown enum {}.".format(
                setting["name"], enum_name
            )

            defaults = setting["defaults"]
            if isinstance(defaults["default"], str):
                defaults["default"] = variant_name_to_int(defaults["default"], enum)

            if "windows" in defaults:
                defaults["windows"] = variant_name_to_int(defaults["windows"], enum)

            if "linux" in defaults:
                defaults["linux"] = variant_name_to_int(defaults["linux"], enum)

def main() -> int:
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="Convert a Settings JSON file to a YAML file that conforms to the Settings 2.0 schema.",
    )

    parser.add_argument(
        "old_file",
        type=str,
        help="A path to an existing Settings JSON file of the old schema.",
    )
    parser.add_argument(
        "new_file",
        type=str,
        help="A file path to store the converted Settings YAML file.",
    )

    args = parser.parse_args()

    settings_json = None
    with open(args.old_file) as file:
        settings_json = json.load(file)
        assert settings_json is not None, "`old_file` invalid: empty JSON input file"

    settings2 = convert_to_settings2(settings_json)
    # convert_settings2_enum_value_to_integer(settings2)

    with open(args.new_file, "w") as file:
        yaml = YAML()
        yaml.default_flow_style = False
        yaml.indent(mapping=2, sequence=4, offset=2)
        yaml.dump(settings2, file)

    return 0

if __name__ == "__main__":
    sys.exit(main())


from django.contrib import admin
from categorias.models import Categoria

# Register your models here.

class CategoriaAdmin(admin.ModelAdmin):
	list_display = ('nome','detalhe')
	fieldsets = [
		('Categoria', {'fields': ['nome']}),
		('Detalhes', {'fields': ['detalhe']})
	]
	search_fields = ['nome']

admin.site.register(Categoria, CategoriaAdmin)


#!/usr/bin/env python3
''' Contains base model for future classes '''

from uuid import uuid4
from datetime import datetime
import models

class BaseModel():
        ''' Sets attributes and methods to be used in all future models '''

        class_att_dict = dict()

        def __init__(self, *args, **kwargs):
                ''' Initializes an object '''
                for key in kwargs.keys():
                        if key == 'created_at' or key == 'updated_at':
                                self.__dict__[key] = datetime.strptime(
                                        kwargs[key], '%Y-%m-%dT%H:%M:%S.%f')
                        else:
                                self.__dict__[key] = kwargs[key]
                if 'id' not in self.__dict__.keys():
                        self.id = str(uuid4())
                if 'created_at' not in self.__dict__.keys():
                        self.created_at = datetime.today()
                models.storage.new(self)
                models.storage.save()

        def __str__(self):
                ''' Printable string of object '''
                return '[{}] ({}) {}'.format(
                        self.__class__.__name__, self.id, self.__dict__)

        def __repr__(self):
                ''' Repr is same as str for formatting reasons '''
                return self.__str__()

        def __setattr__(self, name, value):
                ''' Setattr is customized for updating '''
                self.__dict__[name] = value
                self.save()

        def save(self):
                ''' Updates updated_at attribute and saves to file '''
                self.__dict__['updated_at'] = datetime.today()
                models.storage.save()

        def to_dict(self):
                ''' Creates dictionary for easy saving to JSON '''
                base_dict = dict(self.__dict__)
                base_dict['__class__'] = self.__class__.__name__
                base_dict['created_at'] = self.created_at.isoformat()
                base_dict['updated_at'] = self.updated_at.isoformat()
                return base_dict


# -*- coding: utf-8 -*-

from django.db import models

class Client(models.Model):
    name = models.CharField(max_length=80)
    email = models.CharField(max_length=80)
    telephone = models.CharField(max_length=20, blank=True)
    cellphone = models.CharField(max_length=20, blank=True)
    facebook_profile = models.CharField(max_length=50, blank=True)
    birthdate = models.DateField()
    updated_at = models.DateTimeField(auto_now=True)
    created_at = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return self.name

    class Meta:
        verbose_name = 'Cliente'

class Relation(models.Model):
    RELATIONSHIP = (
        (u'Grandparents', u'avós'),
        (u'Grandmother', u'avó'),
        (u'Grandfather', u'avô'),
        (u'Mother', u'mãe'),
        (u'Father', u'pai'),
        (u'Son', u'filho'),
        (u'Daughter', u'filha'),
        (u'Grandchild', u'neto (sem definir o sexo)'),
        (u'Grandson', u'neto'),
        (u'Granddaughter', u'neta'),
        (u'Granny', u'vovó'),
        (u'Granpa', u'vovô'),
        (u'Daddy', u'papai'),
        (u'Mama', u'mamãe'),
        (u'Sister', u'irmã'),
        (u'Broher', u'irmão'),
        (u'Wife', u'esposa'),
        (u'Husband', u'marido'),
        (u'Girlfriend', u'namorada'),
        (u'Boyfriend', u'namorado'),
        (u'Uncle', u'tio'),
        (u'Aunt', u'tia'),
        (u'Cousin', u'primo'),
        (u'Niece', u'sobrinha'),
        (u'Nephew', u'sobrinho')
    )
    client_1 = models.ForeignKey(Client, related_name='client_1')
    client_2 = models.ForeignKey(Client, related_name='client_2')
    relation = models.CharField(max_length=30, choices=RELATIONSHIP)

    def __str__(self):
        return '%s is %s of %s' % (self.client_1, self.relation, self.client_2)

    class Meta:
        verbose_name = 'relacionamento'


import sys
import pygame
import math

from time import sleep
from random import randint
from random import choice

from graphics import Hit_Decal
from psion import Psion

def update_screen_menu(settings, all_groups):
    """Update images on the screen and flip tot the new screen."""
    # Redraw the MENU screen during each pass through the loop.
    settings.screen.fill(settings.bg_color)
    #settings.screen.blit(settings.background, settings.background_rect)

    pygame.display.flip()

def update_screen_map(settings, worldmap, testplayer, testbutton, all_groups):
    """Update images on the screen and flip tot the new screen."""
    # Redraw the MAP screen during each pass through the loop.
    settings.screen.fill(settings.bg_color_map)
    #settings.screen.blit(settings.background, settings.background_rect)

    worldmap.update()

    testplayer.update()
    testplayer.blitme()

    #worldmap.blitme()
    testbutton.blitme()

    pygame.display.flip()

def update_screen(settings, all_groups):
    """Update images on the screen and flip tot the new screen."""
    # Redraw the screen during each pass through the loop.
    settings.screen.fill(settings.bg_color)
    settings.screen.blit(settings.background, settings.background_rect)

    unit_group_l = all_groups["unit_group_l"]
    unit_group_r = all_groups["unit_group_r"]
    blood = all_groups["blood"]
    hit_decals = all_groups["hit_decals"]

    blood.update()

    #Draw and Update Units
    army_update_order(unit_group_l) #TESTTEST TEST Auto Shoot Order!
    unit_group_l.update()
    army_update_order(unit_group_r) #TESTTEST TEST Auto Shoot Order!
    unit_group_r.update()
    all_groups["unit_group_dead"].update()

    hit_decals.update()

    blood.draw(settings.screen)
    all_groups["unit_group_dead"].draw(settings.screen)
    unit_group_l.draw(settings.screen)
    unit_group_r.draw(settings.screen)

    hit_decals.draw(settings.screen)

    pygame.display.flip()

def check_events_menu(settings, all_groups):
    """Respond to keypresses and mouse events"""
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()

        if event.type == pygame.MOUSEBUTTONDOWN:
            mouse_x, mouse_y = pygame.mouse.get_pos()
            print("x: ", mouse_x)
            print("y: ", mouse_y)

            #TEST
            settings.game_state = "battle"

        if event.type == pygame.KEYDOWN:
            # Quit
            if event.key == settings.control_key_quit:
                sys.exit()

            if event.key == settings.control_key_play:
                settings.game_state = "battle"

def check_events_map(settings, testplayer, all_groups):
    """Respond to keypresses and mouse events"""
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()

        if event.type == pygame.MOUSEBUTTONDOWN:
            mouse_x, mouse_y = pygame.mouse.get_pos()
            print("x: ", mouse_x)
            print("y: ", mouse_y)

            #TEST
            #settings.game_state = "battle"

        if event.type == pygame.KEYDOWN:
            # Quit
            if event.key == settings.control_key_quit:
                sys.exit()

            if event.key == settings.control_key_play:
                settings.game_state = "battle"

            if event.key == pygame.K_LEFT:
                testplayer.move(dx=-1)
            if event.key == pygame.K_RIGHT:
                testplayer.move(dx=1)
            if event.key == pygame.K_UP:
                testplayer.move(dy=-1)
            if event.key == pygame.K_DOWN:
                testplayer.move(dy=1)

def check_events(settings, all_groups):

    terran = all_groups["unit_group_l"]

    """Respond to keypresses and mouse events"""
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            sys.exit()

        if event.type == pygame.MOUSEBUTTONDOWN:
            mouse_x, mouse_y = pygame.mouse.get_pos()
            print("x: ", mouse_x)
            print("y: ", mouse_y)

        if event.type == pygame.KEYDOWN:
            # Quit
            if event.key == settings.control_key_quit:
                sys.exit()

            if event.key == settings.control_key_up:
                for unit in terran:
                    unit.move(0, -5)

            if event.key == settings.control_key_down:
                for unit in terran:
                    unit.move(0, 5)

            if event.key == settings.control_key_left:
                for unit in terran:
                    unit.move(-5, 0)

            if event.key == settings.control_key_right:
                for unit in terran:
                    unit.move(5, 0)

def add_unit(settings, group, typ, number_units, all_groups):
    id_number = 0
    for i in range(number_units):
        unit = typ(settings, all_groups)

        #Get ID
        id_number += 1
        unit.id = id_number

        group.add(unit)

"""UPDATE REFRACTOR"""
# (settings, group, all_groups, -1) side = "left" "right Seite wechseln?
def create_army(settings, group, all_groups, side):
    """Create the army."""
    # ...

    army_size = len(group)

    #Define Side Multiplier and flip image
    if side == "left":
        center_multiplier = 3
        side_multiplier = -1

    elif side == "right":
        center_multiplier = 5
        side_multiplier = 1

        for unit in group:
            unit.flip_horizontal()

    #Square = Rows, 3*3 etc. durch Soldatenzahl herausfinden
    army_sqr = int(math.sqrt(army_size))

    #Platzierung Startort
    battle_center_x = ((settings.screen_width / 8) * center_multiplier)
    battle_center_y = (settings.screen_height / 2)

    #unit width und height herausfinden lol
    for i in group:
        unit_width = i.rect.width
        unit_height = i.rect.height
        break

    new_x = []
    new_y = []

    #Change y_step x_step and +1 for formation order (long or wide)
    for x_step in range(army_sqr):
        for y_step in range(army_sqr+1):

            kopfweh_x = battle_center_x
            kopfweh_y = battle_center_y

            kopfweh_x += ((unit_width + 10) * (x_step)) * side_multiplier
            kopfweh_y += (unit_height - 20) * (y_step)

            new_x.append(kopfweh_x)
            new_y.append(kopfweh_y)

    #Apply x and y to units in group

    for unit, range_army in zip(group, range(army_size)):
        unit.rect.x = new_x[range_army]
        unit.rect.y = new_y[range_army]

        """TEST"""
        unit.army_side = side

        """Placement Step by Step
        sleep(0.1)
        update_screen(settings, all_groups)
        """

    #Move army up (to the middle)
    for unit in group:
        unit.rect.y -= unit_height * (army_sqr / 4)

"""BATTLE""""""BATTLE""""""BATTLE""""""BATTLE""""""BATTLE"""

#ALT
def battle(settings, all_groups):

    unit_group_l = all_groups["unit_group_l"]
    unit_group_r = all_groups["unit_group_r"]

    """TEST ROUND COUNTER"""
    i=0

    #Start loop unit_l -> unit_r, unit_r -> unit_l
    for unit_l, unit_r in zip(unit_group_l, unit_group_r):

        # Unit on the LEFT shoots Random Unit on the RIGHT
        random_unit_r = choice(unit_group_r.sprites())
        hit_calc(settings, unit_l, random_unit_r, all_groups)

        """TEST Step through each Round"""
        print("Round Counter", unit_l.name, i, "group len: ", len(unit_group_l))
        i+=1
        #sleep(0.05)
        update_screen(settings, all_groups)
        """"""

        # Unit on the RIGHT shoots Random Unit on the LEFT
        random_unit_l = choice(unit_group_l.sprites())
        hit_calc(settings, unit_r, random_unit_l, all_groups)

        """TEST Step through each Round"""
        print("Round Counter:", unit_r.name , i, " group len: ", len(unit_group_r))
        i+=1
        #sleep(0.05)
        update_screen(settings, all_groups)
        """"""

        # Check if an army is defeated (not checks for FALSE if group is empty)
        ##### RETURN WINNER?
        if not unit_group_l:
            # TEST Kill Counter
            test_kill_counter(all_groups)
            print("ENEMY WON WE LOST ")
            break

        if not unit_group_r:
            #TEST Kill Counter
            test_kill_counter(all_groups)
            print("WE WON LOL")
            break

def hit_calc(settings, unit, target, all_groups):

    # if target != None:
    #     graphics.rect = target.rect
    #
    # if unit != None:
    #     graphics.rect2 = unit.rect

    add_graphic(settings, unit.rect.x, unit.rect.y, 1, all_groups["hit_decals"])

    rand = randint(1, 100)
    if rand <= unit.accuracy:
        add_graphic(settings, target.rect.x, target.rect.y, 2, all_groups["hit_decals"])
        if randint(0, 100) <= 50 and target.type == 1:
            wound_unit(target)

        else:
            #kill_unit(target)
            kill_unit_pop(target, all_groups)
            unit.kill_count += 1

#image gib oder pew, (2 = gib)
def add_graphic(settings, x, y, image, group):
    graphic = Hit_Decal(settings)
    graphic.rect.x = x #+ randint(-5, 5)
    graphic.rect.y = y #+ randint(-5, 5)

    if image == 2:
        graphic.image = graphic.image2

    #rect?
    group.add(graphic)

def kill_unit(unit):
    unit.type = 0
    unit.combat_active = 0
    unit.draw_wound()

def kill_unit_pop(unit, all_groups):
    unit.type = 0
    unit.combat_active = 0
    unit.draw_wound()

    unit.kill()
    all_groups["unit_group_dead"].add(unit)

def wound_unit(unit):
    unit.type = 2
    unit.draw_wound()

def test_kill_counter(all_groups):
    """TEST KILLCOUNTER"""
    print("Kill Counter:")
    for unit in all_groups["unit_group_l"]:
        print(unit.name, unit.type, " killed: ", unit.kill_count)

    for unit in all_groups["unit_group_r"]:
        print(unit.name, unit.type, " killed: ", unit.kill_count)

    for unit in all_groups["unit_group_dead"]:
        print(unit.name, unit.type, " killed: ", unit.kill_count)

#UPDATEN!!!
def army_update_order(group):
    """Update shooting order for auto_shoot_group"""
    if group:

        #Ganze Gruppe last_update now, als hätten geschossen
        now = pygame.time.get_ticks()

        #herausfinden ob eh nur einer in der Gruppe bereit zu shooten ist
        behindert = 0
        for unit in group:
            if unit.ready_to_fire == 1:
                behindert = 1

        #Alle außer ready to fire auf last_update now
        for unit in group:
            if unit.ready_to_fire == 0:
                unit.last_update = now

        #Wenn keiner ready ist -> random auf ready stellen
        if behindert != 1:
            unit = choice(group.sprites())
            unit.ready_to_fire = 1


from modules.vrs.helpers.vrs_utils import VrsUtil
from utility.configs import EnvConst
from .constants import AppConst, AppKeys

class AppUtil(VrsUtil):

    def get_module_static_key_dict(self, env):
        """
        Only static data will return from this method
        No Dynamic data (i.e: by function call data will not provide from here)
        """
        return AppConst.APP_STATIC_KEY_DIC[env]

    def get_common_info(self):
        common_info_merge_dict = {
            **self.get_vrs_static_key_dict("common"),
            **self.get_vrs_static_key_dict(EnvConst.ACTIVE),
            **self.get_module_static_key_dict("common"),
            **self.get_module_static_key_dict(EnvConst.ACTIVE),
        }

        return common_info_merge_dict

    def get_dynamic_info(self, request, current_route):
        dynamic_info_merge_dict = {'controller_id': current_route, }

        dynamic_info_merge_dict['controller_name'] = current_route + " name"

        return dynamic_info_merge_dict

    def get_curreny_details(self, currency):
        """
        Only static data will return from this method
        No Dynamic data (i.e: by function call data will not provide from here)
        """
        return AppConst.CURRENCY_RATE_DICT[currency]

    def get_logo_name(self, feed, sub_feed):
        logo_name = ""
        partner = AppConst.PARTNER_DETAILS_KEY_VALUE_DICT[feed]

        if sub_feed:
            logo_name = partner[AppKeys.SUB_FEED][sub_feed][
                AppKeys.Name]
        else:
            logo_name = partner[AppKeys.Name]

        return logo_name


# ===============================================================================
# Copyright 2011 Jake Ross
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============================================================================

# ============= enthought library imports =======================
from traits.api import Float, Tuple
# ============= standard library imports ========================
# ============= local library imports  ==========================
# from pychron.config_loadable import ConfigLoadable
from pychron.hardware.axis import Axis
class NewportGroup(Axis):
    # acceleration = Float
    # deceleration = Float
    emergency_deceleration = None
    jerk = Float
    # velocity = Float
    name = 'GroupedAxes'
    machine_velocity = Float
    machine_acceleration = Float
    machine_deceleration = Float
    axes = Tuple

#    calculate_parameters = Bool(True)
    id = None

    MAPPING = dict(acceleration='HA',
                 deceleration='HD',
                 # emergency_deceleration = 'HE',
                 jerk='HJ',
                 velocity='HV',
                 axes='HN'
                 )

#    def traits_view(self):
#        v = View(Item('calculate_parameters'),
#                 Item('velocity', enabled_when = 'not calculate_parameters'),
#                    Item('acceleration', enabled_when = 'not calculate_parameters'),
#                    Item('deceleration', enabled_when = 'not calculate_parameters')
#                    )
#        return v
    def _set_acceleration(self, v):
        self._acceleration = v
        self.nominal_acceleration = v

    def _set_deceleration(self, v):
        self._deceleration = v
        self.nominal_deceleration = v

    def _set_velocity(self, v):
        self._velocity = v
        self.nominal_velocity = v

    def load(self, path):
        config = self.get_configuration(path)
        for attr in ['acceleration',
                     'deceleration',
                     # 'emergency_deceleration',
                     'jerk',
                     'velocity',
                     ]:
            self.set_attribute(config, attr, 'General', attr, cast='float')

        self.set_attribute(config, 'id', 'General', 'id', cast='int')
        self.axes = tuple(map(int, self.config_get(config, 'General', 'axes').split(',')))

        self.nominal_velocity = self.velocity
        self.nominal_acceleration = self.acceleration
        self.nominal_deceleration = self.deceleration

#    def load_from_file(self, fobj):
#
#        def set_params():
#            self.acceleration = 300
#            self.deceleration = 300
#            self.emergency_deceleration = 300
#            self.jerk = 150
#            self.velocity = 3.81 * 50
#            self.axes = 1, 2
#
#        if isinstance(fobj, str):
#            with open(fobj, 'r') as fobj:
#                set_params()
#        else:
#            set_params()

    def build_command(self, new_group):
        cmds = []
        for key, value in self.MAPPING.iteritems():
            if key is not 'axes':

                cmds.append('{}{}{:0.5f}'.format(self.id, value, getattr(self, key)))

        if new_group:
            gid = '{:n}HN{}'.format(self.id, ','.join(map(str, self.axes)))
            cmds = [gid] + cmds

        return ';'.join(cmds)

# ============= EOF ==============================================


from faker import Faker
from flask import Flask
from flask_restful import Api, Resource, fields, marshal_with
from flask_sqlalchemy import SQLAlchemy
from flask_rest_paginate import Pagination

"""
Initialize the app
"""
app = Flask(__name__)
api = Api(app)

app.config['SQLALCHEMY_DATABASE_URI'] = "sqlite:///paginate-test.db"
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)

# Possible configurations for Paginate
# app.config['PAGINATE_PAGE_SIZE'] = 20
# app.config['PAGINATE_PAGE_PARAM'] = "pagenumber"
# app.config['PAGINATE_SIZE_PARAM'] = "pagesize"
# app.config['PAGINATE_RESOURCE_LINKS_ENABLED'] = False
# app.config['PAGINATE_PAGINATION_OBJECT_KEY'] = "pagination"
# app.config['PAGINATE_DATA_OBJECT_KEY'] = "data"
pagination = Pagination(app, db)

"""
Models
"""
class AuthorModel(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String)
    posts = db.relationship('PostModel', backref='author', lazy=True)

class PostModel(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    title = db.Column(db.String(64))
    content = db.Column(db.String(255))
    author_id = db.Column(db.Integer, db.ForeignKey('author_model.id'), nullable=False)

"""
Resource fields for marshalling
"""
post_fields = {
    'title': fields.String,
    'content': fields.String,
    'author_id': fields.Integer
}

author_fields = {
    'name': fields.String,
    'posts': fields.List(fields.Nested(post_fields))
}

"""
Controllers
"""
class Post(Resource):
    @marshal_with(post_fields)
    def get(self, post_id):
        return PostModel.query.filter_by(id=post_id).first()

class PostList(Resource):
    @marshal_with(post_fields)
    def get(self):
        return PostModel.query.all()

class Author(Resource):
    # No need to decorate with `marshal_with` as the function will marshal it automatically
    # @marshal_with(author_fields)
    def get(self, author_id):
        # Can pass the query object with filters
        return pagination.paginate(AuthorModel.query.filter_by(id=author_id), author_fields)

class AuthorList(Resource):
    def get(self):
        # Can also pass the model directly, in which case, no filters can be attached
        return pagination.paginate(AuthorModel, author_fields,  pagination_schema_hook=lambda current_page, page_obj: {
            "next": page_obj.has_next,
            "prev": page_obj.has_prev,
            "current": current_page,
            "pages": page_obj.pages,
            "per_page": page_obj.per_page,
            "total": page_obj.total,
        })

"""
Register the resources
"""
api.add_resource(Post, '/posts/<post_id>')
api.add_resource(PostList, '/posts')
api.add_resource(Author, '/authors/<author_id>')
api.add_resource(AuthorList, '/authors')

"""
Seed database
"""
def seed():
    fake = Faker()
    for i in range(100):
        db.session.add(AuthorModel(name=fake.name()))
        for j in range(10):
            db.session.add(PostModel(title=fake.sentence(), content=fake.text(), author_id=i))

    db.session.commit()

"""
Recreate the DB
"""
def recreate_db():
    db.drop_all()
    db.create_all()
    db.session.commit()

"""
Run the app
"""
if __name__ == '__main__':
    recreate_db()
    seed()
    app.run(debug=True)


from copy import deepcopy
import numpy as np
import logging
import os
import json
import scipy.io.wavfile as wavfile
import librosa
from PIL import Image

DEBUG = False
END = '</s>'
NULL = 'NULL'

if os.path.exists("*.log"):
  os.system("rm *.log")

class XNMTPostprocessor():
  def __init__(self, input_dir, is_phoneme=True):
    self.input_dir = input_dir
    self.is_phoneme = is_phoneme

  # Convert the predicted alignment files to the right format for evaluation
  def convert_alignment_file(self, out_file='pred_alignment.json', feat2wav_file=None):
    all_files = os.listdir(self.input_dir)
    files = []
    for fn in all_files:
      if fn.endswith(".json"):
        files.append(fn)

    files = sorted(files, key=lambda x:int(x.split('.')[-2]))
    alignments = []

    feat2wavs = []
    if feat2wav_file:
      with open(feat2wav_file, 'r') as f:
        feat2wavs = json.load(f)

    for i, f in enumerate(files):
      if f.split('.')[-1] == 'json':
        fp = open(self.input_dir + f, 'r')
        ali = json.load(fp)
        if type(ali) == list:
          ali = ali[0]

        fp.close()

        if END in ali['src_sent']:
          ali['src_sent'].remove(END)
          # XXX: This will make the file format inconsistent with the one generated by the preprocess.py
          # convert_xnmt_text function; fix this later
          ali['trg_sent'].remove(END)
          ali['alignment'] = ali['alignment'][:-1]
          if NULL not in ali['trg_sent']:
            ali['trg_sent'] = [NULL] + ali['trg_sent']
            ali['alignment'] = [a+1 if a<len(ali['trg_sent'])-1 else 0 for a in ali['alignment']]
          else:
            ali['alignment'] = [a if a<len(ali['trg_sent'])-1 else 0 for a in ali['alignment']]

        new_ali = {}
        if not feat2wav_file:
          new_ali = {'index': ali['index'],
                     'is_phoneme': self.is_phoneme,
                     'caption': ali['src_sent'],
                     'image_concepts': ali['trg_sent'],
                     'alignment': ali['alignment'],
                     'attentions': ali['attentions']
                    }
        else:
          feat_len = len(feat2wavs[i])
          ali_len = len(ali['alignment'])
          reduce_factor = ali['reduce_factor']
          new_feat2wav = []
          for i_a in range(ali_len):
            for i_f in range(feat_len):
              st = feat2wavs[i]['feat2wav'][i_f * reduce_factor][0]
              end = feat2wavs[i]['feat2wav'][max(i_f * reduce_factor + 1, feat_len - 1)][1]
              new_feat2wav.append([st, end])

          new_ali = {'index': ali['index'],
                     'is_phoneme': self.is_phoneme,
                     'caption': ali['src_sent'],
                     'image_concepts': ali['trg_sent'],
                     'alignment': ali['alignment'],
                     'attentions': ali['attentions'],
                     'feat2wav': new_feat2wav
                    }

        alignments.append(new_ali)

    with open(out_file, 'w') as fp:
      json.dump(alignments, fp, indent=4, sort_keys=True)

  def convert_retrieval_file(self, in_file, out_file='retrieval_results.txt'):
    fp = open(in_file)
    ret_indices = []
    for line in fp:
      idx_scores = line.strip().split('), (')
      top_indices = []
      for idx_score in idx_scores:
        idx_score = idx_score.replace('[', '')
        idx_score = idx_score.replace('(', '')
        idx_score = idx_score.replace(')', '')
        idx_score = idx_score.replace(']', '')
        idx_score = idx_score.replace(',', '')
        if DEBUG:
          print(idx_score)
        top_indices.append(idx_score.split()[0])
      ret_indices.append(' '.join(top_indices))
    fp.close()
    fp = open(out_file, 'w')
    fp.write('\n'.join(ret_indices))
    fp.close()

def alignment_to_cluster(ali_file, out_file='cluster.json'):
  def _find_distinct_tokens(data):
    tokens = set()
    for datum in data:
      if 'image_concepts' in datum:
        tokens = tokens.union(set(datum['image_concepts']))
      elif 'foreign_sent' in datum:
        tokens = tokens.union(set(datum['foreign_sent']))

    return list(tokens)

  fp = open(ali_file, 'r')
  align_info_all = json.load(fp)
  fp.close()

  classes = _find_distinct_tokens(align_info_all)
  clusters = {c:[] for c in classes}
  for align_info in align_info_all:
    sent = align_info['caption']
    concepts = align_info['image_concepts']
    alignment = align_info['alignment']

    if align_info['is_phoneme']:
      sent, alignment = _findPhraseFromPhoneme(sent, alignment)

    for w_i, a_i in zip(sent, alignment):
      if a_i >= len(concepts):
        if DEBUG:
          print('alignment index: ', align_info['index'])
          print('a_i out of range: ', a_i, concepts)
        a_i = 0
      clusters[concepts[a_i]].append(w_i)
      clusters[concepts[a_i]] = list(set(clusters[concepts[a_i]]))

  with open(out_file, 'w') as fp:
    json.dump(clusters, fp, indent=4, sort_keys=True)

def alignment_to_word_units(alignment_file, phone_corpus,
                                     concept_corpus,
                                     landmark_file = None,
                                     word_unit_file='word_units.wrd',
                                     phone_unit_file='phone_units.phn',
                                     concept2id_file=None,
                                     include_null = False):
  with open(phone_corpus, 'r') as f_p,\
       open(concept_corpus, 'r') as f_c:
    a_corpus, v_corpus = [], []
    for line in f_p:
      a_corpus.append(line.strip().split())

    for line in f_c:
      v_corpus.append(line.strip().split())

  if concept2id_file:
    with open(concept2id_file, 'r') as f:
      concept2id = json.load(f)
  else:
    concept2id = {}

  with open(alignment_file, 'r') as f:
    alignments = json.load(f)

  word_units = []
  phn_units = []
  # XXX
  for align_info, a_sent, v_sent in zip(alignments, a_corpus, v_corpus):
    if len(concept2id) > 0:
      image_concepts = [concept2id[c] for c in v_sent]
      print(image_concepts)
    else:
      image_concepts = align_info['image_concepts']
    alignment = align_info['alignment']
    pair_id = 'pair_' + str(align_info['index'])
    print(pair_id)
    prev_align_idx = -1
    start = 0
    for t, align_idx in enumerate(alignment):
      if t == 0:
        prev_align_idx = align_idx

      phn_units.append('%s %d %d %s\n' % (pair_id, t, t + 1, a_sent[t]))
      if prev_align_idx != align_idx:
        if not include_null and prev_align_idx == 0:
          prev_align_idx = align_idx
          start = t
          continue
        word_units.append('%s %d %d %s\n' % (pair_id, start, t, image_concepts[prev_align_idx]))
        prev_align_idx = align_idx
        start = t
      elif t == len(alignment) - 1:
        if not include_null and prev_align_idx == 0:
          continue
        word_units.append('%s %d %d %s\n' % (pair_id, start, t + 1, image_concepts[prev_align_idx]))

  with open(word_unit_file, 'w') as f:
    f.write(''.join(word_units))

  with open(phone_unit_file, 'w') as f:
    f.write(''.join(phn_units))

def alignment_to_word_classes(alignment_file, phone_corpus,
                                   word_class_file='words.class',
                                   landmark_file=None,
                                   include_null = False):
  f = open(phone_corpus, 'r')
  a_corpus = []
  for line in f:
    a_corpus.append(line.strip().split())
  f.close()
  with open(alignment_file, 'r') as f:
    alignments = json.load(f)

  word_units = {}
  for align_info, a_sent in zip(alignments, a_corpus):
    image_concepts = align_info['image_concepts']
    alignment = align_info['alignment']
    pair_id = 'pair_' + str(align_info['index'])
    print(pair_id)
    # print(image_concepts)
    prev_align_idx = -1
    start = 0
    if len(alignment) != len(a_sent):
      print('Warning: length of the alignment not equal to the length of sentence: %d != %d' % (len(alignment), len(a_sent)))
      gap = len(a_sent) - len(alignment)
      if gap > 0:
        print('Extend the alignment by %d ...' % gap)
        last_align_idx = alignment[-1]
        alignment.extend([last_align_idx]*gap)
        print(len(a_sent), len(alignment))
    for t, (align_idx, phn) in enumerate(zip(alignment, a_sent)):
      if t == 0:
        prev_align_idx = align_idx

      if prev_align_idx != align_idx:
        if not include_null and prev_align_idx == 0:
          prev_align_idx = align_idx
          start = t
          continue
        if image_concepts[prev_align_idx] not in word_units:
          word_units[image_concepts[prev_align_idx]] = ['%s %d %d\n' % (pair_id, start, t)]
        else:
          word_units[image_concepts[prev_align_idx]].append('%s %d %d\n' % (pair_id, start, t))
        prev_align_idx = align_idx
        start = t
      elif t == len(alignment) - 1:
        if not include_null and prev_align_idx == 0:
          continue
        if image_concepts[prev_align_idx] not in word_units:
          word_units[image_concepts[prev_align_idx]] = ['%s %d %d\n' % (pair_id, start, t + 1)]
        else:
          word_units[image_concepts[prev_align_idx]].append('%s %d %d\n' % (pair_id, start, t + 1))

  with open(word_class_file, 'w') as f:
    for i_c, c in enumerate(word_units):
      #print(i_c, c)
      f.write('Class %d:\n' % i_c)
      f.write(''.join(word_units[c]))
      f.write('\n')

def _findPhraseFromPhoneme(sent, alignment):
  if not hasattr(sent, '__len__') or not hasattr(alignment, '__len__'):
    raise TypeError('sent and alignment should be list')
  if DEBUG:
    print(len(sent), len(alignment))
    print(sent, alignment)
  assert len(sent) == len(alignment)
  cur = alignment[0]
  ws = []
  w_align = []
  w = ''
  for i, a_i in enumerate(alignment):
    if cur == a_i:
      w = w + ' ' + sent[i]
    else:
      ws.append(w)
      w_align.append(cur)
      w = sent[i]
      cur = a_i

  ws.append(w)
  w_align.append(cur)

  return ws, w_align

def resample_alignment(alignment_file, src_feat2wavs_file, trg_feat2wavs_file, out_file):
  with open(alignment_file, "r") as f:
    alignments = json.load(f)

  with open(src_feat2wavs_file, "r") as f:
    src_feat2wavs = json.load(f)
  with open(trg_feat2wavs_file, "r") as f:
    trg_feat2wavs = json.load(f)

  src_ids = sorted(src_feat2wavs, key=lambda x:int(x.split("_")[-1]))
  trg_ids = sorted(trg_feat2wavs, key=lambda x:int(x.split("_")[-1]))

  new_alignments = []
  for i_ali, ali in enumerate(alignments):
    # TODO: make this faster by making the feat_id convention more consistent
    trg_feat2wav = trg_feat2wavs[trg_ids[i_ali]]
    src_feat2wav = src_feat2wavs[src_ids[i_ali]]
    wavLen = max(src_feat2wav[-1][1], trg_feat2wav[-1][1])

    # Frames are automatically assigned to the last frame (convenient if the two feat2wavs have different lengths)
    trg_wav2feat = [-1]*wavLen

    alignment = ali["alignment"]

    if DEBUG:
      logging.debug("i_ali: " + str(i_ali))
      logging.debug("src_ids, trg_ids: %s %s" % (src_ids[i_ali], trg_ids[i_ali]))
      logging.debug("# of wav frames in src_feat2wav, # of feat frames: %d %d" % (src_feat2wav[-1][1], len(src_feat2wav)))
      logging.debug("# pf wav frames in trg_feat2wav, # of feat frames: %d %d" % (trg_feat2wav[-1][1], len(trg_feat2wav)))
      logging.debug("# of frames in alignment: " + str(len(alignment)))
      logging.debug("# of frames in trg_feat2wav: " + str(len(trg_feat2wav)))

    for i_seg, seg in enumerate(trg_feat2wav):
      start = seg[0]
      end = seg[1]
      for t in range(start, end):
        trg_wav2feat[t] = i_seg

    new_alignment = [0]*len(trg_feat2wav)
    for i_src in range(len(alignment)):
      for i_wav in range(src_feat2wav[i_src][0], src_feat2wav[i_src][1]):
        if i_wav > len(trg_wav2feat) - 1:
          logging.warning("inconsistent wav lens: %d %d" % (src_feat2wav[-1][1], len(trg_wav2feat)))
          break
        i_trg = trg_wav2feat[i_wav]
        new_alignment[i_trg] = alignment[i_src]

    new_align_info = deepcopy(ali)
    new_align_info["alignment"] = new_alignment
    new_alignments.append(new_align_info)

  with open(out_file, "w") as f:
    json.dump(new_alignments, f, sort_keys=True, indent=4)

def convert_boundary_to_segmentation(binary_boundary_file, frame_boundary_file):
  binary_boundaries = np.load(binary_boundary_file)
  frame_boundaries = []
  for i, b_vec in enumerate(binary_boundaries):
    # print("segmentation %d" % i)
    end_frames = np.nonzero(b_vec)[0]
    frame_boundary = [[0, end_frames[0]]]
    for st, end in zip(end_frames[:-1], end_frames[1:]):
      frame_boundary.append([st, end+1])

    # if i < 5:
    #   print("end_frames: ", end_frames)
    # print("frame_boundary: ", frame_boundary)
    frame_boundaries.append(np.asarray(frame_boundary))

  np.save(frame_boundary_file, frame_boundaries)

def convert_landmark_segment_to_10ms_segmentation(landmark_segment_file, landmarks_file, frame_segment_file):
  lm_segments = np.load(landmark_segment_file)

  lms = np.load(landmarks_file)
  utt_ids = sorted(lms.keys(), key=lambda x:int(x.split('_')[-1]))
  frame_segments = []
  for i, utt_id in enumerate(utt_ids):
    # print(i, utt_id)
    # print('lm_segment: ', lm_segments[i])
    # print('lms: ', lms['arr_'+str(i)])
    # print('len(lm_segment), len(lms): ', len(lm_segments[i]), len(lms['arr_'+str(i)]))
    lm_segment = lm_segments[i]
    lm = np.insert(lms, 0, 0)
    f_seg = []
    for lm_seg in lm_segment:
      lm = np.insert(lms[utt_id], 0, 0)
      f_seg.append(lm[lm_seg])
    frame_segments.append(np.asarray(f_seg))
  np.save(frame_segment_file, frame_segments)

def convert_10ms_segmentation_to_landmark(segmentation_file, ids_to_utterance_label_file, landmarks_file):
  segmentations = np.load(segmentation_file)
  with open(ids_to_utterance_label_file, "r") as f:
    ids_to_utterance_labels = json.load(f)

  landmarks = {}
  assert len(segmentations) == len(ids_to_utterance_labels)
  for (utt, segmentation) in zip(ids_to_utterance_labels, segmentations):
    if int(utt.split('_')[-1]) >= 1 and int(utt.split('_')[-1]) <= 4:
      print(utt, segmentation[-1])
    landmark = [0]
    for seg in segmentation:
      if seg[1] == seg[0]:
        print("Overlapped boundaries: ", seg)
        continue
      landmark.append(int(seg[1]))
    landmarks[utt] = landmark

  np.savez(landmarks_file, **landmarks)

def convert_sec_to_10ms_landmark(real_time_segment_file, feat2wav_file, frame_segment_file, fs=16000, max_feat_len=2000):
  real_time_segments = np.load(real_time_segment_file)
  with open(feat2wav_file, 'r') as f:
    feat2wavs = json.load(f)

  wav2feats = []
  utt_ids = []
  for utt_id, feat2wav in sorted(feat2wavs.items(), key=lambda x:int(x[0].split('_')[-1])):
    feat_len = len(feat2wav)
    wav_len = feat2wav[-1][1]
    print('utt_id, wav_len: ', utt_id, wav_len)
    wav2feat = np.zeros((wav_len,), dtype=int)
    for i in range(feat_len):
      wav2feat[feat2wav[i][0]:feat2wav[i][1]] = i
    wav2feats.append(wav2feat)
    utt_ids.append(utt_id)

  frame_segments = {}
  max_gap = [0, 0.]
  max_gap_segment = []
  for i_seg, r_seg in enumerate(real_time_segments.tolist()):
    feat_len = len(feat2wavs[utt_ids[i_seg]])
    wav2feat = wav2feats[i_seg]
    wav_len = len(wav2feat)

    n_segs = len(r_seg)
    f_seg = [0]
    for i in range(n_segs):
      # XXX: Need thresholding since some syllable segmentation is slightly longer than the actual wavform
      wav_frame_i = min(int(r_seg[i] * fs), wav_len - 1)
      feat_frame_i = wav2feat[wav_frame_i]
      prev_feat_frame = f_seg[-1]

      # XXX: Prevent the segmentation to move backward
      if feat_frame_i <= 0 or feat_frame_i - prev_feat_frame <= 0:
        continue
      else:
        if feat_frame_i - prev_feat_frame > max_gap[1]:
          max_gap = [i_seg, wav2feat[wav_frame_i] - f_seg[-1]]
          max_gap_segment = [f_seg[-1], wav2feat[wav_frame_i]]
        # XXX: Prevent the segmentation to exceends maxlen / length of the actual feature
        if feat_frame_i >= min(feat_len - 1, max_feat_len):
          continue

        f_seg.append(feat_frame_i + 1)
    f_seg.append(min(feat_len, max_feat_len))

    if i_seg >= 1 and i_seg <= 4:
      print('utt_id: ', utt_ids[i_seg])
      print('feat_len, last segmentation frame: ', feat_len)
      print('wav_len from segmentation, wav_len from wav2feats: ', r_seg[-1] * fs, len(wav2feats[i_seg]))
      print("feat_len from segmentation: ", f_seg[-1])

    frame_segments[utt_ids[i_seg]] = f_seg

  print("max_gap: ", max_gap)
  print("max_gap landmark: ", max_gap_segment)
  np.savez(frame_segment_file, **frame_segments)

def convert_txt_to_npy_segment(txt_segment_file, npy_segment_file):
  with open(txt_segment_file, 'r') as f:
    lines = f.read().strip().split('\n\n')
    segmentations = []
    for seg_txt in lines:
      seg = []
      for line in seg_txt.split('\n'):
        t = float(line.split()[1])
        seg.append(t)
      #print(seg)
      segmentations.append(np.asarray(seg))

    np.save(npy_segment_file, segmentations)

def extract_concept_segments(cluster, feat2wav, wav_dir, ids_to_utterance_labels, out_dir=None):
  segments = []
  for seg_info in cluster:
    wav_id = ids_to_utterance[seg_info[0]]
    print(wav_id)
    filename = '_'.join(wav_id.split('_')[:-1]) + ".wav"
    y, sr = wavfile.read(filename)

    t_start, t_end = feat2wav[seg_info[1][0]][0], feat2wav[seg_info[1][1]][1]
    if t_end - t_start <= 1600:
      continue
    segment = y[t_start:t_end]
    segment_melspectrogram = librosa.feature.melspectrogram(segment, sr=sr)

    segments.append(segment)
    segment_features.append(segment_melspectrogram)
    if out_dir:
      segment_file = "%s%s_%d_%d.wav" % (np.out_dir, wav_id, t_start, t_end)
      segment_feat_file = "%s%s_%d_%d.png" % (np.out_dir, wav_id, t_start, t_end)
      wavfile.write(segment_file, sr, y)
      img = Image.fromarray(segment_melspectrogram)
      img.save(segment_feat_file)

  return segments, segment_features

def extract_top_concept_segments(audio_cluster_file, feat2wav_file, wav_dir, ids_to_utterance_label_file, concept_priors_file, n_top=20):
  with open(audio_cluster_file, "r") as f:
    clusters = json.load(f)

  with open(concept_priors_file, "r") as f:
    concept_priors = json.load(f)

  with open(ids_to_utterance_label_file, "r") as f:
    ids_to_utterance_labels = json.load(f)

  with open(feat2wav_file, "r") as f:
    feat2wav = json.load(f)

  top_concepts = sorted(concept_priors, key=lambda x:concept_priors[x], reverse=True)[1:n_top+1]

  for c in top_concepts:
    c_dir = "%s%s/" % (out_dir, c)
    os.mkdir(c_dir)
    _, _ = extract_concept_segments(clusters[c], feat2wav, wav_dir, ids_to_utterance_labels, out_dir=c_dir)

if __name__ == '__main__':
  '''
  postproc = XNMTPostprocessor('../nmt/exp/feb28_phoneme_level_clustering/output/report/')
  postproc.convert_alignment_file('../nmt/exp/feb28_phoneme_level_clustering/output/alignment.json')
  alignment_to_cluster('../nmt/exp/feb28_phoneme_level_clustering/output/alignment.json', '../nmt/exp/feb28_phoneme_level_clustering/output/cluster.json')
  postproc.convert_retrieval_file('../nmt/exp/mar19_phoneme_to_image_norm_over_time/output/phoneme_to_concept.hyp')
  '''
  alignment_file = "../smt/exp/june_24_mfcc_kmeans_mixture=3/flickr30k_pred_alignment.json"
  src_feat2wavs_file = "../data/flickr30k/audio_level/flickr_mfcc_feat2wav.json"
  ref_feat2wavs_file = "../data/flickr30k/audio_level/flickr30k_gold_alignment.json_feat2wav.json"
  out_file = "../smt/exp/june_24_mfcc_kmeans_mixture=3/pred_alignment_resample.json"
  binary_boundary_file = "../comparison_models/exp/july_20_multimodal_kmeans/boundaries_multimodal-kmeans.npy"
  frame_boundary_file = "../data/flickr30k/audio_level/frame_boundaries_semkmeans.npy"
  txt_syl_segment_file = "../data/flickr30k/audio_level/syllable_boundaries.txt"
  npy_syl_segment_file = "../data/flickr30k/audio_level/syllable_segmentations_osc.npy"
  feat_type = "bn"
  if feat_type == "bn":
    landmark_file = "flickr_landmarks_mbn.npz" #"flickr_landmarks_osc_mbn.npz" #"../data/flickr30k/audio_level/flickr_landmarks.npz" #"flickr_landmarks_mbn.npz" #"../data/flickr30k/audio_level/flickr_landmarks.npz"
    feat2wavs_htk_file = "../data/flickr30k/audio_level/flickr30k_gold_alignment.json_feat2wav.json"
  elif feat_type == "mfcc":
    landmark_file = "flickr_landmarks_osc.npz"
    feat2wavs_htk_file = "../data/flickr30k/audio_level/flickr_mfcc_cmvn_htk_feat2wav.json"
  audio_cluster_file = "../comparison_models/exp/aug1_mkmeans/pred_clusters.json"
  ids_to_utterance_file = "../data/flickr30k/audio_level/ids_to_utterance_labels.json"
  wav_dir = "/home/lwang114/data/flickr/flickr_audio/wavs/"
  gold_seg_file = "../data/flickr30k/audio_level/flickr30k_gold_segmentation_mbn.npy"
  #"../data/flickr30k/audio_level/flickr30k_gold_segmentation_mfcc_htk.npy"
  #convert_boundary_to_segmentation(binary_boundary_file, frame_boundary_file)
  #resample_alignment(alignment_file, src_feat2wavs_file, ref_feat2wavs_file, out_file)
  #convert_txt_to_npy_segment(txt_syl_segment_file, npy_syl_segment_file)
  convert_sec_to_10ms_landmark(npy_syl_segment_file, feat2wavs_htk_file, landmark_file)
  #convert_10ms_segmentation_to_landmark(gold_seg_file, ids_to_utterance_file, "flickr30k_gold_landmarks_mbn.npz")


# Copyright (c) [2022] Huawei Technologies Co.,Ltd.ALL rights reserved.
# This program is licensed under Mulan PSL v2.
# You can use it according to the terms and conditions of the Mulan PSL v2.
#          http://license.coscl.org.cn/MulanPSL2
# THIS PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
# See the Mulan PSL v2 for more details.
####################################
# @Author  :
# @email   :
# @Date    :
# @License : Mulan PSL v2

#####################################

from flask import jsonify
from flask_restful import Resource
from flask_pydantic import validate

from server.utils.auth_util import auth
from server.utils.response_util import RET
from server.utils.response_util import response_collect
from server.model.framework import Framework, GitRepo
from server.model.template import Template
from server.utils.db import Insert, Delete, Edit, Select
from server.schema.framework import FrameworkBase, FrameworkQuery, GitRepoBase, GitRepoQuery, GitRepoScopedQuery
from server.utils.resource_utils import ResourceManager
from server.utils.permission_utils import GetAllByPermission
from server import casbin_enforcer
from server.apps.framework.handlers import GitRepoHandler

class FrameworkEvent(Resource):
    @auth.login_required()
    @response_collect
    @validate()
    def post(self, body: FrameworkBase):
        return ResourceManager("framework").add("api_infos.yaml", body.__dict__)

    @auth.login_required()
    @response_collect
    @validate()
    def get(self, query: FrameworkQuery):
        filter_params = GetAllByPermission(Framework).get_filter()
        if query.name:
            filter_params.append(
                Framework.name.like(f'%{query.name}%')
            )
        if query.url:
            filter_params.append(
                Framework.url.like(f'%{query.url}%')
            )
        if query.adaptive:
            filter_params.append(
                Framework.adaptive == query.adaptive
            )

        frameworks = Framework.query.filter(*filter_params).all()
        if not frameworks:
            return jsonify(error_code=RET.OK, error_msg="OK", data=[])

        return jsonify(
            error_code=RET.OK,
            error_msg="OK",
            data=[
                _framework.to_json() for _framework in frameworks
            ]
        )

class FrameworkItemEvent(Resource):
    @auth.login_required
    @response_collect
    @validate()
    @casbin_enforcer.enforcer
    def delete(self, framework_id):
        return ResourceManager("framework").del_cascade_single(framework_id, GitRepo,
                                                               [GitRepo.framework_id == framework_id], False)

    @auth.login_required
    @response_collect
    @validate()
    @casbin_enforcer.enforcer
    def put(self, framework_id, body: FrameworkQuery):
        _body = {
            **body.__dict__,
            "id": framework_id,
        }

        return Edit(Framework, _body).single(Framework, "/framework")

    @auth.login_required
    @response_collect
    @validate()
    @casbin_enforcer.enforcer
    def get(self, framework_id):
        framework = Framework.query.filter_by(id=framework_id).first()
        if not framework:
            return jsonify(
                error_code=RET.NO_DATA_ERR,
                error_msg="the framework does not exist"
            )

        return jsonify(
            error_code=RET.OK,
            error_msg="OK",
            data=framework.to_json()
        )

class GitRepoScopedEvent(Resource):
    @auth.login_required()
    @response_collect
    @validate()
    def get(self, query: GitRepoScopedQuery):
        filter_params = [
            GitRepo.permission_type == query.type
        ]
        if query.type == "group":
            filter_params.append(GitRepo.group_id == query.group_id)
        elif query.type == "org":
            filter_params.append(GitRepo.org_id == query.org_id)

        return GitRepoHandler.get_git_repo(query, filter_params)

class GitRepoEvent(Resource):
    @auth.login_required()
    @response_collect
    @validate()
    def post(self, body: GitRepoBase):
        return ResourceManager("git_repo").add_v2("framework/api_infos.yaml", body.__dict__)

    @auth.login_required()
    @response_collect
    @validate()
    def get(self, query: GitRepoQuery):
        filter_params = GetAllByPermission(GitRepo).get_filter()
        return GitRepoHandler.get_git_repo(query, filter_params)

class GitRepoItemEvent(Resource):
    @auth.login_required
    @response_collect
    @validate()
    @casbin_enforcer.enforcer
    def delete(self, git_repo_id):
        return ResourceManager("git_repo").del_cascade_single(git_repo_id, Template,
                                                              [Template.git_repo_id == git_repo_id], False)

    @auth.login_required
    @response_collect
    @validate()
    @casbin_enforcer.enforcer
    def put(self, git_repo_id, body: GitRepoQuery):
        _body = {
            **body.__dict__,
            "id": git_repo_id,
        }

        return Edit(GitRepo, _body).single(GitRepo, "/git_repo")

    @auth.login_required
    @response_collect
    @casbin_enforcer.enforcer
    def get(self, git_repo_id):
        git_repo = GitRepo.query.filter_by(id=git_repo_id).first()
        if not git_repo:
            return jsonify(
                error_code=RET.NO_DATA_ERR,
                error_msg="the git repo does not exist"
            )

        return jsonify(
            error_code=RET.OK,
            error_msg="OK",
            data=git_repo.to_json()
        )