import sys, os
parent = os.path.dirname(os.path.realpath(__file__))
sys.path.append(parent + '/MITIE/mitielib')

from nltk.corpus import stopwords
spanish_stopwords = stopwords.words('spanish')

from collections import defaultdict
import editdistance
from pyner import ner
from mitie import *
from string import punctuation
from expand_entities_names import query_name
punctuation = punctuation.replace('-','')

parent = os.path.dirname(os.path.realpath(__file__))
ner = named_entity_extractor(parent+'/MITIE/MITIE-models/spanish/ner_model.dat')

def get_entities_mit(body,index):
    # Lógica para comunicarse con MITIE:

    body = body.encode('utf-8')
    tokens = tokenize(body)
    entities = []
    for x in ner.extract_entities(tokens):
        entity = {}
        entity['name'] = " ".join(tokens[i] for i in x[0])
        entity['tag'] = x[1]
        entity['token_initial_index'] = x[0][0]
        length = len(entity['name'].split(' '))
        entity['token_final_index'] = x[0][0]+length
        entity['freq'] = 1
        entity['aliases'] = []
        entities.append(entity)

    # Los cuatro procedimientos para mejorar los nombres de entidades.
    entities = normalize_entities(entities)
    entities = merge_initials(entities,tokens)
    entities = merge_names(entities)
    entities = complete_names(entities, tokens,index)
    entities_dict = {}

    # Build the dictionary.
    for entity in entities:
        if entity['name'] == '':
            continue

        if entity['name'] not in entity['aliases']:
            entity['aliases'].append(entity['name'])

        name = entity['name']
        if name not in entities_dict:
            entity['tags'] = {entity['tag']:entity['freq']}
            entity.pop('tag')
            entity.pop('token_initial_index')
            entity.pop('token_final_index')
            entities_dict[name] = entity

        else:
            entities_dict[name]['freq']+=entity['freq']

            # Merge aliases:
            for alias in entity['aliases']:
                if alias not in entities_dict[name]['aliases']:
                    entities_dict[name]['aliases'].append(alias)

            # Merge tag counts.
            if entity['tag'] in entities_dict[name]['tags']:
                entities_dict[name]['tags'][entity['tag']]+=entity['freq']
            else:
                entities_dict[name]['tags'][entity['tag']]=entity['freq']

    return entities_dict

'''
To address the issue of truncated names (eg ‘Word Life Fund’ may be truncated and processed as ‘World’ and ‘Life Fund’) we:
i) look up every name and its surrounding context in the corpus
ii) extract sentences in the top articles and
iii) find the longest substring matching these sentences.
'''
def complete_names(entities,tokens,index,window_size = 15):
    new_entities = []
    for i in range(0,len(entities)):
        entity = entities[i]
        if entity['tag'] in ['PERSON','LOCATION']:
            new_entities.append(entity)
            continue

	# Extraer los contextos anteriores y posteriores del nombre de la entidad.

        initial_pos = max(0,entity['token_initial_index']-3)
        [tokens[k] for k in range(initial_pos,entity['token_initial_index']) if tokens[k].istitle()]

        final_pos = min(len(tokens),entity['token_initial_index']+3)
        if len([tokens[k] for k in range(initial_pos,entity['token_initial_index']) if tokens[k].istitle()]) == 0 or len([tokens[k] for k in range(entity['token_final_index'],final_pos)if tokens[k].istitle()]) == 0:
            continue

        initial_pos = max(0,entity['token_initial_index']-window_size)
        final_pos = min(len(tokens),entity['token_initial_index']+window_size)
        name_with_context = [tokens[k] for k in range(initial_pos,final_pos)]
        pre_context = ' '.join([tokens[k] for k in range(initial_pos,entity['token_initial_index'])])
        post_context = ' '.join([tokens[k] for k in range(entity['token_final_index'],final_pos)])

	# Intenar expandir el nombre utilizando el índice.
        whole_name = query_name(pre_context,entity['name'],post_context,index)

	# Si hubo un match, hacer un merge con una entidad cercana si corresponde el caso.
        if whole_name is not None:
            #print 'EXPANDING {} to {} based on {}'.format(entity['name'],whole_name, ' '.join(name_with_context))
            entity['name'] = whole_name
            whole_name_words = whole_name.split(' ')
            # Merge it with neighboring entities.
            limit = min([i-k for k in range(0,4) if i-k>=0])
            for j in range(limit,i):
                # Check if it's close enough
                #print i, j,len(entities)
                if entities[i]['token_initial_index']-entities[j]['token_final_index'] > window_size:
                    continue
                # Check if the neighboring entity is contained in the complete name
                not_contained = False
                for word in entities[j]['name'].split(' '):
                    if word not in whole_name_words:
                        not_contained = True
                        break
                if not_contained:
                    break
                # If it's close enough and it's contained in the complete name, delete it
                #print 'REMOVING {} based on {} and {}'.format(entities[j]['name'],entity['name'], ' '.join(name_with_context))
                if entities[j] in new_entities:
                    new_entities.remove(entities[j])
        new_entities.append(entity)

    return new_entities

'''
    Given a list of entities, attempts to match partial names (1 word)
    with the nearest previous complete name (2 words or more) in the article.
    For instance, if the entity Robert Patrick is found, followed by the entity Patrick, we can infer
    that these names relate to the same entity.

    When such a match is found, the entities are merged and the partial name is added as an alias.

    This process is done only on entities of the type 'PERSON'
'''
def merge_names(entities):
    full_persons = []
    new_entities = []

    # Make sure that the entities are sorted according to the order they are mentioned in the article.
    sorted_entities = [(entity['token_initial_index'],entity) for entity in entities]
    sorted_entities.sort()
    entities = [entity for (pos, entity) in sorted_entities]

    for entity in entities:
        if entity['tag'] == 'PERSON':
            tokens = entity['name'].split(' ')
            if len(tokens)>1: # It's a full name.
                full_persons.append(entity)
                new_entities.append(entity)
            else: # It's a partial name (first name or last name)
                found_whole_name = False
                # Attempt to match it to nearest previous entity
                for i in range(len(full_persons)-1,-1,-1):
                    full_person = full_persons[i]

                    '''
                    We need to match the lowercase of names since the camelize function
                    does not change strings of one word
                    '''

                    all_names = [x.lower() for x in full_person['name'].split()]
                    if entity['name'].lower() in all_names:
                        found_whole_name= True
                        full_person['freq']+=entity['freq']
                        if entity['name'] not in full_person['aliases']:
                            #print('ALIASES: {} FOUND: {}'.format(entity['name'],full_person['name']))
                            full_person['aliases'].append(entity['name'])
                            break
                        else:
                            pass
                            #print('ALIASES: {} FOUND: {} already there'.format(entity['name'],full_person['name']))
                if not found_whole_name:
                    new_entities.append(entity)
                    #print('ALIASES: {} NOT FOUND'.format(entity['name']))

        else: # ORG, LOC or MISC
            new_entities.append(entity)

    return new_entities
'''
    Given a list of entities, attempt to match organizations names with their initials.
    To do so, the following pattern is exploited:

    Usually in articles after an organization's full name is first used, the writer writes the initials
    of that organization inside parenthesis. EG: Hewlett Packard (HP).

    This function goes through the list of entities and attempts to exploit that pattern; in case a match
    is found the entities are merged and the initials are added as an alias.
'''
def merge_initials (entities,tokens):
    new_entities = []
    full_organizations = []

    # Make sure that the entities are sorted according to the order they are mentioned in the article.
    sorted_entities = [(entity['token_initial_index'],entity) for entity in entities]
    sorted_entities.sort()
    entities = [entity for (pos, entity) in sorted_entities]
    for entity in entities:
        if entity['tag'] in ['ORGANIZATION','MISC']:
            found = False
            '''
            Verify if it's been found before. If so, update the frequency count and discard it.

            Otherwise, verify if we can match it to another organization.
            '''
            for full_organization in full_organizations:
                if entity['name'] in full_organization['aliases'] or entity['name'] == full_organization['name']:
                   full_organization['freq'] += entity['freq']
                   found = True
                   break
            if found:
                continue
            # Verify the name can be an alias according to the <(INITIALS)> pattern.

            if entity['name'].isupper() and tokens[entity['token_initial_index']-1]=='(' and tokens[entity['token_final_index']]==')' and len(full_organizations)>0:

                # Check the previous organization and the distance to it.

                previous_organization = full_organizations[len(full_organizations)-1]
                if entity['token_initial_index'] - previous_organization['token_final_index'] <3:
                    # It's match for sure.
                    previous_organization['aliases'].append(entity['name'])
                    previous_organization['freq'] += entity['freq']
                    #print 'INITIALS: {} matched with {}'.format(entity['name'],previous_organization['name'])
                    continue
                # If it doesn't match, then add id as an entity and add it to the list of organizations.

                full_organizations.append(entity)
                new_entities.append(entity)
            else: # It's a normal organization
                full_organizations.append(entity)
                new_entities.append(entity)
        else:
            new_entities.append(entity)

    return new_entities
'''
    Given a string, return it's camelized version (meaning every letter of every word is in lowercase,
    except for the first word which is in uppercase).
    If the string only contains one word, or if the word is completely in uppercase, it is not modified.
'''
def camelize(string):
    if string.isupper() or len(string.split(' '))==0:
        return string

    to_upper_case = True
    new_string = ''
    for x in string:
        if to_upper_case:
            new_string+=x.upper()
        else:
            new_string+=x.lower()

        to_upper_case = x == ' '
    return new_string

'''
    Given a string, remove trailing and leading uncapitalized stopwords.
'''
def remove_trailing_stopwords(string):

    words = string.split(' ')
    wordsd = []
    found_upper = False

     # Remove trailing stopwords

    found_non_stopword = False
    for i in range(len(words)):
        word = words[i]
        if len(word)==0:
            continue
        found_non_stopword = found_non_stopword or word not in spanish_stopwords
        if found_non_stopword:
            wordsd.append(word)

    words = [x for x in wordsd]

    wordsd = []
    found_non_stopword = False
    for i in range(len(words)-1,-1,-1):
        word = words[i]
        if len(word)==0:
            continue
        found_non_stopword = found_non_stopword or word not in spanish_stopwords
        if found_non_stopword:
            wordsd = [word]+wordsd

    return ' '.join(wordsd)

'''
Given a string with the name of an entity, normalize it by doing the following steps:
0) Remove names which do not have at least one uppercase letter.
1) Punctuation sign removal.
2) Remove double, leading and trailing whitespaces.
3) Remove leading and trailing stopwords.
4) Camelize the string.
'''
def normalize_text(text):

    # Remove names which are completely undercase.
    if text.islower():
        return ''
    # Clean punctuation

    text = text.replace('.','')
    for punct in punctuation:
        text = text.replace(punct, ' ')

    # Remove double whitespces
    text = text.replace('   ',' ')
    text = text.replace('  ',' ')
    # And remove trailing spaces
    text = text.strip()
    # Filter trailing stopwords.
    text = remove_trailing_stopwords(text)

    # Camelize.
    text = camelize(text)

    return text

def normalize_entities(entities):
    for ent in entities:
        ent['name'] = normalize_text(ent['name'])

    return entities

def get_entities(body,index):
    return get_entities_mit(body,index)

'''
    No se usa la libreria NER de Standford. En su lugar, usamos MITIE.
'''
def get_entities_standford(body):
    tagger = ner.SocketNER(host='localhost', port=9192)
    results = tagger.get_entities(body)
    entities = {}

    for ent_type in ['ORG','PERS','OTROS']:
        if ent_type in results:
            for entity in results[ent_type]:
                if entity in entities:
                    entities[entity]+=1
                else:
                    entities[entity]=1
    return entities


from django import forms

class FormMultivarLSTM(forms.Form):
	epocas = forms.IntegerField(
		label="Épocas",
		widget=forms.TextInput(attrs={
			'class':'form-control border-input',
			'placeholder': 'Cantidad',
			'id':'inp-epocas',
			'value': '20'
			})
		)
	lotes = forms.IntegerField(
		label="Long. de Lotes",
		widget=forms.TextInput(attrs={
			'class':'form-control border-input',
			'placeholder': 'Longitud',
			'value': '50'
			})
		)

	CHOICES = (('1', 'SGD'),('2', 'RMSprop'),('3', 'Adagrad'),('4', 'Adadelta'),('5', 'Adam'),('6', 'Adamax'),('7', 'Nadam') )
	alg_optimizacion = forms.ChoiceField(
	initial='2',
	choices=CHOICES,
	label="Alg. de Optimización",
	widget=forms.Select(attrs={
		'class':'form-control'
		})
	)

	cost_func = forms.ChoiceField(
	choices=CHOICES,
	label="Func. objetivo",
	widget=forms.Select(attrs={
		'class':'form-control'
		})
	)


import traceback
from datetime import datetime, timedelta, date

from django.test import TestCase
from django.utils import timezone

from keymaster.models import Messages, EncryptedConnection, Keymaster, Gatekeeper

from cryptography.fernet import Fernet

class EncryptedConnectionTestCase(TestCase):

    def test_encryption(self):
        key = Fernet.generate_key()
        connection = EncryptedConnection(key)
        message = "This is a test"
        encrypted_message = connection.encrypt_message(message)
        decrypted_message = connection.decrypt_message(encrypted_message)
        self.assertNotEqual(message, encrypted_message)
        self.assertEqual(message, decrypted_message)

class KeymasterTestCase(TestCase):
    def setUp(self):
        self.ip_address = "127.0.0.1"
        new_key = Fernet.generate_key()
        Gatekeeper.objects.create(ip_address=self.ip_address, encryption_key=new_key, is_enabled=True)

    def test_handshake(self):
        message = Messages.TEST_QUESTION
        gatekeeper = Gatekeeper.objects.by_ip(self.ip_address)
        keymaster = Keymaster(gatekeeper)
        response = keymaster.process_message(message)
        self.assertEqual(response, Messages.TEST_RESPONSE)

    def test_mark_success(self):
        start_ts = timezone.now()
        gatekeeper = Gatekeeper.objects.by_ip(self.ip_address)
        self.assertTrue(gatekeeper.sync_ts == None)
        message = Messages.MARK_SUCCESS
        keymaster = Keymaster(gatekeeper)
        response = keymaster.process_message(message)
        self.assertEqual(response, Messages.SUCCESS_RESPONSE)
        self.assertFalse(gatekeeper.sync_ts == None)
        self.assertTrue(gatekeeper.sync_ts > start_ts)


from django.core import signing
from django.db import models
from django.db.models import Q
from django.urls import reverse
from django.utils.translation import gettext_lazy as _
from django.utils.functional import cached_property
from oscar.core.loading import get_model
from oscar.models.fields import PhoneNumberField, NullCharField
from ..core.constants import (
    PREQUAL_ENTRY_POINT_WEB,
    PREQUAL_ENTRY_POINT_CHOICES,
    PREQUAL_TRANS_STATUS_APPROVED,
    PREQUAL_TRANS_STATUS_REJECTED,
    PREQUAL_TRANS_STATUS_CHOICES,
    PREQUAL_CUSTOMER_RESP_NONE,
    PREQUAL_CUSTOMER_RESP_CHOICES,
    get_prequal_trans_status_name,
)
from ..core.fields import USStateField, USZipCodeField
from .transfers import TransferMetadata
from .utils import _max_len
import uuid
import urllib.parse

class PreQualificationRequest(models.Model):
    uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)
    entry_point = models.CharField(
        _("Entry Point"),
        max_length=_max_len(PREQUAL_ENTRY_POINT_CHOICES),
        choices=PREQUAL_ENTRY_POINT_CHOICES,
        default=PREQUAL_ENTRY_POINT_WEB,
    )
    customer_initiated = models.BooleanField(
        _("Check was deliberately initiated by customer action"), default=False
    )
    email = models.EmailField(_("Email"), null=True, blank=True)
    first_name = models.CharField(_("First Name"), max_length=15)
    middle_initial = models.CharField(
        _("Middle Initial"), null=True, blank=True, max_length=1
    )
    last_name = models.CharField(_("Last Name"), max_length=20)
    line1 = models.CharField(_("Address Line 1"), max_length=26)
    line2 = models.CharField(_("Address Line 2"), max_length=26, null=True, blank=True)
    city = models.CharField(_("City"), max_length=15)
    state = USStateField(_("State"))
    postcode = USZipCodeField(_("Postcode"))
    phone = PhoneNumberField(_("Phone"))
    ip_address = models.GenericIPAddressField(null=True, blank=True)
    merchant_name = NullCharField(_("Merchant Name"), max_length=200)
    merchant_num = NullCharField(_("Merchant Number"), max_length=200)
    created_datetime = models.DateTimeField(auto_now_add=True)
    modified_datetime = models.DateTimeField(auto_now=True)

    class Meta:
        verbose_name = _("Pre-Qualification Request")
        verbose_name_plural = _("Pre-Qualification Requests")
        ordering = ("-created_datetime", "-id")
        indexes = [
            models.Index(fields=["-created_datetime", "-id"]),
        ]

    @property
    def entry_point_name(self):
        return dict(PREQUAL_ENTRY_POINT_CHOICES).get(self.entry_point, self.entry_point)

    @property
    def status(self):
        response = getattr(self, "response", None)
        if response:
            return response.status
        return PREQUAL_TRANS_STATUS_REJECTED

    @property
    def status_name(self):
        response = getattr(self, "response", None)
        if response:
            return response.status_name
        return get_prequal_trans_status_name(
            PREQUAL_TRANS_STATUS_REJECTED, self.customer_initiated
        )

    @property
    def credit_limit(self):
        resp = getattr(self, "response", None)
        if resp is None:
            return None
        return resp.credit_limit

    @property
    def customer_response(self):
        resp = getattr(self, "response", None)
        if resp is None:
            return None
        return resp.customer_response

    @property
    def sdk_application_result(self):
        resp = getattr(self, "response", None)
        if resp is None:
            return None
        app_result = getattr(resp, "sdk_application_result", None)
        if app_result is None:
            return None
        return app_result.application_status

    @cached_property
    def resulting_order(self):
        resp = getattr(self, "response", None)
        if resp and resp.customer_order:
            return resp.customer_order
        # Look for other orders which might have been placed by this customer
        Order = get_model("order", "Order")
        email_matches = Q(guest_email=self.email) | Q(user__email=self.email)
        date_matches = Q(date_placed__gt=self.created_datetime)
        order = (
            Order.objects.filter(email_matches & date_matches)
            .order_by("date_placed")
            .first()
        )
        return order

    @property
    def order_total(self):
        return self.resulting_order.total_incl_tax if self.resulting_order else None

    @property
    def order_date_placed(self):
        return self.resulting_order.date_placed if self.resulting_order else None

    @cached_property
    def order_merchant_name(self):
        Transaction = get_model("payment", "Transaction")
        order = self.resulting_order
        if not order:
            return None
        transfers = []
        for source in order.sources.filter(source_type__name="Wells Fargo").all():
            for transaction in source.transactions.filter(
                txn_type=Transaction.AUTHORISE
            ).all():
                transfer = TransferMetadata.get_by_oscar_transaction(transaction)
                if transfer:
                    transfers.append(transfer)
        if len(transfers) <= 0:
            return None
        return transfers[0].merchant_name

    @property
    def response_reported_datetime(self):
        resp = getattr(self, "response", None)
        if resp is None:
            return None
        return resp.reported_datetime

    def get_signed_id(self):
        return signing.Signer().sign(self.pk)

    def get_resume_offer_url(self, next_url="/"):
        url = reverse("wfrs-api-prequal-resume", args=[self.get_signed_id()])
        qs = urllib.parse.urlencode(
            {
                "next": next_url,
            }
        )
        return "{}?{}".format(url, qs)

class PreQualificationResponse(models.Model):
    request = models.OneToOneField(
        PreQualificationRequest, related_name="response", on_delete=models.CASCADE
    )
    status = models.CharField(
        _("Transaction Status"), max_length=10, choices=PREQUAL_TRANS_STATUS_CHOICES
    )
    message = models.TextField(_("Message"))
    offer_indicator = models.CharField(_("Offer Indicator"), max_length=20)
    credit_limit = models.DecimalField(
        _("Credit Limit"), decimal_places=2, max_digits=12
    )
    response_id = models.CharField(_("Unique Response ID"), max_length=8)
    application_url = models.CharField(_("Application URL"), max_length=200)
    customer_response = models.CharField(
        _("Customer Response"),
        max_length=_max_len(PREQUAL_CUSTOMER_RESP_CHOICES),
        choices=PREQUAL_CUSTOMER_RESP_CHOICES,
        default=PREQUAL_CUSTOMER_RESP_NONE,
    )

    customer_order = models.ForeignKey(
        "order.Order",
        null=True,
        blank=True,
        related_name="prequalification_responses",
        on_delete=models.CASCADE,
    )
    reported_datetime = models.DateTimeField(
        null=True,
        blank=True,
        help_text=_("Date customer response was reported to Wells Fargo."),
    )
    created_datetime = models.DateTimeField(auto_now_add=True)
    modified_datetime = models.DateTimeField(auto_now=True)

    class Meta:
        verbose_name = _("Pre-Qualification Response")
        verbose_name_plural = _("Pre-Qualification Responses")
        ordering = ("-created_datetime", "-id")

    @property
    def is_approved(self):
        return self.status == PREQUAL_TRANS_STATUS_APPROVED

    @property
    def status_name(self):
        return get_prequal_trans_status_name(
            self.status, self.request.customer_initiated
        )

    @property
    def customer_response_name(self):
        return dict(PREQUAL_CUSTOMER_RESP_CHOICES).get(
            self.customer_response, self.customer_response
        )

    @property
    def full_application_url(self):
        # Take the given base URL and append a query param with the last 10 digits of the merchant account number
        merchant_num = self.request.merchant_num[-10:]
        return "{base_url}&mn={merchant_num}".format(
            base_url=self.application_url, merchant_num=merchant_num
        )

    def check_account_status(self):
        from ..connector import AccountsAPIClient

        client = AccountsAPIClient()
        inquiry = client.lookup_account_by_prequal_offer_id(
            first_name=self.request.first_name,
            last_name=self.request.last_name,
            unique_id=self.response_id,
        )
        inquiry.prequal_response_source = self
        inquiry.save()
        return inquiry

class PreQualificationSDKApplicationResult(models.Model):
    prequal_response = models.OneToOneField(
        PreQualificationResponse,
        verbose_name=_("PreQualification Response"),
        related_name="sdk_application_result",
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
    )
    application_id = models.CharField(_("Unique Response ID"), max_length=8)
    first_name = models.CharField(_("First Name"), max_length=15)
    last_name = models.CharField(_("Last Name"), max_length=20)
    application_status = models.CharField(_("Application Status"), max_length=20)
    created_datetime = models.DateTimeField(auto_now_add=True)
    modified_datetime = models.DateTimeField(auto_now=True)

    class Meta:
        verbose_name = _("Pre-Qualification SDK Application Result")
        verbose_name_plural = _("Pre-Qualification SDK Application Results")

    @property
    def prequal_request_uuid(self):
        if self.prequal_response and self.prequal_response.request:
            return self.prequal_response.request.uuid
        return None


import argparse
from os import listdir
from os.path import join, isdir
import pandas as pd

parser = argparse.ArgumentParser()
parser.add_argument('--path', type=str,
                    default='../../../model_results/mixup_250_run1')
args = parser.parse_args()

out_path = getattr(args, 'path')

def list_folders_only(path):
    return [x for x in listdir(path) if isdir(join(path, x)) and 'csv' not in x]

model_folders = sorted(list_folders_only(out_path))
for model_folder in model_folders:
    param_folders = list_folders_only(join(out_path, model_folder))
    param_folders = sorted(param_folders)

    print(model_folder)
    print('(contains', len(param_folders), 'folders)')

    # assume true and try to prove false
    epochs_consistent = True
    epochs_mod_10 = True

    for param_folder in param_folders:
        df = pd.read_csv(join(out_path,
                              model_folder,
                              param_folder,
                              'scores.csv'))
        try:
            if df.epoch.max() != last_epoch:
                epochs_consistent = False
            else:
                last_epoch = df.epoch.max()
        except:
            last_epoch = df.epoch.max()
        if df.epoch.max() % 10 != 0:
            epochs_mod_10 = False
        print('   ', param_folder, 'EPOCHS DONE:', df.epoch.max())
    del last_epoch

    if not epochs_consistent:
        print('    WARNING: NOT ALL MODELS HAVE SAME # OF EPOCHS')
    else:
        print('    SUCCESS: ALL MODELS HAVE SAME # OF EPOCHS')
    if not epochs_mod_10:
        print('    WARNING: MODEL SET PROBABLY NOT DONE (epochs finished not a multiple of 10)')
    else:
        print('    SUCCESS: IT APPEARS THAT THIS MODEL SET FINISHED')
    print('')


"""create tables

Revision ID: d5e7168efc2b
Revises: 143983f32e7b
Create Date: 2019-05-16 18:07:55.518092

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = 'd5e7168efc2b'
down_revision = '143983f32e7b'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('models',
    sa.Column('model_id', sa.String(), nullable=False),
    sa.Column('class_path', sa.String(), nullable=True),
    sa.Column('feature_groups', sa.ARRAY(sa.String(), as_tuple=True), nullable=True),
    sa.Column('hyperparameters', sa.JSON(), nullable=True),
    sa.Column('feature_importance', sa.JSON(), nullable=True),
    sa.Column('training_dates', sa.ARRAY(sa.DateTime()), nullable=True),
    sa.Column('train_label_1', sa.String(), nullable=True),
    sa.Column('train_label_0', sa.Boolean(), nullable=True),
    sa.Column('threshold_for_nonbinary_target_variable', sa.Numeric(), nullable=True),
    sa.Column('train_base_number', sa.Integer(), nullable=True),
    sa.Column('train_total_number', sa.Integer(), nullable=True),
    sa.PrimaryKeyConstraint('model_id'),
    schema='results'
    )
    op.create_table('evaluation',
    sa.Column('evaluation_id', sa.String(), nullable=False),
    sa.Column('model_id', sa.String(), nullable=False),
    sa.Column('test_label_1', sa.String(), nullable=False),
    sa.Column('test_label_0', sa.Boolean(), nullable=False),
    sa.Column('test_base_number', sa.Integer(), nullable=True),
    sa.Column('test_total_number', sa.Integer(), nullable=True),
    sa.Column('testing_dates', sa.ARRAY(sa.DateTime()), nullable=False),
    sa.Column('metric', sa.String(), nullable=False),
    sa.Column('parameter', sa.String(), nullable=False),
    sa.Column('value', sa.Numeric(), nullable=True),
    sa.Column('pr_curve', postgresql.JSONB(astext_type=sa.ARRAY(sa.Numeric())), nullable=True),
    sa.ForeignKeyConstraint(['model_id'], ['results.models.model_id'], ),
    sa.PrimaryKeyConstraint('evaluation_id'),
    schema='results'
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('evaluation', schema='results')
    op.drop_table('models', schema='results')
    # ### end Alembic commands ###


# coding=utf-8
import codecs
import random

def word_and_idx(vocab_file):
    # get the dic for word and idx

    vocab = [vb for vb in codecs.open(
        vocab_file, 'r').read().split('\n')]
    word2idx = {word: idx for idx, word in enumerate(vocab)}
    idx2word = {idx: word for idx, word in enumerate(vocab)}
    return word2idx, idx2word

def split_sent(sent):
    # To split a sentence into list
    # if the sent is not chinese then split it by space
    # if the sent is chinse then just split it

    sent = sent.strip()
    if (' ' in sent):
        split_sent = sent.split(' ')
    else:
        split_sent = [word for word in sent]

    return split_sent

def word_2_idx(sents, word2idx):
    # change word to idx
    # if not find the word then reutrn 0--unk

    idx_sents = []
    for sent in sents:
        idx_sent = []
        for word in sent:
            idx_sent.append(word2idx.get(word, 0))
        idx_sents.append(idx_sent)
    return idx_sents

def idx_2_word(sents, idx2word):
    # change idx to word
    # if not find the idx then return <unk>

    word_sents = []
    for sent in sents:
        word_sent = []
        for idx in sent:
            word_sent.append(idx2word.get(idx, '<unk>'))
        word_sents.append(word_sent)
    return word_sents

def yield_data(data, batch_size, shuffle=False):

    if shuffle:
        random.shuffle(data)

    for i in range(0, len(data), batch_size):
        yield data[i, i + batch_size]

if __name__ == '__main__':
    vocab_file = '../corpus/data/vocab.data'
    word2idx, idx2word = word_and_idx(vocab_file)
    sents = [['select', 'movie', 'yes']]
    w2id = idx_2_word(sents, word2idx)


import osmnx as ox
import networkx as nx
from collections import deque
import colorsys
from folium.features import DivIcon
import folium
import random

# # =================================
# STEP 1: LOAD AND SANITY CHECK DATA

# load graph from GML (created via load_map.py)
graph = ox.io.load_graphml('graph_ubc.gml')

# ...................................
# Visualize map for sanity check
fig, ax = ox.plot_graph(graph)
fig.savefig('ubc_map.png')

# ...................................
# Visualize map with elevation for sanity check

nc = ox.plot.get_node_colors_by_attr(graph, 'elevation', cmap='plasma')
fig, ax = ox.plot_graph(graph, node_color=nc, node_size=5, edge_color='#333333', bgcolor='k')
fig.savefig('ubc_map_elevation.png')

# =================================
# Workout planning with length, bearing, and elevation
# You will debug and complete our implementation, including the following features:
# 1) find any path in the UBC graph whose total distance is > target using dfs
# 2) above plus: take the "straightest" direction out of any vertex
# 3) above plus: report total elevation gain

# Helper function that determines if edge (v,w) is a valid candidate for adding to the graph
def good(gst, d, v, w):
    return v not in list(gst.adj[w]) and \
           w not in list(gst.adj[v]) and \
           graph.edges[v, w, 0]['length'] > 0 and \
           d + graph.edges[v, w, 0]['length'] < goal_dist*1.1

# Helper function that returns the absolute difference between any 2 given directions.
# Note that the value should never be more than 180, since a left turn of x is
# equivalent to a right turn of (360 - x).
def get_bearing_diff(b1,b2):
    bdiff = abs(b1-b2) % 360 # allows for neg and large bearings
    return bdiff if bdiff <= 180 else 360-bdiff

# Main dfs function. Given a start node, goal distance, and graph of distances,
# solve these 2 related questions:
# Part 1: return a subgraph whose edges are a trail with distance at least goal_distance
# Part 2: return a subgraph with the characteristics from Part 1, but change the definition
# of "neighbors" so that at every node, the direction of the next edge is as close as possible
# to the current direction. This feature changes the order in which the neighbors are considered.

def find_route(start, goal_dist, graph):
    # distances and feasible edges will come from 'graph', solution built in 'gstate'
    gstate = nx.DiGraph()
    gstate.add_nodes_from(graph)

    # need stack of: (gstate, prev node, curr node, totlen so far, number of edges in route so far)
    # init stack & push start vertex
    stack = deque()
    stack.append((gstate, start, start, 0, 0))
    # next two lines are necessary for part 2) so that every current bearing has a previous bearing to compare against
    graph.add_edge(start, start, 0)
    graph.edges[start, start, 0]['bearing'] = random.randint(0,360) # grab a random initial direction

    while stack:
        gst, prev, curr, lensofar, clock = stack.pop()  # gst, previous node, curr node, dist so far, edges so far

        if curr not in list(gst.neighbors(prev)):
            gst.add_edge(prev, curr)
            gst.edges[prev, curr]['time'] = clock # need this for path drawing

            # stopping criteria: if we've gone far enough, return our solution graph and the number of edges
            if lensofar >= goal_dist:
                return gst, clock

            # neighbors for part 1)
            #neighbors = graph.neighbors(curr)
            # neighbors for part 2)
            neighbors = sorted(graph.neighbors(curr),
                               key=lambda x: get_bearing_diff(graph.edges[prev, curr, 0]['bearing'],
                                                              graph.edges[curr, x, 0]['bearing']),
                               reverse=True)

            for w in neighbors:
                if good(gst, lensofar, curr, w):
                    gstnew = gst.copy() # copy the path so we don't have to deal w backtracking. ok for small graphs.
                    stack.append((gstnew, curr, w, lensofar + graph.edges[curr, w, 0]['length'], clock + 1))

# returns the total elevation gain in gr, over the route described by rt (list of vertices).
# edges whose elevation gain is negative should be ignored.
# you can refer to a node's elevation by: gr.nodes[rt[k]]['elevation'], where k is the kth element
# of the rt list.
def total_elevation_gain(gr,rt):
    eg = 0
    for k in range(len(rt)):
        if k == 0:
            if gr.nodes[rt[k]]['elevation'] > 0:
                eg += gr.nodes[rt[k]]['elevation']
        else:
            if gr.nodes[rt[k]]['elevation'] > gr.nodes[rt[k-1]]['elevation']:
                eg += gr.nodes[rt[k]]['elevation']-gr.nodes[rt[k-1]]['elevation']
    return eg

# =======================================================
# Main driving code starts here
#
# Choose a starting location. For the final deliverable, please choose your favorite place on campus as the start.
# # location of the reconciliation pole
#lat, lon = 49.2600154, -123.2510869
# Wreck Beach
lat, lon = 49.255, -123.255

start = ox.get_nearest_node(graph, (lat, lon)) # start is a node number -- not very useful to us
startlat, startlon = graph.nodes[start]['y'], graph.nodes[start]['x']

goal_dist = 5000  # meters, must go at least this far

route, time = find_route(start, goal_dist, graph) # calls the main dfs function

# variable 'route' is a DiGraph, but we want a sequence of vertices along the solution path.
# take a look at these variables to see what's going on.
sorted_lst = sorted(route.edges(), key=lambda x: route.edges[x[0], x[1]]['time'])
routevertices = [vert[1] for vert in sorted_lst] #list comprehension to assemble the list of vertices in order

# find coordinates of stopping point: last node on the route
endlat, endlon = graph.nodes[routevertices[-1]]['y'], graph.nodes[routevertices[-1]]['x']

# Part 3
# add an accumulator that sums the total elevation gain over the course of the
# workout. If an edge (u,v) in the graph corresponds to a downhill segment (difference
# in elevations from u to v is negative), then it is ignored.
eg = total_elevation_gain(graph,routevertices) # sums the elevation gain over the route

# =================================
# VISUALIZATION!!
# There is only one known bug below, in the function that chooses a rainbow color. Please
# complete the visualization by fixing the edge colorer and completing the code for the
# end marker.
#

# Plotting is strange. In order to get the rainbow colors, we have to plot one edge of the
# route at a time, calculating the color of each.

# Initialize the map plot
kwargs = {'opacity': 0}
m = ox.plot_graph_folium(graph, tiles='openstreetmap', **kwargs)

# hsv color representation gives a rainbow from red and back to red over values 0 to 1.
# this function returns the color in rgb hex, given the current and total edge numbers
def shade_given_time(k, n):
    col = colorsys.hsv_to_rgb(k / n, 1, 1)
    tup = tuple((int((x * 255)) for x in col)) # why doesn't this work???
    st = f"#{tup[0]:02x}{tup[1]:02x}{tup[2]:02x}"
    return st

# Add the edges in the graph one at a time so you can specify a
# different color for each edge. route_map=m plots on map m
for k,x in enumerate(sorted_lst[1:]):
    kwargs = {'opacity': 1.0}
    ox.plot_route_folium(graph, x, route_map=m, weight=5, color=shade_given_time(k, time), **kwargs)

# need to get the right zoom level for the whole route. Lots of ways of doing this more dynamically. We're just
# going to take a 1200m radius from the start location. bbox stands for bounding box
bbox = ox.utils_geo.bbox_from_point((startlat, startlon), dist=1200)
swne = [(min(bbox[0:2]),min(bbox[2:4])),(max(bbox[0:2]),max(bbox[2:4]))]
m.fit_bounds(swne)

# Place the elevation gain on the map at the end point of the workout.
folium.map.Marker(
    [endlat, endlon],
    icon=DivIcon(
        icon_size=(250,36),
        icon_anchor=(0,0),
        html=f'
Elevation Gain: {eg}m
',
    )
).add_to(m)

# add green start circle
folium.CircleMarker((startlat,startlon),
                    color='green',radius=5,fill=True).add_to(m)
# add red end circle
folium.CircleMarker((endlat,endlon),
                    color='red',radius=5,fill=True).add_to(m)

filepath = "route_graph_workout.html"
m.save(filepath)


from flask import *
from extensions import mysql
from globals import route_prefix

pic = Blueprint('pic', __name__, template_folder='templates')

# /pic
# View Picture with Prev/Next Links
@pic.route(route_prefix + '/pic')
def pic_route():
	cur = mysql.connection.cursor()

	# Get the current picture id
	currentPicId = request.args.get('id')

	#404 if bad pic id
	checkIDStatement = "SELECT * FROM Photo WHERE picid = %(id)s"
	cur.execute(checkIDStatement, {'id': currentPicId})
	if (cur.rowcount <= 0):
		return render_template('404.html'), 404

	# Find this specific picture
	select_stmt = "SELECT * FROM Photo WHERE picid = %(picid)s"
	cur.execute(select_stmt, { 'picid': currentPicId })

	picture = cur.fetchall();
	attributed_picture = {
		"picid": picture[0][0],
		"format": picture[0][1],
		"date": picture[0][2]
	}

	# Get the albumid of this specific picture
	select_stmt = "SELECT albumid, sequencenum FROM Contain WHERE picid = %(picid)s"
	cur.execute(select_stmt, { 'picid': currentPicId })

	albumid_sequencenum = cur.fetchall();
	currentAlbumId = albumid_sequencenum[0][0];
	currentSequenceNum = albumid_sequencenum[0][1];

	# Get the photos that are before and after this one (in sequencenum order)
	select_stmt = "SELECT Contain.sequencenum, Contain.picid, Photo.format "
	select_stmt += "FROM Contain INNER JOIN Photo "
	select_stmt += "ON Contain.picid = Photo.picid "
	select_stmt += "WHERE (Contain.albumid = %(abi)s AND "
	select_stmt += "(Contain.sequencenum = (%(csn)s - 1) OR Contain.sequencenum = (%(csn)s + 1))) "
	select_stmt += "ORDER BY Contain.sequencenum ASC"
	cur.execute(select_stmt, { 'abi': currentAlbumId, 'csn': currentSequenceNum })
	prev_and_next = cur.fetchall();
	numOfPrevAndNext = len(prev_and_next)
	previousPicInfo = {
		"picid": 'make america great again',
		"format": 'make america great again',
	}
	nextPicInfo = {
		"picid": 'make america great again',
		"format": 'make america great again',
	}

	# If numOfPrevAndNext == 1, then there is either only one num (first or last picture in album)
	if numOfPrevAndNext == 1:
		if currentSequenceNum == 0:
			# If currentSequenceNum == 1, then only a "next row" showed up
			nextPicInfo['picid'] = prev_and_next[0][1]
			nextPicInfo['format'] = prev_and_next[0][2]
		else:
			# Else, only a "previous row" showed up
			previousPicInfo['picid'] = prev_and_next[0][1]
			previousPicInfo['format'] = prev_and_next[0][2]

	# The query pulled both a previous and next picture (neither first nor last picture in album with 3+ pictures)
	elif numOfPrevAndNext == 2:
		previousPicInfo['picid'] = prev_and_next[0][1]
		previousPicInfo['format'] = prev_and_next[0][2]
		nextPicInfo['picid'] = prev_and_next[1][1]
		nextPicInfo['format'] = prev_and_next[1][2]

	# The query pulled neither (1 picture in album)
		# Don't have to do anything in this case

	options = {
		"picture": attributed_picture,
		"albumId": currentAlbumId,
		"previousPicInfo": previousPicInfo,
		"nextPicInfo": nextPicInfo
	}
	return render_template("pic.html", **options)


# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#  change the label algorithm   ljqu  2017.4.28
# ==============================================================================

"""Functions for downloading and reading MNIST data."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy
from six.moves import xrange  # pylint: disable=redefined-builtin

import csv
from glob import glob
import os
import struct

import numpy as np
from config import cfg
from tensorflow.python.platform import gfile

from tensorflow.contrib.learn.python.learn.datasets import base
from tensorflow.python.framework import dtypes
import tensorflow as tf

DATALEN = cfg.data_period  # 125
LABELNUM = cfg.label_num  # 19
LBLPOSTNUM = cfg.label_post_num  # 20        #利用后面多少个数据来计算标签
loss_ratio = cfg.loss_ratio  # 0.05
profit_ratio = cfg.profit_ratio  # 0.20
image_size = cfg.image_size

def _read32(bytestream):
    dt = numpy.dtype(numpy.uint32).newbyteorder('>')
    return numpy.frombuffer(bytestream.read(4), dtype=dt)[0]

# %%
def extract_images(filequeue):
    """Extract the images into a 4D uint8 numpy array [index, y, x, depth].

    Args:
        f: A file object that can be passed into a gzip reader.

    Returns:
        train_iamges: A 4D float32 numpy array [index, x, y, depth].
        train_labels: A 1D int numpy  array [index]

    Raises:
        ValueError: If the bytestream does not start with 2051.

    """
    features_dtype = "float32"
    target_dtype = "int"
    for i, filename in enumerate(filequeue):
        print('Extracting', filename)
        train_image, train_label = get_csv_data1(filename, target_dtype, features_dtype, image_size)

        #write down the feature and label data to file. extensive is .bin
        if cfg.is_feature_store:
            # change the 4 dimensions data to 2 dimensions
            train1_image = train_image.reshape([train_image.shape[0],-1])
            #csv2bin(filename, train1_image, train_label)
            csv2tfrecord(filename, train1_image, train_label)
        if i == 0:
            train_images = train_image
            train_labels = train_label
        else:
            train_images = np.concatenate((train_images, train_image), axis=0)
            train_labels = np.concatenate((train_labels, train_label), axis=0)
    return train_images, train_labels

def csv2tfrecord(filename, feature, label):
    """
        convert the csv data to tfrecord,  and output to a file
        Args:
            filename:  the given file(the full name)
            feature:    the feature columns
            label:      label data
    """
    if filename.split('.')[-1] != 'csv':
        return
    binfilename = filename.replace('csv', 'tfrecord')
    if not os.path.exists(binfilename):
        writer = tf.python_io.TFRecordWriter(binfilename)
        for i, data in enumerate(feature):
            data = data.astype(numpy.float32)
            data_raw = data.tostring()
            labeldata = label[i]

            features = {'images':tf.train.Feature(bytes_list = tf.train.BytesList(value=[data_raw])),
                        'labels':tf.train.Feature(int64_list = tf.train.Int64List(value=[labeldata]))}
            # Create an example protocol buffer
            example = tf.train.Example(features=tf.train.Features(feature=features))
            # Serialize to string and write on the file
            writer.write(example.SerializeToString())
        writer.close()
        print('Write down feature data, file:', binfilename)
    else:
        print("%s has change." % binfilename)

def csv2bin(filename, feature, label):
    """
        convert the csv data to hex,  and output to a file
        Args:
            filename:  the given file(the full name)
            feature:    the feature columns
            label:      label data
            dtype:     the feature and label data output the hex file data type
    """
    if filename.split('.')[-1] != 'csv':
        return
    binfilename = filename.replace('csv', 'bin')
    binfile = open(binfilename, 'wb')

    for i, data in enumerate(feature):
        for j in range(feature.shape[1]):
            parsedata = struct.pack("f", float(data[j]))
            binfile.write(parsedata)
        labeldata = struct.pack("f", float(label[i]))
        binfile.write(labeldata)
    binfile.close()
# %%
def get_csv_data1(filename,
                  target_dtype,
                  features_dtype,
                  datalen=DATALEN,
                  fileout=cfg.is_true_out):
    """  get data from given file(.txt or .csv)
        Args:
            file: the given file (the full name)
            target_dtype:  datatype of the label
            features_dtype:  datatype of the features
            datalen:  the length of data
        Returns:
            train_images:  feature 4D array [Index,25,25,1]
            train_labels:  label 1D array  [Index]
            :param fileout:
    """
    idx = -1
    with gfile.Open(filename) as csv_file:
        # read data according to the file name
        data = numpy.loadtxt(open(filename), dtype=features_dtype, delimiter=",", skiprows=1, usecols=(2,3, 4, 5))  #H,L,C
        n_samples = data.shape[0]  # int(header[0])
        n_features = data.shape[1]  # header.shape[0] - 2
        # print("data:", data.shape, ",n_sampes:",n_samples,",n_features:",n_features )
        train_images = np.zeros((n_samples - 2*datalen - LBLPOSTNUM + 1, datalen,datalen,3), dtype=features_dtype)
        train_labels = np.zeros(n_samples - 2*datalen - LBLPOSTNUM + 1, dtype=target_dtype)

        # deal the data one by one
        for i, d in enumerate(data):
            if i >= 2*datalen and i <= data.shape[0] - LBLPOSTNUM:
                idx += 1
                for j in range(datalen):
                    fArr1 = diverse_standard(data[i - datalen-j:i-j, 1])  # High
                    fArr2 = diverse_standard(data[i - datalen-j:i-j, 2])  # Low
                    fArr3 = diverse_standard(data[i - datalen-j:i-j, 3])  # Close

                    f1 = np.transpose(fArr1)
                    f2 = np.transpose(fArr2)
                    f3 = np.transpose(fArr3)
                    train_images[idx,j,:,0] = f1  #np.concatenate((f1, f2, f3), axis=1)
                    train_images[idx, j, :,1] = f2
                    train_images[idx, j, :, 2] = f3

                # deal the target category (one-shot = ture)
                tArr = data[i - 2*datalen:i + LBLPOSTNUM, :]  # the column to calculate the profit
                label_data = deal_label(tArr, 2*datalen)
                train_labels[idx] = label_data
    if fileout:
        out_indi_data(filename, train_labels, cfg.label_path, datalen)
    # trn_images = train_images[:idx + 1, ]
    # trn_images = trn_images.reshape([-1, image_size, image_size, 3])
    # trn_labels = train_labels[:idx + 1, ]
    return train_images, train_labels

# %%
def diverse_standard_comp(x, k):
    x_max = max(x[:, 1])
    x_min = min(x[:, 2])
    if (x_max - x_min) < 1e-10:  # when in top limit or lower limit,
        # print(x_max,",",x_min)
        x1 = np.ones(x.shape[0])
    else:
        x1 = np.round((x[:, k] - x_min) / (x_max - x_min), 4)  # round the data to 1e-4
    x1 = x1.reshape(x.shape[0], 1)
    return x1

# %%
def diverse_standard(x):
    x_max = max(x)
    x_min = min(x)
    if (x_max - x_min) < 1e-10:  # when in top limit or lower limit,
        # print(x_max,",",x_min)
        x1 = np.ones(x.shape[0])
    else:
        x1 = np.round((x - x_min) / (x_max - x_min), 4)  # round the data to 1e-4
    x1 = x1.reshape(x.shape[0], 1)
    return x1

# %%
def deal_label(x, len1=DATALEN):
    """According to the pre 100 bars and post 20 bars
        to calculate the label.

    Args:
        x: An array that include the pre 100 and post 20 bars.
        len1: data length

    Returns:
        lbl: A int scalar. Represent the current bar's label

    """
    x1 = x[0:len1, :]
    x2 = x[len1 + 1:len1 + LBLPOSTNUM, :]  # the old value is :x2=x[len1,len1+20,:]   ,update 2017.5.13
    close = x[len1, 3]  # the old value is:close = x[len1 - 1, 3],  because of the sign is in the dataLen Bar,
    # so define the openprice is the close of the dataLen Bar. update 2017.5.13

    # calculate the 100 bars range
    x1_max = max(x1[:, 1])  # high
    x1_min = min(x1[:, 2])  # low
    x1_range = x1_max - x1_min

    x2_max = max(x2[:, 1])  # high
    x2_min = min(x2[:, 2])  # low
    x2_hc = max(x2[:, 3])  # highest close
    x2_lc = min(x2[:, 3])  # lowest close

    long_loss = close - x2_min
    long_profit = x2_hc - close

    long_loss_std = long_loss / x1_range
    long_profit_std = long_profit / x1_range

    short_loss = x2_max - close
    short_profit = close - x2_lc
    short_loss_std = short_loss / x1_range
    short_profit_std = short_profit / x1_range

    long_lbl = 0
    short_lbl = 0
    lbl = 0

    if long_loss_std >= loss_ratio:
        long_lbl = 0
    elif long_profit_std < profit_ratio:
        long_lbl = 0
    else:
        long_lbl = int((long_profit_std + 0.000001 - profit_ratio) / 0.2) + 1
    if long_lbl > 9:
        long_lbl = 9

    if short_loss_std >= loss_ratio:
        short_lbl = 0
    elif short_profit_std < profit_ratio:
        short_lbl = 0
    else:
        short_lbl = -int((short_profit_std + 0.000001 - profit_ratio) / 0.2) - 1
    if short_lbl < -9:
        short_lbl = -9

    if long_lbl >= abs(short_lbl):
        lbl = long_lbl
    else:
        lbl = abs(short_lbl) + 9
    return lbl

def dense_to_one_hot(labels_dense, num_classes):
    """Convert class labels from scalars to one-hot vectors."""
    num_labels = labels_dense.shape[0]
    index_offset = numpy.arange(num_labels) * num_classes
    labels_one_hot = numpy.zeros((num_labels, num_classes))
    labels_one_hot.flat[index_offset + labels_dense.ravel()] = 1
    return labels_one_hot

def extract_labels(labels, one_hot=False, num_classes=LABELNUM):
    """Extract the labels into a 1D uint8 numpy array [index].

    Args:
      labels: a 1D unit8 numpy array.
      one_hot: Does one hot encoding for the result.
      num_classes: Number of classes for the one hot encoding.

    Returns:
      labels: a 2D unit8 numpy array [index,19].

    Raises:
      ValueError: If the bystream doesn't start with 2049.
    """
    if one_hot:
        return dense_to_one_hot(labels, num_classes)
    return labels

class DataSet(object):
    def __init__(self,
                 images,
                 labels,
                 fake_data=False,
                 one_hot=False,
                 dtype=dtypes.float32,
                 reshape=True):
        """Construct a DataSet.
        one_hot arg is used only if fake_data is true.  `dtype` can be either
        `uint8` to leave the input as `[0, 255]`, or `float32` to rescale into
        `[0, 1]`.
        """
        dtype = dtypes.as_dtype(dtype).base_dtype
        if dtype not in (dtypes.uint8, dtypes.float32):
            raise TypeError('Invalid image dtype %r, expected uint8 or float32' %
                            dtype)
        if fake_data:
            self._num_examples = 10000
            self.one_hot = one_hot
        else:
            assert images.shape[0] == labels.shape[0], (
                'images.shape: %s labels.shape: %s' % (images.shape, labels.shape))
            self._num_examples = images.shape[0]

            # Convert shape from [num examples, rows, columns, depth]
            # to [num examples, rows*columns] (assuming depth == 1)
            if reshape:
                assert images.shape[3] == 1
                images = images.reshape(images.shape[0],
                                        images.shape[1] * images.shape[2])
            if dtype == dtypes.float32:
                # Convert from [0, 255] -> [0.0, 1.0].
                images = images.astype(numpy.float32)
                # images = numpy.multiply(images, 1.0 / 255.0)
        self._images = images
        self._labels = labels
        self._epochs_completed = 0
        self._index_in_epoch = 0

    @property
    def images(self):
        return self._images

    @property
    def labels(self):
        return self._labels

    @property
    def num_examples(self):
        return self._num_examples

    @property
    def epochs_completed(self):
        return self._epochs_completed

    def next_batch(self, batch_size, fake_data=False):
        """Return the next `batch_size` examples from this data set."""
        if fake_data:
            fake_image = [1] * 500
            if self.one_hot:
                fake_label = [1] + [0] * 9
            else:
                fake_label = 0
            return [fake_image for _ in xrange(batch_size)], [
                fake_label for _ in xrange(batch_size)
            ]
        start = self._index_in_epoch
        self._index_in_epoch += batch_size
        if self._index_in_epoch > self._num_examples:
            # Finished epoch
            self._epochs_completed += 1
            # Shuffle the data
            perm = numpy.arange(self._num_examples)
            numpy.random.shuffle(perm)
            self._images = self._images[perm]
            self._labels = self._labels[perm]
            # Start next epoch
            start = 0
            self._index_in_epoch = batch_size
            assert batch_size <= self._num_examples
        end = self._index_in_epoch
        return self._images[start:end], self._labels[start:end]

# %%
def read_data_sets(train_dir,
                   fake_data=False,
                   one_hot=False,
                   dtype=dtypes.float32,
                   reshape=False,
                   validation_size=10):
    if fake_data:
        def fake():
            return DataSet([], [], fake_data=True, one_hot=one_hot, dtype=dtype)

        train = fake()
        validation = fake()
        test = fake()
        return base.Datasets(train=train, validation=validation, test=test)

    train_file = glob(train_dir)
    # print(train_file)
    train_images, train_labels = extract_images(train_file)
    train_labels = extract_labels(train_labels, one_hot=one_hot)
    print(train_images.shape)

    # TEST_IMAGES = (['test_data/rb.HOT.15m(1).csv'])
    test_file = glob(cfg.test_dataset)
    test_images, test_labels = extract_images(test_file)
    test_labels = extract_labels(test_labels, one_hot=one_hot)

    if not 0 <= validation_size <= len(train_images):
        raise ValueError(
            'Validation size should be between 0 and {}. Received: {}.'
                .format(len(train_images), validation_size))

    validation_images = train_images[:validation_size]
    validation_labels = train_labels[:validation_size]
    train_images = train_images[validation_size:]
    train_labels = train_labels[validation_size:]

    train = DataSet(train_images, train_labels, dtype=dtype, reshape=reshape)
    validation = DataSet(validation_images,
                         validation_labels,
                         dtype=dtype,
                         reshape=reshape)
    test = DataSet(test_images, test_labels, dtype=dtype, reshape=reshape)

    return base.Datasets(train=train, validation=validation, test=test)

# %%
def read_test_sets(filename,
                   one_hot=False,
                   dtype=dtypes.float32,
                   reshape=False,
                   validation_size=100):
    local_file = [filename]  # TRAIN_IMAGES
    train_images, train_labels = extract_images(local_file)
    train_labels = extract_labels(train_labels, one_hot=one_hot)

    train = DataSet(train_images, train_labels, dtype=dtype, reshape=reshape)

    return train

# %%

def load_data(train_dir='train_data'):
    return read_data_sets(train_dir)

# %%
def get_csv_data2(filename,
                  features_dtype="float32",
                  datalen=image_size):
    """  get data from given file(.txt or .csv)
        Args:
            file: the given file (the full name)
            features_dtype:  datatype of the features
            datalen:  the length of data
        Returns:
            train_images:  feature 2D array [Index,900]
            train_labels:  label 1D array  [Index]
    """
    idx = -1
    with gfile.Open(filename) as csv_file:
        # get Open,High,Low,Close,Volume
        data = numpy.loadtxt(open(filename), dtype=features_dtype, delimiter=",", skiprows=1, usecols=(2, 3, 4, 5))
        n_samples = data.shape[0]  # int(header[0])
        n_features = data.shape[1]  # header.shape[0] - 2
        train_images = np.zeros((n_samples - datalen, datalen,datalen,3), dtype=features_dtype)

        #get the datetime info
        dt1 = numpy.loadtxt(open(filename),dtype=str,delimiter=",",skiprows=1,usecols=(0,1))
        # get the end record [date & time ]
        dt = dt1[len(dt1)-1]

        for i, d in enumerate(data):
            if i >= 2 * datalen and i <= data.shape[0]:
                idx +=1
                for j in range(datalen):
                    fArr1 = diverse_standard(data[i - datalen - j:i - j, 1])  # High
                    fArr2 = diverse_standard(data[i - datalen - j:i - j, 2])  # Low
                    fArr3 = diverse_standard(data[i - datalen - j:i - j, 3])  # Close

                    f1 = np.transpose(fArr1)
                    f2 = np.transpose(fArr2)
                    f3 = np.transpose(fArr3)
                    train_images[idx, j, :, 0] = f1  # np.concatenate((f1, f2, f3), axis=1)
                    train_images[idx, j, :, 1] = f2
                    train_images[idx, j, :, 2] = f3

        # trn_images = train_images[:idx + 1, ]
        # trn_images = trn_images.reshape([-1, image_size, image_size, 1])
    return train_images, dt

# %%   output the original data and indi data to a csv file
def out_indi_data(infile,
                  indi,
                  outpath=cfg.indi_path,
                  datalen=image_size):
    """  get data from given file(.txt or .csv)
        Args:
            infile: the given file (the full name)
            indi: calculated indicator values(array)[None]
            datalen:  the length of data
        Returns:
    """
    if not os.path.exists(outpath):
        os.mkdir(outpath)
    #path = outpath  # os.path.split(os.path.realpath(__file__))[0] + "/"+outpath+"/"
    filename = infile
    tempfile = filename.split("/")
    outfile = tempfile[len(tempfile) - 1]
    outfile = outfile.replace("csv", "indi.csv")
    outfile = outpath + outfile

    fp = open(outfile, 'w')
    writer = csv.writer(fp)

    with gfile.Open(filename) as csv_file:
        data_file = csv.reader(csv_file)  # DictReader
        header = next(data_file)
        header1 = header + [' <Indi>']
        #header1[0] = ' <Date>'
        writer.writerow(header1)  # write the header
        for i, row in enumerate(data_file):
            if i < 2*datalen or i>=np.shape(indi)[0]+2*datalen:
                idx = '0'
            else:
                idx = str(indi[i - 2*datalen])
            writer.writerow(row + [idx])
            # print(" i = %d row = %s " % (i,row),type(row))

    # close the file pointer
    fp.close()

if __name__ == "__main__":
    data = read_data_sets(cfg.dataset, one_hot=False)


import numpy as np

def correct(name_file, q_out):
    print(name_file)
    load_file = '{}.dat'.format(name_file)
    with open(load_file, 'rb') as f:
        a = f.read()

    bytes_ = [0b00100010, 0b00100000, 0b00100100]
    d = bytearray()
    i = 0
    while i < len(a):
        pack = a[i:i+7]
        if len(pack) == 7:
            c = pack[0] in bytes_
            if c == True:
                d.extend(pack)
            else:
                while not a[i] in bytes_:
                    i+= 1
        i+= 7

    dtype = np.dtype([('signals','uint8'),('value', 'uint16'),('time','uint32')])
    data = np.frombuffer(d, dtype)
    diff = np.diff(data['time'])
    ind = np.where(diff < np.median(diff)*2)
    data = data[ind]
    print(type(data))
    np.save(name_file, data)
    with open('{}.dat'.format(name_file),'wb') as f:
        f.write(d)

#     q_out.put(data)
    print('correct')


"""Complex HTTP Server.

This module builds on BaseHTTPServer by implementing the standard GET
and HEAD requests in a fairly straightforward manner.

"""

__version__ = "0.6"

__all__ = ["ComplexHTTPRequestHandler"]

import os
import posixpath
import http.server
import urllib.request, urllib.parse, urllib.error
import cgi
import sys
import shutil
import mimetypes
import time,datetime
try:
    from io import StringIO
except ImportError:
    from io import StringIO

class ComplexHTTPRequestHandler(http.server.BaseHTTPRequestHandler):

    """Complex HTTP request handler with GET and HEAD commands.

    This serves files from the current directory and any of its
    subdirectories.  The MIME type for files is determined by
    calling the .guess_type() method.

    The GET and HEAD requests are identical except that the HEAD
    request omits the actual contents of the file.

    """

    server_version = "ComplexHTTP/" + __version__

    def do_GET(self):
        """Serve a GET request."""
        f = self.send_head()
        if f:
            self.copyfile(f, self.wfile)
            f.close()

    def do_HEAD(self):
        """Serve a HEAD request."""
        f = self.send_head()
        if f:
            f.close()

    def send_head(self):
        """Common code for GET and HEAD commands.

        This sends the response code and MIME headers.

        Return value is either a file object (which has to be copied
        to the outputfile by the caller unless the command was HEAD,
        and must be closed by the caller under all circumstances), or
        None, in which case the caller has nothing further to do.

        """
        path = self.translate_path(self.path)
        f = None
        if os.path.isdir(path):
            if not self.path.endswith('/'):
                # redirect browser - doing basically what apache does
                self.send_response(301)
                self.send_header("Location", self.path + "/")
                self.end_headers()
                return None
            for index in "index.html", "index.htm":
                index = os.path.join(path, index)
                if os.path.exists(index):
                    path = index
                    break
            else:
                return self.list_directory(path)
        ctype = self.guess_type(path)
        try:
            # Always read in binary mode. Opening files in text mode may cause
            # newline translations, making the actual size of the content
            # transmitted *less* than the content-length!
            f = open(path, 'rb')
        except IOError:
            self.send_error(404, "File not found")
            return None
        self.send_response(200)
        self.send_header("Content-type", ctype)
        fs = os.fstat(f.fileno())
        self.send_header("Content-Length", str(fs[6]))
        self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
        self.end_headers()
        return f

    def sizeof_fmt(self, num, suffix='B'):
        for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
            if abs(num) < 1024.0:
                return "%3.1f%s%s" % (num, unit, suffix)
            num /= 1024.0
        return "%.1f%s%s" % (num, 'Yi', suffix)

    def get_size(self, start_path = '.'):
        if os.path.isfile(start_path): return os.path.getsize(start_path)
        total_size = 0
        for dirpath, dirnames, filenames in os.walk(start_path):
            for f in filenames:
                fp = os.path.join(dirpath, f)
                total_size += os.path.getsize(fp)
        return total_size

    def list_directory(self, path):
        """Helper to produce a directory listing (absent index.html).

        Return value is either a file object, or None (indicating an
        error).  In either case, the headers are sent, making the
        interface the same as for send_head().

        """
        try:
            list = os.listdir(path)
        except os.error:
            self.send_error(404, "No permission to list directory")
            return None
        list.sort(key=lambda a: a.lower())
        list.sort(key=lambda x: os.path.getmtime(os.path.join(path,x)), reverse=True)
        f = StringIO()
        displaypath = cgi.escape(urllib.parse.unquote(self.path))
        f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
        f.write("\n<title>Directory listing for %s</title>\n" % displaypath)
        f.write("\n
Directory listing for %s
\n" % displaypath)
        f.write('

Name	Date Modified	Size

\n')
        for name in list:
            fullname = os.path.join(path, name)
            displayname = linkname = name
            # Append / for directories or @ for symbolic links
            if os.path.isdir(fullname):
                displayname = name + "/"
                linkname = name + "/"
            if os.path.islink(fullname):
                displayname = name + "@"
                # Note: a link to a directory displays with @ and links with /
            filename=os.path.join(path,name)
            modified = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getmtime(filename)))
            size = self.sizeof_fmt(self.get_size(filename))
            f.write('
%s	%s	%s	\n'
                    % (urllib.parse.quote(linkname), cgi.escape(displayname), cgi.escape(modified), cgi.escape(size)))
        f.write("
\n
\n\n\n")
        length = f.tell()
        f.seek(0)
        self.send_response(200)
        encoding = sys.getfilesystemencoding()
        self.send_header("Content-type", "text/html; charset=%s" % encoding)
        self.send_header("Content-Length", str(length))
        self.end_headers()
        return f

    def translate_path(self, path):
        """Translate a /-separated PATH to the local filename syntax.

        Components that mean special things to the local file system
        (e.g. drive or directory names) are ignored.  (XXX They should
        probably be diagnosed.)

        """
        # abandon query parameters
        path = path.split('?',1)[0]
        path = path.split('#',1)[0]
        path = posixpath.normpath(urllib.parse.unquote(path))
        words = path.split('/')
        words = [_f for _f in words if _f]
        path = os.getcwd()
        for word in words:
            drive, word = os.path.splitdrive(word)
            head, word = os.path.split(word)
            if word in (os.curdir, os.pardir): continue
            path = os.path.join(path, word)
        return path

    def copyfile(self, source, outputfile):
        """Copy all data between two file objects.

        The SOURCE argument is a file object open for reading
        (or anything with a read() method) and the DESTINATION
        argument is a file object open for writing (or
        anything with a write() method).

        The only reason for overriding this would be to change
        the block size or perhaps to replace newlines by CRLF
        -- note however that this the default server uses this
        to copy binary data as well.

        """
        shutil.copyfileobj(source, outputfile)

    def guess_type(self, path):
        """Guess the type of a file.

        Argument is a PATH (a filename).

        Return value is a string of the form type/subtype,
        usable for a MIME Content-type header.

        The default implementation looks the file's extension
        up in the table self.extensions_map, using application/octet-stream
        as a default; however it would be permissible (if
        slow) to look inside the data to make a better guess.

        """

        base, ext = posixpath.splitext(path)
        if ext in self.extensions_map:
            return self.extensions_map[ext]
        ext = ext.lower()
        if ext in self.extensions_map:
            return self.extensions_map[ext]
        else:
            return self.extensions_map['']

    if not mimetypes.inited:
        mimetypes.init() # try to read system mime.types
    extensions_map = mimetypes.types_map.copy()
    extensions_map.update({
        '': 'application/octet-stream', # Default
        '.py': 'text/plain',
        '.c': 'text/plain',
        '.h': 'text/plain',
	'.txt': 'text/plain',
        })

def test(HandlerClass = ComplexHTTPRequestHandler,
         ServerClass = http.server.HTTPServer):
    http.server.test(HandlerClass, ServerClass)

if __name__ == '__main__':
    test()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Project Euler Problem 248
=======================

The first number n for which φ(n)=13! is 6227180929.

Find the 150,000th such number.

"""

def main():
    return "unimplemented"

if __name__ == "__main__":
    import ntpath
    import time
    from common.shared_functions import verify_solution

    problem_number = int(ntpath.basename(__file__).replace("euler", "").replace(".py", ""))
    print("Retrieving my answer to Euler Problem {0} ...".format(problem_number))

    ts = time.time()
    my_answer = main()
    te = time.time()

    print("My answer: {1}".format(problem_number, my_answer))

    verification_type = verify_solution(problem_number, my_answer)
    print("Verification: {0}".format(verification_type.name))
    print("Took {0} seconds.".format(te - ts))


import sys
import pygame
from Game import *
from Start_menu import *
from End_menu import *

pygame.init()

width = 1200
height = 980

# on set la fenetre
pygame.display.set_caption("Sweet Night Declaration")
screen = pygame.display.set_mode((width, height))

game = Game()
start = Start_menu(game)
clock = pygame.time.Clock()
end = End_menu(game)

# image du background
background = pygame.image.load('assets/background/background.png').convert_alpha()
background = pygame.transform.scale(background, (1200, 980))

# boucle principale
while game.is_running:

    screen.blit(background, (0, 0))
    clock.tick(60)

    # boucle de jeu
    if game.is_playing and not game.is_dead:
        game.update(screen)
    # boucle du menu
    elif not game.is_playing and not game.is_dead:
        start.start_menu(screen)
    else:
        end.end_menu(screen)

    pygame.display.flip()


from pyspark import SparkContext, SparkConf
from pyspark.sql import SQLContext
from pyspark.sql import HiveContext
from pyspark.sql import SparkSession
import findspark
findspark.init()
from pyspark.sql import Row
from pyspark.sql.types import StringType, ArrayType
from pyspark.sql.functions import udf, col, max as max, to_date, date_add, \
    add_months
from datetime import datetime, timedelta
import os
from os.path import join, abspath
from typing import Optional
import logging
import random
import string
import math
import mathOperations as mo
import UsedFunctions as uf

def main():
  rec = {}
  settings = [
                ("hive.exec.dynamic.partition", "true"),
                ("hive.exec.dynamic.partition.mode", "nonstrict"),
                ("spark.sql.orc.filterPushdown", "true"),
                ("hive.msck.path.validation", "ignore"),
                ("spark.sql.caseSensitive", "true"),
                ("spark.speculation", "false"),
                ("hive.metastore.authorization.storage.checks", "false"),
                ("hive.metastore.client.connect.retry.delay", "5s"),
                ("hive.metastore.client.socket.timeout", "1800s"),
                ("hive.metastore.connect.retries", "12"),
                ("hive.metastore.execute.setugi", "false"),
                ("hive.metastore.failure.retries", "12"),
                ("hive.metastore.schema.verification", "false"),
                ("hive.metastore.schema.verification.record.version", "false"),
                ("hive.metastore.server.max.threads", "100000"),
                ("hive.metastore.authorization.storage.checks", "/apps/hive/warehouse")
]
  configs = {"DB":"pycharm",
           "tableName":"randomDataPy"}
  DB = "pycharm"
  tableName = "randomDataPy"
  fullyQualifiedTableName = DB +"."+tableName
  spark = SparkSession.builder \
          .appName("app1") \
          .enableHiveSupport() \
          .getOrCreate()

  spark.sparkContext._conf.setAll(settings)

  sc = SparkContext.getOrCreate()
  print(sc.getConf().getAll())
  sqlContext = SQLContext(sc)
  HiveContext = HiveContext(sc)
  lst = (spark.sql("SELECT FROM_unixtime(unix_timestamp(), 'dd/MM/yyyy HH:mm:ss.ss') ")).collect()
  print("\nStarted at");uf.println(lst)

  ##global numRows
  numRows = 10   ## do in increment of 50K rows otherwise you blow up driver memory!
  #
  ## Check if table exist otherwise create it

  rows = 0
  sqltext  = ""
  ##if (spark.sql(f"SHOW TABLES IN {DB} like '{tableName}'").count() == 1):
  if (spark.sql("SHOW TABLES IN pycharm like 'randomDataPy'").count() == 1):
    rows = spark.sql("""SELECT COUNT(1) FROM pycharm.randomDataPy""").collect()[0][0]
    print ("number of rows is ",rows)
  else:
    print("\nTable pycharm.randomDataPy does not exist, creating table ")
    sqltext = """
    CREATE TABLE pycharm.randomDataPy(
    ID INT
    , CLUSTERED INT
    , SCATTERED INT
    , RANDOMISED INT
    , RANDOM_STRING VARCHAR(50)
    , SMALL_VC VARCHAR(50)
    , PADDING  VARCHAR(4000)
    )
    STORED AS PARQUET
    """
    spark.sql(sqltext)

  start = 0
  if (rows == 0):
    start = 1
    maxID= 0
  else:
    maxID = spark.sql("SELECT MAX(id) FROM pycharm.randomDataPy").collect()[0][0]
  start = maxID + 1
  end = start + numRows - 1
  print ("starting at ID = ",start, ",ending on = ",end)
  Range = range(start, end+1)
  ## This traverses through the Range and increment "x" by one unit each time, and that x value is used in the code to generate random data through Python functions in a class

  rdd = sc.parallelize(Range). \
           map(lambda x: (x, uf.clustered(x,numRows), \
                             uf.scattered(x,numRows), \
                             uf.randomised(x, numRows), \
                             uf.randomString(50), \
                             uf.padString(x," ",50), \
                             uf.padSingleChar("x",4000)))
  df = rdd.toDF(). \
       withColumnRenamed("_1","ID"). \
       withColumnRenamed("_2", "CLUSTERED"). \
       withColumnRenamed("_3", "SCATTERED"). \
       withColumnRenamed("_4", "RANDOMISED"). \
       withColumnRenamed("_5", "RANDOM_STRING"). \
       withColumnRenamed("_6", "SMALL_VC"). \
       withColumnRenamed("_7", "PADDING")
  df.write.mode("overwrite").saveAsTable("pycharm.ABCD")
  df.printSchema()
  df.explain()
  df.createOrReplaceTempView("tmp")
  sqltext = """
    INSERT INTO TABLE pycharm.randomDataPy
    SELECT
            ID
          , CLUSTERED
          , SCATTERED
          , RANDOMISED
          , RANDOM_STRING
          , SMALL_VC
          , PADDING
    FROM tmp
    """
  spark.sql(sqltext)
  spark.sql("SELECT MIN(id) AS minID, MAX(id) AS maxID FROM pycharm.randomDataPy").show(n=20,truncate=False,vertical=False)
  ##sqlContext.sql("""SELECT * FROM pycharm.randomDataPy ORDER BY id""").show(n=20,truncate=False,vertical=False)
  lst = (spark.sql("SELECT FROM_unixtime(unix_timestamp(), 'dd/MM/yyyy HH:mm:ss.ss') ")).collect()
  print("\nFinished at");uf.println(lst)

  ##print(os.listdir(warehouseLocation))
  spark.sql("show databases").show()

if __name__ == "__main__":
  print("\ncpc Michboy")
  import run_oracle as to


from django.conf.urls import url, include
from events.models import Event
from contacts.models import Speaker, Organisation
from rest_framework import routers
from api.views import *

router = routers.DefaultRouter()
router.register(r'speakers', SpeakerViewSet)
router.register(r'events', EventViewSet)
router.register(r'organisations', OrganisationViewSet)
router.register(r'partners', PartnerViewSet)

urlpatterns = [
    url(r'^', include(router.urls)),
    url(r'^events/past/(?P<limit>[0-9]+)$', pastEvents ),
    url(r'^events/future/(?P<limit>[0-9]+)$', futureEvents ),
    url(r'^events/current$', currentEvent ),
    url(r'^events/(?P<eventId>[0-9]+)/keynote$', keynote ),
    url(r'^events/(?P<eventId>[0-9]+)/panel$', panel ),
    url(r'^events/(?P<eventId>[0-9]+)/demo$', demo ),
    url(r'^contacts/keynote$', speaker_set ),

]


from django.conf.urls import include, url
from . import views

urlpatterns = [
    url(r'^home/$', views.home,name='home'),
    url(r'^product/$', views.product_list,name='product_list'),
    url(r'^product/([\d]+)/$', views.product,name='product'),
    url(r'^add_to_cart/$',views.add_to_cart,name='add_to_cart'),
    url(r'^cart/$',views.cart,name='cart'),
    url(r'^test/$',views.test,name='test'),
    url(r'^signup/$',views.signup,name='signup'),
    url(r'^login/$',views.logging_in,name='logging_in'),
    url(r'^register/$',views.register,name='register'),
    url(r'^logout/$',views.logging_out,name='logging_out')
]


import tensorflow as tf
from params import Params

class DPNet():
    def __init__(self, action_space):

        self.IMG_X = Params['IMG_X']
        self.IMG_Y = Params['IMG_Y']
        self.IMG_Z = Params['IMG_Z']
        self.entropy_penalty = Params['ENTROPY_PENALTY']
        self.action_space = action_space
        self.learning_rate = Params['LEARNING_RATE']

        self.reward_discount = 0.99

        self.policy_nn()

    def policy_nn(self):

        self.state_in, self.policy_out = self.build_nn()

        self.predict_action = tf.argmax(self.policy_out, axis = 1)

        self.exp_reward = tf.placeholder(tf.float32, shape=[None])

        self.actions = tf.placeholder(tf.int32, shape=[None])

        r_mean, r_var = tf.nn.moments(self.exp_reward, axes = [0])

        normalized_reward = (self.exp_reward  - r_mean)/tf.sqrt(r_var)

        #normalized_reward = self.exp_reward

        actions_onehot = tf.one_hot(self.actions, self.action_space, dtype=tf.float32)

        neg_log_prob =  -tf.multiply( actions_onehot * tf.log(self.policy_out + 1e-6), tf.reshape(normalized_reward, [-1, 1]))

        entropy = self.policy_out * tf.log(self.policy_out + 1e-6)

        loss = tf.reduce_mean(neg_log_prob) + self.entropy_penalty * tf.reduce_mean(entropy)

        self.update = tf.train.AdamOptimizer(learning_rate = self.learning_rate).minimize(loss)

    def build_nn(self):

        # [batch, in_height, in_width, in_channels]
        # assuming input to be batch_size*84*84*4
        state_in = tf.placeholder(tf.float32, shape=[None, self.IMG_X, self.IMG_Y, self.IMG_Z])
        state_resized = tf.image.resize_images(state_in, [80, 80])

        ##########################################################
        #[filter_height, filter_width, in_channels, out_channels]
        # conv layer 1, 8*8*32 filters, 4 stride
        conv1_W = tf.Variable(tf.truncated_normal([8, 8, self.IMG_Z, 32], stddev = 0.01))
        conv1_b = tf.Variable(tf.truncated_normal([1, 20, 20, 32], stddev = 0.01))
        conv1_strides = [1, 4, 4, 1]
        #output 20*20*32
        conv1_out = tf.nn.conv2d(state_resized, conv1_W, conv1_strides,
                                          padding = 'SAME') + conv1_b
        conv1_out = tf.nn.relu(conv1_out)

        ###########################################################
        # conv layer 2, 4*4*64 filters, 2 stride
        conv2_W = tf.Variable(tf.truncated_normal([4, 4, 32, 64], stddev = 0.01))
        conv2_b = tf.Variable(tf.truncated_normal([1, 9, 9, 64], stddev = 0.01))
        conv2_strides = [1, 2, 2, 1]
        # output 9*9*64
        conv2_out = tf.nn.conv2d(conv1_out, conv2_W, conv2_strides,
                                          padding = 'VALID') + conv2_b
        conv2_out = tf.nn.relu(conv2_out)

        ###########################################################
        # fully connected layer 1, (7*7*64 = 3136) * 512
        ff1_input = tf.reshape(conv2_out, [-1, 5184])
        ff1_W = tf.Variable(tf.truncated_normal([5184, 256], stddev = 0.01))
        ff1_b = tf.Variable(tf.truncated_normal([1, 256], stddev = 0.01))
        # output batch_size * 512
        ff1_out = tf.matmul(ff1_input, ff1_W) + ff1_b
        ff1_out = tf.nn.relu(ff1_out)

        ##################################################################
        ff2_W = tf.Variable(tf.truncated_normal([ 256, self.action_space],
                                                         stddev = 0.01))
        ff2_b = tf.Variable(tf.truncated_normal([ 1, self.action_space],
                                                         stddev = 0.01))
        # final output, batch_size * action_space
        ff2_out = tf.matmul(ff1_out, ff2_W) + ff2_b

        policy_out = tf.nn.softmax(ff2_out)

        return state_in, policy_out

    def train(self, sess, state, action, reward):

        sess.run(self.update, feed_dict={self.state_in: state,
                                         self.actions: action,
                                         self.exp_reward: reward})

    def predict_policy(self, sess, state):
        # 1X80X80X4 single image
        policy = sess.run(self.policy_out,
                          feed_dict = {self.state_in: state})
        return policy


from blog.models import Post
from blog.models import Tag

# ./manage.py shell

p = Post(title='New Post', slug='new-slug', body='new post body')
p.save()
# w/o save()
p1 = Post.objects.create(title='New Post2', slug='new-slug2', body='new post body2')

Post.objects.all()
post = Post.objects.get(slug='New-slug2')
post2 = Post.objects.get(slug__iexact='New-slug2')  # <Post: New Post2>
post3 = Post.objects.filter(slug__contains='new')  # <QuerySet [<Post: New Post>, <Post: New Post2>]>

p2 = Post.objects.create(title='New Post3', slug='new-slug3', body='new post body3')
p3 = Post.objects.create(title='New Post4', slug='new-slug4', body='new post body4')
p4 = Post.objects.create(title='New Post5', slug='new-slug5', body='new post body5')
p5 = Post.objects.create(title='New Post6', slug='new-slug6', body='new post body6')

p_1 = Post.objects.get(slug='new-slug')
p_1.slug = 'new-post'
p_1.save()
p_2 = Post.objects.get(slug='new-slug2')
p_2.slug = 'new-post2'
p_2.save()
p_3 = Post.objects.get(slug='new-slug3')
p_3.slug = 'new-post3'
p_3.save()
p_4 = Post.objects.get(slug='new-slug4')
p_4.slug = 'new-post4'
p_4.save()
p_5 = Post.objects.get(slug='new-slug5')
p_5.slug = 'new-post5'
p_5.save()
p_6 = Post.objects.get(slug='new-slug6')
p_6.slug = 'new-post6'
p_6.save()

t1 = Tag.objects.create(title='python', slug='python')
p_1.tags.add(t1)
t2 = Tag.objects.create(title='django', slug='django')
t3 = Tag.objects.create(title='flask', slug='flask')
t4 = Tag.objects.create(title='skills', slug='skills')
t5 = Tag.objects.create(title='development', slug='development')
p_1.tags.add(t2)
p_1.tags.add(t3)
p_1.tags.add(t4)
p_1.tags.add(t5)


import pytest
import shutil

try:
    import urllib.parse as urlparse
except ImportError:
    import urlparse

from poetry.packages import Dependency
from poetry.repositories.exceptions import PackageNotFound
from poetry.repositories.legacy_repository import LegacyRepository
from poetry.repositories.legacy_repository import Page
from poetry.utils._compat import PY35
from poetry.utils._compat import Path

class MockRepository(LegacyRepository):

    FIXTURES = Path(__file__).parent / "fixtures" / "legacy"

    def __init__(self):
        super(MockRepository, self).__init__(
            "legacy", url="http://foo.bar", disable_cache=True
        )

    def _get(self, endpoint):
        parts = endpoint.split("/")
        name = parts[1]

        fixture = self.FIXTURES / (name + ".html")

        with fixture.open() as f:
            return Page(self._url + endpoint, f.read(), {})

    def _download(self, url, dest):
        filename = urlparse.urlparse(url).path.rsplit("/")[-1]
        filepath = self.FIXTURES.parent / "pypi.org" / "dists" / filename

        shutil.copyfile(str(filepath), dest)

def test_page_relative_links_path_are_correct():
    repo = MockRepository()

    page = repo._get("/relative")

    for link in page.links:
        assert link.netloc == "foo.bar"
        assert link.path.startswith("/relative/poetry")

def test_page_absolute_links_path_are_correct():
    repo = MockRepository()

    page = repo._get("/absolute")

    for link in page.links:
        assert link.netloc == "files.pythonhosted.org"
        assert link.path.startswith("/packages/")

def test_sdist_format_support():
    repo = MockRepository()
    page = repo._get("/relative")
    bz2_links = list(filter(lambda link: link.ext == ".tar.bz2", page.links))
    assert len(bz2_links) == 1
    assert bz2_links[0].filename == "poetry-0.1.1.tar.bz2"

def test_missing_version(mocker):
    repo = MockRepository()

    with pytest.raises(PackageNotFound):
        repo._get_release_info("missing_version", "1.1.0")

def test_get_package_information_fallback_read_setup():
    repo = MockRepository()

    package = repo.package("jupyter", "1.0.0")

    assert package.name == "jupyter"
    assert package.version.text == "1.0.0"
    assert (
        package.description
        == "Jupyter metapackage. Install all the Jupyter components in one go."
    )

    if PY35:
        assert package.requires == [
            Dependency("notebook", "*"),
            Dependency("qtconsole", "*"),
            Dependency("jupyter-console", "*"),
            Dependency("nbconvert", "*"),
            Dependency("ipykernel", "*"),
            Dependency("ipywidgets", "*"),
        ]


# Copyright (c) 2015 OpenStack Foundation.  All rights reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import collections

from neutron.db import common_db_mixin as common_db_api
from neutron.db.quota import models as quota_models

class QuotaUsageInfo(collections.namedtuple(
    'QuotaUsageInfo', ['resource', 'tenant_id', 'used', 'reserved', 'dirty'])):

    @property
    def total(self):
        """Total resource usage (reserved and used)."""
        return self.reserved + self.used

def get_quota_usage_by_resource_and_tenant(context, resource, tenant_id,
                                           lock_for_update=False):
    """Return usage info for a given resource and tenant.

    :param context: Request context
    :param resource: Name of the resource
    :param tenant_id: Tenant identifier
    :param lock_for_update: if True sets a write-intent lock on the query
    :returns: a QuotaUsageInfo instance
    """

    query = common_db_api.model_query(context, quota_models.QuotaUsage)
    query = query.filter_by(resource=resource, tenant_id=tenant_id)

    if lock_for_update:
        query = query.with_lockmode('update')

    result = query.first()
    if not result:
        return
    return QuotaUsageInfo(result.resource,
                          result.tenant_id,
                          result.in_use,
                          result.reserved,
                          result.dirty)

def get_quota_usage_by_resource(context, resource):
    query = common_db_api.model_query(context, quota_models.QuotaUsage)
    query = query.filter_by(resource=resource)
    return [QuotaUsageInfo(item.resource,
                           item.tenant_id,
                           item.in_use,
                           item.reserved,
                           item.dirty) for item in query]

def get_quota_usage_by_tenant_id(context, tenant_id):
    query = common_db_api.model_query(context, quota_models.QuotaUsage)
    query = query.filter_by(tenant_id=tenant_id)
    return [QuotaUsageInfo(item.resource,
                           item.tenant_id,
                           item.in_use,
                           item.reserved,
                           item.dirty) for item in query]

def set_quota_usage(context, resource, tenant_id,
                    in_use=None, reserved=None, delta=False):
    """Set resource quota usage.

    :param context: instance of neutron context with db session
    :param resource: name of the resource for which usage is being set
    :param tenant_id: identifier of the tenant for which quota usage is
                      being set
    :param in_use: integer specifying the new quantity of used resources,
                   or a delta to apply to current used resource
    :param reserved: integer specifying the new quantity of reserved resources,
                     or a delta to apply to current reserved resources
    :param delta: Specififies whether in_use or reserved are absolute numbers
                  or deltas (default to False)
    """
    query = common_db_api.model_query(context, quota_models.QuotaUsage)
    query = query.filter_by(resource=resource).filter_by(tenant_id=tenant_id)
    usage_data = query.first()
    with context.session.begin(subtransactions=True):
        if not usage_data:
            # Must create entry
            usage_data = quota_models.QuotaUsage(
                resource=resource,
                tenant_id=tenant_id)
            context.session.add(usage_data)
        # Perform explicit comparison with None as 0 is a valid value
        if in_use is not None:
            if delta:
                in_use = usage_data.in_use + in_use
            usage_data.in_use = in_use
        if reserved is not None:
            if delta:
                reserved = usage_data.reserved + reserved
            usage_data.reserved = reserved
        # After an explicit update the dirty bit should always be reset
        usage_data.dirty = False
    return QuotaUsageInfo(usage_data.resource,
                          usage_data.tenant_id,
                          usage_data.in_use,
                          usage_data.reserved,
                          usage_data.dirty)

def set_quota_usage_dirty(context, resource, tenant_id, dirty=True):
    """Set quota usage dirty bit for a given resource and tenant.

    :param resource: a resource for which quota usage if tracked
    :param tenant_id: tenant identifier
    :param dirty: the desired value for the dirty bit (defaults to True)
    :returns: 1 if the quota usage data were updated, 0 otherwise.
    """
    query = common_db_api.model_query(context, quota_models.QuotaUsage)
    query = query.filter_by(resource=resource).filter_by(tenant_id=tenant_id)
    return query.update({'dirty': dirty})

def set_resources_quota_usage_dirty(context, resources, tenant_id, dirty=True):
    """Set quota usage dirty bit for a given tenant and multiple resources.

    :param resources: list of resource for which the dirty bit is going
                      to be set
    :param tenant_id: tenant identifier
    :param dirty: the desired value for the dirty bit (defaults to True)
    :returns: the number of records for which the bit was actually set.
    """
    query = common_db_api.model_query(context, quota_models.QuotaUsage)
    query = query.filter_by(tenant_id=tenant_id)
    if resources:
        query = query.filter(quota_models.QuotaUsage.resource.in_(resources))
    # synchronize_session=False needed because of the IN condition
    return query.update({'dirty': dirty}, synchronize_session=False)

def set_all_quota_usage_dirty(context, resource, dirty=True):
    """Set the dirty bit on quota usage for all tenants.

    :param resource: the resource for which the dirty bit should be set
    :returns: the number of tenants for which the dirty bit was
              actually updated
    """
    query = common_db_api.model_query(context, quota_models.QuotaUsage)
    query = query.filter_by(resource=resource)
    return query.update({'dirty': dirty})


from PyQt5.QtWidgets import QTabWidget, QTextEdit, QMenu, QToolButton, QTabBar

from lib.UI.TabsViews.TabStatus import TabStatus
from lib.UI.TabsViews.TabMeasuring import TabMeasuring
from lib.UI.TabsViews.TabProgramms import TabProgramms

class Tabs(QTabWidget):
    """ The basic architecture of the surface is defined here
    """

    # tab names
    TAB_STATUS = "Tab Status"
    TAB_MEASURING = "Tab Measuring"
    TAB_PROGRAMMS = "Tab Programms"

    def __init__(self, main_toolbar):
        super().__init__()

        self.main_toolbar = main_toolbar

        status_index = 0
        self.tabStatus = TabStatus()
        self.addTab(self.tabStatus, self.TAB_STATUS)
        self.setTabText(status_index, self.TAB_STATUS)
        self.main_toolbar.add_link(
            self, status_index, self.TAB_STATUS, "multimedia-volume-control"
        )

        measuring_index = 1
        self.tabMeasuring = TabMeasuring()
        self.addTab(self.tabMeasuring, self.TAB_MEASURING)
        self.setTabText(measuring_index, self.TAB_MEASURING)
        self.main_toolbar.add_link(
            self, measuring_index, self.TAB_MEASURING, "x-office-spreadsheet"
        )

        programms_index = 2
        self.tabProgramms = TabProgramms()
        self.addTab(self.tabProgramms, self.TAB_PROGRAMMS)
        self.setTabText(programms_index, self.TAB_PROGRAMMS)
        self.main_toolbar.add_link(
            self, programms_index, self.TAB_PROGRAMMS, "x-office-spreadsheet"
        )

        # utilities-system-monitor   -- Window with EKG
        # utilities-terminal  -- Window with terminal