# -*- coding: utf-8 -*-
from django.http import HttpResponse
from django.utils import simplejson as json
from django.conf import settings
import csv, os
from backend.models import Product, Vendor, ItemProduct
from decimal import *
from threading import Thread
from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
from datetime import datetime

def index(request):

    vendor_id = int(request.POST['vendor'])

    #start upload file async
    Thread(target=handle_uploaded_file, kwargs={"file": request.FILES['product'], "vendor_id": vendor_id, "request": request}).start()

    return HttpResponse(json.dumps({ "success": True, "file": request.FILES['product'].name }), mimetype='text/html')

def handle_uploaded_file(**kwargs):

    file      = kwargs['file']
    vendor_id = kwargs['vendor_id']
    request   = kwargs['request']
    user      = request.user.id

    path = "%s/%s" % (settings.UPLOAD_PRODUCT_DIR, file.name,)

    destination = open(path, 'wb+')

    for chunk in file.chunks():
        destination.write(chunk)
    destination.close()

    csvfile = csv.reader(open(path, 'rU'), delimiter=settings.CSV_DELIMITER)

    #header exist -> skip header
    csvfile.next()

    for row in csvfile:
        price = None

        try:
            product = Product.objects.get(product_num=row[0], vendor__id=vendor_id)
        except ObjectDoesNotExist:
            #there is no such product
            #cleaning
            row = clean(row)

            if len(row) >= 8 and row[7]:
                price = Decimal(row[7].replace(',','.').replace('$','').replace(' ', ''))
            else:
                price = None

            if len(row) >= 9:
                notes = row[8]
            else:
                notes = None

            Product.objects.create(product_num=row[0], upc=row[1], category=row[2], \
                                    name1=row[3], name2=row[4], package=row[5], brand=row[6], \
                                    price=price, notes=notes, date_modified=datetime.now(), vendor_id=vendor_id, user_id=user)
        else:
            #product already exist -> update price and date_modified
            if len(row) >= 8 and row[7]:
                price = Decimal(row[7].replace(',', '.').replace('$', '').replace(' ', ''))
            else:
                price = None

            product.price = price
            product.date_modified = datetime.now()

            product.save()

        updateItemProduct(row[0], vendor_id, price)

    #reindex sphinx, takes many resources, let's not do it here, sphinx reindexes itself daily
    #Thread(target=reindex).start()

    #rename filename
    rename(path, vendor_id, request)

def updateItemProduct(product_num, vendor_id, price):
    try:
        product = ItemProduct.objects.get(product_num=product_num, vendor__id=vendor_id)
    except ObjectDoesNotExist:
        pass
    except MultipleObjectsReturned:
        product = ItemProduct.objects.filter(product_num=product_num, vendor__id=vendor_id).order_by('-date_modified')[0]
    else:
        if price is not None:
            product.price = price
            product.date_modified = datetime.now()

            product.save()

def rename(old_filename, vendor_id, request):

    try:
        vendor = Vendor.objects.get(pk=vendor_id)
    except ObjectDoesNotExist:
        vendor_name = 'unknown_vendor'
    else:
        vendor_name = vendor.name

    #renaming file
    new_filename = "%s/%s__%s__%s.csv" % (settings.UPLOAD_PRODUCT_DIR, request.user, vendor_name, datetime.now().strftime("%Y-%m-%d %H_%M_%S"),)

    os.rename(old_filename, new_filename)

def clean(row):
    clean_row = []

    for col in row:
        if col == '':
            col = None
        else:
            col = cut_none_unicode(col)

        clean_row.append(col)

    return clean_row

def cut_none_unicode(col):
    clean_col = []

    for letter in col:
        try:
            letter = unicode(letter)
        except UnicodeDecodeError:
            pass
        else:
            clean_col.append(letter)

    return ''.join(clean_col)

def reindex():
    import subprocess

    command = "/bin/indexer --config %s/sphinx.conf --rotate --all" % (settings.PROJECT_ROOT,)
    subprocess.call(command, shell = True)


from django.conf import settings
from django.core.mail import EmailMultiAlternatives
from django.shortcuts import render, redirect
from django.views.generic import TemplateView

from .forms import ContactForm
from .models import Contact

class Contactview(TemplateView):
    """ View for contact us
    """
    template_name = 'pages/contact.html'

    def get(self, *args, **kwargs):
        form = ContactForm()
        return render(self.request, self.template_name, {'form': form})

    def post(self, *args, **kwargs):
        form = ContactForm(self.request.POST)
        if form.is_valid():
            email = form.cleaned_data['email']
            name = form.cleaned_data['name']
            subject_title = form.cleaned_data['subject']

            subject, from_email, to = subject_title, settings.EMAIL_HOST_USER, email
            text_content = 'Thank you for reaching us.'
            html_content = '
Hello, thank you for reaching swiftkind
'
            msg = EmailMultiAlternatives(subject, text_content, from_email, [to])
            msg.attach_alternative(html_content, "text/html")
            msg.send()
            form.save()
            return redirect('contact')
        return render(self.request, self.template_name, {'form': form})


from django.core import serializers
import json

class Japi:
	"""API simple responder class"""

	response = { 'status': 'ok', 'action':'' }

	def __init__( self, action ):
		self.response[ 'action' ] = action
		if not hasattr( self, action ):
			self.response[ 'status' ] = "ko"
			self.response[ 'error'] = action + " not found!"
		else:
			func = getattr( self, action )
			func()

	def __str__( self ):
		return json.dumps( self.response )

	def help( self ):
		self.response[ 'available_methods' ] = {
			'help':'this action, provide some basic snippets'
		}


import logging

from flask import Blueprint, render_template, request, redirect, url_for
from sqlalchemy.orm.exc import NoResultFound

from mantis.model import Corpus, Message, Document, Annotation, Pagination, serialize
from mantis.utils import ctx
from mantis.views.login import check_login

__all__ = [
    'page',
]

page = Blueprint('explorer_page', __name__, template_folder='templates')

logger = logging.getLogger(__name__)

@page.route('/', methods=['GET'])
def render_explorer():
    with ctx.SessionContext() as app:
        user = check_login(app)
        if user:
            corpora = app.database.query(Corpus).all()
            html = render_template('pages/explorer.html', messages=app.messages, corpora=corpora, selected_corpus=None)
            app.clear('messages')
        else:
            html = redirect(url_for('index_page.render_index'))
        return html

def serialize_documents(documents):
    return {'documents': [{**item.serialize(), 'annotations': serialize(item.annotations)} for item in documents]}

@page.route('/corpus', methods=['GET'])
def render_explorer_corpus():
    with ctx.SessionContext() as app:
        user = check_login(app)
        if user:
            id_ = request.args.get('id', None, int)
            index = request.args.get('page', 1, int)
            if id_ is not None:
                corpora = app.database.query(Corpus).all()
                try:
                    sel_corpus = app.database.query(Corpus).filter_by(id=id_).one()
                except NoResultFound as _:
                    msg = 'Corpus not found in the database. You will be redirected to a default corpus if exists'
                    app.messages.append(Message(Message.Type.ERROR, msg))
                else:
                    qry_documents = app.database.query(Document).filter_by(corpus_id=id_)
                    pagination = Pagination(qry_documents)
                    page_ = pagination.get(index)
                    sel_corpus = {**sel_corpus.serialize(), **serialize_documents(page_.items)}
                    html = render_template(
                        'pages/explorer.html',
                        messages=app.messages, corpora=corpora, selected_corpus=sel_corpus, pagination=pagination,
                        page=page_
                    )
                    app.clear('messages')
                    return html
            else:
                msg = 'Invalid value for parameter ID. ' \
                      'Found \'{}\' expected an integer'.format(request.args.get('id', None))
                app.messages.append(Message(Message.Type.ERROR, msg))
        return redirect(url_for('.render_explorer'))

@page.route('/annotation/create', methods=['POST'])
def render_explorer_annotation_create():
    with ctx.SessionContext() as app:
        user = check_login(app)
        if user:
            id_ = request.form.get('id', None, int)
            if id_ is not None:
                # -- construct annotation --
                owner_id = user.id
                start = request.form.get('span_start')
                length = request.form.get('span_length')
                features = dict(zip(request.form.getlist('feature_key'), request.form.getlist('feature_value')))
                annotation = Annotation(owner_id=owner_id, document_id=id_, start=start, length=length,
                                        features=features)
                # -- add to database --
                app.database.add(annotation)
                app.database.commit()
                msg = 'Annotation created successfully'
                app.messages.append(Message(Message.Type.SUCCESS, msg))
                document = app.database.query(Document).get(id_)
                return redirect(url_for('.render_explorer_corpus', id=document.corpus.id))
            else:
                msg = 'Invalid value for the document ID. Found \'{}\' expected an integer'.format(
                    request.form.get('id', None)
                )
                app.messages.append(Message(Message.Type.ERROR, msg))
        return redirect(url_for('.render_explorer'))

@page.route('/annotation/update', methods=['POST'])
def render_explorer_annotation_update():
    with ctx.SessionContext() as app:
        user = check_login(app)
        if user:
            id_ = request.args.get('id', None, int)
            if id_ is not None:
                owner_id = user.id
                annotation = app.database.query(Annotation).get(id_)
                id_ = request.form.get('id', None, int)
                if owner_id == annotation.owner_id:
                    if id_ == annotation.document.id:
                        start = request.form.get('span_start')
                        length = request.form.get('span_length')
                        feature_keys = request.form.getlist('feature_key')
                        feature_values = request.form.getlist('feature_value')
                        features = dict(zip(feature_keys, feature_values))
                        annotation.start = start
                        annotation.length = length
                        annotation.features = features
                        app.database.commit()
                        corpus_id = annotation.document.corpus.id
                        msg = 'Annotation (id=\'{}\') updated successfully'.format(id_)
                        app.messages.append(Message(Message.Type.SUCCESS, msg))
                        return redirect(url_for('.render_explorer_corpus', id=corpus_id))
                    else:
                        msg = 'Can\'t alter the document of the annotation'
                        app.messages.append(Message(Message.Type.ERROR, msg))
                else:
                    msg = 'You are not authorized to delete annotation with ID \'{}\''.format(id_)
                    app.messages.append(Message(Message.Type.ERROR, msg))
            else:
                msg = 'Invalid value for the document ID. Found \'{}\' expected an integer'.format(
                    request.args.get('id', None)
                )
                app.messages.append(Message(Message.Type.ERROR, msg))
        return redirect(url_for('.render_explorer'))

@page.route('/annotation/delete', methods=['GET', 'POST'])
def render_explorer_annotation_delete():
    with ctx.SessionContext() as app:
        user = check_login(app)
        if user:
            id_ = request.args.get('id', None, int)
            if id_ is not None:
                owner_id = user.id
                annotation = app.database.query(Annotation).get(id_)
                if annotation is not None:
                    corpus_id = annotation.document.corpus.id
                    if owner_id == annotation.owner_id:
                        app.database.delete(annotation)
                        app.database.commit()
                        msg = 'Annotation (id=\'{}\') deleted successfully'.format(id_)
                        app.messages.append(Message(Message.Type.SUCCESS, msg))
                        return redirect(url_for('.render_explorer_corpus', id=corpus_id))
                    else:
                        msg = 'You are not authorized to delete annotation with ID \'{}\''.format(id_)
                        app.messages.append(Message(Message.Type.ERROR, msg))
                else:
                    msg = 'Unable to find an annotation with the provided ID \'{}\''.format(id_)
                    app.messages.append(Message(Message.Type.ERROR, msg))
            else:
                msg = 'Invalid value for the document ID. Found \'{}\' expected an integer'.format(
                    request.args.get('id', None)
                )
                app.messages.append(Message(Message.Type.ERROR, msg))
        return redirect(url_for('.render_explorer'))


import Utils.globals, Utils.keyUtils.keys
from threading import Thread
from PyQt5 import QtCore, QtGui, QtWidgets
from UI.pyQt5_GUI.mainwindow import Ui_MainWindow
import sys
from Manager import core_manager

# To generate the gui python code form the .ui file, use this command:
# pyuic5 UI/pyQt5_GUI/mainwindow.ui  -o UI/pyQt5_GUI/mainwindow.py

# Used to update the UI when receving incoming message
class UI_updater(Thread):

    def __init__(self,ui):
        Thread.__init__(self)
        self.ui = ui
        self.receivingQueue = Utils.globals.receivingQueue

    def run(self):
        i = 0
        while True:
            message = self.receivingQueue.get()
            cmd = str(i) + "- Message: " + str(message["command"])
            self.ui.listWidget.addItem(cmd)
            i += 1

        print("Exit pyQt5_GUI Thread")

class Ui_manager:

    def __init__(self):
        self.receivingQueue = Utils.globals.receivingQueue
        self.sendingQueue = Utils.globals.sendingQueue

        self.app = QtWidgets.QApplication(sys.argv)
        self.MainWindow = QtWidgets.QMainWindow()
        self.ui = Ui_MainWindow()

        # Start UI
        self.ui.setupUi(self.MainWindow)

        # Init components
        self.init_components()

        UI_updater(self.ui).start()

        # CLose UI
        self.MainWindow.show()
        sys.exit(self.app.exec_())

    # define the actions when event triggered by the UI
    def init_components(self):

        # onClick on elt from listWidget, display content in the plainTextEdit
        self.ui.listWidget.itemClicked.connect(lambda: self.onClick_listWidget(self.ui.listWidget.currentItem()) )

        # onClick on Version button, send Version to node
        self.ui.pushButton.toggle()
        self.ui.pushButton.clicked.connect(lambda: self.onClick_version())

        # onClick on Version button, send Verack to node
        self.ui.pushButton_2.toggle()
        self.ui.pushButton_2.clicked.connect(lambda: self.onClick_verack())

        # onClick on Version button, send Ping to node
        self.ui.pushButton_3.toggle()
        self.ui.pushButton_3.clicked.connect(lambda: self.onClick_ping())

        # ...

        #... keyUtils tab

        self.ui.createAddressButton.toggle()
        self.ui.createAddressButton.clicked.connect(lambda: self.onClick_createAddress())

        # ...

    def onClick_listWidget(self, item):
        id = int(item.text().split('-')[0])
        self.ui.plainTextEdit.clear()
        self.ui.plainTextEdit.setPlainText(Utils.globals.node_messages[id]["header"] + "\n" + Utils.globals.node_messages[id]["payload"])

    def onClick_version(self):
        version = core_manager.get_version_pkt()
        self.sendingQueue.put(version)

    def onClick_verack(self):
        verack = core_manager.get_verack_pkt()
        self.sendingQueue.put(verack)

    def onClick_ping(self):
        ping = core_manager.get_ping_pkt()
        self.sendingQueue.put(ping)

    def onClick_createAddress(self):
        self.ui.KeysDisplay.clear()
        private_key = self.ui.privateKeyInsert.text()
        if (len(private_key)<1):
            print ("here")
            nk = Utils.keyUtils.keys.Key()
        else:
            print ("there")
            nk = Utils.keyUtils.keys.Key(private_key)
        self.ui.KeysDisplay.setPlainText(str(
              "Private key: " + nk.printable_pk) + "\n"
             + "Public key: " + str(nk.public_key, "ascii") + "\n"
             + "Hashes public key: " + str(nk.hashed_public_key, "ascii") + "\n"
             + "Address: " + str(nk.addr))


import os
from datetime import datetime
from decimal import Decimal

from influxdb import InfluxDBClient

from ..acoustics import acoustic_analysis

from ..sql.models import SoundFile, Discourse

from ..graph.discourse import DiscourseInspecter

def sanitize_formants(value):
    try:
        f1 = value['F1'][0]
    except TypeError:
        f1 = value['F1']
    if f1 is None:
        f1 = 0
    try:
        f2 = value['F2'][0]
    except TypeError:
        f2 = value['F2']
    if f2 is None:
        f2 = 0
    try:
        f3 = value['F3'][0]
    except TypeError:
        f3 = value['F3']
    if f3 is None:
        f3 = 0
    return f1, f2, f3

def to_nano(seconds):
    if not isinstance(seconds,Decimal):
        seconds = Decimal(seconds).quantize(Decimal('0.001'))
    return int(seconds * Decimal('1e9'))

def to_seconds(timestring):
    try:
        d = datetime.strptime(timestring, '%Y-%m-%dT%H:%M:%S.%fZ')
    except:
        d = datetime.strptime(timestring, '%Y-%m-%dT%H:%M:%SZ')

    s = 60*60*d.hour + 60*d.minute + d.second + d.microsecond / 1e6
    s = Decimal(s).quantize(Decimal('0.001'))
    return s

class AudioContext(object):
    """
    Class that contains methods for dealing with audio files for corpora
    """
    def analyze_acoustics(self, stop_check = None, call_back = None):
        """
        Runs all acoustic analyses for the corpus.
        """
        if not self.has_sound_files:
            raise(NoSoundFileError)
        acoustic_analysis(self, stop_check = stop_check, call_back = call_back)

    def genders(self):

        res = self.execute_cypher('''MATCH (s:Speaker:{corpus_name}) RETURN s.gender as gender'''.format(corpus_name = self.cypher_safe_name))
        genders = set()
        for s in res:
            g = s['gender']
            if g is None:
                g = ''
            genders.add(g)
        return sorted(genders)

    def reset_acoustics(self, call_back = None, stop_check = None):
        self.acoustic_client().drop_database(self.corpus_name)

    def acoustic_client(self):
        client = InfluxDBClient(**self.config.acoustic_conncetion_kwargs)
        databases = client.get_list_database()
        if self.corpus_name not in databases:
            client.create_database(self.corpus_name)
        return client

    def inspect_discourse(self, discourse, begin = None, end = None):
        """
        Get a discourse inspecter object for a discourse

        Parameters
        ----------
        discourse : str
            Name of the discourse
        begin : float, optional
            Beginning of the initial cache
        end : float, optional
            End of the initial cache

        Returns
        -------
        :class:`~polyglotdb.graph.discourse.DiscourseInspecter`
            DiscourseInspecter for the specified discourse
        """
        return DiscourseInspecter(self, discourse, begin, end)

    def discourse_sound_file(self, discourse):
        """
        Gets the sound file object for the discourse

        Parameters
        ----------
        discourse : str
            Name of the discourse

        Returns
        -------
        :class:`~polyglotdb.sql.models.SoundFile`
            the first soundfile
        """
        q = self.sql_session.query(SoundFile).join(SoundFile.discourse)
        q = q.filter(Discourse.name == discourse)
        sound_file = q.first()
        return sound_file

    def discourse_audio_directory(self, discourse):
        """
        Return the directory for the stored audio files for a discourse
        """
        return os.path.join(self.config.audio_dir, discourse)

    def has_all_sound_files(self):
        """
        Check whether all discourses have a sound file

        Returns
        -------
        bool
            True if a sound file exists for each discourse name in corpus,
            False otherwise
        """
        if self._has_all_sound_files is not None:
            return self._has_all_sound_files
        discourses = self.discourses
        for d in discourses:
            sf = self.discourse_sound_file(d)
            if sf is None:
                break
            if not os.path.exists(sf.filepath):
                break
        else:
            self._has_all_sound_files = True
        self._has_all_sound_files = False
        return self._has_all_sound_files

    @property
    def has_sound_files(self):
        """
        Check whether any discourses have a sound file

        Returns
        -------
        bool
            True if there are any sound files at all, false if there aren't
        """
        if self._has_sound_files is None:
            self._has_sound_files = self.sql_session.query(SoundFile).first() is not None
        return self._has_sound_files

    def get_formants(self, discourse, begin, end, channel = 0, source = None):
        """
        Get formants for a given discourse and time range

        Parameters
        ----------
        discourse : str
            Name of the discourse
        begin : float
            Beginning of time range
        end : float
            End of time range
        channel : int, defaults to 0
            Specify a channel of the audio file
        source : str, defaults to None
            Specify a source of the formants, if None, use the formant_algorithm
            attribute of the CorpusContext

        Returns
        -------
        list
            List of results with fields for ``time``, ``F1``, ``F2``, and ``F3``
        """
        begin = Decimal(begin).quantize(Decimal('0.001'))
        end = Decimal(end).quantize(Decimal('0.001'))
        if source is None:
            source = self.config.formant_algorithm
        client = self.acoustic_client()
        result = client.query('''select "time", "F1", "F2", "F3" from "formants"
                        WHERE "discourse" = '{}' AND "source" = '{}'
                        AND "time" >= {}
                        AND "time" <= {};'''.format(discourse, source,
                                                to_nano(begin),to_nano(end)))
        listing = []
        for r in result.get_points('formants'):
            s = to_seconds(r['time'])
            listing.append((s, r['F1'], r['F2'], r['F3']))
        return listing

    def get_pitch(self, discourse, begin, end, channel = 0, source = None):
        """
        Get pitch for a given discourse and time range

        Parameters
        ----------
        discourse : str
            Name of the discourse
        begin : float
            Beginning of time range
        end : float
            End of time range
        channel : int, defaults to 0
            Specify a channel of the audio file
        source : str, defaults to None
            Specify a source of the formants, if None, use the pitch_algorithm
            attribute of the CorpusContext

        Returns
        -------
        list
            List of results with fields for ``time`` and ``F0``
        """
        begin = Decimal(begin).quantize(Decimal('0.001'))
        end = Decimal(end).quantize(Decimal('0.001'))
        if source is None:
            source = self.config.pitch_algorithm
        client = self.acoustic_client()
        query = '''select "time", "F0" from "pitch"
                        WHERE "discourse" = '{}' AND "source" = '{}'
                        AND "time" >= {}
                        AND "time" <= {};'''.format(discourse, source,
                                                to_nano(begin),to_nano(end))
        result = client.query(query)
        listing = []
        for r in result.get_points('pitch'):
            s = to_seconds(r['time'])
            listing.append((s, r['F0']))
        return listing

    def save_formants(self, sound_file, formant_track, channel = 0, source = None):
        """
        Save a formant track for a sound file

        Parameters
        ----------
        sound_file : str or :class:`~polyglotdb.sql.models.SoundFile`
            Discourse name or SoundFile object
        formant_track : dict
            Dictionary with times as keys and tuples of F1, F2, and F3 values as values
        channel : int, defaults to 0
            Specify a channel of the audio file
        source : str, defaults to None
            Specify a source of the formants, if None, use the formant_algorithm
            attribute of the CorpusContext
        """
        if isinstance(sound_file, str):
            sound_file = self.discourse_sound_file(sound_file)
        if sound_file is None:
           return
        if source is None:
            source = self.config.formant_algorithm
        data = []
        for timepoint, value in formant_track.items():
            f1, f2, f3 = sanitize_formants(value)
            d = {'measurement': 'formants',
                'tags': {'discourse': sound_file.discourse.name,
                        'channel': channel,
                        'source': source},
                "time": to_nano(timepoint),
                "fields": {'F1': f1,'F2':f2,'F3':f3}
                }
            data.append(d)
        self.acoustic_client().write_points(data)

    def save_pitch(self, sound_file, pitch_track, channel = None, speaker = None, source = None):
        """
        Save a pitch track for a sound file

        Parameters
        ----------
        sound_file : str or :class:`~polyglotdb.sql.models.SoundFile`
            Discourse name or SoundFile object
        pitch_track : dict
            Dictionary with times as keys and F0 values as values
        channel : int, defaults to 0
            Specify a channel of the audio file
        source : str, defaults to None
            Specify a source of the formants, if None, use the pitch_algorithm
            attribute of the CorpusContext
        """
        if isinstance(sound_file, str):
            sound_file = self.discourse_sound_file(sound_file)
        if sound_file is None:
           return
        if source is None:
            source = self.config.pitch_algorithm
        data = []
        tag_dict = {}
        if isinstance(sound_file, str):
            tag_dict['discourse'] = sound_file
        else:
            tag_dict['discourse'] = sound_file.discourse.name
        if speaker is not None:
            tag_dict['speaker'] = speaker
        if source is not None:
            tag_dict['source'] = source
        if channel is not None:
            tag_dict['channel'] = channel
        for timepoint, value in pitch_track.items():
            try:
                value = float(value['F0'])
            except TypeError:
                try:
                    value = float(value[0])
                except TypeError:
                    try:
                        value = float(value)
                    except ValueError:
                        continue

            d = {'measurement': 'pitch',
                'tags': tag_dict,
                "time": to_nano(timepoint),
                "fields": {'F0': value}
                }
            data.append(d)
        self.acoustic_client().write_points(data)

    def has_formants(self, discourse, source = None):
        """
        Return whether a discourse has any formant values associated with it
        """
        client = self.acoustic_client()
        if source is None:
            source = self.config.pitch_algorithm
        result = client.query('''select "F1" from "formants" WHERE "discourse" = '{}' AND "source" = '{}' LIMIT 1;'''.format(discourse, source))
        if len(result) == 0:
            return False
        return True

    def has_pitch(self, discourse, source = None):
        """
        Return whether a discourse has any pitch values associated with it
        """
        client = self.acoustic_client()
        if source is None:
            source = self.config.pitch_algorithm
        result = client.query('''select "F0" from "pitch" WHERE "discourse" = '{}' AND "source" = '{}' LIMIT 1;'''.format(discourse, source))
        if len(result) == 0:
            return False
        return True


from django.core.management.base import BaseCommand

from PotionsCraftHelperApp.models import Potion
import json
class Command(BaseCommand):

    def handle(self, *args, **options):
        potions = Potion.objects.all()
        data = []
        for potion in potions:
            data.append({
                'name': potion.name,
                'ingredients': potion.ingredients,
                'time': potion.time,
                'time_plus': potion.time_plus,
                'time_up_level': potion.time_up_level,
                'up_level': potion.up_level,
                'up_time': potion.up_time,
                'image': str(potion.image)
            })
        with open('potions_data.json', 'w') as file:
            json.dump(data, file, indent=4)


import logging
import time

from useful.resource.readers import ResourceURL, readers
from useful.resource.parsers import parsers

_log = logging.getLogger(__name__)

def cached_load(timeout=300):
    """
    Define timeout to be used in `load()` function.

    Args:
        timeout (int, optional): Number of seconds to cache data without
            checking if it has changed in any way. Defaults to 300.

    Returns:
        function: A function using timeout variable
    """
    memory = {}

    def load(url, mimetype=None, parser=None, hook=None):
        """
        Load resource from uri or cache if already used before.

        Args:
            url (str): String represeting URL specified in RFC 1738.
            mimetype (str, optional): Forced MIME type if not None. Defaults to
                None.
            parser (useful.resource.parsers.Parser, optional): A parser class
                to use instead of parsers from useful.resource.parsers.parsers.
                Defaults to None.
            hook (callable, optional): An optional function to call after
                reading and parsing the data. Defaults to None.

        Raises:
            ValueError: No reader supports provided url scheme
            ValueError: No parser supports provided mimetype

        Returns:
            object: Final data after running reader, parser and hook on the
                resource url
        """
        hash_ = None

        # get the reader from url
        resource_url = ResourceURL(url, mimetype=mimetype)
        try:
            reader = readers[resource_url.scheme]
        except KeyError:
            raise ValueError(
                f"Unsupported reader scheme '{resource_url.scheme}'")

        reader = reader(url=resource_url)

        # if url has been cached for less than `timeout` or hash sum of the
        # resource is still equal, return cached value
        if url in memory:
            now = time.time()
            if now - memory[url]['time'] < timeout:
                _log.debug(
                    f"Url '{url}' in memory for less then {timeout} seconds",
                    extra={"url": url, "timeout": timeout})
                return memory[url]['data']
            else:
                hash_ = reader.hash()
                if hash_ == memory[url]['hash']:
                    _log.debug(
                        f"Url '{url}' in memory hasn't changed hash sum",
                        extra={"url": url, "hash": hash_})
                    # update object timestamp in memory
                    memory[url]['time'] = now
                    return memory[url]['data']

        # if url has been cached but needs to update use cached hook as a
        # hook, otherwise use function parameter hook as hook object
        hook = memory.get(url, {}).get("hook", hook)
        # use already calculated above hash sum or calculate hash sum if it was
        # never calculated
        hash_ = hash_ or reader.hash()

        # use user-provided parser or get parser from mimetype
        try:
            parser = parser or parsers[resource_url.mimetype]
            parser = parser(reader=reader)
        except KeyError:
            raise ValueError(
                f"Unsupported parser mimetype {resource_url.mimetype}")

        # parse data provided by reader
        data = parser.parse()

        # call hook on data
        if hook is not None:
            data = hook(data)

        # cache results and other relevant data
        memory[url] = {
            'time': time.time(),
            'hash': hash_,
            'data': data,
            'hook': hook
        }
        _log.debug(f"Upserting url '{url}' in memory",
                   extra={"url": url, "hash": hash_})
        return data

    return load

load = cached_load(timeout=300)


import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import MeanShift
from sklearn.cluster import KMeans
from astroML.stats import binned_statistic_2d
FileType='NGC'
num='0821'

data=u'../DATA/%s%s.csv' %(FileType,num)
ID,RA,Dec,Magg,Errg,Magr,Errr=np.loadtxt(data, delimiter=',', usecols=(0,1,2,3,4,5,6), unpack=True)
ra=list()
dec=list()
for i in range(len(RA)):
    if Errr[i]<0.1 and Errg[i]<0.1:
        ra.append(RA[i])
        dec.append(Dec[i])

RA_range=[min(RA),max(RA)]
Dec_range=[min(Dec),max(Dec)]

ra_=np.array(ra).reshape(1,len(ra)).T
dec_=np.array(dec).reshape(1,len(dec)).T
X=np.hstack((ra_,dec_))
ms=MeanShift(bandwidth=0.4,cluster_all=False)
ms.fit(X)
centers=ms.cluster_centers_

N, xedges, yedges = binned_statistic_2d(ra, dec, ra ,'count', bins=30, range=[RA_range,Dec_range])
plt.figure(figsize=(9,9))
plt.imshow(N.T,origin='lower',extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]],cmap = plt.cm.gray_r,interpolation='gaussian')
plt.scatter(centers[:,0],centers[:,1])
plt.xlim(RA_range[1],RA_range[0])
plt.ylim(Dec_range)
plt.show()


import os
import sys
import zipfile

xmls = [f.replace('.xml', '') for f in os.listdir(sys.argv[1]) if f.endswith('.xml')]
d = sys.argv[1] + '_zips'
os.makedirs(d)
for x in xmls:
    zipf = zipfile.ZipFile(d + '/' + x + '.zip', 'w')
    for f in os.listdir(sys.argv[1]):
        if f.startswith(x):
            zipf.write(sys.argv[1] + '/' + f)
    zipf.close()


import pyglet
from .. import character

class TestEnemy(character.Character):
    def __init__(self, core):
        super().__init__(core, pyglet.resource.image("test_enemy.png"), -0.5, -0.5, -3.5)
        self.just_faught = False

    def update(self, overworld, dt):
        super().update(overworld, dt)
        if self.x - self.w < overworld.character.x < self.x + self.w:
            if self.z - self.w < overworld.character.z < self.z + self.w:
                if not self.just_faught:
                    overworld.game.start_mode(self.core.combat_arena)
                    self.just_faught = True
                return
        self.just_faught = False


# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/topics/items.html

from scrapy.item import Item, Field

class iTunesItem(Item):
	genreId = Field()
	genreUrl = Field()
	genreName = Field()
	subgenreId = Field()
	subgenreUrl = Field()
	subgenreName = Field()

class PodcastItem(Item):
	uri = Field()
	id = Field()
	audioType = Field()
	indexedDate = Field()
	mediaType = Field()
	images = Field(default=['dummyUrl'])

	itunesArtistId = Field()
	itunesArtistName = Field()
	itunesArtistViewUrl = Field()
	itunesArtworkUrl100 = Field()
	itunesArtworkUrl30 = Field()
	itunesArtworkUrl55 = Field()
	itunesArtworkUrl60 = Field()
	itunesArtworkUrl170 = Field()
	itunesArtworkUrl600 = Field()
	itunesCollectionCensoredName = Field()
	itunesCollectionExplicitness = Field()
	itunesCollectionId = Field()
	itunesCollectionName = Field()
	itunesCollectionPrice = Field()
	itunesCollectionViewUrl = Field()
	itunesCurrency = Field()
	itunesGenreIds = Field()
	itunesGenres = Field()
	itunesHtmlChannel = Field()
	itunesHtmlOwner = Field()
	itunesHtmlDescription = Field()
	itunesPopular = Field()
	itunesPopularInGenre = Field()
	itunesPrimaryGenreName = Field()
	itunesRelated = Field()
	itunesReleaseDate = Field()
	itunesSimilar = Field()
	itunesTrackCensoredName = Field()
	itunesTrackCount = Field()
	itunesTrackExplicitness = Field()
	itunesTrackId = Field()
	itunesTrackName = Field()
	itunesTrackPrice = Field()
	itunesTrackViewUrl = Field()

	rssChannelTitle = Field()
	rssChannelLink = Field()
	rssChannelDescription = Field()
	rssChannelCopyright = Field()
	rssChannelLanguage = Field()
	rssChannelGenerator = Field()
	rssChannelPubDate = Field()
	rssChannelitunesSummary = Field()
	rssChannelitunesSubtitle = Field()
	rssChannelitunesKeywords = Field()
	rssChannelitunesAuthor = Field()
	rssChannelitunesOwnerEmail = Field()
	rssChannelitunesOwnerName = Field()
	rssChannelitunesCategory = Field()
	rssChannelitunesImage = Field()
	rssChannelitunesExplicit = Field()
	rssChannelImageUrl = Field()
	rssChannelImageTitle = Field()
	rssChannelImageLink = Field()
	rssChannelLastBuildDate = Field()

	rssItemTitle = Field()
	rssItemDescription = Field()
	rssItemPubDate = Field(default=['1900-01-01T00:00:00Z'])
	rssItemLink = Field()
	rssItemGuid = Field()
	rssItemAuthor = Field()
	rssItemitunesSubtitle = Field()
	rssItemitunesSummary = Field()
	rssItemitunesKeywords = Field()
	rssItemitunesDuration = Field()
	rssItemitunesExplicit = Field()
	rssItemEnclosureUrl = Field()
	rssItemEnclosureLength = Field()
	rssItemEnclosureType = Field()
	rssItemMediaUrl = Field()
	rssItemMediaFileSize = Field()
	rssItemMediaType = Field()

	episodeId = Field()
	episodeTitle = Field()
	episodeTitleN = Field()
	episodeSubtitle = Field()
	episodeDescription = Field()
	episodeStart = Field()
	episodeEnd = Field()
	episodeDuration = Field()
	episodePublishDate = Field()
	episodePublishYear = Field()
	# Solr has episodePublishYear which is derived from above in DIH
	episodeMimeType = Field()
	episodeMedia = Field()
	episodeImage = Field()
	episodeHomepage = Field()
	episodeFirstBroadcast = Field()
	episodeAvailableUntil = Field()
	episodeTags = Field()
	episodeRelatedTitle = Field()
	episodeRelatedUrl = Field()

	seriesId = Field()
	seriesName = Field()
	seriesNameN = Field()
	seriesDescription = Field()
	seriesImage = Field()
	seriesFeed = Field()
	seriesCurrentItem = Field()
	seriesHomepage = Field()
	seriesTags = Field()

	brandAvgDuration = Field()
	brandCategory = Field()
	brandCurrentItem = Field()
	brandDescription = Field()
	brandExplicit = Field()
	brandFeed = Field() # The on demand feed for the content
	brandFrequency = Field()
	brandGenreIds = Field()
	brandGenres = Field(default='unknown')
	brandHomepage = Field()
	brandId = Field()
	brandIds = Field()
	brandImage = Field()
	brandLanguage = Field()
	brandCulture = Field()
	brandLanguage = Field()
	brandName = Field()
	brandNameN = Field()
	brandPublishDate = Field()
	brandRelated = Field()
	brandRegions = Field()
	brandShortName = Field()
	brandSimilar = Field()
	brandTags = Field()
	brandTimes = Field()

	channelId = Field()
	channelName = Field()
	channelNameN = Field()
	channelTitle = Field()
	channelSubTitle = Field()
	channelDescription = Field()
	channelImage = Field()
	channelFeed = Field() # The live stream for a channel
	channelHomepage = Field()
	channelLocation = Field()
	channelAddress = Field()
	channelTuneInId = Field()
	channelCategory = Field()
	channelGenres = Field()
	channelGenreIds = Field()
	channelPlaylist = Field()
	channelPlaylistHeaders = Field()
	channelFormats = Field()
	channelRelatedGenres = Field()
	channelRelatedGenreIds = Field()
	channelRecommended = Field()
	channelRecommendedDescription = Field()
	channelRecommendedId = Field()
	channelRecommendedFormats = Field()
	channelRecommendedType = Field()
	channelRecommendedImage = Field()
	channelCallSign = Field()
	channelSlogan = Field()
	channelUrl = Field()
	channelLocation = Field()
	channelEmail = Field()
	channelPhone = Field()
	channelLanguage = Field()

	channelTuneInReportUrl = Field()
	channelTuneInDetailUrl = Field()
	channelTuneInIsPreset = Field()
	channelTuneInIsAvailable = Field()
	channelTuneInIsMusic = Field()
	channelTuneInHasSong = Field()
	channelTuneInHasSchedule = Field()
	channelTuneInHasTopics = Field()
	channelTuneInTwitterId = Field()
	channelTuneInItem = Field()
	channelTuneInNowPlayingId = Field()
	channelTuneInPresetId = Field()
	channelTuneInType = Field()
	channelTuneInBitrate = Field()
	channelTuneInReliability = Field()
	channelTuneInGuideId = Field()
	channelTuneInShowId = Field()
	channelTuneInCurrentTrack = Field()
	channelTuneInUrl = Field()
	channelTuneInId = Field()
	channelTuneInDesc = Field()
	channelTuneInLogo = Field()
	channelTuneInAddress = Field()
	channelTuneInEmail = Field()
	channelTuneInLocation = Field()
	channelTuneInName = Field()
	channelTuneInHomepage = Field()
	channelTuneInStreamIds = Field()
	channelTuneInGenres = Field()
	channelTuneInLang = Field()
	channelTuneInContact = Field()
	channelTuneInNetworks = Field()
	channelTuneInNetworkIds = Field()

	#live streams
	#all stream* should be multivalue in Solr
	streamId = Field()
	streamName = Field()
	streamReliability = Field()
	streamUrl = Field()
	streamMediaType = Field()
	streamMedia = Field()
	streamBandwidth = Field()
	streamData = Field() #anything else that dont know where to put but may be useful
	streamRecentlyPlayed = Field()
	streamHistory = Field()
	streamServerName = Field()
	streamGenre = Field()
	streamContentType = Field()
	streamTitle = Field()
	streamHomepage = Field()
	streamStatus = Field()
	streamStatusUrl = Field()

	#audio books
	audiobookProjectName = Field()
	audiobookLibrivoxURL = Field()
	audiobookRssURL = Field()
	audiobookCategory = Field()
	audiobookGenre = Field()
	audiobookAuthor1 = Field()
	audiobookAuthor2 = Field()
	audiobookAuthor3 = Field()
	audiobookAuthor4 = Field()
	audiobookTranslator = Field()
	audiobookLanguage = Field()
	audiobookType = Field()

	ownerId = Field()
	ownerName = Field()
	ownerNameN = Field()
	ownerEmail = Field()
	ownerKey = Field()
	ownerImage = Field()
	ownerHomepage = Field()
	ownerDescription = Field()

	#some meta data fields created locally in spider
	#persisted here to be used later in pipelines
	metaBrandName = Field()
	metaBrandDescription = Field()
	metaGenreName = Field()
	metaChannelName = Field()
	metaOwnerName = Field()


#!/usr/local/bin/python3
##        (C) COPYRIGHT Ingenic Limited.
##             ALL RIGHTS RESERVED
##
## File       : plt.py
## Authors    : ydwu@ydwu-white
## Create Time: 2019-05-07:13:46:08
## Description:
##
##
import matplotlib.pyplot as plt

plt.imshow()
#是不显示图像的

#解决方法:在后面加一句：
plt.show()

#原理：plt.imshow()函数负责对图像进行处理，并显示其格式，而plt.show()则是将plt.imshow()处理后的函数显示出来。

# 在 plt.show() 之前调用 plt.savefig()；
plt.savefig("filename.png")
plt.show()

import cv2
import matplotlib.pyplot as plt

img = cv2.imread("sample_dog.jpg")
plt.subplot(121)
plt.imshow(img)

# convert BGR to RGB
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
plt.subplot(122)
plt.imshow(img)
plt.show()

## automatically close
time.sleep(5)

## close by key
plt.close()

############################

import plotly.plotly
import plotly.graph_objs as go

data = [
    go.Box(
        y=[0, 1, 1, 2, 3, 5, 8, 13, 21]  # 9个数据
    )
]
plotly.offline.plot(data)  # 离线绘图

############################

# First create some toy data:
x = np.linspace(0, 2*np.pi, 400)
y = np.sin(x**2)

# Creates just a figure and only one subplot
fig, ax = plt.subplots()
ax.plot(x, y)
ax.set_title('Simple plot')

# Creates two subplots and unpacks the output array immediately
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True)
ax1.plot(x, y)
ax1.set_title('Sharing Y axis')
ax2.scatter(x, y)

# Creates four polar axes, and accesses them through the returned array
fig, axes = plt.subplots(2, 2, subplot_kw=dict(polar=True))
axes[0, 0].plot(x, y)
axes[1, 1].scatter(x, y)

# Share a X axis with each column of subplots
plt.subplots(2, 2, sharex='col')

# Share a Y axis with each row of subplots
plt.subplots(2, 2, sharey='row')

# Share both X and Y axes with all subplots
plt.subplots(2, 2, sharex='all', sharey='all')

# Note that this is the same as
plt.subplots(2, 2, sharex=True, sharey=True)

# Creates figure number 10 with a single subplot
# and clears it if it already exists.
fig, ax=plt.subplots(num=10, clear=True)


# This file is part of Tryton.  The COPYRIGHT file at the top level of
# this repository contains the full copyright notices and license terms.
from trytond import backend
from trytond.model import ModelSQL, fields
from trytond.pyson import Eval
from trytond.pool import PoolMeta, Pool
from trytond.tools.multivalue import migrate_property
from trytond.modules.company.model import (
    CompanyMultiValueMixin, CompanyValueMixin)

class Party(CompanyMultiValueMixin, metaclass=PoolMeta):
    __name__ = 'party.party'
    sale_price_list = fields.MultiValue(fields.Many2One(
            'product.price_list', "Sale Price List",
            help="The default price list for new sales.",
            domain=[
                ('company', '=', Eval('context', {}).get('company', -1)),
                ],
            states={
                'invisible': ~Eval('context', {}).get('company'),
                }))
    sale_price_lists = fields.One2Many(
        'party.party.sale_price_list', 'party', "Sale Price Lists")

    @classmethod
    def default_sale_price_list(cls, **pattern):
        pool = Pool()
        Configuration = pool.get('sale.configuration')
        config = Configuration(1)
        price_list = config.get_multivalue('sale_price_list', **pattern)
        return price_list.id if price_list else None

class PartySalePriceList(ModelSQL, CompanyValueMixin):
    "Party Sale Price List"
    __name__ = 'party.party.sale_price_list'
    party = fields.Many2One(
        'party.party', "Party", ondelete='CASCADE', select=True)
    sale_price_list = fields.Many2One(
        'product.price_list', "Sale Price List",
        domain=[
            ('company', '=', Eval('company', -1)),
            ],
        depends=['company'])

    @classmethod
    def __register__(cls, module_name):
        exist = backend.TableHandler.table_exist(cls._table)

        super(PartySalePriceList, cls).__register__(module_name)

        if not exist:
            cls._migrate_property([], [], [])

    @classmethod
    def _migrate_property(cls, field_names, value_names, fields):
        field_names.append('sale_price_list')
        value_names.append('sale_price_list')
        fields.append('company')
        migrate_property(
            'party.party', field_names, cls, value_names,
            parent='party', fields=fields)


import numpy as np
import matplotlib.pyplot as plt

# Ordinary Least Squares: A method for finding the best-fit curve, in this case a polynomial, to a set of (x, y) pairs.
# If the data can be described as y = X * w where y is the output data, X is the data matrix, and w is the weight vector, then w = (X.T * X)^-1 * X.T * y

def predict(x, weights):
    val = 0
    n = 0
    for w in weights:
        val += w * x ** n
        n += 1
    return val

def OLS(data_matrix, ys):
    meany = sum(ys) / len(ys)
    SStot = sum([(y - meany) ** 2 for y in ys])
    weights = np.dot(np.dot(np.linalg.inv(np.dot(data_matrix.T, data_matrix)), data_matrix.T), ys)
    SSres = sum([(ys[i] - predict(data_matrix[i][1], weights)) ** 2 for i in range(len(ys))])
    r2 = 1 - SSres/SStot
    return weights, r2

xs = [1, 2, 4, 7, 10, 14]
ys = np.asarray([3, 5, 6, 8, 12, 21])
degree = 3
data_matrix = np.array([[x ** n for n in range(degree + 1)] for x in xs])
weights, r2 = OLS(data_matrix, ys)
plt.scatter(xs, ys, c = "black")
model_xs = np.linspace(min(xs), max(xs), 100)
model_ys = [predict(x, weights) for x in model_xs]
plt.plot(model_xs, model_ys, "r-")
equation = "y = "
n = len(weights) - 1
for w in weights[::-1]:
    if n == len(weights) - 1: equation += "%.3fx^%d" % (w, n)
    else:
        if n == 0:
            if w >= 0: equation += " + %.3f" % w
            else: equation += " - %.3f" % abs(w)
        else:
            if w >= 0: equation += " + %.3fx^%d" % (w, n)
            else: equation += " - %.3fx^%d" % (abs(w), n)
    n -= 1
plt.suptitle("Ordinary Least Squares Regression")
plt.title(equation + ". R^2 = %.4f" % r2)
plt.show()


import numpy as np
import powerlaw
import random
import math

from mesa import Agent, Model
from mesa.time import RandomActivation
from directions import Directions, Direction
#from mesa.space import MultiGrid
from mesa.datacollection import DataCollector

class Pedestrian(Agent):

    DIRECTION_DEGREES = 15
    #NR_OF_DIRECTIONS = 24

    def __init__(self, unique_id, model, pos, exp, seed=None):
        self.unique_id = unique_id
        self.model = model
        self.pos = pos
        self.traversable=False

        self.area_traversed = np.zeros((model.width, model.height))
        self.traversable = True
        self.trip_lengths = powerlaw.Power_Law(xmin=7, parameters=[exp]) #1.5-2.0 #xmin 17?
        self.direction_range=3
        self.directions = Directions(self.direction_range)
        self.trip_lengths_covered = []
        self.steps_covered = []

        self.area_traversed[self.pos[0], self.pos[1]] = 1
        self.on_trip = False
        self.current_direction = 0
        self.remaining_steps = 0

    def plan_trip(self):
        '''
        Plan new tip:
        i. Determine trip length
        ii. Determine new direction based on current direction (correlated RW)
        '''

        # Direction
        if self.current_direction == 0:  # first trip
            direction = random.choice(self.directions.directions)
            self.current_direction = Direction(direction, self.direction_range)
        else:  # correlated random walk
            radians = np.random.vonmises(0, kappa=4)
            degrees = math.degrees(radians)
            jumps = math.floor(degrees/Pedestrian.DIRECTION_DEGREES)
            self.current_direction = self.directions.correlated(self.current_direction, jumps)

        # Trip length
        length_of_trip =  int(self.trip_lengths.generate_random(1)[0]) #generate_random ignores xmax
        self.remaining_steps = self.current_direction.calculate_steps(length_of_trip)
        self.trip_lengths_covered.append(length_of_trip)
        self.steps_covered.append(self.remaining_steps)

    def contact(self):
        '''
        Find all other agents within range (radius = 5) and update contacts.
        '''
        neighbors_in_contact = self.model.grid.get_neighbors(self.pos, moore=True, radius = 5)
        if len(neighbors_in_contact) > 0:
            for neighbor in neighbors_in_contact:
                if neighbor.unique_id != self.unique_id:
                    self.model.contact_update((self.unique_id, neighbor.unique_id))

    def check_if_traversable(self, coordinate):
        '''
        Check if location is traversable.
        '''

        traversable = True
        if ( 0 <= coordinate[0] < self.model.width) and (0 <= coordinate[1] < self.model.height):
            contents = self.model.grid.get_cell_list_contents(coordinate)

            for content in contents:
                if not content.traversable:
                    traversable = False
        else:
            traversable = False

        return traversable

    def move(self):
        '''
        Move pedestrian one step
        '''
        # Plan new trip, if current trip length reached
        if self.remaining_steps == 0:
            self.plan_trip()

        # Determine move based on current direction
        shift = self.current_direction.move()
        move = (self.pos[0] + shift[0], self.pos[1] + shift[1])

        # Check if move is possible (can't traverse walls)
        if self.check_if_traversable(move):
            new_position = move
            self.model.grid.move_agent(self, new_position)
            self.remaining_steps -=1
        else:
            '''
            If not traversable, determine the two alternative moves (i. bounce
            of vertical wall, ii. bounce of horizontal wall). Determine
            which bouncing move works, then move and update direction accordingly
            '''
            # Bouncing moves
            alternative_shifts = [
            [-shift[0], shift[1]],
            [shift[0], -shift[1]],
            [-shift[0], -shift[1]]
            ]

            # If bouncing move possible --> move and update direction
            # TODO: Possible bias for objects??? Always checks vertical wall shift first
            for bounce_shift in alternative_shifts:

                move = (self.pos[0] + bounce_shift[0], self.pos[1] + bounce_shift[1])
                if self.check_if_traversable(move):

                    # Bounce: move + new direction
                    new_position = move
                    self.model.grid.move_agent(self, new_position)
                    self.remaining_steps -=1
                    self.current_direction.change(bounce_shift)

                    break

    def step(self):
        self.move()
        #self.contact()


import ast
import asyncio
import httpreq
import time
from logging import info as printf

proxy_list = []
index = 0

async def Init():
   printf("init")
   await GetHttpsProxyList()

async def GetHttpsProxyList():
    proxies = await httpreq.GetHtml('https://raw.githubusercontent.com/fate0/proxylist/master/proxy.list')
    if proxies is None:
        printf("get raw proxy list error")
        return None
    # 使用'\n' 切分字符串,得到一个list
    resp = str(proxies['resp'], encoding='utf-8')
    lines = resp.splitlines(False)
    for line in lines:
        # 只保留高密代理,https的一定是高密的
        if (line.find('https') > 0):
            try:
                proxy = ast.literal_eval(line)
                proxy['used'] = False
                proxy['url'] = proxy['type'] + '://' + \
                               proxy['host']+ ':' + str(proxy['port'])
                proxy['fail_cnt'] = 0
                proxy_list.append(proxy)
            except Exception as e:
                printf(str(e))
    printf("got %d proxies", len(proxy_list))

def GetUnusedProxy():
    global index
    for proxy in proxy_list[index:]:
        if (proxy['used'] == False):
            index = index + 1
            if index == len(proxy_list):
                index = 0
            return proxy
    return None

def GetProxy():
    for i in range(10):
        proxy = GetUnusedProxy()
        if (proxy):
            return proxy
        printf("no available proxy,sleep 1s to retry")
        time.sleep(1)
    printf("no available proxy, retry 10 times")
    return None

def SetUnused(proxy):
    proxy['used'] = False

def SetUsed(proxy):
    proxy['used'] = True

def RemoveProxy(proxy):
    proxy_list.remove(proxy)

def IncFailCnt(proxy):
    proxy['fail_cnt'] = proxy['fail_cnt'] + 1
    if proxy['fail_cnt'] >= 3:
       proxy_list.remove(proxy)


# -*- coding: UTF-8 -*-
from il.Congestion_controller.congestionController import CongestionController
# import matplotlib.pyplot as plt
import numpy as np
import datetime
# from main_multi import stop_time
stop_time = 10000000

S_LEN = 4
INPUT_LEN = 15
ROW_DATA_LEN = INPUT_LEN * (S_LEN * 2 + 1)

FLOOR_BITRATE_M = 0.01
CEIL_BITRATE_M = 2.5

def rectificate_bitrate(bitrate):
        """
        矫正速率
        :param bitrate:
        :return:
        """
        bitrate = bitrate // 1e5 * 0.1
        bitrate = min(CEIL_BITRATE_M, bitrate)
        bitrate = max(FLOOR_BITRATE_M, bitrate)
        return bitrate

class DaggerMasterController(CongestionController):
    """
    收集数据
    """

    def __init__(self):
        self.start_bitrate_bps = 1e6
        self.rate, self.lbrates, self.dbrates, self.delay, self.loss, self.delay_diff, self.bandwidth = \
            [], [], [], [], [], [], []
        self.ts_delta, self.t_delta, self.trendline, self.mt, self.threashold = [], [], [], [], []
        self.time = []
        self.inuse = []
        self.rate.append(self.start_bitrate_bps)

        # mz
        # 获取当前时间
        self.start_time = datetime.datetime.now()

        # 初始化
        # 保存路径
        self.save_path = './data_ver/D.npy'

        # 初始配置
        self.start_bitrate = 0.7  # 初始速率
        # self.codec_bitrate = 0  # 初始编码速率
        self.send_time_last_state = 0  # 上一次发送时间的状态
        self.arrival_time_last_state = 0  # 达到发送时间的状态
        self.start_arrival_time = 0  # 开始到达的时间

        self.count = 0  # 计数
        # 三输入的数组维度为  1 X (INPUT_LEN * (S_LEN * 2 + 1) + 1)
        raw = INPUT_LEN * (S_LEN * 2 + 1) + 1
        self.save_data_one = np.zeros((1, raw))  # len+1  初始化 save_data_one 数组维度为

        # 数组维度为 (stop_time // 1000) X (INPUT_LEN * (S_LEN))
        self.save_data = np.zeros((stop_time // 1000, raw))  # len+1

        # 丢包窗口 (INPUT_LEN X SLEN)
        self.save_loss_windows = np.zeros((INPUT_LEN, S_LEN))
        # 吞吐量窗口   INPUT_LEN X 1
        self.save_throughput_windows = np.zeros((INPUT_LEN, 1))
        # 延迟窗口      INPUT_LEN X S_LEN
        self.save_delay_interval_windows = np.zeros((INPUT_LEN, S_LEN))

        self.start_time = 0
        self.bitrate_lable = self.start_bitrate
        self.loss_windows = []
        self.throughput_windows = []
        self.delay_interval_windows = []

    def estimate(self, feedback):
        if not feedback:
            return self.start_bitrate_bps
        target_bitrate = feedback.bandwidth[-1]

        self.time.append(feedback.arrival_time_ms[-1] / 1000.0)
        self.rate.append(target_bitrate)
        self.loss.extend(feedback.loss)
        self.delay.extend([feedback.arrival_time_ms[i] - feedback.send_time_ms[i]
                           for i in range(len(feedback.arrival_time_ms))])
        self.bandwidth.append(feedback.average_bandwidth)
        self.save_master_data(feedback)
        return target_bitrate

    def compute_throught(self, feedbackPacket):
        """
        计算吞吐量
        :param feedbackPacket:
        :return:
        """
        arrival_time = feedbackPacket.arrival_time_ms
        payload_size = feedbackPacket.payload_size
        # 计算吞吐量
        intervals = arrival_time[-1] - self.start_arrival_time
        throughput = np.sum(payload_size) / intervals / 1000.0
        self.start_arrival_time = arrival_time[-1]
        return throughput

    def compute_delay_interval(self, feedbackPacket):
        """
        计算时延
        :param feedbackPacket:
        :return:
        """
        send_time = feedbackPacket.send_time_ms
        arrival_time = feedbackPacket.arrival_time_ms

        # 需要计算时延，这些是输出
        delay_send = []
        delay_arrival = []
        delay_interval = []

        # 初始时间间隔
        # delay_interval.append(
        #     (send_time[0] - self.send_time_last_state) - (arrival_time[0] - self.arrival_time_last_state))
        delay_interval.append((arrival_time[0] - self.arrival_time_last_state) -(send_time[0] -
                                                                                self.send_time_last_state))

        # 计算时间间隔
        for i in range(1, S_LEN):
            delay_send.append(send_time[i] - send_time[i - 1])
            delay_arrival.append(arrival_time[i] - arrival_time[i - 1])
            delay_interval.append(delay_arrival[i - 1] - delay_send[i - 1])  # 2

        self.send_time_last_state = send_time[-1]
        self.arrival_time_last_state = arrival_time[-1]
        # print(self.send_time_last_state)
        # print(self.arrival_time_last_state)
        return delay_interval

    def save_master_data(self, feedbackPacket):
        """
        保存数据
        :param feedbackPacket:
        :return:
        """
        self.bitrate_lable = feedbackPacket.average_bandwidth // 1e6  # M

        loss = feedbackPacket.loss[:S_LEN]  # 1
        now_time = (feedbackPacket.arrival_time_ms[-1] - 4) // 1000

        # 如果发送时间是 0 ，那么初始化这些列表
        if self.send_time_last_state == 0:
            self.start_time = now_time

        delay_interval = self.compute_delay_interval(feedbackPacket)
        throughput = self.compute_throught(feedbackPacket)

        # 保存
        if now_time != self.start_time:
            np_loss_windows = np.array(self.loss_windows)
            np_throughput_windows = np.array(self.throughput_windows)
            np_throughput_windows = np_throughput_windows[:, np.newaxis]
            np_delay_interval_windows = np.array(self.delay_interval_windows)

            #  如果累计的值达到 INPUT_LEN，开始
            #  shape[0] 存放了数组的一维大小
            if np_delay_interval_windows.shape[0] >= INPUT_LEN:
                self.save_delay_interval_windows = np_delay_interval_windows[-INPUT_LEN:, :]
                self.save_throughput_windows = np_throughput_windows[-INPUT_LEN:, :]
                self.save_loss_windows = np_loss_windows[-INPUT_LEN:, :]
            else:
                # row_stack() 列数不变，合并
                self.save_loss_windows = np.row_stack(
                    (self.save_loss_windows[-(INPUT_LEN - np_delay_interval_windows.shape[0]):, :], np_loss_windows))
                self.save_throughput_windows = np.row_stack((self.save_throughput_windows[
                                                             -(INPUT_LEN - np_delay_interval_windows.shape[0]):, :],
                                                             np_throughput_windows))
                self.save_delay_interval_windows = np.row_stack((self.save_delay_interval_windows[
                                                                 -(INPUT_LEN - np_delay_interval_windows.shape[0]):, :],
                                                                 np_delay_interval_windows))

            # print(self.bitrate_lable)
            # 保存一行
            self.save_one_row()

            self.start_time = now_time
            self.loss_windows = []
            self.throughput_windows = []
            self.delay_interval_windows = []

            self.count = self.count + 1

            # 每过一段时间缓存或者是快到达临界值的时候
            if now_time % 3600 == 0 or stop_time // 1000 - now_time < 5:
                print(stop_time // 1000 - now_time)
                print(self.save_data[0:self.count, :].shape)
                np.save(self.save_path, self.save_data[0:self.count, :])

        self.loss_windows.append(loss)  # 1  append
        self.delay_interval_windows.append(delay_interval)  # 2 append
        self.throughput_windows.append(throughput)  # 3 append

    def save_one_row(self):
        """
        保存一行数据，[loss, delay, throughput, bitrate_label]
        :return:
        """
        # reshape(-1) 将二维数组展开成一维数组
        self.save_data_one[0, 0:INPUT_LEN * S_LEN] = self.save_loss_windows.reshape(-1)
        self.save_data_one[0, INPUT_LEN * S_LEN:INPUT_LEN * S_LEN * 2] = self.save_delay_interval_windows.reshape(
            -1)
        self.save_data_one[0,
        INPUT_LEN * S_LEN * 2:INPUT_LEN * (S_LEN * 2 + 1)] = self.save_throughput_windows.reshape(-1)
        self.save_data_one[0, -1] = self.bitrate_lable
        self.save_data[self.count, :] = self.save_data_one

    def plot_target_send_rate(self, ip):
        """
        用来绘制图像
        :return:
        """
        pass
        # print(self.time[-1])
        # plt.figure()
        # plt.suptitle(ip + ', avg_loss:' + str(round(np.mean(self.loss), 2)) + '%'
        #              + ', delay:' + str(round(np.mean(self.delay), 2)) + 'ms'
        #              )
        #
        # plt.subplot(511)
        # # plt.plot(self.time, self.bandwidth)
        # plt.plot(self.bandwidth)
        # plt.ylabel('Bitrate (bps)')
        # # plt.plot(self.time, self.rate[:-1])
        # plt.plot(self.rate[:-1])
        # plt.legend(['bandwidth', 'target send rate'])
        #
        # plt.subplot(512)
        # plt.ylabel('Delay (ms)')
        # plt.ylim((0, 200))
        # plt.plot(self.delay)
        #
        # plt.subplot(513)
        # plt.ylabel('Loss (%)')
        # plt.plot(self.loss)
        #
        # plt.subplot(514)
        # plt.ylabel('Deltas(ms)')
        # plt.plot(self.ts_delta)
        # plt.plot(self.t_delta)
        # plt.legend(['send deltas', 'arrival deltas'])
        #
        # plt.subplot(515)
        # plt.ylabel('m(t)')
        # plt.ylim((-10, 10))
        # plt.plot(self.mt)
        # plt.plot(self.threashold)
        # plt.legend(['m(t)', 'threashold'])
        # plt.show()


from django.shortcuts import render, redirect
from django.http import HttpResponse
import requests

def get_api(request):
    rdata = requests.get('https://covid19.th-stat.com/api/open/timeline')
    data = rdata.json()
    total = [num for num in range(len(data['Data']))]
    num = str(len(total)-1)
    if request.POST:
        num = request.POST.get('date')
        print(num)

    context = {'datas': zip(reversed(total),
        reversed(data['Data'])), 'current': data['Data'][int(num)]}

    return render(request, 'timeline/timeline.html', context)


'''
Created on Mar 7, 2015

@author: Adrian
'''
def getJson(url):
    import requests, json
    r = requests.get(url)
    obj = json.loads(r.content)
    results = obj['json']
    return results
def sendPost(url, obj):
    import requests
    import urllib
    payload = urllib.urlencode(obj,doseq=True)
    headers = {'Content-Type':'application/x-www-form-urlencoded'}
    r = requests.post(url, data=payload, headers=headers)
    return r

def getRemovedHostNames():
    results = getJson('http://localhost:3000/bad_links')

    removed_domains = []

    for result in results:
        removed_domains.append(result['DOMAIN_NAME'])

    return removed_domains


import sys
import face_recognition as fr
import numpy as np

storedPath = open(sys.argv[1], 'r')
uploadedPath = open(sys.argv[2], 'r')

storedString = storedPath.read().replace('[', '').replace(']', '').replace('\\n', '').replace('\n', '').replace('\"', '').replace(',', '')
uploadedString = uploadedPath.read().replace('[', '').replace(']', '').replace('\n', '')

storedImg = np.fromstring(storedString, dtype=np.float64, sep=' ')
uploadedImg = np.fromstring(uploadedString, dtype=np.float64, sep=' ')

enset = [storedImg, uploadedImg]
# ff = open('compresult.txt', 'w')
dist = fr.face_distance(enset, uploadedImg)[0]

if (0.4 > dist):
    print('True\n' + str(float(dist)))
else:
    print('False\n' + str(float(dist)))


from datetime import date

class Person():
    '''A class representing a person, used to show a simple example
    of OOP concepts'''

    def __init__(self, name, dob, gender):
        '''(Person, str, str, str) -> NoneType
        Set up a new person with a name dob (date of birth) and gender
        REQ: dob is formatted as "YYYY-MM-DD" all numeric values
        REQ: gender is one of "Male","Female"
        '''
        self._name = name
        year = int(dob[0:4])
        month = int(dob[5:7])
        day = int(dob[8:10])
        self._dob = date(year, month, day)
        self.is_male = (gender == "Male")

    def __str__(self):
        '''(Person) -> str
        Return a string representing this person
        '''
        return ("Hi, my name is " + self._name + " and I am " +
                str(self.get_age()) + " years old")

    def get_age(self):
        '''(Person) -> int
        Return this person's current age in years
        '''
        total_age = date.today() - self._dob
        age_years = total_age.total_seconds() // (365 * 24 * 60 * 60)
        return int(age_years)

class Student(Person):
    '''A class representing a student, created by inheritence
    inheriting attributes/methods from person and only extending
    the ones we need to make this a student'''

    def __init__(self, name, dob, gender, student_num, gpa):
        '''(Student, str, str, float) -> NoneType
        Set up a new student with a name dob (date of birth), gender,
        student_num, and their current gpa
        REQ: dob is formatted as "YYYY-MM-DD" all numeric values
        REQ: gender is one of "Male","Female"
        REQ: 0 <= gpa <= 4
        '''
        self._name = name
        year = int(dob[0:4])
        month = int(dob[5:7])
        day = int(dob[8:10])
        self._dob = date(year, month, day)
        self.is_male = (gender == "Male")
        self._student_num = student_num
        self._gpa = gpa

    def __str__(self):
        '''(Student) -> NoneType
        Return a string representing this student
        '''
        # get the result of my parent class's string method, and add to it
        ret = ("Hi, my name is " + self._name + " and I am " +
               str(self.get_age()) + " years old")
        if(self._gpa >= 3.5):
            ret += " and I'm on the honours list"
        elif(self._gpa < 2):
            ret += " and I need to study harder"
        return ret

    def raise_gpa(self, raise_amount):
        '''(Student, float) -> NoneType
        Increase the student's GPA by raise_amount
        '''
        self._gpa += raise_amount
        # Cap the GPA at 4
        if(self._gpa > 4):
            self._gpa = 4

if (__name__ == "__main__"):
    alice = Person("Alice Jones", "1978-03-05", "Female")
    bob = Student("Robert Smith", "1995-01-01", "Male", "1234", 3.6)
    print(alice)
    print(bob)
    # we cah get both of their ages
    print(alice.get_age())
    print(bob.get_age())
    # but only students have a raise_gpa method
    # alice.raise_gpa(0.1) #<-- this will cause a crash
    bob.raise_gpa(0.1)


from telegram import ParseMode
from telegram.ext import Filters, CommandHandler, MessageHandler

from phrases import start_phrase, help_phrase
from tools import get_uid, store_name

def start_callback(update, context):
    """ callback for /start (private) """

    uid = get_uid(update)

    context.bot.send_message(uid,
                             start_phrase,
                             parse_mode=ParseMode.HTML)

start_handler = CommandHandler('start',
                               callback=start_callback,
                               filters=Filters.private)

def help_callback(update, context):
    """ callback for /help (private) """

    uid = get_uid(update)

    context.bot.send_message(uid,
                             help_phrase,
                             parse_mode=ParseMode.HTML)

help_handler = CommandHandler('help',
                              callback=help_callback,
                              filters=Filters.private)

def store_names_callback(update, context):
    """ stores names for each user, if not already present in DB"""

    store_name(update)

store_names_handler = MessageHandler(Filters.all, callback=store_names_callback)


# Parses a CSV with GeoBlacklight Aardvark metadata into multiple JSON files.
# Resources: https://fivestepguide.com/technology/machine-learning/csv-rows-to-multiple-json-python-pandas/

# Import the required libraries
import csv
import json

# Define the CSV to be parsed - should be the output from formatCSVToAardvark.py
file = 'aardvark.csv'

# Define a function to generate boolean values from strings 'True' and 'False'
def makebool(text):
    if text == 'True':
        return True
    else:
        return False

# Open the CSV and make sure to close it when done processing
with open(file, 'r') as f:

    # Read the CSV and load values into dictionary
    reader = csv.DictReader(f)

    # Process one row at a time
    for row in reader:

        # Format the integer fields
        row['gbl_indexYear_im'] = int(row['gbl_indexYear_im'])

        # Format the boolean fields
        row['gbl_suppressed_b'] = makebool(row['gbl_suppressed_b'])
        row['gbl_georeferenced_b'] = makebool(row['gbl_georeferenced_b'])

        # Clean and format the list fields: remove quotes + spaces + brackets, then split into a list
        subject1 = (row['dct_subject_sm']).replace("'", '')
        subject2 = subject1.replace(', ', ',')
        subject3 = subject2.removeprefix('[').removesuffix(']')
        row['dct_subject_sm'] = subject3.split(',')

        theme1 = (row['dcat_theme_sm']).replace("'", '')
        theme2 = theme1.replace(', ', ',')
        theme3 = theme2.removeprefix('[').removesuffix(']')
        row['dcat_theme_sm'] = theme3.split(',')

        geonames1 = (row['umass_geonames_sm']).replace("'", '')
        geonames2 = geonames1.replace(', ', ',')
        geonames3 = geonames2.removeprefix('[').removesuffix(']')
        row['umass_geonames_sm'] = geonames3.split(',')

        # Clean and format a list field with commas: remove spaces + selected quotes + brackets, then split into a list
        coverage1 = (row['dct_spatial_sm']).replace("', ", "',")
        coverage2 = coverage1.replace("['", "[").replace("']", "]").replace(",'", ",")
        coverage3 = coverage2.replace('[', '').replace(']', '')
        row['dct_spatial_sm'] = coverage3.split("',")

        # Retrieve the 'mods_ID' value to use in naming the output file
        mods_ID = row['dct_identifier_sm']

        # Output individual JSON files
        out = json.dumps(row, indent=4)
        jsonoutput = open(mods_ID + '.json', 'w')
        jsonoutput.write(out)


from dataclasses import dataclass
from enum import Enum
from typing import Optional, Tuple, Dict, Union, Any

@dataclass
class Link:
    url: str
    bookmark_id: str
    heading_id: str

def hex_to_rgb(input: str):
    if input.startswith("#"):
        input = input[1:]
    return tuple(int(input[i : i + 2], 16) / 255.0 for i in (0, 2, 4))

def doc_color_to_color(info: Optional[dict]):
    if not info:
        return None
    base = info.get("color")
    if not base:
        return None
    base = base.get("rgbColor")
    if not base:
        return
    return Color(red=base["red"], green=base["green"], blue=base["blue"])

def doc_link_to_link(info: dict):
    if not info:
        return None
    # TODO: identify link format for these:
    # info.get('bookmarkId'), info.get('headingId')
    return info.get("url")  # Link(info.get('url'), info.get('bookmarkId'), info.get('headingId'))

@dataclass
class Color:
    hex: Optional[str] = None
    red: Optional[float] = None
    blue: Optional[float] = None
    green: Optional[float] = None
    alpha: Optional[float] = None

    @classmethod
    def parse(cls, item: Any):
        if isinstance(item, Color):
            return item
        elif isinstance(item, str):
            return Color(item)
        elif isinstance(item, tuple):
            return Color(None, *item)
        elif not item:
            return None
        else:
            raise ValueError(f"Cannot parse {item} to Color")

    def __post_init__(self):

        assert self.hex or (
            self.red is not None and self.blue is not None and self.green is not None
        )
        if self.hex:
            self.red, self.green, self.blue = hex_to_rgb(self.hex)

    def to_json(self):

        return {"color": {"rgbColor": {"red": self.red, "green": self.green, "blue": self.blue}}}

    def to_sheet_style(self):

        base = {"red": self.red, "green": self.green, "blue": self.blue}

        if self.alpha is not None:
            base["alpha"] = self.alpha

        return base

    def to_slide_style(self):

        base = {"color": {"rgbColor": {"red": self.red, "green": self.green, "blue": self.blue}}}

        if self.alpha is not None:
            base["alpha"] = self.alpha

        return base

@dataclass
class TextStyle:
    font_size: Optional[int] = None
    font: Optional[str] = None
    font_weight: Optional[int] = None
    color: Optional[Color] = None
    background_color: Optional[Color] = None
    bold: Optional[bool] = None
    italic: Optional[bool] = None
    strikethrough: Optional[bool] = None
    small_caps: Optional[bool] = None
    underline: Optional[bool] = None
    link: Optional[str] = None
    font_unit: Optional[str] = "PT"

    def to_doc_style(self) -> Tuple[str, Dict]:  # noqa: C901
        base: Dict[str, Union[bool, str, int, dict]] = {}
        masks = []
        if self.font_size is not None:
            base["fontSize"] = {"magnitude": self.font_size, "unit": "PT"}
            masks.append("fontSize")
        if self.bold is not None:
            base["bold"] = self.bold
            masks.append("bold")
        if self.italic is not None:
            base["italic"] = self.italic
            masks.append("italic")
        if self.underline is not None:
            base["underline"] = self.underline
            masks.append("underline")
        if self.small_caps is not None:
            base["smallCaps"] = self.small_caps
            masks.append("smallCaps")
        if self.background_color is not None:
            base["backgroundColor"] = self.background_color.to_json()
            masks.append("backgroundColor")
        if self.color is not None:
            base["foregroundColor"] = self.color.to_json()
            masks.append("foregroundColor")
        if self.font is not None:
            if not base.get("weightedFontFamily"):
                base["weightedFontFamily"] = {}
            base["weightedFontFamily"]["fontFamily"] = self.font  # type: ignore
            masks.append("weightedFontFamily.fontFamily")
        if self.font_weight is not None:
            if not base.get("weightedFontFamily"):
                base["weightedFontFamily"] = {}
                base["weightedFontFamily"]["fontFamily"] = "Arial"  # type: ignore
            base["weightedFontFamily"]["weight"] = self.font_weight  # type: ignore
            masks.append("weightedFontFamily.fontFamily")
            masks.append("weightedFontFamily.weight")

        return ",".join(masks), base

    def to_sheet_style(self) -> Tuple[list, Dict]:  # noqa: C901

        base: Dict[str, Union[str, bool, int, dict]] = {}
        masks = []

        if self.font_size is not None:
            base["fontSize"] = {"magnitude": self.font_size, "unit": "PT"}
            masks.append("fontSize")
        if self.bold is not None:
            base["bold"] = self.bold
            masks.append("bold")
        if self.italic is not None:
            base["italic"] = self.italic
            masks.append("italic")
        if self.underline is not None:
            base["underline"] = self.underline
            masks.append("underline")
        if self.color is not None:
            base["foregroundColor"] = self.color.to_sheet_style()
            masks.append("foregroundColor")
        if self.font is not None:
            base["fontFamily"] = self.font
            masks.append("fontFamily")

        return masks, base

    def to_slide_style(self):

        base = {}
        masks = []

        if self.font_size is not None:
            base["fontSize"] = {"magnitude": self.font_size, "unit": "PT"}
            masks.append("fontSize")
        if self.bold is not None:
            base["bold"] = self.bold
            masks.append("bold")
        if self.italic is not None:
            base["italic"] = self.italic
            masks.append("italic")
        if self.underline is not None:
            base["underline"] = self.underline
            masks.append("underline")
        if self.color is not None:
            base["foregroundColor"] = self.color.to_sheet_style()
            masks.append("foregroundColor")
        if self.font is not None:
            base["fontFamily"] = self.font
            masks.append("fontFamily")
        if self.link is not None:
            base["link"] = {"url": self.link}
            masks.append("link")
        return masks, base

    @classmethod
    def from_doc_style(cls, info):
        if info.get("namedStyleType"):
            return DefaultFonts[info.get("namedStyleType")]
        return TextStyle(
            font_size=info.get("fontSize", {}).get("magnitude"),
            font_unit=info.get("fontSize", {}).get("unit"),
            font=info.get("font"),
            font_weight=info.get("fonWeight"),
            color=doc_color_to_color(info.get("color")),
            background_color=doc_color_to_color(info.get("backgroundColor")),
            bold=info.get("bold"),
            italic=info.get("italic"),
            strikethrough=info.get("strikethrough"),
            small_caps=info.get("smallCaps"),
            underline=info.get("underline"),
            link=doc_link_to_link(info.get("link")),
        )

class BorderStyle(Enum):
    NONE = "NONE"
    DOTTED = "DOTTED"
    DASHED = "DASHED"
    SOLID = "SOLID"
    SOLID_MEDIUM = "SOLID_MEDIUM"
    SOLID_THICK = "SOLID_THICK"
    DOUBLE = "DOUBLE"

class BorderPosition(Enum):
    TOP = "top"
    BOTTOM = "bottom"
    LEFT = "left"
    RIGHT = "right"
    INNER_HORIZONTAL = "innerHorizontal"
    INNER_VERTICAL = "innerVertical"

@dataclass
class Border:
    """Dataclass to represent a Border object for a border along a cell.

    Google documentation: https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/cells#Border
    """

    BORDER_STYLES = ["NONE", "DOTTED", "DASHED", "SOLID", "SOLID_MEDIUM", "SOLID_THICK", "DOUBLE"]
    COLOR_STYLES = ["rgbColor", "themeColor"]
    THEME_COLOR_TYPES = [
        "TEXT",
        "BACKGROUND",
        "ACCENT1",
        "ACCENT2",
        "ACCENT3",
        "ACCENT4",
        "ACCENT5",
        "ACCENT6",
        "LINK",
    ]

    position: BorderPosition
    style: BorderStyle
    color: Color
    color_style: Optional[Union[str, Color]] = None

    def to_json(self):

        # assert self.style in self.BORDER_STYLES

        base = {"style": self.style.value, "color": self.color.to_sheet_style()}

        if self.color_style is not None:
            assert self.color_style in self.COLOR_STYLES
            if self.color_style == "rgbColor":
                assert isinstance(self.color_style, Color)
                base["colorStyle"] = self.color_style.to_sheet_style()
            elif self.color_style == "themeColor":
                assert self.color_style in self.THEME_COLOR_TYPES
                base["themeColor"] = self.color_style

        return base

class DefaultFonts(Enum):
    NORMAL_TEXT = TextStyle(font_size=11, font="arial", color=Color(hex="#000000"))
    TITLE = TextStyle(font_size=26, font="arial", color=Color(hex="#000000"))
    SUBTITLE = TextStyle(font_size=15, font="arial", color=Color(hex="#666666"))
    HEADING1 = TextStyle(font_size=20, font="arial", color=Color(hex="#000000"))
    HEADING2 = TextStyle(font_size=16, font="arial", color=Color(hex="#666666"))
    HEADING3 = TextStyle(font_size=14, font="arial", color=Color(hex="#434343"))
    HEADING4 = TextStyle(font_size=12, font="arial", color=Color(hex="#666666"))
    HEADING5 = TextStyle(font_size=11, font="arial", color=Color(hex="#666666"))
    HEADING6 = TextStyle(font_size=11, font="arial", italic=True, color=Color(hex="#666666"))

    def to_doc_style(self):
        return self.value.to_doc_style()

@dataclass
class ParagraphStyle:
    alignment: str
    line_space: int
    direction: str
    spacing_mode: str
    space_above: str
    space_below: str
    border_between: str
    border_top: BorderStyle
    keep_lines_together: bool
    keep_with_next: bool
    avoid_widow_and_orphan: bool
    shading: str

    def to_doc_style(self) -> Tuple[str, Dict]:  # noqa: C901
        base = {}
        masks = []
        if self.alignment is not None:
            base["alignment"] = {"alignment": self.alignment}
            masks.append("alignment")
        # TODO: validate all of these
        for attr in [
            "line_space",
            "direction",
            "spacing_mode",
            "space_above",
            "space_below",
            "border_between",
            "border_top",
            "keep_lines",
            "keep_with_next",
            "avoid_widow_and_orphan",
            "shading",
        ]:
            test = getattr(self, attr)
            if test:
                components = attr.split("_")
                name = components[0] + "".join([component.title() for component in components])
                base[name] = {name: test}
                masks.append(name)
        return ",".join(masks), base

    def to_sheet_style(self) -> Tuple[list, Dict]:  # noqa: C901
        raise NotImplementedError

    @classmethod
    def from_doc_style(cls, info):
        if info.get("namedStyleType"):
            return DefaultFonts[info.get("namedStyleType")]
        return ParagraphStyle(**info)


import argparse
import re
import threading
import time
from datetime import datetime

import numpy as np
import tensorflow as tf

if hasattr(tf, 'enable_eager_execution'):
    tf.enable_eager_execution()

from waymo_open_dataset.utils import frame_utils
from waymo_open_dataset import dataset_pb2 as open_dataset

from custom_utils import poolingOverlap, projected_points_to_image, turbo_rgba
import matplotlib.pyplot as plt
from custom_utils import colorize_np_arr

import errno
import os

from concurrent.futures import ThreadPoolExecutor, as_completed

executor = None

camera_names = {
    'front': open_dataset.CameraName.FRONT,
    # 'front_left': open_dataset.CameraName.FRONT_LEFT,
    # 'front_right': open_dataset.CameraName.FRONT_RIGHT,
    # 'side_left': open_dataset.CameraName.SIDE_LEFT,
    # 'side_right': open_dataset.CameraName.SIDE_RIGHT
}

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:  # Python ≥ 2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

def get(object_list, name):
    """ Search for an object by name in an object list. """

    object_list = [obj for obj in object_list if obj.name == name]
    return object_list[0]

def depth_map_by_camera_name(frame, camera, points_all, cp_points_all):
    """

    :param frame: frame camera
    :param camera: camera name
    :param points_all:  ndarray
    :param cp_points_all: ndarray
    :return:
    """
    camera_image = get(frame.images, camera)
    # get the image
    img_np = np.zeros(shape=(1280, 1920))
    # img_tensor = tf.image.decode_jpeg(camera_image.image)
    #
    # img_np = np.zeros_like(img_tensor.numpy())[:, :, 0].astype(np.float32)
    # img_np[:] = np.nan

    # The distance between lidar points and vehicle frame origin.
    points_all_tensor = tf.norm(points_all, axis=-1, keepdims=True)
    cp_points_all_tensor = tf.constant(cp_points_all, dtype=tf.int32)

    mask = tf.equal(cp_points_all_tensor[..., 0], camera_image.name)
    cp_points_all_tensor = tf.cast(
        tf.gather_nd(
            cp_points_all_tensor, tf.where(mask)), dtype=tf.float32)
    points_all_tensor = tf.gather_nd(points_all_tensor, tf.where(mask))

    projected_points_all_from_raw_data = tf.concat(
        [cp_points_all_tensor[..., 1:3], points_all_tensor],
        axis=-1).numpy()

    coordinates = projected_points_all_from_raw_data[:, 0:2]. \
        astype(np.int32)

    img_np[coordinates[:, 1], coordinates[:, 0]] = \
        projected_points_all_from_raw_data[:, 2]

    dense_matrix = \
        poolingOverlap(img_np, ksize=(15, 10), stride=(1, 1), method=None,
                       pad=True)

    return dense_matrix

def pool_points_project_to_image(dense_matrix, img_np):

    # TODO: this is super costly
    (width, height) = img_np.shape
    dense_matrix = np.nanmean(dense_matrix, axis=(2, 3))

    xv, yv = np.meshgrid(range(0, width), range(0, height), indexing='ij')
    xv, yv = xv.reshape(-1, ), yv.reshape(-1, )
    dense_vector = dense_matrix.reshape(-1, )
    valid_coordinates = dense_vector > 0.
    xv, yv = xv[valid_coordinates], yv[valid_coordinates]
    dense_vector = dense_vector[valid_coordinates]
    projected_matrix = np.vstack([yv, xv, dense_vector]).transpose()

    # project the points to an depth_np_arr
    depth_np_arr = projected_points_to_image(
        img_np.shape, projected_matrix
    )

    return depth_np_arr

def project_image_lidar_to_image_plane(frame, camera_names):
    (range_images, camera_projections, range_image_top_pose) = \
        frame_utils.parse_range_image_and_camera_projection(frame)
    # # convert the points to a point cloud from the first laser return
    points, cp_points = frame_utils.convert_range_image_to_point_cloud(
        frame, range_images, camera_projections, range_image_top_pose,
        ri_index=0)
    # ... and the second return
    points_second, cp_points_second = frame_utils.convert_range_image_to_point_cloud(
        frame, range_images, camera_projections, range_image_top_pose,
        ri_index=1)
    #
    points_all = np.concatenate(points, axis=0)
    cp_points_all = np.concatenate(cp_points, axis=0)
    points_second_all = np.concatenate(points_second, axis=0)
    cp_points_second_all = np.concatenate(cp_points_second, axis=0)
    #
    points_all = np.concatenate([points_all, points_second_all])
    cp_points_all = np.concatenate([cp_points_all, cp_points_second_all])

    # go through all the camera images
    _image_results = {}
    for camera_name, camera_dataset in camera_names.items():
        # img_np = np.zeros(shape=(1280, 1920))
        depth_np_matrix = depth_map_by_camera_name(
            frame, camera_dataset, points_all, cp_points_all
        )

        if camera_name not in _image_results:
            _image_results[camera_name] = depth_np_matrix
        else:
            raise BaseException('Camera-Name appear twice {}'.format(camera_name))

    return _image_results

def colorize_depth_map_save_img(depth_map, output_path, frame_id):
    print('[{}] {} Start colorizing: {}'.format(
        datetime.now(), threading.get_ident(), os.path.join(
            output_path, 'frame_{:04d}'.format(frame_id))
    ))
    img_np = np.zeros(shape=(1280, 1920))
    depth_np_arr = pool_points_project_to_image(depth_map, img_np)
    print(np.max(depth_np_arr))
    colorized_depth_image = colorize_np_arr(
        depth_np_arr, turbo_rgba
    )
    _save_path = os.path.join(output_path,
                              'frame_{:05d}.png'.format(frame_id))
    colorized_depth_image.save(_save_path)

def handle_frame(frame, output_data_folder, frame_id):
    print('[{}] {} Start handling frame'.format(
        datetime.now(),
        threading.get_ident(),
        os.path.join(output_data_folder, 'frame_{:04d}'.format(frame_id))))
    _s = datetime.now()
    lidar = project_image_lidar_to_image_plane(frame, camera_names)

    for camera_name, depth_np_matrix in lidar.items():
        _fpath = os.path.join(output_data_folder, camera_name)
        if not os.path.exists(_fpath):
            mkdir_p(_fpath)

        colorize_depth_map_save_img(depth_np_matrix, _fpath, frame_id)
        # future = executor.submit(
        #     colorize_depth_map_save_img, depth_np_matrix, _fpath, frame_id
        # )
        # futures.append(future)

    print('[{}] {} End handling frame'.format(datetime.now() - _s, threading.get_ident()))

def handle_file(input_filename, output_dir):
    assert executor is not None, 'executor has to be set to None'
    input_path, input_fname = os.path.split(input_filename)
    segment_m = re.search(
        '(.+_with_camera_labels).tfrecord$', input_fname
    )
    if segment_m is None:
        return
    # create the output folder
    if not os.path.exists(os.path.join(output_dir, 'lidar_rgb')):
        os.mkdir(os.path.join(output_dir, 'lidar_rgb'))

    output_data_folder = os.path.join(output_dir, 'lidar_rgb',
                                      segment_m.group(1))
    if not os.path.exists(output_data_folder):
        os.mkdir(output_data_folder)

    try:
        dataset = tf.data.TFRecordDataset(input_filename, compression_type='')
    except BaseException as e:
        print('Error while reading file: {}'.format(e))
        return

    read_frames = []
    print('[{}] Reading all frames'.format(datetime.now()))
    _s = datetime.now()
    for data in dataset:
        frame = open_dataset.Frame()
        try:
            frame.ParseFromString(bytearray(data.numpy()))
        except BaseException as e:
            print('\tError: {}'.format(e))
        read_frames.append(frame)
    print('[{}] Read all frames. Took {}'.format(
        datetime.now(), datetime.now() - _s))

    # # TODO: dropping frames for development purposes
    # read_frames = read_frames[:5]

    print('[{}] Extract dense matrices for all frames'.format(datetime.now()))
    _s = datetime.now()
    image_matrices = []
    futures = []

    for frame_id, frame in enumerate(read_frames):
        _f = executor.submit(handle_frame, frame, output_data_folder, frame_id)
        futures.append(_f)

    #
    #
    #
    # for i, frame in enumerate(read_frames):
    #     if i % 10 == 0:
    #         print('Frame {}/{}'.format(i, len(read_frames)))
    #         continue
    #     image_matrices.append(project_image_lidar_to_image_plane(frame, camera_names))
    # print('[{}] Extracted all dense-matrices for all frames ({} / sample)'.format(
    #     datetime.now() - _s, (datetime.now() - _s) / len(read_frames)))
    #
    # print('[{}] Colorize all frames'.format(datetime.now()))
    # futures = []
    # for frame_id, image_dict in enumerate(image_matrices):
    #     for camera_name, depth_np_matrix in image_dict.items():
    #         _fpath = os.path.join(output_data_folder, camera_name)
    #         if not os.path.exists(_fpath):
    #             mkdir_p(_fpath)
    #
    #         # colorize_depth_map_save_img(depth_np_matrix, _fpath, frame_id)
    #         future = executor.submit(
    #             colorize_depth_map_save_img, depth_np_matrix, _fpath, frame_id
    #         )
    #         futures.append(future)
    #
    # print('[{}] Colorized all frames'.format(datetime.now() - _s))

    return futures

if __name__ == '__main__':
    parser = argparse.ArgumentParser('export_data')
    parser.add_argument('--input_dir', type=str, help='Input directory')
    parser.add_argument('--output_dir', type=str, help='Output directory',
                        default='./output')
    parser.add_argument('--max_workers', type=int, default=1)
    parser.add_argument('--files_to_handle', type=int, default=-1)
    args = parser.parse_args()

    max_workers = 1
    if args.max_workers != -1:
        max_workers = args.max_workers
    executor = ThreadPoolExecutor(max_workers=max_workers)

    from turbo_map import RGBToPyCmap, turbo_colormap_data

    mpl_data = RGBToPyCmap(turbo_colormap_data)
    plt.register_cmap(name='turbo', data=mpl_data,
                      lut=turbo_colormap_data.shape[0])

    mpl_data_r = RGBToPyCmap(turbo_colormap_data[::-1, :])
    plt.register_cmap(name='turbo_r', data=mpl_data_r,
                      lut=turbo_colormap_data.shape[0])

    if not os.path.exists(args.output_dir):
        print('Output directory does not exist')
        exit(1)

    input_directory = os.path.join(
        os.path.expanduser('~'), 'Downloads', 'waymodataset'
    ) if args.input_dir is None else args.input_dir
    if not os.path.exists(input_directory):
        print('Input directory does not exist')
        exit(1)

    # get the files in the directory
    files = sorted(os.listdir(input_directory))
    if args.files_to_handle > 0:
        files = files[:args.files_to_handle]

    num_files = len(files)
    print(f'Reading from {input_directory} found {num_files} files')

    # go through them
    collected_futures = {}
    for i, f in enumerate(files):
        print(f'({i}/{num_files}) Extracting lidar data from {f}')
        _input_filename = os.path.join(input_directory, f)
        futures = handle_file(_input_filename, args.output_dir)
        collected_futures[f] = futures

    for key, futures in collected_futures.items():
        print(key, sum([f.done() for f in futures]))

    executor.shutdown(wait=True)


from django.http import HttpResponse
from django.shortcuts import render, redirect, get_object_or_404
from django.views.decorators.http import require_POST
from coupons.forms import CouponApplyForm
from Books.models import Book
from .cart import Cart
from .forms import CartAddBookForm

@require_POST
def cart_add(request, book_id):
    """
    add books to basket
    :param request:
    :param book_id:
    :return:
    """
    cart = Cart(request)
    book = get_object_or_404(Book, id=book_id)
    form = CartAddBookForm(request.POST, id)
    if book.inventory > 0:
        if form.is_valid():
            cd = form.cleaned_data
            if cd['quantity'] < book.inventory:

                cart.add(book=book,
                         quantity=cd['quantity'],
                         update_quantity=cd['update'])
            else:
                return HttpResponse("کتاب مورد نظر موجود نیست")
        return redirect('cart_detail')

    else:
        return HttpResponse("کتاب مورد نظر موجود نیست")

def cart_remove(request, book_id):
    """
        remove books from basket
        :param request:
        :param book_id:
        :return:
    """
    cart = Cart(request)
    book = get_object_or_404(Book, id=book_id)
    cart.remove(book)
    return redirect('cart_detail')

def cart_detail(request):
    """
        final basket view
        :param request:
        :param book_id:
        :return:
        """
    cart = Cart(request)
    for item in cart:
        item['update_count_form'] = CartAddBookForm(initial={'quantity': item['quantity'],
                                                             'update': True})
    discount_apply_form = CouponApplyForm()
    return render(request, 'cart/cart_detail.html', {'cart': cart, 'discount_apply_form': discount_apply_form})