import pygame
import math

class Menu:
    """Menu class returns an interactive text-based menu
        drawn on a dynamic surface"""

    def __init__(self, image_name, style, items=None, header=None, selected=0):
        self.image_name     = image_name
        self.items          = items if not items == None else list()
        self.header         = header
        self.selected       = selected
        self.style          = style
        self.image          = None
        self.surface        = None
        self.rendered_items = list()

        self.image = pygame.image.load(self.image_name)

    def render_menu(self):
        width  = 0
        height = 0
        self.rendered_items = list() # clear rendered items

        # render the items and store them
        # start with the header
        if self.header is not None:
            divider = '' + ('-' * len(self.header))
            rendered_header = self.style.h_font.render(
                self.header, True, self.style.h_color)
            rendered_divider = self.style.h_font.render(
                divider, True, self.style.h_color)
            width = rendered_header.get_width()
            height += rendered_header.get_height() + rendered_divider.get_height()
            self.rendered_items.append(rendered_header)
            self.rendered_items.append(rendered_divider)
        for index in range(len(self.items)):
            if index == self.selected:
                rendered_item = self.style.s_font.render(
                    self.items[index], True, self.style.s_color)
            else:
                rendered_item = self.style.n_font.render(
                    self.items[index], True, self.style.n_color)
            self.rendered_items.append(rendered_item)
            this_width = rendered_item.get_width()
            width = this_width if this_width > width else width
            height += rendered_item.get_height()

        # calculate dimensions for rendered surface
        i = int(math.ceil(height / self.style.c_rect.height))
        j = int(math.ceil(width / self.style.c_rect.width))
        surface_width = j * self.style.c_rect.width + self.style.l_rect.width + self.style.r_rect.width
        surface_height = i * self.style.c_rect.height + self.style.t_rect.height + self.style.b_rect.height

        # create and render the surface
        self.surface = pygame.Surface((surface_width, surface_height))
        self.surface.blit(self.image, (0,0), self.style.tl_rect)
        for x in range(0, j):
            self.surface.blit(self.image, (x * self.style.c_rect.width + self.style.tl_rect.width, 0), self.style.t_rect)
        self.surface.blit(self.image, (j * self.style.c_rect.width + self.style.tl_rect.width, 0), self.style.tr_rect)
        for y in range(0, i):
            self.surface.blit(self.image, (0, y * self.style.c_rect.height + self.style.t_rect.height), self.style.l_rect)
            for x in range(0, j):
                self.surface.blit(self.image,
                    (x * self.style.c_rect.width + self.style.l_rect.width,
                    y * self.style.c_rect.height + self.style.t_rect.height), self.style.c_rect)
            self.surface.blit(self.image,
                (j * self.style.c_rect.width + self.style.l_rect.width,
                y * self.style.c_rect.height + self.style.t_rect.height), self.style.r_rect)
        self.surface.blit(self.image, (0, i * self.style.c_rect.height + self.style.t_rect.height), self.style.bl_rect)
        for x in range(0, j):
            self.surface.blit(self.image,
                (x * self.style.b_rect.width + self.style.l_rect.width,
                i * self.style.c_rect.height + self.style.t_rect.height), self.style.b_rect)
        self.surface.blit(self.image,
            (j * self.style.c_rect.width + self.style.bl_rect.width,
            i * self.style.c_rect.height + self.style.t_rect.height), self.style.br_rect)

        # draw the items on the surface
        y_offset = self.style.t_rect.height
        for item in self.rendered_items:
            x_offset = (self.surface.get_width() - item.get_width()) / 2
            self.surface.blit(item, (x_offset, y_offset))
            y_offset += item.get_height()

        return self.surface

    def move_selected(self, up=False):
        if up:
            self.selected -= 1
            if self.selected < 0:
                self.selected = len(self.items) - 1
        else:
            self.selected += 1
            if self.selected == len(self.items):
                self.selected = 0

class Item:
    """Item represents one menu item.  An Item's handler property
        is a callback function which is called when the Item is selected"""

    def __init__(self, text, handler):
        self.text       = text
        self.handler    = handler

class Style:
    """Style defines the fonts and colors used for normal and highlighted
        menu items"""

    def __init__(self, n_font, n_color, s_font, s_color,
                 tl_rect, tr_rect, bl_rect, br_rect,
                 t_rect, b_rect, l_rect, r_rect, c_rect, h_font=None, h_color=None):

        self.n_font     = n_font
        self.n_color    = n_color
        self.s_font     = s_font
        self.s_color    = s_color
        self.h_font     = h_font
        self.h_color    = h_color
        self.tl_rect    = tl_rect
        self.tr_rect    = tr_rect
        self.bl_rect    = bl_rect
        self.br_rect    = br_rect
        self.t_rect     = t_rect
        self.b_rect     = b_rect
        self.l_rect     = l_rect
        self.r_rect     = r_rect
        self.c_rect     = c_rect


from django.conf import settings
from django.contrib.sessions.middleware import SessionMiddleware
from django.http import HttpResponse
from django.utils import timezone
from django.utils.deprecation import MiddlewareMixin

from users.models import User

class LoginRequiredMiddleware(MiddlewareMixin):
    admin_path = '/admin/'

    def process_view(self, request, callback, callback_args, callback_kwargs):
        login_required = not (
                getattr(callback, 'login_not_required', False) or request.path.startswith(self.admin_path),
        )

        if not (request.user.is_authenticated or getattr(request, '_force_auth_user', False)) and login_required:
            return HttpResponse(status=401, content='The user must be authorized.')

class HeaderSessionMiddleware(SessionMiddleware):
    cookie_name = 'HTTP_' + settings.SESSION_COOKIE_NAME.upper()

    def process_request(self, request):
        session_key = request.META.get(self.cookie_name, None)
        if session_key:
            request.session = self.SessionStore(session_key)

    def process_response(self, request, response):
        try:
            modified = request.session.modified
            empty = request.session.is_empty()
        except AttributeError:
            pass
        else:
            # First check if we need to delete this cookie.
            # The session should be deleted only if the session is entirely empty.
            if self.cookie_name in request.META and empty:
                pass
            else:
                if (modified or settings.SESSION_SAVE_EVERY_REQUEST) and not empty:
                    if response.status_code != 500:
                        request.session.save()
                        response[settings.SESSION_COOKIE_NAME] = request.session.session_key
        return response

class UpdateLastActivityMiddleware(MiddlewareMixin):

    def process_request(self, request):
        assert hasattr(request, 'user'), ('The UpdateLastActivityMiddleware requires '
                                          'authentication middleware to be installed.')

        if request.user.is_authenticated:
            User.objects.filter(id=request.user.id).update(last_activity=timezone.now())


"""
Tests for each of the workflow components try to avoid specific openeye functions.
"""
from typing import Dict, List

import pytest
from openff.toolkit.topology import Molecule
from openff.toolkit.utils.toolkits import OpenEyeToolkitWrapper, RDKitToolkitWrapper

from openff.qcsubmit import workflow_components
from openff.qcsubmit.datasets import ComponentResult
from openff.qcsubmit.exceptions import (
    ComponentRegisterError,
    InvalidWorkflowComponentError,
)
from openff.qcsubmit.utils import check_missing_stereo, get_data
from openff.qcsubmit.validators import check_torsion_connection
from openff.qcsubmit.workflow_components import (
    CustomWorkflowComponent,
    ToolkitValidator,
    deregister_component,
    get_component,
    list_components,
    register_component,
)

def get_container(molecules: [Molecule]) -> ComponentResult:
    """
    Make and return a new molecule container.
    """

    return ComponentResult(
        component_name="intial", component_description={"description": "initial filter"}, molecules=molecules,
        component_provenance={"test": "test component"})

def get_stereoisomers():
    """
    Get a set of molecules that all have some undefined stereochemistry.
    """
    mols = Molecule.from_file(get_data("stereoisomers.smi"), allow_undefined_stereo=True)

    return mols

def get_tautomers():
    """
    Get a set of molecules that all have tautomers
    """

    mols = Molecule.from_file(get_data("tautomers_small.smi"), allow_undefined_stereo=True)

    return mols

def test_list_components():
    """
    Make sure that all registered components are returned.
    """
    components = list_components()
    for component in components:
        assert component.component_name == component.__class__.__name__

def test_register_component_replace():
    """
    Test registering a component that is already registered with and without using replace.
    """
    # get the standard conformer generator
    gen = workflow_components.StandardConformerGenerator(max_conformers=1)

    with pytest.raises(ComponentRegisterError):
        register_component(component=gen, replace=False)

    # now register using replace with a new default
    register_component(component=gen, replace=True)

    gen2 = get_component(gen.component_name)
    assert gen2.max_conformers == gen.max_conformers

    # now return th list back to normal
    register_component(component=workflow_components.StandardConformerGenerator(), replace=True)

def test_register_component_error():
    """
    Make sure an error is raised if we try and register a component that is not a sub class of CustomWorkflowComponent.
    """
    # fake component
    charge_filter = {"component_name": "charge_filter"}

    with pytest.raises(InvalidWorkflowComponentError):
        register_component(component=charge_filter)

@pytest.mark.parametrize("component", [
    pytest.param(workflow_components.SmartsFilter(), id="Class instance"),
    pytest.param("SmartsFilter", id="Class name")
])
def test_deregister_component(component):
    """
    Make sure we can deregister components via name or class.
    """
    # deregister the component
    deregister_component(component=component)

    assert workflow_components.SmartsFilter() not in list_components()

    # now add it back
    register_component(component=workflow_components.SmartsFilter())

def test_deregister_component_error():
    """
    Make sure an error is raised when we try to remove a component that was not registered first.
    """

    with pytest.raises(ComponentRegisterError):
        deregister_component(component="ChargeFilter")

def test_get_component():
    """
    Make sure the correct component is returned when requested.
    """
    gen = get_component("standardconformergenerator")

    assert gen == workflow_components.StandardConformerGenerator()

def test_get_component_error():
    """
    Make sure an error is rasied when we try to get a component that was not registered.
    """
    with pytest.raises(ComponentRegisterError):
        get_component(component_name="ChargeFilter")

def test_custom_component():
    """
    Make sure users can not use custom components unless all method are implemented.
    """

    with pytest.raises(TypeError):
        test = CustomWorkflowComponent()

    class TestComponent(CustomWorkflowComponent):

        component_name = "Test component"
        component_description = "Test component"
        component_fail_message = "Test fail"

        def _apply(self, molecules: List[Molecule]) -> ComponentResult:
            pass

        def provenance(self) -> Dict:
            return {"test": "version1"}

        @staticmethod
        def is_available() -> bool:
            return True

    test = TestComponent()
    assert test.component_name == "Test component"
    assert test.component_description == "Test component"
    assert test.component_fail_message == "Test fail"
    assert {"test": "version1"} == test.provenance()

@pytest.mark.parametrize(
    "toolkit",
    [
        pytest.param(("openeye", OpenEyeToolkitWrapper), id="openeye"),
        pytest.param(("rdkit", RDKitToolkitWrapper), id="rdkit"),
    ],
)
def test_toolkit_mixin(toolkit):
    """
    Make sure the pydantic ToolkitValidator mixin is working correctly, it should provide provenance and toolkit name
    validation.
    """

    toolkit_name, toolkit_class = toolkit
    if toolkit_class.is_available():

        class TestClass(ToolkitValidator, CustomWorkflowComponent):
            """Should not need to implement the provenance."""

            component_name = "ToolkitValidator"
            component_description = "ToolkitValidator test class."
            component_fail_message = "Test fail"

            def _apply(self, molecules: List[Molecule]) -> ComponentResult:
                pass

        test = TestClass()
        with pytest.raises(ValueError):
            test.toolkit = "ambertools"

        test.toolkit = toolkit_name
        prov = test.provenance()
        assert toolkit_name in prov
        assert "openff-toolkit" in prov

    else:
        pytest.skip(f"Toolkit {toolkit_name} not avilable.")

@pytest.mark.parametrize(
    "toolkit",
    [
        pytest.param(("openeye", OpenEyeToolkitWrapper), id="openeye"),
        pytest.param(("rdkit", RDKitToolkitWrapper), id="rdkit"),
    ],
)
def test_standardconformer_generator_validators(toolkit):
    """
    Test the standard conformer generator which calls the OFFTK.
    """

    toolkit_name, toolkit_class = toolkit
    if toolkit_class.is_available():

        conf_gen = workflow_components.StandardConformerGenerator()

        # make sure we are casting to ints
        conf_gen.max_conformers = 1.02
        assert conf_gen.max_conformers == 1

        # test the toolkit validator
        with pytest.raises(ValueError):
            conf_gen.toolkit = "ambertools"
        conf_gen.toolkit = toolkit_name
        assert toolkit_name in conf_gen.provenance()

        conf_gen.clear_existing = "no"
        assert conf_gen.clear_existing is False

        assert "openff-toolkit" in conf_gen.provenance()

    else:
        pytest.skip(f"Toolkit {toolkit} not available.")

def test_element_filter_validators():
    """
    Make sure the element filter validators are working.
    """

    elem_filter = workflow_components.ElementFilter()

    with pytest.raises(KeyError):
        elem_filter.allowed_elements = ["carbon", "hydrogen"]

    elem_filter.allowed_elements = [1.02, 2.02, 3.03]

    assert elem_filter.allowed_elements == [1, 2, 3]

    assert "openmm_elements" in elem_filter.provenance()

def test_weight_filter_validator():
    """
    Make sure the weight filter simple validators work.
    """
    weight = workflow_components.MolecularWeightFilter()

    weight.minimum_weight = 0.0
    assert weight.minimum_weight == 0

    assert "openmm_units" in weight.provenance()
    assert "openff-toolkit" in weight.provenance()

def test_weight_filter_apply():
    """
    Make sure the weight filter returns molecules within the limits.
    """

    weight = workflow_components.MolecularWeightFilter()
    weight.minimum_weight = 0
    weight.maximum_weight = 80

    molecules = get_tautomers()

    result = weight.apply(molecules, processors=1)
    assert result.n_molecules == 14
    assert result.n_filtered == 36

@pytest.mark.parametrize(
    "data",
    [
        pytest.param((workflow_components.ElementFilter, "allowed_elements", [1, 10, 100]), id="ElementFilter"),
        pytest.param((workflow_components.MolecularWeightFilter, "minimum_weight", 1), id="WeightFilter"),
        pytest.param((workflow_components.StandardConformerGenerator, "max_conformers", 1), id="StandardConformers"),
        pytest.param((workflow_components.EnumerateTautomers, "max_tautomers", 2), id="EnumerateTautomers"),
        pytest.param((workflow_components.EnumerateStereoisomers, "undefined_only", True), id="EnumerateStereoisomers"),
        pytest.param((workflow_components.RotorFilter, "maximum_rotors", 3), id="RotorFilter"),
        pytest.param((workflow_components.SmartsFilter, "allowed_substructures", ["[C:1]-[C:2]"]), id="SmartsFilter"),
        pytest.param((workflow_components.WBOFragmenter, "threshold", 0.5), id="WBOFragmenter"),
        pytest.param((workflow_components.EnumerateProtomers, "max_states", 5), id="EnumerateProtomers"),
        pytest.param((workflow_components.RMSDCutoffConformerFilter, "cutoff", 1.2), id="RMSDCutoffConformerFilter")
    ],
)
def test_to_from_object(data):
    """
    Test changing a attribute of the class and making another class by parsing the object, this replicates building
    the workflow from file which is also tested in test_factories.py
    """

    component, attribute, value = data

    active_comp = component()
    # set the attribute
    setattr(active_comp, attribute, value)
    # make a new comp from parsing
    copy_comp = component.parse_obj(active_comp)

    assert copy_comp == active_comp

def test_rmsd_filter():
    """
    Test the RMSD conformer filter method.
    """
    import copy

    from simtk import unit

    rmsd_filter = workflow_components.RMSDCutoffConformerFilter(cutoff=1)
    mol = Molecule.from_smiles("CCCC")
    # make a lot of conformers for the molecule
    mol.generate_conformers(n_conformers=1000, rms_cutoff=0.5 * unit.angstrom, toolkit_registry=RDKitToolkitWrapper())
    ref_mol = copy.deepcopy(mol)
    result = rmsd_filter.apply([mol, ], processors=1)
    # now make sure the number of conformers is different
    assert result.molecules[0].n_conformers != ref_mol.n_conformers

def test_rmsd_filter_no_conformers():
    """
    Make sure the molecule is failed when no conformers are present.
    """
    rmsd_filter = workflow_components.RMSDCutoffConformerFilter(cutoff=1)
    mol = Molecule.from_smiles("CCCC")
    result = rmsd_filter.apply([mol, ], processors=1)
    # now make sure the number of conformers is different
    assert result.n_molecules == 0
    assert result.n_filtered == 1

@pytest.mark.parametrize(
    "toolkit",
    [
        pytest.param(("openeye", OpenEyeToolkitWrapper), id="openeye"),
        pytest.param(("rdkit", RDKitToolkitWrapper), id="rdkit"),
    ],
)
def test_conformer_apply(toolkit):
    """
    Test applying the standard conformer generator to a workflow.
    """

    toolkit_name, toolkit_class = toolkit
    if toolkit_class.is_available():

        conf_gen = workflow_components.StandardConformerGenerator()
        conf_gen.toolkit = toolkit_name
        conf_gen.max_conformers = 1
        conf_gen.clear_existing = True

        mols = get_stereoisomers()
        # remove duplicates from the set
        molecule_container = get_container(mols)

        result = conf_gen.apply(molecule_container.molecules, processors=1)

        assert result.component_name == conf_gen.component_name
        assert result.component_description == conf_gen.dict()
        # make sure each molecule has a conformer that passed
        for molecule in result.molecules:
            assert molecule.n_conformers == 1, print(molecule.conformers)

        for molecule in result.filtered:
            assert molecule.n_conformers == 0

    else:
        pytest.skip(f"Toolkit {toolkit_name} not available.")

def test_elementfilter_apply():
    """
    Test applying the element filter to a workflow.
    """

    elem_filter = workflow_components.ElementFilter()
    elem_filter.allowed_elements = [1, 6, 7, 8]

    mols = get_tautomers()

    result = elem_filter.apply(mols, processors=1)

    assert result.component_name == elem_filter.component_name
    assert result.component_description == elem_filter.dict()
    # make sure there are no unwanted elements in the pass set
    for molecue in result.molecules:
        for atom in molecue.atoms:
            assert atom.atomic_number in elem_filter.allowed_elements

    for molecue in result.filtered:
        elements = set([atom.atomic_number for atom in molecue.atoms])
        assert sorted(elements) != sorted(elem_filter.allowed_elements)

@pytest.mark.parametrize(
    "toolkit",
    [
        pytest.param(("openeye", OpenEyeToolkitWrapper), id="openeye"),
        pytest.param(("rdkit", RDKitToolkitWrapper), id="rdkit"),
    ],
)
def test_enumerating_stereoisomers_validator(toolkit):
    """
    Test the validators in enumerating stereoisomers.
    """

    toolkit_name, toolkit_class = toolkit
    if toolkit_class.is_available():

        enumerate_stereo = workflow_components.EnumerateStereoisomers()
        with pytest.raises(ValueError):
            enumerate_stereo.toolkit = "ambertools"

        enumerate_stereo.toolkit = toolkit_name
        assert toolkit_name in enumerate_stereo.provenance()

        enumerate_stereo.undefined_only = "y"
        assert enumerate_stereo.undefined_only is True

        enumerate_stereo.max_isomers = 1.1
        assert enumerate_stereo.max_isomers == 1

    else:
        pytest.skip(f"Toolkit {toolkit_name} is not available.")

@pytest.mark.parametrize(
    "toolkit",
    [
        pytest.param(("openeye", OpenEyeToolkitWrapper), id="openeye"),
        pytest.param(("rdkit", RDKitToolkitWrapper), id="rdkit"),
    ],
)
def test_enumerating_stereoisomers_apply(toolkit):
    """
    Test the stereoisomer enumeration.
    """

    toolkit_name, toolkit_class = toolkit
    if toolkit_class.is_available():

        enumerate_stereo = workflow_components.EnumerateStereoisomers()
        # set the options
        enumerate_stereo.toolkit = toolkit_name
        enumerate_stereo.undefined_only = True
        enumerate_stereo.rationalise = True

        mols = get_stereoisomers()

        result = enumerate_stereo.apply(mols, processors=1)
        for mol in mols:
            assert mol in result.molecules

        # make sure no molecules have undefined stereo
        for molecule in result.molecules:
            assert Molecule.from_smiles(molecule.to_smiles(), toolkit_registry=RDKitToolkitWrapper()) == molecule
            assert molecule.n_conformers >= 1

    else:
        pytest.skip(f"Toolkit {toolkit_name} is not available.")

@pytest.mark.parametrize(
    "toolkit",
    [
        pytest.param(("openeye", OpenEyeToolkitWrapper), id="openeye"),
        pytest.param(("rdkit", RDKitToolkitWrapper), id="rdkit"),
    ],
)
def test_enumerating_stereoisomers_poor_input(toolkit):
    """
    Test stereoisomer enumeration with an impossible stereochemistry.
    """

    toolkit_name, toolkit_class = toolkit
    if toolkit_class.is_available():
        molecule = Molecule.from_smiles("C=CCn1c([C@@H]2C[C@@H]3CC[C@@H]2O3)nnc1N1CCN(c2ccccc2)CC1")
        enumerate_stereo = workflow_components.EnumerateStereoisomers()
        # the molecule should fail conformer generation
        enumerate_stereo.toolkit = toolkit_name
        enumerate_stereo.undefined_only = True
        enumerate_stereo.rationalise = True

        result = enumerate_stereo.apply(molecules=[molecule, ], processors=1)
        assert result.n_molecules == 0
        assert result.n_filtered == 1

        # now turn of rationalise
        enumerate_stereo.rationalise = False
        result = enumerate_stereo.apply([molecule, ], processors=1)
        assert molecule in result.molecules
        assert result.n_molecules == 1

        # now enumerate all stereo and rationalise
        enumerate_stereo.rationalise = True
        enumerate_stereo.undefined_only = False
        # make sure the input is missing and new isomers are found

        result = enumerate_stereo.apply([molecule, ], processors=1)
        assert molecule not in result.molecules
        assert molecule in result.filtered

    else:
        pytest.skip(f"Toolkit {toolkit_name} is not available.")

@pytest.mark.parametrize("data", [
    pytest.param(("[H]C(=C([H])Cl)Cl", True), id="Molecule with missing stereo"),
    pytest.param(("[H]c1c(c(c(c(c1N([C@@]([H])(O[H])SC([H])([H])[C@]([H])(C(=O)N([H])C([H])([H])C(=O)O[H])N([H])C(=O)C([H])([H])C([H])([H])[C@@]([H])(C(=O)O[H])N([H])[H])O[H])[H])[H])I)[H]", False), id="Molecule with good stereo")
])
def test_check_missing_stereo(data):
    """
    Make sure that molecules with missing stereo chem are correctly identified.
    """
    smiles, result = data
    molecule = Molecule.from_smiles(smiles=smiles, allow_undefined_stereo=True)
    assert result is check_missing_stereo(molecule=molecule)

@pytest.mark.parametrize(
    "toolkit",
    [
        pytest.param(("openeye", OpenEyeToolkitWrapper), id="openeye"),
        pytest.param(("rdkit", RDKitToolkitWrapper), id="rdkit"),
    ],
)
def test_enumerating_tautomers_apply(toolkit):
    """
    Test enumerating tautomers and make sue the input molecule is also returned.
    """

    toolkit_name, toolkit_class = toolkit
    if toolkit_class.is_available():

        enumerate_tauts = workflow_components.EnumerateTautomers()
        enumerate_tauts.toolkit = toolkit_name
        enumerate_tauts.max_tautomers = 2

        mols = get_tautomers()

        result = enumerate_tauts.apply(mols, processors=1)

        # check the input molecule is present
        for mol in mols:
            assert mol in result.molecules

        assert result.n_molecules > len(mols)

    else:
        pytest.skip(f"Toolkit {toolkit_name} is not available.")

@pytest.mark.parametrize(
    "toolkit",
    [
        pytest.param(("openeye", OpenEyeToolkitWrapper), id="openeye"),
        pytest.param(("rdkit", RDKitToolkitWrapper), id="rdkit"),
    ],
)
def test_enumerating_tautomers_validator(toolkit):
    """
    Test the validators in enumerating tautomers.
    """

    toolkit_name, toolkit_class = toolkit

    if toolkit_class.is_available():

        enumerate_tautomers = workflow_components.EnumerateTautomers()

        with pytest.raises(ValueError):
            enumerate_tautomers.toolkit = "ambertools"

        enumerate_tautomers.toolkit = toolkit_name
        enumerate_tautomers.max_tautomers = 1.1
        assert enumerate_tautomers.max_tautomers == 1

        assert toolkit_name in enumerate_tautomers.provenance()

    else:
        pytest.skip(f"Toolkit {toolkit_name} not available.")

def test_enumerating_protomers_apply():
    """
    Test enumerating protomers which is only availabe in openeye.
    """

    enumerate_protomers = workflow_components.EnumerateProtomers(max_states=2)
    assert enumerate_protomers.is_available()

    with pytest.raises(ValueError):
        # make sure rdkit is not allowed here
        enumerate_protomers.toolkit = "rdkit"

    mol = Molecule.from_smiles('Oc2ccc(c1ccncc1)cc2')
    result = enumerate_protomers.apply([mol, ], processors=1)

    assert mol in result.molecules
    # this means that the parent molecule was included
    assert result.n_molecules == 3

def test_coverage_filter():
    """
    Make sure the coverage filter removes the correct molecules.
    """
    from openff.toolkit.typing.engines.smirnoff import ForceField

    coverage_filter = workflow_components.CoverageFilter()
    coverage_filter.allowed_ids = ["b83"]
    coverage_filter.filtered_ids = ["b87"]

    mols = get_stereoisomers()

    # we have to remove duplicated records
    # remove duplicates from the set
    molecule_container = get_container(mols)
    result = coverage_filter.apply(molecule_container.molecules, processors=1)

    forcefield = ForceField("openff_unconstrained-1.0.0.offxml")
    # now see if any molecules do not have b83
    parameters_by_id = {}
    for molecule in result.molecules:
        labels = forcefield.label_molecules(molecule.to_topology())[0]
        covered_types = set(
            [label.id for types in labels.values() for label in types.values()]
        )
        # now store the smiles under the ids
        for parameter in covered_types:
            parameters_by_id.setdefault(parameter, []).append(molecule.to_smiles())

    expected = parameters_by_id["b83"]
    for molecule in result.molecules:
        assert molecule.to_smiles() in expected
        assert "dihedrals" not in molecule.properties

    # we now need to check that the molecules passed contain only the allowed atoms
    # do this by running the component again
    result2 = coverage_filter.apply(result.molecules, processors=1)
    assert result2.n_filtered == 0
    assert result.n_molecules == result.n_molecules

def test_coverage_filter_tag_dihedrals():
    """
    Make sure the coverage filter tags dihedrals that we request.
    """

    coverage_filter = workflow_components.CoverageFilter()
    coverage_filter.allowed_ids = ["t1"]
    coverage_filter.tag_dihedrals = True

    mols = get_tautomers()

    # we have to remove duplicated records
    # remove duplicates from the set
    molecule_container = get_container(mols)

    result = coverage_filter.apply(molecule_container.molecules, processors=1)

    for molecule in result.molecules:
        assert "dihedrals" in molecule.properties
        torsion_indexer = molecule.properties["dihedrals"]
        assert torsion_indexer.n_torsions > 0, print(molecule)
        assert torsion_indexer.n_double_torsions == 0
        assert torsion_indexer.n_impropers == 0

def test_fragmentation_settings():
    """
    Make sure the settings are correctly handled.
    """

    fragmenter = workflow_components.WBOFragmenter()
    with pytest.raises(ValueError):
        fragmenter.functional_groups = get_data("functional_groups_error.yaml")

    fragmenter.functional_groups = get_data("functional_groups.yaml")

    assert fragmenter.functional_groups is not None

def test_fragmentation_apply():
    """
    Make sure that fragmentation is working.
    """
    fragmenter = workflow_components.WBOFragmenter()
    assert fragmenter.is_available()
    # check that a molecule with no rotatable bonds fails if we dont want the parent back
    benzene = Molecule.from_file(get_data("benzene.sdf"), "sdf")
    result = fragmenter.apply([benzene, ], processors=1)
    assert result.n_molecules == 0

    # now try ethanol
    ethanol = Molecule.from_file(get_data("methanol.sdf"), "sdf")
    fragmenter.include_parent = True
    result = fragmenter.apply([ethanol, ], processors=1)
    assert result.n_molecules == 1

    # now try a molecule which should give fragments
    diphenhydramine = Molecule.from_smiles("O(CCN(C)C)C(c1ccccc1)c2ccccc2")
    fragmenter.include_parent = False
    result = fragmenter.apply([diphenhydramine, ], processors=1)
    assert result.n_molecules == 4
    for molecule in result.molecules:
        assert "dihedrals" in molecule.properties

def test_rotor_filter_pass():
    """
    Make sure the rotor filter removes the correct molecules.
    """

    rotor_filter = workflow_components.RotorFilter()
    rotor_filter.maximum_rotors = 3

    mols = get_tautomers()

    # we have to remove duplicated records
    molecule_container = get_container(mols)
    result = rotor_filter.apply(molecule_container.molecules, processors=1)
    for molecule in result.molecules:
        assert len(molecule.find_rotatable_bonds()) <= rotor_filter.maximum_rotors

def test_rotor_filter_fail():
    """
    Test filtering out molecules with too many rotatable bonds.
    """

    rotor_filter = workflow_components.RotorFilter()
    rotor_filter.maximum_rotors = 1

    mols = get_tautomers()

    molecule_container = get_container(mols)
    result = rotor_filter.apply(molecule_container.molecules, processors=1)
    for molecule in result.molecules:
        assert len(molecule.find_rotatable_bonds()) <= rotor_filter.maximum_rotors
    for molecule in result.filtered:
        assert len(molecule.find_rotatable_bonds()) > rotor_filter.maximum_rotors

def test_smarts_filter_validator():
    """
    Make sure the validator is checking the allowed and filtered fields have valid smirks strings.
    """

    from openff.toolkit.typing.chemistry import SMIRKSParsingError

    filter = workflow_components.SmartsFilter()

    with pytest.raises(SMIRKSParsingError):
        filter.allowed_substructures = [1, 2, 3, 4]

    with pytest.raises(SMIRKSParsingError):
        # bad string
        filter.allowed_substructures = ["fkebfsjb"]

    with pytest.raises(SMIRKSParsingError):
        # make sure each item is checked
        filter.allowed_substructures = ["[C:1]-[C:2]", "ksbfsb"]

    with pytest.raises(SMIRKSParsingError):
        # good smarts with no tagged atoms.
        filter.allowed_substructures = ["[C]=[C]"]

    # a good search string
    filter.allowed_substructures = ["[C:1]=[C:2]"]
    assert len(filter.allowed_substructures) == 1

def test_smarts_filter_apply_parameters():
    """
    Make sure the environment filter is correctly identifying substructures.
    """

    filter = workflow_components.SmartsFilter()

    # this should only allow C, H, N, O containing molecules through similar to the element filter
    filter.allowed_substructures = ["[C:1]", "[c:1]", "[H:1]", "[O:1]", "[N:1]"]
    filter.filtered_substructures = ["[Cl:1]", "[F:1]", "[P:1]", "[Br:1]", "[S:1]", "[I:1]", "[B:1]"]
    allowed_elements = [1, 6, 7, 8]

    molecules = get_tautomers()

    result = filter.apply(molecules, processors=1)

    assert result.component_name == filter.component_name
    assert result.component_description == filter.dict()
    # make sure there are no unwanted elements in the pass set
    for molecule in result.molecules:
        for atom in molecule.atoms:
            assert atom.atomic_number in allowed_elements, print(molecule)

    for molecule in result.filtered:
        elements = set([atom.atomic_number for atom in molecule.atoms])
        assert sorted(elements) != sorted(allowed_elements)

def test_smarts_filter_apply_none():
    """
    Make sure we get the expected behaviour when we supply None as the filter list.
    """

    filter = workflow_components.SmartsFilter()

    filter.allowed_substructures = None

    molecules = get_tautomers()

    # this should allow all molecules through
    result = filter.apply(molecules=molecules, processors=1)

    assert len(result.molecules) == len(molecules)

    # now filter the molecule set again removing aromatic carbons
    filter.filtered_substructures = ["[c:1]"]

    result2 = filter.apply(molecules=result.molecules, processors=1)

    assert len(result2.molecules) != len(result.molecules)

@pytest.mark.parametrize("tag_dihedrals", [
    pytest.param(True, id="Tag dihedrals"),
    pytest.param(False, id="Do not tag Dihedrals")
])
def test_smarts_filter_apply_tag_torsions(tag_dihedrals):
    """
    Make sure that torsions in molecules are tagged if we supply a torsion smarts a pattern.
    """
    filter = workflow_components.SmartsFilter(tag_dihedrals=tag_dihedrals)

    # look for methyl torsions here
    filter.allowed_substructures = ["[*:1]-[*:2]-[#6H3:3]-[#1:4]"]

    molecules = get_tautomers()
    # this should filter and tag the dihedrals
    result = filter.apply(molecules, processors=1)
    for molecule in result.molecules:
        if tag_dihedrals:
            assert "dihedrals" in molecule.properties
            # make sure the torsion is connected
            torsions = molecule.properties["dihedrals"]
            for torsion in torsions.get_dihedrals:
                _ = check_torsion_connection(torsion=torsion.torsion1, molecule=molecule)
        else:
            assert "dihedrals" not in molecule.properties


"""Execute raw SQL queries against an SQLAlchemy engine."""
from typing import List, Optional

from sqlalchemy import text
from sqlalchemy.engine.base import Engine

from logger import LOGGER

def fetch_job_listings(engine: Engine) -> Optional[List[dict]]:
    """
    Select rows from database and parse as list of dicts.

    :param engine: Database engine to handle raw SQL queries.
    :type engine: engine

    :return: Optional[List[dict]]
    """
    result = engine.execute(
        text(
            "SELECT job_id, agency, business_title, \
            salary_range_from, salary_range_to \
            FROM nyc_jobs ORDER BY RAND() LIMIT 10;"
        )
    )
    rows = [dict(row) for row in result.fetchall()]
    LOGGER.info(f"Selected {result.rowcount} rows: {rows}")
    return rows

def update_job_listing(engine: Engine) -> Optional[List[dict]]:
    """
    Update row in database with problematic characters escaped.

    :param engine: Engine object representing a SQL database.
    :type engine: engine

    :return: Optional[List[dict]]
    """
    result = engine.execute(
        text(
            "UPDATE nyc_jobs SET business_title = 'Senior QA Scapegoat 🏆', \
            job_category = 'Information? <>!#%%Technology!%%#^&%* & Telecom' \
            WHERE job_id = 229837;"
        )
    )
    LOGGER.info(
        f"Selected {result.rowcount} row: \
        {result}"
    )
    return result.rowcount


#!/usr/bin/python3
# -*- coding: utf-8 -*-
'''
RENDER_MODEL_VIEWS.py
brief:
    render projections of a 3D model from viewpoints specified by an input parameter file
usage:
	blender blank.blend --background --python render_model_views.py -- <shape_obj_filename> <shape_category_synset> <shape_model_md5> <shape_view_param_file> <syn_img_output_folder>

inputs:
       <shape_obj_filename>: .obj file of the 3D shape model
       <shape_category_synset>: synset string like '03001627' (chairs)
       <shape_model_md5>: md5 (as an ID) of the 3D shape model
       <shape_view_params_file>: txt file - each line is '<azimith angle> <elevation angle> <in-plane rotation angle> <distance>'
       <syn_img_output_folder>: output folder path for rendered images of this model

author: hao su, charles r. qi, yangyan li
'''

import os
import bpy
import sys
import random
import numpy as np
import bpy
from bpy_extras.object_utils import world_to_camera_view
import bmesh
from mathutils import Vector, Matrix
from mathutils.geometry import intersect_ray_tri
import math
from collections import OrderedDict

# Load rendering light parameters
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.dirname(BASE_DIR))
from global_variables import *
from render_pipeline.blender_utils import *

light_num_lowbound = g_syn_light_num_lowbound
light_num_highbound = g_syn_light_num_highbound
light_dist_lowbound = g_syn_light_dist_lowbound
light_dist_highbound = g_syn_light_dist_highbound

render_scale = bpy.context.scene.render.resolution_percentage / 100
render_size = (
            int(bpy.context.scene.render.resolution_x * render_scale),
            int(bpy.context.scene.render.resolution_y * render_scale),
            )

def project_by_object_utils(cam, point):
    scene = bpy.context.scene
    co_2d = world_to_camera_view(scene, cam, point)
    return Vector((co_2d.x * render_size[0], render_size[1] - co_2d.y * render_size[1]))

# Input parameters
#'%s %s --background --python %s -- %s %s %s %s %s %s' % (g_blender_executable_path, blank_file, render_code, args.model_file, 'xxx', 'xxx', view_file, temp_dirname, args.scale)
shape_file = sys.argv[-6]
keypoint_file = sys.argv[-5]
#shape_synset = sys.argv[-5]
shape_md5 = sys.argv[-4]
shape_view_params_file = sys.argv[-3]
syn_images_folder = sys.argv[-2]
scale = float(sys.argv[-1])

if not os.path.exists(syn_images_folder):
    os.mkdir(syn_images_folder)
#syn_images_folder = os.path.join(g_syn_images_folder, shape_synset, shape_md5)
view_params = [[float(x) if i < 4 else x for i,x in enumerate(line.strip().split(' '))] for line in open(shape_view_params_file).readlines()]

if not os.path.exists(syn_images_folder):
    os.makedirs(syn_images_folder)

if shape_file[-3:] == 'ply':
    bpy.ops.import_mesh.ply(filepath=shape_file)
    bpy.ops.transform.resize(value=(scale, scale, scale))
elif shape_file[-3:] == 'dae':
    bpy.ops.wm.collada_import(filepath=shape_file)
elif shape_file[-3:] == 'obj':
    bpy.ops.import_scene.obj(filepath=shape_file)

scene = bpy.context.scene
#obj = bpy.data.meshes[shape_file.split('/')[-1][:-4]]
#print(obj.vertices[0].co)
# needed to rescale 2d coordinates
render = scene.render

bpy.context.scene.render.alpha_mode = 'TRANSPARENT'
#bpy.context.scene.render.use_shadows = False
#bpy.context.scene.render.use_raytrace = False

bpy.data.objects['Lamp'].data.energy = 0

#m.subsurface_scattering.use = True

camObj = bpy.data.objects['Camera']
# camObj.data.lens_unit = 'FOV'
# camObj.data.angle = 0.2

# Recipe to merge meshes as CAD models may have more than one mesh
objs = []
for ob in bpy.context.scene.objects:
    ob.select = False
    if ob.type == 'MESH':
        objs.append(ob)
objs = sorted(objs, key=lambda ob: ob.name)
for ob in objs:
    ob.select = True
    bpy.context.scene.objects.active = ob

bpy.ops.object.join()

# Find mesh name
mesh_name = [x.name for x in bpy.data.objects if x.name not in ['Camera','Lamp']][0]

# Push all vertices in a dict
vert_visibility_dict = {}
obj = bpy.data.objects[mesh_name]
verts = [obj.matrix_world*vert.co for vert in obj.data.vertices]
for vert in verts:
    vert_str = "{:.40f},{:.40f},{:.40f}".format(vert[0],vert[1], vert[2])
    vert_visibility_dict[vert_str] = (False, None)
vert_visibility_dict = OrderedDict(sorted(vert_visibility_dict.items(), key=lambda x: x[0]))
# set lights
bpy.ops.object.select_all(action='TOGGLE')
if 'Lamp' in list(bpy.data.objects.keys()):
    bpy.data.objects['Lamp'].select = True # remove default light
bpy.ops.object.delete()

# YOUR CODE START HERE
with open(keypoint_file) as f:
    kp_list =  [int(x.strip()) for x in f.readlines()]

for param in view_params:
    azimuth_deg = param[0]
    elevation_deg = param[1]
    theta_deg = -1 * param[2] # ** multiply by -1 to match pascal3d annotations **
    rho = param[3]
    img_file = param[4]
    lbl_file = param[5]
    # clear default lights
    bpy.ops.object.select_by_type(type='LAMP')
    bpy.ops.object.delete(use_global=False)

    # set environment lighting
    #bpy.context.space_data.context = 'WORLD'
    bpy.context.scene.world.light_settings.use_environment_light = True
    bpy.context.scene.world.light_settings.environment_energy = np.random.uniform(g_syn_light_environment_energy_lowbound, g_syn_light_environment_energy_highbound)
    bpy.context.scene.world.light_settings.environment_color = 'PLAIN'
    # set point lights
    for i in range(random.randint(light_num_lowbound,light_num_highbound)):
        light_azimuth_deg = np.random.uniform(g_syn_light_azimuth_degree_lowbound, g_syn_light_azimuth_degree_highbound)
        light_elevation_deg  = np.random.uniform(g_syn_light_elevation_degree_lowbound, g_syn_light_elevation_degree_highbound)
        light_dist = np.random.uniform(light_dist_lowbound, light_dist_highbound)
        lx, ly, lz = obj_centened_camera_pos(light_dist, light_azimuth_deg, light_elevation_deg)
        bpy.ops.object.lamp_add(type='POINT', view_align = False, location=(lx, ly, lz))
        bpy.data.objects['Point'].data.energy = np.random.normal(g_syn_light_energy_mean, g_syn_light_energy_std)

    cx, cy, cz = obj_centened_camera_pos(rho, azimuth_deg, elevation_deg)
    q1 = camPosToQuaternion(cx, cy, cz)
    q2 = camRotQuaternion(cx, cy, cz, theta_deg)
    q = quaternionProduct(q2, q1)
    camObj.location[0] = cx
    camObj.location[1] = cy
    camObj.location[2] = cz
    camObj.rotation_mode = 'QUATERNION'
    camObj.rotation_quaternion[0] = q[0]
    camObj.rotation_quaternion[1] = q[1]
    camObj.rotation_quaternion[2] = q[2]
    camObj.rotation_quaternion[3] = q[3]
    bpy.context.scene.update()
    vert_list_from_dict = list(vert_visibility_dict.keys())
    # use generator expressions () or list comprehensions []
    for obj in [bpy.data.objects[mesh_name]]:
        coords_2d = [project_by_object_utils(camObj, coord) for coord in verts]
        eye_location = camObj.location
        # Find visible points in current camera
        vert_indices = []
        for polygon in obj.data.polygons:
            vert_index = [p for p in polygon.vertices]
            vert_indices.append(vert_index)
        random.shuffle(vert_indices)
        for i, vert in enumerate(verts):
            vert_str = "{:.40f},{:.40f},{:.40f}".format(vert[0],vert[1], vert[2])
            if vert_list_from_dict.index(vert_str) not in kp_list:
                continue
            vis = True
            for vert_index in vert_indices:
                if i in vert_index:
                    continue
                v1 = Vector(verts[vert_index[0]])
                v2 = Vector(verts[vert_index[1]])
                v3 = Vector(verts[vert_index[2]])
                ray = Vector(eye_location-vert)
                orig = Vector(vert)

                int_pt = intersect_ray_tri(v1, v2, v3, ray, orig)
                if int_pt != None:
                    vis = False
                    break

            if vis:
                vert_visibility_dict[vert_str] = (True, coords_2d[i])
            else:
                vert_visibility_dict[vert_str] = (False, coords_2d[i])

        # 2d data printout:
        rnd = lambda i: round(i)
        #syn_label_file = 'syn_a%03d_e%03d_t%03d_d%03d.txt' % (round(azimuth_deg), round(elevation_deg), round(theta_deg), round(rho))
        render_scale = bpy.context.scene.render.resolution_percentage / 100
        render_size = (
            int(bpy.context.scene.render.resolution_x * render_scale),
            int(bpy.context.scene.render.resolution_y * render_scale),
            )
        with open(lbl_file, 'w') as f:
             for i, vert in enumerate(vert_visibility_dict.items()):
                if vert[1][1] == None:
                    key_px = "{} {} {}".format(i, -1, -1)
                else:
                    x, y = vert[1][1]
        #        key_px = "{} {} {}".format(i, rnd(x), rnd(y))
                    if vert[1][0] and (0 <= x < render_size[0]) and (0 <= y < render_size[1]):
                        key_px = "{} {} {}".format(i, rnd(x), rnd(y))
                    else:
                        key_px = "{} {} {}".format(i, -1, -1)
                f.write("%s\n"%key_px)
    #syn_image_file = 'syn_a%03d_e%03d_t%03d_d%03d.png' % (round(azimuth_deg), round(elevation_deg), round(theta_deg), round(rho))
    bpy.data.scenes['Scene'].render.filepath = os.path.join(img_file)
    bpy.ops.render.render( write_still=True )


import os
import torch.utils.data
import torchvision.transforms as transforms
import torchvision.datasets as datasets

"""
The data is available from https://ai.stanford.edu/~jkrause/cars/car_dataset.html
use bounding boxes provided to extract cars from background
"""

class StanfordCarsDataProvider:

    def __init__(self, save_path=None, train_batch_size=32, test_batch_size=200, valid_size=None, n_worker=32,
                 resize_scale=0.08, distort_color=None, image_size=224,
                 num_replicas=None, rank=None):

        norm_mean = [0.4105214534294453, 0.38574356611082533, 0.3959628699849632]
        norm_std = [0.28611458811352136, 0.2801378084154138, 0.28520087594365295]

        valid_transform = transforms.Compose([
            transforms.Resize((image_size, image_size), interpolation=3),
            transforms.CenterCrop(image_size),
            transforms.ToTensor(),
            transforms.Normalize(norm_mean, norm_std),
        ])

        valid_data = datasets.ImageFolder(os.path.join(save_path, 'valid'), valid_transform)

        self.test = torch.utils.data.DataLoader(
            valid_data, batch_size=test_batch_size, shuffle=False,
            pin_memory=True, num_workers=n_worker)


#Based on: https://nostarch.com/RaspberryPiProject

import io
import picamera
import logging
import socketserver
from threading import Condition
from http import server

PAGE="""\

<title>Raspberry Surveillance Camera</title>

Raspberry Pi - Surveillance Camera

"""

class StreamingOutput(object):
    def __init__(self):
        self.frame = None
        self.buffer = io.BytesIO()
        self.condition = Condition()

    def write(self, buf):
        if buf.startswith(b'\xff\xd8'):
            #new frame, copy the existing buffer's content and notify all
            #clients it's available
            self.buffer.truncate()
            with self.condition:
                self.frame = self.buffer.getvalue()
                self.condition.notify_all()
            self.buffer.seek(0)
        return self.buffer.write(buf)

class StreamingHandler(server.BaseHTTPRequestHandler):
    def do_GET(self):
        if self.path == '/':
            self.send_response(301)
            self.send_header('Location', '/index.html')
            self.end_headers()
        elif self.path == '/index.html':
            content = PAGE.encode('utf-8')
            self.send_response(200)
            self.send_header('Content-Type', 'text/html')
            self.send_header('Content-Length', len(content))
            self.end_headers()
            self.wfile.write(content)
        elif self.path == '/stream.mjpg':
            self.send_response(200)
            self.send_header('Age', 0)
            self.send_header('Cache-Control', 'no-cache, private')
            self.send_header('Pragma', 'no-cache')
            self.send_header('Content-Type', 'multipart/x-mixed-replace; boundary=FRAME')
            self.end_headers()
            try:
                while True:
                    with output.condition:
                        output.condition.wait()
                        frame = output.frame
                    self.wfile.write(b'--FRAME\r\n')
                    self.send_header('Content-Type', 'image/jpeg')
                    self.send_header('Content-Length', len(frame))
                    self.end_headers()
                    self.wfile.write(frame)
                    self.wfile.write(b'\r\n')
            except Exception as e:
                logging.warning(
                    'Removed streaming client %s: %s',
                    self.client_address, str(e))
        else:
            self.send_error(404)
            self.end_headers()

class StreamingServer(socketserver.ThreadingMixIn, server.HTTPServer):
    allow_reuse_address = True
    daemon_threads = True

with picamera.PiCamera(resolution='640x480', framerate=24) as camera:
    output = StreamingOutput()
    camera.start_recording(output, format='mjpeg')
    try:
        address = ('', 8000)
        server = StreamingServer(address, StreamingHandler)
        server.serve_forever()
    finally:
        camera.stop_recording()


import pygame, sys

# const
SCREEN_SIZE = (800,800)
FPS = 120

# pygame setup
pygame.init()
screen = pygame.display.set_mode(SCREEN_SIZE)
clock = pygame.time.Clock()

while True:
    # track events
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
            sys.exit()

    # while body
    screen.fill((217,217,217))
    pygame.display.update()
    clock.tick(FPS) # frame per sec


import logging
import os
import tweepy

logging.basicConfig(level=logging.INFO, format='%(asctime)s | %(levelname)s | %(funcName)s |%(message)s')

TWITTER_CONSUMER_KEY = os.environ.get('TWITTER_CONSUMER_KEY')
TWITTER_CONSUMER_SECRET = os.environ.get('TWITTER_CONSUMER_SECRET')
TWITTER_ACCESS_TOKEN = os.environ.get('TWITTER_ACCESS_TOKEN')
TWITTER_ACCESS_SECRET = os.environ.get('TWITTER_ACCESS_SECRET')

def authenticate():
    consumer_key = TWITTER_CONSUMER_KEY
    consumer_secret = TWITTER_CONSUMER_SECRET
    access_token = TWITTER_ACCESS_TOKEN
    access_token_secret = TWITTER_ACCESS_SECRET
    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    api = tweepy.API(auth, wait_on_rate_limit=True,
                     wait_on_rate_limit_notify=True)
    try:
        api.verify_credentials()
        logging.info("Verified. Logging ins")
    except Exception as e:
        logging.error("Error creating API", exc_info=True)
        raise e
    print("API created")

    return api


import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import pandas as pd
import itertools

from matplotlib import cm

def plot_color_table(df,
                     axis=1,
                     title=None,
                     rev_index=None,
                     color='RdYlGn',
                     ):
    """
    Creates color coded comparison table from dataframe values (green high, red low)

    Parameters
    ----------
    df: pd.DataFrame
        DataFrame of values
    axis: int, default=1
        axis to normalize data upon
    title: str
        title for table
    rev_index: list(str)
        list of column names to reverse color coding
    color: str, default='RdYlGn'
        color palette for table

    Returns
    -------
        plot of color coded table
    """

    labels = df.values
    cdf = df.copy()
    if axis == 1:
        cdf -= cdf.mean(axis=0)
        cdf /= cdf.std(axis=0)
    else:
        cdf = cdf.transpose()
        cdf -= cdf.mean(axis=0)
        cdf /= cdf.std(axis=0)
        cdf = cdf.transpose()

    if rev_index:
        for i in rev_index:
            cdf.iloc[:, i] = 1 - cdf.iloc[:, i]

    plt.figure()
    if title:
        plt.title(title)
    sns.heatmap(cdf, cmap='RdYlGn', linewidths=0.5, annot=labels,
                fmt='0.1f', cbar=False)
    plt.xticks(rotation=0)
    plt.yticks(rotation=0)

def plot_explained_variance(ev,
                            type='bar',
                            toff=0.02,
                            boff=0.08,
                            yoff=None,
                            **kwargs,
                            ):
    """
    Plot explained variance of PCA

    Parameters
    ----------
    ev: nd.array
        explained variance values
    type: str, default='bar'
        - 'bar': bar plots for cumulative variance
        - 'line': line plot for cumulative variance
    toff: float
        top x offset for cumulative variance values on plot
    boff: float
        bottom x offset for individual variance values on plot
    yoff: float
        tottom y offset for individual variance values on plot
    **kwargs:
        kwargs for plt.plot() if type==line for cumulative variance

    Returns
    -------
    plot of explained variance
    """

    ax = pd.Series(100*ev).plot(kind='bar', figsize=(10,5),
                   color='steelblue', fontsize=13)

    ax.set_ylabel('Explained Variance')
    ax.set_yticks([0, 20, 40, 60, 80, 100])
    ax.set_xlabel('Principal Component')

    # Create a list to collect the plt.patches data.
    totals = []

    # Find the values and append to list.
    for i in ax.patches:
        totals.append(i.get_height())

    # Set individual bar lables using patch list.
    total = sum(totals)

    yoff_d = {'bar': 0, 'line': 0.2}
    yoff_d[type] = yoff_d[type] if yoff is None else yoff

    # Set individual bar lables using patch list.
    for j, i in enumerate(ax.patches):
        # Get_x pulls left or right; get_height pushes up or down.
        if j > 0:
            ax.text(i.get_x()+boff, i.get_height()+1,
                    str(round(100*ev[j], 1))+'%', fontsize=11,
                    color='dimgrey')
        ax.text(i.get_x()+toff, 100*np.cumsum(ev)[j]+1+yoff_d[type],
                str(round(100*np.cumsum(ev)[j], 1))+'%', fontsize=12,
                color='dimgrey')

    xlab = [f'PC-{i+1}' for i, _ in enumerate(ax.patches)]
    if type == 'bar':
        ax2 = pd.Series(np.cumsum(100*ev)).plot(kind='bar', figsize=(10,5),
                   color='steelblue', fontsize=8, alpha=0.25)
        for i in ax2.patches:
            totals.append(i.get_height())
        ax2.set_xticklabels(xlab, rotation='horizontal')
        ax2.xaxis.grid(False)
    elif type == 'line':
        plt.plot(np.cumsum(100*ev), ':o', ms=8, c='steelblue', alpha=0.75, **kwargs)
        ax.xaxis.grid(False)
        ax.set_xticklabels(xlab, rotation='horizontal')

def correlation(df, sort_col=None, plot=False):
    """
    Find correlation of DataFrame, plot results and/or
    return sorted correlations for a single column of
    the DataFrame.

    Parameters
    ----------
    df: pd.DataFrame
        DataFrame of input values
    sort_col: str, default=None
        column to sort correlations on. If provided,
        function returns DataFrame of results
    plot: bool, default=False
        if True plot correlation matrix

    Returns
    -------
    ret_col: pd.Series
        results for one column if sort_col is given
    plot of correlation matrix if plot=True

    """

    corr = df.corr().abs()

    if plot:
        # Plot results
        fig, ax = plt.subplots(figsize=(12, 10))
        ax.matshow(corr)
        cmap = cm.get_cmap('coolwarm', 300)
        cax = ax.imshow(corr, interpolation="nearest", cmap=cmap)
        plt.xticks(range(len(corr.columns)), corr.columns)
        plt.yticks(range(len(corr.columns)), corr.columns)
        for tick in ax.get_xticklabels():
            tick.set_rotation(45)
        cax.set_clim([0, 1])
        fig.colorbar(cax, ticks=[0, .25, .5, .75, 1])

    if ret_col:
        return corr.sort_values([col], ascending=False)[col][1:]

def plot_ROC_curve(roc,
                   fpr,
                   tpr,
                   color_palette='muted',
                   colors=None,
                   figsize=(10, 10),
                   title='ROC Curves for Studied Models',
                   ):
    """
    Plot ROC curve with AUC for multliple models.

    Parameters
    ----------
    roc: dict
        dictionary of ROC AUC values
        with model names as keys
    fpr: dict
        dictionary of false positive rate
        values with model names as keys
    roc: dict
        dictionary of true positive rate
        values with model names as keys
    color_palette: str, defualt='muted'
        name of seaborn color palette to use
    colors: list(str), default=None
        if given, use provided colors instead
        of a color palette
    figsize: tuple(int), default=(10,10)
        figure size
    title: str
        figure title

    Returns
    -------
    plot of ROC curve
    """

    plt.figure(figsize=figsize)
    sns.set_palette(color_palette, len(roc))
    for m in roc.keys():
        plt.plot(fpr[m], tpr[m], alpha=0.7, label=f'{m} (area = {roc[m]:0.3f})')
    plt.plot([0, 1], [0, 1], color='dimgrey', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    if title:
        plt.title(title)
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.legend(loc='lower right')

def plot_confusion_matrix(cm,
                          classes,
                          prec=0,
                          title='Confusion Matrix',
                          cbar=True,
                          cmap=plt.cm.Blues,
                          ):
    """
    Plots the confusion matrix with count and normalzied values.

    Parameters
    ----------
    cm: nd.array
        confusiton matrix count values
    classes: list(str)
        class labels for x-axis
    prec: int, default=0
        precision of normalized values
    title: str, default='Confusion Matrix'
        figure title
    cbar: bool, default=True
        if True include color bar in figure
    cmap: matplotlib colormap, default=plt.cm.Blues
        colormap for confusion matrix

    Returns
    -------
    plot of confusion matrix
    """

    plt.imshow(cm, interpolation='nearest', cmap=cmap)

    if title:
        plt.title(title)
    if cbar:
        plt.colorbar()

    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=0)
    plt.yticks(tick_marks, classes)

    # Normalize counts to %
    cm_norm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    thresh = cm.max() / 2.  # threshold for text color so it is visible
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, f'{cm[i, j]:.0f}\n({100*cm_norm[i, j]:.{prec}f}%)',
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True Label')
    plt.xlabel('Predicted Label')
    plt.grid(False)


# -*- coding: utf-8 -*-
"""
Created on Tue Mar 10 14:26:57 2020

@author: USER
"""
import os
import gc
import matplotlib as mpl
import matplotlib.pyplot as plt
from IPython.display import display
import cv2

import pandas as pd
import numpy as np

from PIL import Image

from skimage.feature import hog
from skimage.color import rgb2grey

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.svm import SVC

from sklearn.metrics import roc_curve, auc

def create_features(img):
    # flatten three channel color image
    color_features = img.flatten()
    # convert image to greyscale
    grey_image = rgb2grey(img)
    # get HOG features from greyscale image
    hog_features = hog(grey_image, block_norm='L2-Hys', pixels_per_cell=(16, 16))
    # combine color and hog features into a single array
    flat_features = np.hstack([color_features, hog_features])
    return flat_features

def create_feature_matrix():
    features_list = []
    directory='D:/Transformations/test'
    i=1
    for filename in os.listdir(directory):
        print(i)
        img = Image.open(os.path.join(directory, filename))
        img=np.array(img)

        # get features for image
        image_features = create_features(img)
        features_list.append(image_features)
        i+=1
    # convert list of arrays into a matrix
    gc.collect()
    feature_matrix = np.array(features_list)
    return feature_matrix

# run create_feature_matrix on our dataframe of images
feature_matrix = create_feature_matrix()
print(feature_matrix.shape)
scaler=StandardScaler()#instantiate
scaler.fit(feature_matrix) # compute the mean and standard which will be used in the next command
X_scaled=scaler.transform(feature_matrix)
pca=PCA(n_components=6)
#pca=PCA()
pca.fit(X_scaled)
X_pca=pca.transform(X_scaled)
pca_variance = pca.explained_variance_

plt.figure(figsize=(8, 6))
plt.bar(range(6), pca_variance, alpha=0.5, align='center', label='individual variance')
plt.legend()
plt.ylabel('Variance ratio')
plt.xlabel('Principal components')
plt.show()
pca=PCA(n_components=3)
#pca=PCA() z
pca.fit(X_scaled)
X_pca=pca.transform(X_scaled)
pca_variance = pca.explained_variance_
print("shape of X_pca", X_pca.shape)

ex_variance=np.var(X_pca,axis=0)
ex_variance_ratio = ex_variance/np.sum(ex_variance)
print(ex_variance_ratio )

'''plt.figure(figsize=(8,6))
plt.scatter(X_pca[:,0], X_pca[:,5], c=('red', 'blue', 'black', 'green', 'yellow', 'cyan'))
plt.show()'''

plt.matshow(pca.components_[:,:7],cmap='viridis')
plt.yticks([0,1,2],['1st Comp','2nd Comp','3rd Comp'],fontsize=10)
plt.colorbar()
#plt.xticks(range(),cancer.feature_names,rotation=65,ha='left')
plt.tight_layout()
plt.show()


import sqlite3
conect = sqlite3.connect("example.db")

# them student cach 1
sql = "INSERT INTO STUDENT(NAME, AGE, ADDRESS) VALUES('TRAN QUOC Toan',22,'Ba Dinh,tp Ho Chi Minh')"
conect.execute(sql)
conect.commit()

# them data theo cach 2
sql = "INSERT INTO STUDENT(NAME, AGE, ADDRESS) VALUES(?,?,?)"
conect.execute(sql,('VO TRUONG TOAN', 24, 'Hiep Thanh-TP Ho Chi Minh'))
conect.commit()

# # UPDAT'E data
# sql = "UPDATE STUDENT SET AGE = 200, NAME = 'NGO QUOC TUN' WHERE [ID=(1)]"
# conect.execute(sql)
# conect.commit()

# Doc sach
sql = "SELECT * FROM STUDENT"
Cursor = conect.execute(sql)
for row in Cursor :
    print("ID : ",row[0 ])
    print("NAME : ",row[1])
    print("AGE : ",row[2])
    print("ADDRESS : ",row[3])
    print("\n")

conect.close()


# Mean, Median, STD branches (Summary of deposits)
''' This function reads in the summary of deposits data and the mean, median or
    standard deviation of the distance between branches and the head quarters
    per year.

    Packages needed: Pandas, Geocoder, Numpy, Numba, Math
    '''

# Import Packages
import pandas as pd
import numpy as np
import geocoder

import os
os.chdir(r'X:\My Documents\Data\Data_SOD')

#--------------------------------------------------
# Prelim
begin_year = 2001
end_year = 2019

#--------------------------------------------------

# Function
from numba import njit, jit

@jit
def readSODFiles(use_cols,begin_year,end_year):
    '''This method reads in all the data from the Summary of Deposits and returns
        a dataframe with the selected columns only. Requires numba,jit'''
    path = r'X:/My Documents/Data/Data_SOD/'
    df_sod = pd.DataFrame()

    #Read data
    for i in range(begin_year,end_year):
        df_load = pd.read_csv(path + 'ALL_{}.csv'.format(i), encoding = 'ISO-8859-1', usecols = use_cols)
        df_sod = df_sod.append(df_load,ignore_index = True)

    return (df_sod)

use_cols = ['YEAR','RSSDID','ADDRESBR','CITYBR','STALPBR','ADDRESS','CITY',\
            'STALP','BKMO','SIMS_LATITUDE','SIMS_LONGITUDE']
df_sod = readSODFiles(use_cols,begin_year,end_year)

#--------------------------------------------------

@jit
def lookupLatLong(x):
    ''' This method looks up the addresses of all branches and head quarters in the SOD.
    To speed up the process, we will only look up the coordinates for the branches that have missing
    information for latitude and longitude, but have an address.'''
    x_nan = x[np.isnan(x.SIMS_LATITUDE)]
    x_nan.reset_index(inplace = True, drop = True)

    res = pd.DataFrame(x_nan[['YEAR','RSSDID','ADDRESBR','CITYBR','STALPBR']],\
                       columns = ['YEAR','RSSDID','ADDRESBR','CITYBR','STALPBR',\
                                  'lat_arcgis','long_arcgis'])
    address = (x_nan['ADDRESBR'] + ', ' + x_nan['CITYBR'] + ' ' + x_nan['STALPBR'])

    for i in range(x_nan.shape[0]):
        g = geocoder.arcgis(address[i])
        res.lat_arcgis[i], res.long_arcgis[i] = g.lat, g.lng

    return(res)

geo_list = lookupLatLong(df_sod)
geo_list.to_csv('SOD_all_coordinates.csv')

# Add geo_list to df_sod
df_sod = df_sod.merge(geo_list, on = ['YEAR','RSSDID','ADDRESBR','CITYBR','STALPBR'],\
             how = 'left', indicator = True)

df_sod['branch_lat'] = df_sod[['SIMS_LATITUDE','lat_arcgis']].sum(axis=1)
df_sod['branch_lng'] = df_sod[['SIMS_LONGITUDE','long_arcgis']].sum(axis=1)

#-------------------------------------------------
# Make two variables that are the lat and long for the head quarters
## First drop _merge and coordinates that are zero.
df_sod.drop(['_merge'], axis = 1, inplace = True)

## Restrict the latitude to strictly positive and longitude to strictly negative
df_sod = df_sod[(df_sod.branch_lat > 0.0) & (df_sod.branch_lng < 0.0)]

df_head = df_sod[df_sod.BKMO == 1]
df_head.rename(columns = {'branch_lat':'head_lat','branch_lng':'head_lng'}, inplace = True)
df_sod_complete = df_sod.merge(df_head[['YEAR','RSSDID','head_lat','head_lng']], on = ['YEAR','RSSDID'], how = 'left', indicator = True)

#Drop duplicates and _merge and where both lat and long are 0.0
df_sod_complete.drop(['_merge'], axis = 1, inplace = True)
df_sod_complete.drop_duplicates(subset = ['YEAR','RSSDID','ADDRESBR','CITYBR','STALPBR'], inplace = True)
df_sod_complete.reset_index(inplace = True)

df_sod_complete.to_csv('SOD_all_coordinates_trans.csv')

#------------------------------------------------
# Calculate the distance

from math import sin, cos, sqrt, atan2, radians

@jit(nopython = True, fastmath = True)
def distanceCalculator(lat1, lon1, lat2, lon2):
    ''' This method calculates the distance in km between two points on the map (calculated
        by the latitude and longitude). It uses Numby to speed up the process.

        Calculation method used: Haversine formula
    '''
    # Define the method that calculates the distance
    def haversine(lat1, lon1, lat2, lon2):
        R = 6372.8  # Earth radius in km

        phi1, phi2 = radians(lat1), radians(lat2)
        dphi       = radians(lat2 - lat1)
        dlambda    = radians(lon2 - lon1)

        a = sin(dphi/2)**2 + \
            cos(phi1)*cos(phi2)*sin(dlambda/2)**2

        return 2*R*atan2(sqrt(a), sqrt(1 - a))

    # Setup the results vector
    res = np.empty(len(lat1))

    # Loop over the vectors
    for i in range(len(lat1)):
        res[i] = haversine(lat1[i], lon1[i], lat2[i], lon2[i])

    return (res)

df_sod_complete['distance'] = distanceCalculator(np.array(df_sod_complete.branch_lat),\
                                                 np.array(df_sod_complete.branch_lng),\
                                                 np.array(df_sod_complete.head_lat),\
                                                 np.array(df_sod_complete.head_lng))

# Save the df
df_sod_complete.to_csv('SOD_all_distances.csv')


"""
Sub class to Cluster
holds methods to plot the aggregate(s)
no interactive plots
"""

import random

import matplotlib.pyplot as plt
import numpy as np
import shapely.geometry as geom
import shapely.ops as shops
from matplotlib.patches import Ellipse

import ipas.collection_from_db.cluster as clus

def auto_str(cls):
    def __str__(self):
        return "%s(%s)" % (
            type(self).__name__,
            ", ".join("%s=%s" % item for item in vars(self).items()),
        )

    cls.__str__ = __str__
    return cls

@auto_str
class PlotCluster(clus.Cluster):
    def __init__(self, cluster):
        # call parent constructor IceCluster
        super().__init__(cluster)

    def _crystal_projectxy(self, n):
        try:
            return geom.MultiPoint(self.points[n][["x", "y"]]).convex_hull
        except IndexError:
            return None

    def _crystal_projectxz(self, n):
        try:
            return geom.MultiPoint(self.points[n][["x", "z"]]).convex_hull
        except IndexError:
            return None

    def _crystal_projectyz(self, n):
        try:
            return geom.MultiPoint(self.points[n][["y", "z"]]).convex_hull
        except IndexError:
            return None

    def projectxy(self):

        polygons = [
            self._crystal_projectxy(n)
            for n in range(self.ncrystals)
            if self._crystal_projectxy(n) is not None
        ]
        return shops.cascaded_union(polygons)

    def projectxz(self):

        polygons = [
            self._crystal_projectxz(n)
            for n in range(self.ncrystals)
            if self._crystal_projectxz(n) is not None
        ]
        return shops.cascaded_union(polygons)

    def projectyz(self):

        polygons = [
            self._crystal_projectyz(n)
            for n in range(self.ncrystals)
            if self._crystal_projectyz(n) is not None
        ]
        return shops.cascaded_union(polygons)

    def _plot_crystal(self, ncrys, ax, color):
        # plots individual monomers

        x = np.zeros(27)
        y = np.zeros(27)
        z = np.zeros(27)

        X = self.points["x"][ncrys]
        Y = self.points["y"][ncrys]
        Z = self.points["z"][ncrys]

        prismind = [0, 6, 7, 1, 2, 8, 9, 3, 4, 10, 11, 5]  # prism lines
        i = 0
        for n in prismind:
            x[i] = X[n]
            y[i] = Y[n]
            z[i] = Z[n]
            i += 1

        ax.plot(x[0:12], y[0:12], z[0:12], color=color)

        i = 0
        for n in range(0, 6):  # basal face lines

            x[i + 12] = X[n]
            y[i + 12] = Y[n]
            z[i + 12] = Z[n]
            i += 1

        x[18] = X[0]
        y[18] = Y[0]
        z[18] = Z[0]

        ax.plot(x[12:19], y[12:19], z[12:19], color=color)

        i = 0
        for n in range(6, 12):  # basal face lines

            x[i + 19] = X[n]
            y[i + 19] = Y[n]
            z[i + 19] = Z[n]
            i += 1

        x[25] = X[6]
        y[25] = Y[6]
        z[25] = Z[6]

        ax.plot(x[19:26], y[19:26], z[19:26], color=color)

    #         ax.set_zticklabels([])
    #         ax.set_yticklabels([])
    #         ax.set_xticklabels([])
    #         ax.grid(False)
    #         for pos in ['right','top','bottom','left']:
    #             ax.spines[pos].set_visible(False)
    #         plt.axis('off')

    def plot_ellipsoid_surface(self, ax):

        A = self.fit_ellipsoid()
        rx, ry, rz = self.ellipsoid_axes_lengths(A)
        x, y, z = self.ellipsoid_axes_coords(rx, ry, rz)
        xell, yell, zell = self.ellipsoid_surface(A, x, y, z)
        ax.plot_surface(xell, yell, zell, cstride=1, rstride=1, alpha=0.1)
        maxXe = np.max(xell)
        minXe = np.min(xell)
        maxYe = np.max(yell)
        minYe = np.min(yell)
        maxZe = np.max(zell)
        minZe = np.min(zell)

        maxxyz = max(maxXe, maxYe, maxZe)
        minxyz = min(minXe, minYe, minZe)

        ax.set_xlim(minxyz, maxxyz)
        ax.set_ylim(minxyz, maxxyz)
        ax.set_zlim(minxyz, maxxyz)

        return xell, yell, zell

    def plot_ellipsoid_axes(self, ax, xell, yell, zell):

        ax.plot(xell[0, :], yell[0, :], zell[0, :], color="b", marker="o")
        ax.plot(xell[20, :], yell[20, :], zell[20, :], color="b", marker="o")

        ax.plot(xell[10, :], yell[10, :], zell[10, :], color="r", marker="o")
        ax.plot(xell[30, :], yell[30, :], zell[30, :], color="r", marker="o")

        ax.plot(xell[:, 20], yell[:, 20], zell[:, 20], color="g", marker="o")

    def init_plot_view(self, ax, view):

        # 90, 0 for z orientation, 0, 90 for y orientation, 0, 0 for x orientation
        # ax.view_init(elev=90, azim=270)
        # SET UP VIEW
        if view == "x":  # default
            ax.view_init(elev=0, azim=90)
        elif view == "y":
            ax.view_init(elev=0, azim=0)
        elif view == "z":
            ax.view_init(elev=90, azim=0)
        else:
            ax.view_init(elev=0, azim=40)
        return ax

    def plot_ellipsoid_aggs(
        self,
        clusters,
        nearest_geoms_xz=None,
        nearest_geoms_yz=None,
        nearest_geoms_xy=None,
        view="x",
        circle=None,
        agg_agg=True,
        add_ellipsoid=True,
        add_axes=True,
    ):
        """
        plot multiple aggregates (3D), each a different color
        """
        fig = plt.figure(figsize=(9, 9), dpi=300)
        ax = fig.add_subplot(111, projection="3d")
        ax = self.init_plot_view(ax, view)

        # PLOT ELLIPSOID SURFACE
        if add_ellipsoid:
            xell, yell, zell = self.plot_ellipsoid_surface(ax)

        if add_axes:
            self.plot_ellipsoid_axes(ax, xell, yell, zell)

        # PLOT MONOMERS
        if agg_agg:
            start_list = [cluster.ncrystals for cluster in clusters]
            start = [0] + start_list
            end = [
                (np.sum([start[i], start[i + 1]]) + 1) for i in range(len(start_list))
            ]
            colors = ["k", "k", "b", "darkgreen"]
            for cluster in range(len(clusters)):
                # lowered color range so that darker colors are generated
                # color = list(np.random.choice(range(10), size=3)/10)
                color = colors[cluster]
                for crys in range(start[cluster], end[cluster] - 1):
                    self._plot_crystal(crys, ax, color)
        else:
            start_list = [cluster.ncrystals - 1 for cluster in clusters]
            start_list = [i + 1 if i == 0 else i for i in start_list]
            start = [0] + start_list
            end = [np.sum([start[i], start[i + 1]]) for i in range(len(start_list))]
            colors = ["k", "k", "b", "darkgreen"]
            for cluster in range(len(clusters)):
                # lowered color range so that darker colors are generated
                # color = list(np.random.choice(range(10), size=3)/10)
                color = colors[cluster]
                for crys in range(start[cluster], end[cluster]):
                    self._plot_crystal(crys, ax, color)

        if nearest_geoms_xz is not None:
            if view == "x":
                ax.scatter(
                    nearest_geoms_xz[0].x,
                    nearest_geoms_yz[0].y,
                    nearest_geoms_xz[0].y,
                    c="red",
                    s=100,
                    zorder=10,
                )
                ax.scatter(
                    nearest_geoms_xz[1].x,
                    nearest_geoms_yz[1].y,
                    nearest_geoms_xz[1].y,
                    c="k",
                    s=100,
                    zorder=10,
                )
            elif view == "y":
                ax.scatter(
                    nearest_geoms_xz[0].x,
                    nearest_geoms_yz[0].x,
                    nearest_geoms_yz[0].y,
                    c="red",
                    s=100,
                    zorder=10,
                )
                ax.scatter(
                    nearest_geoms_xz[1].x,
                    nearest_geoms_yz[1].x,
                    nearest_geoms_yz[1].y,
                    c="k",
                    s=100,
                    zorder=10,
                )
            else:
                ax.scatter(
                    nearest_geoms_xy[0].x,
                    nearest_geoms_xy[0].y,
                    nearest_geoms_yz[0].y,
                    c="red",
                    s=100,
                    zorder=10,
                )
                ax.scatter(
                    nearest_geoms_xy[1].x,
                    nearest_geoms_xy[1].y,
                    nearest_geoms_yz[1].y,
                    c="k",
                    s=100,
                    zorder=10,
                )

        ax.set_xlabel("X")
        ax.set_ylabel("Y")
        ax.set_zlabel("Z")
        #         ax.set_zticklabels([])
        #         ax.set_yticklabels([])
        #         ax.set_xticklabels([])
        #         ax.grid(False)
        #         for pos in ['right','top','bottom','left']:
        #             ax.spines[pos].set_visible(False)
        #         plt.axis('off')

        # ax.view_init(30, i)
        # plt.pause(.001)

        # current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
        # fig.savefig('plot_clusters/'+current_time+'.png',rasterized=True, bbox_inches = 'tight')

        plt.show()

    def ellipse_vertices_rotated(self, params, angle, leftverticex, rightverticex):
        # 2D
        # rotate axis points and reposition if off center

        leftverticey = params["xy"][1]
        rightverticey = params["xy"][1]

        xleft = (
            (leftverticex - params["xy"][0]) * np.cos(angle)
            - (leftverticey - params["xy"][1]) * np.sin(angle)
        ) + params["xy"][0]

        xright = (
            (rightverticex - params["xy"][0]) * np.cos(angle)
            - (rightverticey - params["xy"][1]) * np.sin(angle)
        ) + params["xy"][0]

        yleft = (
            (leftverticex - params["xy"][0]) * np.sin(angle)
            + (leftverticey - params["xy"][1]) * np.cos(angle)
        ) + params["xy"][1]

        yright = (
            (rightverticex - params["xy"][0]) * np.sin(angle)
            + (rightverticey - params["xy"][1]) * np.cos(angle)
        ) + params["xy"][1]

        x = [xleft, xright]
        y = [yleft, yright]

        return x, y

    def plot_axes_ellipse(self, params, ax):
        # 2D
        # major axis
        # orientation angle of ellipse in radians
        angle = params["angle"] * np.pi / 180
        leftverticex = params["xy"][0] - params["width"] / 2
        rightverticex = params["xy"][0] + params["width"] / 2
        x, y = self.ellipse_vertices_rotated(params, angle, leftverticex, rightverticex)
        ax.plot(x, y, color="k", linewidth=3)  # major/minor axis lines
        # minor axis
        leftverticex = params["xy"][0] - params["height"] / 2
        rightverticex = params["xy"][0] + params["height"] / 2
        angle = params["angle"] * np.pi / 180 + np.pi / 2
        x, y = self.ellipse_vertices_rotated(params, angle, leftverticex, rightverticex)
        ax.plot(x, y, color="k", linewidth=3)  # major/minor axis lines

    def plot_ellipse(self, dims, add_axes=True):

        params = self.fit_ellipse(dims)
        ellipse = Ellipse(**params)

        fig, ax = plt.subplots(1, 1)
        ax.add_artist(ellipse)
        ellipse.set_alpha(0.2)  # opacity
        ellipse.set_facecolor("darkorange")

        for l in range(len(dims)):
            for i in range(self.ncrystals):
                hex1pts = self.points[dims[l]][i][0:6]  # first basal face
                poly1 = geom.Polygon([[p[0], p[1]] for p in hex1pts])
                hex2pts = self.points[dims[l]][i][6:12]  # second basal face
                poly2 = geom.Polygon([[p[0], p[1]] for p in hex2pts])
                x1, y1 = poly1.exterior.xy  # array of xy points
                x2, y2 = poly2.exterior.xy

                if i == 1:
                    color = "navy"
                    zorder = 3
                else:
                    color = "darkgreen"
                    zorder = 4
                for n in range(7):  # plot the prism face lines
                    x = [x1[n], x2[n]]
                    y = [y1[n], y2[n]]
                    ax.plot(x, y, color=color, zorder=zorder, linewidth="2")

                ax.plot(
                    x1, y1, color=color, zorder=2, linewidth="2"
                )  # edges of polygons
                ax.plot(x2, y2, color=color, zorder=4, linewidth="2")

        if add_axes:
            self.plot_axes_ellipse(params, ax)

        ax.set_aspect("equal", "datalim")
        plt.show()


#!/usr/bin/env python3
from flask import Flask, request, jsonify
# from algorithm import approx
from worker import integrate

# http GET http://localhost:5000
# http GET http://localhost:5000/0
# http GET http://localhost:5000/1
# http PUT http://localhost:5000 (next line)
# f='sqrt(4 - xs**2)' a:=0 b:=2 c:=0 d:=2 size:=10000
app = Flask(__name__)
TASKS = {}

@app.route('/<int:task_id>', methods=['GET'])
def get_task(task_id):
    task = TASKS.get(task_id)
    if task and task.ready():
        response = {task_id: task.get(timeout=1)}
    else:
        response = {task_id: 'still working...'}
    return jsonify(response)

@app.route('/', methods=['GET'])
def list_task():
    tasks = {task_id: {'ready': task.ready()}
             for task_id, task in TASKS.items()}
    return jsonify(tasks)

@app.route('/', methods=['PUT'])
def put_task():
    f = request.json['f']
    a = request.json['a']
    b = request.json['b']
    c = request.json['c']
    d = request.json['d']
    size = request.json.get('size', 100)

    # response = {
    #     'result': approx(f, a, b, c, d, size),
    # }

    task_id = len(TASKS)
    TASKS[task_id] = integrate.delay(f, a, b, c, d, size)
    response = {'result': task_id}
    return jsonify(response)

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0')


"""
given a labeled and unlabeled datafile, creates a single file, that contains the data
from both. the labels for the unlabeled data file will be UNK
"""
import argparse
from collections import OrderedDict
import csv

def get_col_by_role(role_string):
    col_by_role = {}
    for s in role_string.split(','):
        split_s = s.split('=')
        col_by_role[split_s[0]] = split_s[1]
    print('col_by_role', col_by_role)
    return col_by_role

def run(labeled_files, unlabeled_files, out_file, unlabeled_columns, labeled_columns, no_add_cust_tokens, add_columns, column_order):
    add_columns = add_columns.split(',')
    out_f = open(out_file, 'w')
    dict_writer = csv.DictWriter(out_f, fieldnames=column_order.split(','))
    dict_writer.writeheader()

    labeled_col_by_role = get_col_by_role(labeled_columns)
    unlabeled_col_by_role = get_col_by_role(unlabeled_columns)

    labeled_view1 = labeled_col_by_role['view1']
    labeled_label = labeled_col_by_role['label']

    unlabeled_view1 = unlabeled_col_by_role['view1']
    unlabeled_view2 = unlabeled_col_by_role['view2']

    unlabeled_by_first_utterance = OrderedDict()
    for filename in unlabeled_files:
        print(filename)
        with open(filename, 'r') as in_f:
            dict_reader = csv.DictReader(in_f)
            for row in dict_reader:
                view1 = row[unlabeled_view1]
                if not no_add_cust_tokens and not view1.startswith('<cust__'):
                    view1 = '<cust__ ' + view1 + ' __cust>'
                out_row = {
                    'view1': view1,
                    'view2': row[unlabeled_view2],
                    'label': 'UNK'
                }
                for k in add_columns:
                    out_row[k] = row[k]
                unlabeled_by_first_utterance[view1] = out_row
    print('loaded unlabeled')

    for filename in labeled_files:
        print(filename)
        with open(filename, 'r') as in_f:
            dict_reader = csv.DictReader(in_f)
            for row in dict_reader:
                view1 = row[labeled_view1]
                if view1 not in unlabeled_by_first_utterance:
                    print('warning: not found in unlabelled', view1)
                    continue
                out_row = unlabeled_by_first_utterance[view1]
                out_row['label'] = row[labeled_label]
                for k in add_columns:
                    assert out_row[k] == row[k]

    for row in unlabeled_by_first_utterance.values():
        dict_writer.writerow(row)

    out_f.close()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--labeled-files', type=str, nargs='+', required=True)
    parser.add_argument('--unlabeled-files', type=str, nargs='+', required=True)
    parser.add_argument('--out-file', type=str, required=True)
    parser.add_argument('--unlabeled-columns', type=str, default='view1=first_utterance,view2=context')
    parser.add_argument('--labeled-columns', type=str, default='view1=text,label=tag')
    parser.add_argument('--no-add-cust-tokens', action='store_true')
    parser.add_argument('--add-columns', type=str, default='id,question_body')
    parser.add_argument('--column-order', type=str, default='id,label,view1,view2,question_body')
    args = parser.parse_args()
    run(**args.__dict__)


"""Usage: pongbot.py teamname host port """

from __future__ import division
import logging
import sys
import controller
import estimator
import os
import math
import random
import json
from paddle import Paddle

from connection import JsonOverTcp
from connection import ThrottledConnection

CONTROLLER_LOG_FILE = "controller.log"
LOG_LEVEL = logging.INFO

def mean(list, default = 0.0):
    return float(sum(list))/len(list) if len(list) > 0 else default

class PingPongBot(object):
    def __init__(self, connection, log):
        self._connection = connection
        self._log = log
        self._estimator = estimator.Estimator(log)
        self._estimations = []
        self._paddle = Paddle()
        self._hit_point = 0

        if log.getEffectiveLevel() == logging.DEBUG:
            controller_log_file = open(os.path.join("log", CONTROLLER_LOG_FILE), "w")
        else:
            controller_log_file = None

        self._controller = controller.Controller(log, controller_log_file)

    def run(self, teamname):
        self._connection.send({'msgType': 'join', 'data': teamname})
        self._response_loop()

    def duel(self, teamname, partnername):
        self._connection.send({'msgType': 'requestDuel', 'data': [teamname, partnername]})
        self._response_loop()

    def _response_loop(self):
        response_handlers = {
                'joined': self._game_joined,
                'gameStarted': self._game_started,
                'gameIsOn': self._make_move,
                'gameIsOver': self._game_over
                }
        while True:
            response = self._connection.receive()
            msg_type, data = response['msgType'], response['data']
            try:
                response_handlers[msg_type](data)
            except KeyError:
                self._log.error('Unkown response: %s' % msg_type)

    def _game_joined(self, data):
        self._log.info('Game visualization url: %s' % data)

    def _game_started(self, data):
        self._estimator.resetData()
        self._estimations = []
        self._paddle.reset()
        self._log.info('Game started: %s vs. %s' % (data[0], data[1]))

    def _make_move(self, data):
        ball = data['ball']['pos']['y']
        paddle = data['left']['y'] + data['conf']['paddleHeight'] / 2
        tick_interval = data['conf']['tickInterval']

        self._estimations.append(self._estimator.update(data))
        self._estimations = self._estimations[-10::]

        tiltRange = (0, 0)
        self._paddle.update_direction(paddle)

        if self.direction_changed():
            previous = self._estimations[-5]
            current = self._estimations[-1]
            change = math.fabs(math.fabs(current.k) - math.fabs(previous.k))
            self._log.info("k: %f (%f) change: (%f)" % (current.k, previous.k, change))
        try:
            ballHitPoint = mean(self._estimator.getHitRange()[0], data['ball']['pos']['y'])
            if self.direction_changed() and self._estimator._ballDirectionX == -1:
                opponent = data["right"]["y"] / (data["conf"]["paddleHeight"] / 2)
                attack = self._estimator.attack(self._paddle.tilt_angle()[1], opponent)
                self._hit_point = attack.position().y - ballHitPoint
                self._log.info("Ball at %f" % self._estimator._data[-4]['ball']['pos']['y'])
                self._log.info("target k: %f, offset: %f, hitpoint: %f" % (attack.position().k, self._hit_point, attack.position().estimate(data['conf']['maxWidth'] - 20).y))
                #self._log.info("Hit point %s" % json.dumps(attack._edges()))
            elif self.direction_changed():
                previous = self._estimations[-5]
                current = self._estimations[-1]
                change = math.fabs(math.fabs(current.k) - math.fabs(previous.k))
                control = self._hit_point / (float(data['conf']['paddleHeight']) / 2)
                self._paddle.update(control, change)
                self._hit_point = 0.0
                self._log.info("Paddle tilt angle %f" % self._paddle.tilt_angle()[1])
                self._log.info("New ball estimation: %f" % (self._estimator.position().estimate(data['conf']['maxWidth'] - 20).y + 5))

            ph = ballHitPoint + self._hit_point
            control = self._controller.update(paddle, ph,
                                              tick_interval, self._paddle.has_direction_changed())
        except IndexError:
            control = self._controller.update(paddle, ball,
                                              tick_interval, self._paddle.has_direction_changed())

        self._send_control_msg(control)

    def _send_control_msg(self, control):
        if control is not None:
            self._connection.send({'msgType': 'changeDir', 'data': control})
        else:
            self._log.debug("Control is same as before; not sending update msg")

    def _game_over(self, data):
        self._log.info('Game ended. Winner: %s' % data)

    def direction_changed(self):
        if len(self._estimations) < 6:
            return False
        previous = self._estimations[-5]
        current = self._estimations[-4]
        return previous.direction != current.direction

if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s',
                        level=LOG_LEVEL)
    log = logging.getLogger(__name__)

    teamname, hostname, port = sys.argv[1:]
    PingPongBot(ThrottledConnection(JsonOverTcp(hostname, port),
                                    log), log).run(teamname)


from __future__ import unicode_literals
from mezzanine import template
from bs4 import BeautifulSoup
from PIL import Image
import six
try:
    from StringIO import StringIO
except ImportError:
    from io import BytesIO

import requests
import json
import logging
try:
    import urlparse
    from urllib import urlencode
except: # For Python 3
    import urllib.parse as urlparse
    from urllib.parse import urlencode

logger = logging.getLogger(__name__)
register = template.Library()

@register.filter
def to_normal_url(url):
    return url.replace("amp/","")

@register.filter
def append_amp_optimized_param(url):
    url_parts = list(urlparse.urlparse(url))
    url_parts[2] = "/ampoptimized" + url_parts[2]
    return urlparse.urlunparse(url_parts)

@register.filter
def to_amp_html(html):
    """
    Markdown HTML to AMP HTML
    """
    soup = BeautifulSoup(html, "html5lib")
    # ------------------------------------------------
    # romove style & script tags
    # ------------------------------------------------
    [s.decompose() for s in soup('style')]
    [s.decompose() for s in soup('script')]
    remove_attrs(soup, ("style","script",))
    # ------------------------------------------------
    # amp id replace to "accelerated-mobile-pages"
    # ------------------------------------------------
    for elem in soup.find_all(True, id=lambda x: x and 'amp' in x):
        elem["id"] = elem.get("id").replace("amp", "accelerated-mobile-pages")

    # h2
    for h2 in soup.find_all('h2'):
        h2['class'] = h2.get('class', []) + ['bold', 'mt2', 'mb2']
    # h3
    for h3 in soup.find_all('h3'):
        h3['class'] = h3.get('class', []) + ['bold', 'mt1', 'mb1']
    # h4
    for h4 in soup.find_all('h4'):
        h4['class'] = h4.get('class', []) + ['bold', 'mt1', 'mb1']

    # -----------------------------------------------
    # iframe replace to amp-iframe
    # -----------------------------------------------
    for iframe in soup.find_all('iframe'):
        iframe.name = "amp-iframe"

    # -----------------------------------------------
    # img replace to amp-img
    # -----------------------------------------------
    for img in soup.find_all('img'):
        try:
            amp_img = soup.new_tag("amp-img")
            for attr in img.attrs:
                if "style" != attr:
                    amp_img[attr] = img[attr]
            src = str(img.get("src"))
            if src.startswith("//"):
                src = src.replace("//", "https://")
            req = requests.get(src)
            if six.PY2:
                picture_IO = StringIO(req.content)
            else:
                picture_IO = BytesIO(req.content)
            picture_IO.seek(0)
            im = Image.open(picture_IO)
            amp_img["width"] = im.size[0]
            amp_img["height"] = im.size[1]
            amp_img["layout"] = "responsive"
            img.replace_with(amp_img)
        except IOError:
            logger.warning("something raised an exception: ", exc_info=True)
            amp_img["width"] = "4"
            amp_img["height"] = "3"
            amp_img["layout"] = "responsive"
            img.replace_with(amp_img)

    soup.body.hidden = True
    return str(soup.body)

def remove_attrs(soup, brack_list=tuple()):
    for tag in soup.findAll(True):
        for attr in [attr for attr in tag.attrs if attr in brack_list]:
            del tag[attr]
    return soup

@register.as_tag
def conv_blog_post_to_json_ld(blog=None):
    """
    Get blogpost JSON-LD
    """
    result_dict = {
        "@context": "http://schema.org",
        "@type": "BlogPosting",
        "headline": blog.title,
        "author": {"@type": "Person", "name": blog.user.first_name},
        "publisher": {"@type": "Organization",
                      "url": "https://www.monotalk.xyz",
                      "name": blog.user.first_name,
                      "logo": {"@type": "ImageObject", "url": "https://drive.google.com/uc?export=view&id=0By5O5w7iwOMOVE5pTEcyeE40WlE"}
                      },
        "image": {"@type": "ImageObject", "url": "https://drive.google.com/uc?export=view&id=0By5O5w7iwOMOMDdhaDhHdXBVTHc", "height": 450, "width": 800},
        "mainEntityOfPage": {
            "@type": "WebPage",
            "@id": blog.get_absolute_url_with_host(),
        },
        "genre": ' '.join(map(lambda n: n.title, blog.categories.all())),
        "wordcount": str(len(blog.content)),
        "datePublished": str(blog.publish_date),
        "dateCreated": str(blog.created),
        "dateModified": str(blog.updated),
        "description": blog.description
    }
    json_o = json.dumps(result_dict, ensure_ascii=False)
    return json_o