from odoo import models, fields, api, exceptions
from odoo11.odoo.addons import decimal_precision as dp
from odoo.fields import Date as fDate
from datetime import timedelta
from odoo.exceptions import UserError
from odoo.tools.translate import _

class LibraryBook(models.Model):

    _name = "library.book"
    _description = "Library Book"
    _order = 'date_release desc, name'
    _rec_name = 'short_name'

    name = fields.Char("Title", required=True)
    isbn = fields.Char('ISBN')
    date_release = fields.Date("Release Date")
    author_ids = fields.Many2many('res.partner', string='Authors')
    short_name = fields.Char(string='Short Title', required=True, )
    notes = fields.Text('Internal Notes')
    state = fields.Selection([('draft', 'Not Available'), ('available', 'Available'), ('lost', 'Lost')], 'State')
    description = fields.Html(string='Description',
                              sanitize=True,
                              strip_style=False,
                              translate=False, )
    cover = fields.Binary('Book Cover')
    out_of_print = fields.Boolean('Out of Print?')
    date_release = fields.Date('Release Date')
    date_updated = fields.Datetime('Last Updated')
    pages = fields.Integer(string='Number of Pages', default=0,
                           help='Total book page count',
                           groups='base.group_user',
                           states={'lost': [('readonly', True)]}, copy=True,
                           index=False,
                           readonly=False,
                           required=False,
                           company_dependent=False, )
    reader_rating = fields.Float('Reader Average Rating', digits=(14, 4))
    cost_price = fields.Float('Book Cost', dp.get_precision('Book Price'))
    currency_id = fields.Many2one('res.currency', string='Currency')
    retail_price = fields.Monetary('Retail Price', currency_field='currency_id', )
    publisher_id = fields.Many2one('res.partner', string='Publisher', ondelete='set null', )
    publisher_city = fields.Char('Publisher City',
                                 related='publisher_id.city',
                                 readonly=True)

    age_days = fields.Float(
        string='Days Since Release',
        compute='_compute_age',
        inverse='_inverse_age',
        search='_search_age',
        store=False,
        compute_sudo=False,
    )

    ref_doc_id = fields.Reference(
        selection='_referencable_models',
        string='Reference Document')

    manager_remarks = fields.Text('Manager Remarks')

    _sql_constraints = [
        ('name_unique',
         'UNIQUE (name)',
         'Book title must be unique.')
    ]

    _sql_constraints = [
        ('shortname_unique',
         'UNIQUE (short_name)',
         'Book Short Name must be unique.')
    ]

    @api.constrains('date_release')
    def _check_release_date(self):
        for record in self:
            if (record.date_release and
                    record.date_release > fields.Date.today()):
                raise models.ValidationError('Release date must be in the past')

    @api.depends('date_release')
    def _compute_age(self):
        today = fDate.from_string(fDate.today())
        for book in self.filtered('date_release'):
            try:
                delta = (today - fDate.from_string(book.date_release))
                book.age_days = delta.days
            except (IOError, OSError) as exc:
                message = _('Unable to compute age days') % exc
                raise UserError(message)

    def _inverse_age(self):
        today = fDate.from_string(fDate.context_today(self))
        for book in self.filtered('date_release'):
            try:
                d = today - timedelta(days=book.age_days)
                book.date_release = fDate.to_string(d)
            except (IOError, OSError) as exc:
                message = _('Unable to inverse age') % exc
                raise UserError(message)

    def _search_age(self, operator, value):
        try:
            today = fDate.from_string(fDate.context_today(self))
            value_days = timedelta(days=value)
            value_date = fDate.to_string(today - value_days)
            operator_map = {'>': '<', '>=': '<=', '<': '>', '<=': '>=', }
            new_op = operator_map.get(operator, operator)
            return [('date_release', new_op, value_date)]
        except (IOError, OSError) as exc:
            message = _('Unable to search by age') % exc
            raise UserError(message)

    @api.multi
    def name_get(self):
        result = []
        for book in self:
            authors = book.author_ids.mapped('name')
            name = '%s (%s)' % (book.name, ', '.join(authors))
            result.append((book.id, name))
            return result

    @api.model
    def _name_search(self, name='', args=None, operator='ilike',
                     limit=100, name_get_uid=None):
        args = [] if args is None else args.copy()
        if not (name == '' and operator == 'ilike'):
            args += ['|', '|',
                 ('name', operator, name),
                 ('isbn', operator, name),
                 ('author_ids.name', operator, name)
                 ]
        return super(LibraryBook, self)._name_search(
            name='', args=args, operator='ilike',
            limit=limit, name_get_uid=name_get_uid)

    @api.model
    def _referencable_models(self):
        models = self.env['res.request.link'].search([])
        return [(x.object, x.name) for x in models]

    @api.model
    def is_allowed_transition(self, old_state, new_state):
        allowed = [('draft', 'available'),
                   ('available', 'borrowed'),
                   ('borrowed', 'available'),
                   ('available', 'lost'),
                   ('borrowed', 'lost'),
                   ('lost', 'available')]
        return (old_state, new_state) in allowed

    @api.multi
    def change_state(self, new_state):
        for book in self:
            if book.is_allowed_transition(book.state, new_state):
                book.state = new_state
            else:
                continue

    @api.model
    def get_all_library_members(self):
        library_member_model = self.env['library.member']
        return library_member_model.search([])

    @api.model
    def create(self, values):
        if not self.user_has_groups(
                'library.group_library_manager'):
            if 'manager_remarks' in values:
                raise exceptions.UserError('You are not allowed to modify ''manager_remarks')
            return super(LibraryBook, self).create(values)

    @api.multi
    def write(self, values):
        if not self.user_has_groups('library.group_library_manager'):
            if 'manager_remarks' in values:
                raise exceptions.UserError('You are not allowed to modify ''manager_remarks')
        return super(LibraryBook, self).write(values)

    @api.model
    def fields_get(self,
                   allfields=None,
                   attributes=None):
        fields = super(LibraryBook, self).fields_get(
            allfields=allfields,
            attributes=attributes
        )
        if not self.user_has_groups('library.group_library_manager'):
            if 'manager_remarks' in fields:
                fields['manager_remarks']['readonly'] = True

class ResPartner(models.Model):
    _inherit = 'res.partner'
    _order = 'name'
    _name = 'res.partner'

    name = fields.Char('Name', required=True)
    email = fields.Char('Email')
    date = fields.Date('Date')
    is_company = fields.Boolean(
        'Is a company',
        help="Check if the contact is a company, "
             "otherwise it is a person"
    )
    parent_id = fields.Many2one('res.partner', 'Related Company')
    child_ids = fields.One2many('res.partner', 'parent_id',
                                'Contacts')

    published_book_ids = fields.One2many('library.book', 'publisher_id', string='Published Books')
    authored_book_ids = fields.Many2many('library.book',
                                         string='Authored Books',
                                         relation='library_book_res_partner_rel')
    count_books = fields.Integer('Number of Authored Books',
                                 compute='_compute_count_books')

    def first_val(self):
        today_str = fields.Date.context_today(self)
        val1 = {'name': 'Eric Idle',
                'email': 'eric.idle@example.com',
                'date': today_str}
        val2 = {'name': 'John Cleese',
                'email': 'john.cleese@example.com',
                'date': today_str}
        partner_val = {
            'name': 'Flying Circus',
            'email': 'm.python@example.com',
            'date': today_str,
            'is_company': True,
            'child_ids': [(0, 0, val1),
                          (0, 0, val2),
                          ]}
        record = self.env['res.partner'].create(partner_val)

    @api.model
    def add_contacts(self, partner, contacts):
        partner.ensure_one()
        if contacts:
            partner.date = fields.Date.context_today(self)
            partner.child_ids |= contacts

    @api.depends('authored_book_ids')
    def _compute_count_books(self):
        for r in self:
            r.count_books = len(r.authored_book_ids)

    @api.constrains('date_release')
    def _check_release_date(self):
        for record in self:
            if (record.date_release and
                    record.date_release > fields.Date.today()):
                raise models.ValidationError('Release date must be in the past')

    @api.model
    def find_partners_and_contacts(self, name):
        partner = self.env['res.partner']
        domain = ['|',
                  '&',
                  ('is_company', '=', True),
                  ('name', 'like', name),
                  '&',
                  ('is_company', '=', False),
                  ('parent_id.name', 'like', name)]
        return partner.search(domain)

    @api.model
    def partners_with_email(self, partners):
        def predicate(partner):
            if partner.email:
                return True
            return False
        return partners.filter(predicate)

    @api.model
    def get_email_addresses(self, partner):
        return partner.mapped('child_ids.email')

    @api.model
    def get_companies(self, partners):
        return partners.mapped('parent_id')

class LibraryMember(models.Model):
    _inherit = {'res.partner': 'partner_id'}
    _inherit = 'library.member'
    _name = 'library.member'
    loan_duration = fields.Integer('Loan duration',
                                   default=15,
                                   required=True)
    name = fields.Char('Name', required=True)
    email = fields.Char('Email')
    date = fields.Date('Date')
    is_company = fields.Boolean(
        'Is a company',
        help="Check if the contact is a company, "
             "otherwise it is a person"
    )

    partner_id = fields.Many2one('res.partner',
                                 ondelete='cascade')
    date_start = fields.Date('Member Since')
    date_end = fields.Date('Termination Date')
    member_number = fields.Char()
    date_of_birth = fields.Date('Date of birth')

class LibraryBookLoan(models.Model):
    _inherit = 'library.book.loan'
    expected_return_date = fields.Date('Due for', required=True)

class BaseArchive(models.AbstractModel):
    _name = 'base.archive'
    active = fields.Boolean(default=True)

    def do_archive(self):
        for record in self:
            record.active = not record.active

class LibraryLoanWizard(models.TransientModel):
    _inherit = 'library.load.wizard'

    def _prepare_loan(self, book):
        values = super(LibraryLoanWizard,self)._prepare_loan(book)
        loan_duration = self.member_id.loan_duration
        today_str = fields.Date.context_today(self)
        today = fields.Date.from_string(today_str)
        expected = today + timedelta(days=loan_duration)
        values.update({'expected_return_date':
        fields.Date.to_string(expected)})
        return values

    @api.multi
    def record_loans(self):
        for wizard in self:
            books = wizard.book_ids
            loan = self.env['library.book.loan']
            for book in wizard.book_ids:
                values = wizard._prepare_loan(book)
                loan.create(values)

    @api.multi
    def _prepare_loan(self, book):
        return {'member_id': self.member_id.id,
                'book_id': book.id}


"""
Main plugin module for Report plugin
"""

from flask import Blueprint,request
from sqlalchemy import or_, func

from DivvyBlueprints.v2 import RequiresTokenAuthentication
from DivvyPlugins.plugin_helpers import register_api_blueprint, unregister_api_blueprints
from DivvyPlugins.plugin_metadata import PluginMetadata
from DivvyDb.DivvyDb import SharedSessionScope
from DivvyDb.DivvyCloudGatewayORM import DivvyCloudGatewayORM
from DivvyDb.DivvyDbObjects import ResourceProperty,DivvyBot
from DivvyResource import ResourceIds, Resources
from DivvyPermissions import SessionPermissions, Permissions
from DivvyUtils.flask_helpers import JsonResponse

class metadata(PluginMetadata):
    """
    Information about this plugin
    """
    version = '1.0'
    last_updated_date = '2015-08-06'
    author = 'Divvy Cloud Corp.'
    nickname = 'Global Bot Compliancy Report'
    default_language_description = 'List of resources that have been marked non-compliant froma Bot'
    support_email = 'support@divvycloud.com'
    support_url = 'http://support.divvycloud.com'
    main_url = 'http://www.divvycloud.com'
    category = 'Reports'
    managed = False

# This is our Flask blueprint for exposing the API call
blueprint = Blueprint('compliance_report_by_bot', __name__)

@blueprint.route('/compliance_report/list_bots', methods=['POST'])
@SharedSessionScope(DivvyCloudGatewayORM)
@RequiresTokenAuthentication()
def get_all_bots():

    db = DivvyCloudGatewayORM()
    bots = []
    bot_list = db.session.query(DivvyBot).all()
    for bot in bot_list:
        if bot.state == "ARCHIVED":
            continue

        bots.append({
        "name" : bot.name,
        "id" : bot.resource_id})

    return JsonResponse(bots)

@blueprint.route('/compliance_report/get_report', methods=['POST'])
@SharedSessionScope(DivvyCloudGatewayORM)
@RequiresTokenAuthentication()
def get_compliance_report():
    """
    Generate a resource breakdown by type report.
    """

    bot_resource_id = request.args['bot_resource_id']
    # Get a list of all non-compliant resources
    db = DivvyCloudGatewayORM()
    report = []
    bot_resource_id = ResourceIds.ResourceId.from_string(bot_resource_id)
    bot = Resources.DivvyBot.from_resource_id(bot_resource_id)
    bot_resource_id_str = bot.resource_id.to_string().replace(":",".")
    like_str = "%%%s%%" % (bot_resource_id_str)
    property_list = db.session.query(ResourceProperty.resource_id,func.group_concat(ResourceProperty.name).label('prop_name'),func.group_concat(ResourceProperty.value).label('prop_value')).filter(ResourceProperty.name.like(like_str)).group_by(ResourceProperty.resource_id).all()
    for row in property_list:
        is_compliant,last_change,reason = row.prop_value.split(',')
        resource = Resources.Resource.from_resource_id(ResourceIds.ResourceId.from_string(row.resource_id))
        if(is_compliant == 'true'):
            continue

        report.append({
            'resource_id' : row.resource_id,
            'resource_type' : resource.get_resource_type(),
            'bot_resource_id' : bot.resource_id.to_string(),
            'bot_description' : bot.description,
            'reason' :  reason,
            'last_change' :  last_change,
            'resource_name' :  resource.name
        })
    return JsonResponse({'resources': report})

def load():
    register_api_blueprint(blueprint)

def unload():
    unregister_api_blueprints()


# -*- coding: utf-8 -*-

# third-party packages - discord related
import discord
from discord.ext import commands

# local modules
from pythonbot import PythonBot
from utils import print_context

class Members(commands.Cog, name='members'):
    """assorted commands for guild members"""

    def __init__(self, bot: PythonBot) -> None:
        """initializer"""

        self.bot = bot

    @commands.command()
    @commands.guild_only()
    @print_context
    async def joined(
            self,
            ctx: commands.Context,
            member: discord.Member = None,
    ) -> None:
        """Says when a member joined."""

        if member is None:
            member = ctx.message.author

        if member == ctx.message.author:
            title = 'You joined:'
        else:
            title = f"{member.display_name} joined:"

        formatted_datetime = member.joined_at.strftime('%B %d %Y at %I:%M:%S %p')
        await self.bot.send_titled_info_msg(ctx, title, formatted_datetime)

    @joined.error
    async def joined_handler(
            self,
            ctx: commands.Context,
            error: commands.CommandError,
    ) -> None:
        """error handling for joined command"""

        if isinstance(error, commands.BadArgument):
            msg = 'Member not found'
            await self.bot.send_error_msg(ctx, msg)

    @commands.command(
        help='shame tony for his laziness',
        aliases=['search', 'lmgtfy', 'lmgt',],
    )
    @commands.guild_only()
    @print_context
    async def google(
            self,
            ctx: commands.Context,
            *obvious_keywords: str
    ) -> None:
        """
        generate letmegooglethat links for help vampires/askholes
        """

        if not obvious_keywords:
            msg = '!google OBVIOUS SEARCH KEYWORDS'
            await self.bot.send_usage_msg(ctx, msg)
        else:
            keywords = '+'.join(obvious_keywords)
            msg = ':link: http://letmegooglethat.com/?q=' + keywords
            await self.bot.send_info_msg(ctx, msg)

    @commands.command(hidden=True)
    @commands.guild_only()
    @print_context
    async def hello(self, ctx: commands.Context) -> None:
        """A simple command which only responds to the owner of the bot."""

        if ctx.author.id == self.bot.owner_id:
            msg = 'Hello father'
        else:
            msg = f'Hello {ctx.author.display_name}'
        await ctx.send(msg)

def setup(bot: PythonBot) -> None:
    """
    function the bot uses to load this extension
    """

    bot.add_cog(Members(bot))


#################################################################################
# The Institute for the Design of Advanced Energy Systems Integrated Platform
# Framework (IDAES IP) was produced under the DOE Institute for the
# Design of Advanced Energy Systems (IDAES), and is copyright (c) 2018-2021
# by the software owners: The Regents of the University of California, through
# Lawrence Berkeley National Laboratory,  National Technology & Engineering
# Solutions of Sandia, LLC, Carnegie Mellon University, West Virginia University
# Research Corporation, et al.  All rights reserved.
#
# Please see the files COPYRIGHT.md and LICENSE.md for full copyright and
# license information.
#################################################################################
"""
Tests for ControlVolumeBlockData.

Author: Andrew Lee
"""
import inspect
import pytest
from pyomo.environ import ConcreteModel, Block, Set, units
from pyomo.common.config import ConfigBlock, ConfigValue
from idaes.core import (ControlVolumeBlockData, CONFIG_Template,
                        MaterialBalanceType, EnergyBalanceType,
                        MomentumBalanceType, FlowDirection,
                        declare_process_block_class,
                        FlowsheetBlockData, UnitModelBlockData, useDefault,
                        PhysicalParameterBlock, ReactionParameterBlock,
                        MaterialFlowBasis, StateBlock, StateBlockData,
                        ReactionBlockBase, ReactionBlockDataBase)
from idaes.core.util.exceptions import (ConfigurationError, DynamicError,
                                        PropertyPackageError, BurntToast)

# -----------------------------------------------------------------------------
# Test Enumerators for balance type options
@pytest.mark.unit
def test_material_balance_type():
    assert len(MaterialBalanceType) == 6

    # Test that error is raised when given non-member
    with pytest.raises(AttributeError):
        MaterialBalanceType.foo  # pylint: disable=no-member

@pytest.mark.unit
def test_energy_balance_type():
    assert len(EnergyBalanceType) == 6

    # Test that error is raised when given non-member
    with pytest.raises(AttributeError):
        EnergyBalanceType.foo  # pylint: disable=no-member

@pytest.mark.unit
def test_momentum_balance_type():
    assert len(MomentumBalanceType) == 5

    # Test that error is raised when given non-member
    with pytest.raises(AttributeError):
        MomentumBalanceType.foo  # pylint: disable=no-member

@pytest.mark.unit
def testflow_direction():
    assert len(FlowDirection) == 2

    # Test that error is raised when given non-member
    with pytest.raises(AttributeError):
        FlowDirection.foo  # pylint: disable=no-member

# -----------------------------------------------------------------------------
# Test CONFIG_Template
@pytest.mark.unit
def test_CONFIG_Template():
    c = CONFIG_Template()

    assert len(c) == 18

    for i in c:
        if i == "dynamic":
            assert c[i] == useDefault
        elif i == "material_balance_type":
            assert c[i] == MaterialBalanceType.componentPhase
        elif i == "energy_balance_type":
            assert c[i] == EnergyBalanceType.enthalpyTotal
        elif i == "momentum_balance_type":
            assert c[i] == MomentumBalanceType.pressureTotal
        elif i == "property_package":
            assert c[i] == useDefault
        elif i == "reaction_package":
            assert c[i] is None
        elif i in ["property_package_args", "reaction_package_args"]:
            assert isinstance(c[i], ConfigBlock)
            assert len(c[i]) == 0
        else:
            assert c[i] is False

@pytest.mark.unit
def test_CONFIG_Template_validation_general():
    # No config argument takes a string, float/int or list
    c = CONFIG_Template()

    for i in c:
        with pytest.raises(ValueError):
            c[i] = "foo"
        with pytest.raises(ValueError):
            c[i] = 10.0
        with pytest.raises(ValueError):
            c[i] = [1, 2]

@pytest.mark.unit
def test_CONFIG_Template_true_false():
    # Check arguments that accept True/False as values
    c = CONFIG_Template()

    for i in c:
        if i not in ["material_balance_type", "energy_balance_type",
                     "momentum_balance_type", "property_package",
                     "reaction_package", "property_package_args",
                     "reaction_package_args"]:
            c[i] = True
            c[i] = False

@pytest.mark.unit
def test_CONFIG_Template_material_balance_type():
    c = CONFIG_Template()

    for i in MaterialBalanceType:
        c["material_balance_type"] = i

@pytest.mark.unit
def test_CONFIG_Template_energy_balance_type():
    c = CONFIG_Template()

    for i in EnergyBalanceType:
        c["energy_balance_type"] = i

@pytest.mark.unit
def test_CONFIG_Template_momentum_balance_type():
    c = CONFIG_Template()

    for i in MomentumBalanceType:
        c["momentum_balance_type"] = i

# -----------------------------------------------------------------------------
# Mockup classes for testing
@declare_process_block_class("Flowsheet")
class _Flowsheet(FlowsheetBlockData):
    def build(self):
        super(_Flowsheet, self).build()

@declare_process_block_class("Unit")
class _UnitData(UnitModelBlockData):
    CONFIG = UnitModelBlockData.CONFIG()
    CONFIG.declare("property_package",
                   ConfigValue(default=None))
    CONFIG.declare("property_package_args",
                   ConfigValue(default={}))

    def build(self):
        super(_UnitData, self).build()

# -----------------------------------------------------------------------------
# Testing ControlVolumeBlockData
@declare_process_block_class("CVFrame")
class CVFrameData(ControlVolumeBlockData):
    def build(self):
        super(ControlVolumeBlockData, self).build()

@pytest.mark.unit
def test_config_block():
    cv = CVFrame(concrete=True)

    assert len(cv.config) == 7
    assert cv.config.dynamic == useDefault
    assert cv.config.has_holdup is useDefault
    assert cv.config.property_package == useDefault
    assert isinstance(cv.config.property_package_args, ConfigBlock)
    assert len(cv.config.property_package_args) == 0
    assert cv.config.reaction_package is None
    assert isinstance(cv.config.reaction_package_args, ConfigBlock)
    assert len(cv.config.reaction_package_args) == 0
    assert cv.config.auto_construct is False

# -----------------------------------------------------------------------------
# Test _setup_dynamics
@pytest.mark.unit
def test_setup_dynamics_use_parent_value():
    # Test that dynamic = None works correctly
    m = ConcreteModel()
    m.fs = Flowsheet(default={"dynamic": False})
    m.fs.u = Unit(default={"dynamic": False})
    m.fs.u.cv = CVFrame()

    m.fs.u.cv._setup_dynamics()

    assert m.fs.u.cv.config.dynamic is False
    assert m.fs.u.cv.config.has_holdup is False

@pytest.mark.unit
def test_setup_dynamics_use_parent_value_fail_no_dynamic():
    # Test that default falls back to flowsheet
    fs = Flowsheet(default={"dynamic": False}, concrete=True)

    # Create a Block (with no dynamic attribute)
    fs.b = Block()
    fs.b.cv = CVFrame()
    fs.b.cv._setup_dynamics()

    assert fs.b.cv.config.dynamic is False

@pytest.mark.unit
def test_setup_dynamics_dynamic_in_ss():
    # Test that dynamic = None works correctly
    fs = Flowsheet(default={"dynamic": False}, concrete=True)

    # Create a Block (with no dynamic attribute)
    fs.b = Block()
    # Add a time attribute to make sure the correct failure triggers
    fs.b.time_ref = Set(initialize=[0])

    fs.b.cv = CVFrame(default={"dynamic": True, "has_holdup": True})

    # _setup_dynamics should return DynamicError
    with pytest.raises(DynamicError):
        fs.b.cv._setup_dynamics()

@pytest.mark.unit
def test_setup_dynamics_dynamic_holdup_inconsistent():
    # Test that dynamic = None works correctly
    fs = Flowsheet(default={"dynamic": True, "time_units": units.s},
                   concrete=True)

    # Create a Block (with no dynamic attribute)
    fs.b = Block()
    # Add a time attribute to make sure the correct failure triggers
    fs.b.time_ref = Set(initialize=[0])

    fs.b.cv = CVFrame(default={"dynamic": True, "has_holdup": False})

    # _setup_dynamics should return ConfigurationError
    with pytest.raises(ConfigurationError):
        fs.b.cv._setup_dynamics()

# -----------------------------------------------------------------------------
# Test _get_property_package
@declare_process_block_class("PropertyParameterBlock")
class _PropertyParameterBlock(PhysicalParameterBlock):
    def build(self):
        super(_PropertyParameterBlock, self).build()

        frm = inspect.stack()[1]
        self._package_module = inspect.getmodule(frm[0])

        self.phase_list = Set(initialize=["p1", "p2"])
        self.component_list = Set(initialize=["c1", "c2"])

@pytest.mark.unit
def test_get_property_package_set():
    m = ConcreteModel()
    m.pp = PropertyParameterBlock()
    m.cv = CVFrame(default={"property_package": m.pp})
    m.cv._get_property_package()

@pytest.mark.unit
def test_get_property_package_default_args():
    m = ConcreteModel()
    m.pp = PropertyParameterBlock(
                default={"default_arguments": {"test": "foo"}})
    m.cv = CVFrame(default={"property_package": m.pp})
    m.cv._get_property_package()

    assert m.cv.config.property_package_args["test"] == "foo"

@pytest.mark.unit
def test_get_reaction_package_module_combine_args():
    # Test that local and default args combine correctly
    m = ConcreteModel()
    m.pp = PropertyParameterBlock(
            default={"default_arguments": {"test1": "foo",
                                           "test2": "bar"}})
    m.cv = CVFrame(default={"property_package": m.pp,
                            "property_package_args": {"test2": "baz",
                                                      "test3": "bar"}})
    m.cv._get_property_package()

    assert m.cv.config.property_package_args["test1"] == "foo"
    assert m.cv.config.property_package_args["test2"] == "baz"
    assert m.cv.config.property_package_args["test3"] == "bar"

# -----------------------------------------------------------------------------
# Test _get_default_prop_pack
@pytest.mark.unit
def test_get_default_prop_pack_works():
    m = ConcreteModel()
    m.fs = Flowsheet()
    m.fs.pp = PropertyParameterBlock()
    m.fs.config.default_property_package = m.fs.pp

    m.fs.cv = CVFrame()
    assert m.fs.cv._get_default_prop_pack() == m.fs.pp

# TODO : should test more failure modes
@pytest.mark.unit
def test_get_default_prop_pack_no_default():
    m = ConcreteModel()
    m.fs = Flowsheet()

    m.fs.cv = CVFrame()
    with pytest.raises(ConfigurationError):
        m.fs.cv._get_default_prop_pack()

@pytest.mark.unit
def test_get_property_package_call_to_get_default_prop_pack():
    m = ConcreteModel()
    m.fs = Flowsheet()
    m.fs.pp = PropertyParameterBlock()
    m.fs.config.default_property_package = m.fs.pp

    m.fs.cv = CVFrame()
    m.fs.cv._get_property_package()
    assert m.fs.cv.config.property_package == m.fs.pp

# -----------------------------------------------------------------------------
# Test _get_indexing_sets
@pytest.mark.unit
def test_get_indexing_sets_missing_phase_list():
    m = ConcreteModel()
    m.pp = PropertyParameterBlock()
    m.pp.del_component(m.pp.phase_list)
    m.cv = CVFrame(default={"property_package": m.pp})
    m.cv._get_property_package()

    with pytest.raises(PropertyPackageError):
        m.cv._get_indexing_sets()

@pytest.mark.unit
def test_get_indexing_sets_missing_component_list():
    m = ConcreteModel()
    m.pp = PropertyParameterBlock()
    m.pp.del_component(m.pp.component_list)
    m.cv = CVFrame(default={"property_package": m.pp})
    m.cv._get_property_package()

    with pytest.raises(PropertyPackageError):
        m.cv._get_indexing_sets()

# -----------------------------------------------------------------------------
# Test _get_reaction_package
@pytest.mark.unit
def test_get_reaction_package_none():
    m = ConcreteModel()
    m.r = CVFrame()

    m.r._get_reaction_package()

    assert hasattr(m.r, "reaction_module") is False

@declare_process_block_class("ReactionParameterTestBlock")
class _ReactionParameterBlock(ReactionParameterBlock):
    def build(self):
        super(ReactionParameterBlock, self).build()

        frm = inspect.stack()[1]
        self._package_module = inspect.getmodule(frm[0])

@pytest.mark.unit
def test_get_reaction_package_module():
    m = ConcreteModel()
    m.rp = ReactionParameterTestBlock(
                default={"default_arguments": {"test": "foo"}})
    m.cv = CVFrame(default={"reaction_package": m.rp})

    m.cv._get_reaction_package()

    assert m.cv.config.reaction_package == m.rp
    assert m.cv.config.reaction_package_args["test"] == "foo"

@pytest.mark.unit
def test_get_reaction_package_module_default_args():
    # Test that local and default args combine correctly
    m = ConcreteModel()
    m.rp = ReactionParameterTestBlock(
            default={"default_arguments": {"test1": "foo",
                                           "test2": "bar"}})
    m.cv = CVFrame(default={"reaction_package": m.rp,
                            "reaction_package_args": {"test2": "baz",
                                                      "test3": "bar"}})

    m.cv._get_reaction_package()

    assert m.cv.config.reaction_package_args["test1"] == "foo"
    assert m.cv.config.reaction_package_args["test2"] == "baz"
    assert m.cv.config.reaction_package_args["test3"] == "bar"

# -----------------------------------------------------------------------------
# Test build and auto_construct methods
@pytest.mark.unit
def test_build():
    m = ConcreteModel()
    m.fs = Flowsheet()
    m.fs.pp = PropertyParameterBlock()
    m.fs.cv = CVFrame(default={"property_package": m.fs.pp})

    super(CVFrameData, m.fs.cv).build()

@pytest.mark.unit
def test_add_geometry():
    m = ConcreteModel()
    m.fs = Flowsheet()
    m.fs.cv = CVFrame()

    with pytest.raises(NotImplementedError):
        m.fs.cv.add_geometry()

@pytest.mark.unit
def test_auto_construct():
    m = ConcreteModel()
    m.fs = Flowsheet()
    m.fs.pp = PropertyParameterBlock()
    m.fs.cv = CVFrame(default={"property_package": m.fs.pp,
                               "auto_construct": True})

    with pytest.raises(NotImplementedError):
        super(CVFrameData, m.fs.cv).build()

# -----------------------------------------------------------------------------
# Test NotImplementedErrors for all property and balance type methods
@pytest.mark.unit
def test_add_state_blocks():
    m = ConcreteModel()
    m.cv = CVFrame()

    with pytest.raises(NotImplementedError):
        m.cv.add_state_blocks()

@pytest.mark.unit
def test_add_reaction_blocks():
    m = ConcreteModel()
    m.cv = CVFrame()

    with pytest.raises(NotImplementedError):
        m.cv.add_reaction_blocks()

@pytest.mark.unit
def test_add_material_balances():
    m = ConcreteModel()
    m.cv = CVFrame()

    for t in MaterialBalanceType:
        if t == MaterialBalanceType.none:
            assert m.cv.add_material_balances(t) is None
        elif t == MaterialBalanceType.useDefault:
            with pytest.raises(ConfigurationError):
                m.cv.add_material_balances(t)
        else:
            with pytest.raises(NotImplementedError):
                m.cv.add_material_balances(t)

@pytest.mark.unit
def test_add_energy_balances():
    m = ConcreteModel()
    m.cv = CVFrame()

    for t in EnergyBalanceType:
        if t == EnergyBalanceType.none:
            assert m.cv.add_energy_balances(t) is None
        elif t == EnergyBalanceType.useDefault:
            with pytest.raises(ConfigurationError):
                m.cv.add_energy_balances(t)
        else:
            with pytest.raises(NotImplementedError):
                m.cv.add_energy_balances(t)

@pytest.mark.unit
def test_add_momentum_balances():
    m = ConcreteModel()
    m.cv = CVFrame()

    for t in MomentumBalanceType:
        if t == MomentumBalanceType.none:
            assert m.cv.add_momentum_balances(t) is None
        else:
            with pytest.raises(NotImplementedError):
                m.cv.add_momentum_balances(t)


#! /usr/bin/env python
# -*- coding:utf-8 -*-

from __future__ import print_function, division
import rospy
import numpy as np
import numpy
import tf
import math
import cv2
import time
from nav_msgs.msg import Odometry
from sensor_msgs.msg import Image, CompressedImage
from cv_bridge import CvBridge, CvBridgeError
from numpy import linalg
from tf import transformations
from tf import TransformerROS
import tf2_ros
from geometry_msgs.msg import Twist, Vector3, Pose, Vector3Stamped
from ar_track_alvar_msgs.msg import AlvarMarker, AlvarMarkers
from nav_msgs.msg import Odometry
from sensor_msgs.msg import Image
from std_msgs.msg import Header
from numpy import linalg
from tf import transformations
from tf import TransformerROS
import tf2_ros
import math
from geometry_msgs.msg import Twist, Vector3, Pose, Vector3Stamped
from ar_track_alvar_msgs.msg import AlvarMarker, AlvarMarkers
from nav_msgs.msg import Odometry
from sensor_msgs.msg import Image
from std_msgs.msg import Header
from sensor_msgs.msg import LaserScan

import visao_module_eiki

color = input("Cor:")
station = input("Estacao: ")
ID = input("ID: ")

instruction = [color, ID, station]

bridge = CvBridge()

cv_image = None
media_color = []
media_verde = []
media_amarelo = []
centro = []
atraso = 1.5E9 # 1 segundo e meio. Em nanossegundos

area = 0.0 # Variavel com a area do maior contorno

# Só usar se os relógios ROS da Raspberry e do Linux desktop estiverem sincronizados.
# Descarta imagens que chegam atrasadas demais
check_delay = False

resultados = [] # Criacao de uma variavel global para guardar os resultados vistos

x = 0
y = 0
z = 0
id = 0

# INTERSECÇÃO
xii = 0
yii = 0

frame = "camera_link"
# frame = "head_camera"  # DESCOMENTE para usar com webcam USB via roslaunch tag_tracking usbcam
scan_frente = []
tfl = 0

tf_buffer = tf2_ros.Buffer()

def scaneou(dado):
	#print("Faixa valida: ", dado.range_min , " - ", dado.range_max )
	#print("Leituras:")
	#print(np.array(dado.ranges).round(decimals=2))
	k = np.array(dado.ranges).round(decimals=2)
	global scan_frente
	scan_frente = [k[-8],k[-7],k[-6],k[-5],k[-4],k[-3],k[-2],k[-1],k[0],k[1],k[2],k[3],k[4],k[5],k[6],k[7],k[8]]
	#print("Intensities")
	#print(np.array(dado.intensities).round(decimals=2))

def recebe(msg):
	global x # O global impede a recriacao de uma variavel local, para podermos usar o x global ja'  declarado
	global y
	global z
	global id

	for marker in msg.markers:
		id = marker.id
		marcador = "ar_marker_" + str(id)

		print(tf_buffer.can_transform(frame, marcador, rospy.Time(0)))
		header = Header(frame_id=marcador)
		# Procura a transformacao em sistema de coordenadas entre a base do robo e o marcador numero 100
		# Note que para seu projeto 1 voce nao vai precisar de nada que tem abaixo, a
		# Nao ser que queira levar angulos em conta
		trans = tf_buffer.lookup_transform(frame, marcador, rospy.Time(0))

		# Separa as translacoes das rotacoes
		x = trans.transform.translation.x
		y = trans.transform.translation.y
		z = trans.transform.translation.z
		# ATENCAO: tudo o que vem a seguir e'  so para calcular um angulo
		# Para medirmos o angulo entre marcador e robo vamos projetar o eixo Z do marcador (perpendicular)
		# no eixo X do robo (que e'  a direcao para a frente)
		t = transformations.translation_matrix([x, y, z])
		# Encontra as rotacoes e cria uma matriz de rotacao a partir dos quaternions
		r = transformations.quaternion_matrix([trans.transform.rotation.x, trans.transform.rotation.y, trans.transform.rotation.z, trans.transform.rotation.w])
		m = numpy.dot(r,t) # Criamos a matriz composta por translacoes e rotacoes
		z_marker = [0,0,1,0] # Sao 4 coordenadas porque e'  um vetor em coordenadas homogeneas
		v2 = numpy.dot(m, z_marker)
		v2_n = v2[0:-1] # Descartamos a ultima posicao
		n2 = v2_n/linalg.norm(v2_n) # Normalizamos o vetor
		x_robo = [1,0,0]
		cosa = numpy.dot(n2, x_robo) # Projecao do vetor normal ao marcador no x do robo
		angulo_marcador_robo = math.degrees(math.acos(cosa))

		# Terminamos
		print("id: {} x {} y {} z {} angulo {} ".format(id, x,y,z, angulo_marcador_robo))

# A função a seguir é chamada sempre que chega um novo frame
def roda_todo_frame(imagem):
    print("frame")
    global cv_image
    global media_verde
    global media_color
    global media_amarelo
    global centro

    global resultados

    global xii
    global yii

    #global maior_area_verde

    #Variaveis para intersecção

    now = rospy.get_rostime()
    imgtime = imagem.header.stamp
    lag = now-imgtime # calcula o lag
    delay = lag.nsecs
    # print("delay ", "{:.3f}".format(delay/1.0E9))
    if delay > atraso and check_delay==True:
        print("Descartando por causa do delay do frame:", delay)
        return
    try:
        antes = time.clock()
        cv_image = bridge.compressed_imgmsg_to_cv2(imagem, "bgr8")
        # Note que os resultados já são guardados automaticamente na variável
        # chamada resultados
        centro, imagem, resultados =  visao_module_eiki.processa(cv_image)

        media_color, maior_area_color =  visao_module_eiki.identifica_cor(cv_image, color)

        media_amarelo, maior_area_amarelo =  visao_module_eiki.identifica_cor_amarelo(cv_image, color)

        #media_verde, maior_area_color =  visao_module_marcelo.identifica_cor(cv_image)

        #COR AMARELO
        #media_amarelo, maior_area_amarelo =  visao_module_marcelo.identifica_cor_amarelo(cv_image)

        for r in resultados:
            # print(r) - print feito para documentar e entender
            # o resultado
            pass

        depois = time.clock()
        # Desnecessário - Hough e MobileNet já abrem janelas
        #cv2.imshow("Camera", cv_image)

    except CvBridgeError as e:
        print('ex', e)

if __name__=="__main__":
    rospy.init_node("cor")

    topico_imagem = "/camera/rgb/image_raw/compressed"

    recebedor = rospy.Subscriber(topico_imagem, CompressedImage, roda_todo_frame, queue_size=4, buff_size = 2**24)
    recebedor = rospy.Subscriber("/ar_pose_marker", AlvarMarkers, recebe) # Para recebermos notificacoes de que marcadores foram vistos

    recebe_scan = rospy.Subscriber("/scan", LaserScan, scaneou) #SCAN

    print("Usando ", topico_imagem)

    velocidade_saida = rospy.Publisher("/cmd_vel", Twist, queue_size = 1)

    tolerancia = 25
    procura_animal = False
    pegou_color = False
    # Exemplo de categoria de resultados
    # [('chair', 86.965459585189819, (90, 141), (177, 265))]

    try:
        # Inicializando - por default gira no sentido anti-horário
        # vel = Twist(Vector3(0,0,0), Vector3(0,0,math.pi/10.0))

        while not rospy.is_shutdown():
            vel = Twist(Vector3(0,0,0), Vector3(0,0,0))
            if len(scan_frente) > 0:
                minimo = min(scan_frente)
            else:
                minimo = 5
            if minimo < 0.2: #Valor mínimo do scan (Saber se o robo bateu no verde)
                pegou_color = True
            #if achou_verde == True:
                #tolerancia = 25
                #vel = Twist(Vector3(0,0,0), Vector3(0,0,0))
                # len(media_verde) != 0 quando camera encontra verde
            if pegou_color == False:
                if len(media_color) != 0 and len(centro) != 0:
                    print ("PROCURANDO: ", color)
                    tolerancia = 15
                    vel = Twist(Vector3(0,0,0), Vector3(0,0,0))
                    print("Média do ", color, ": {0}, {1}".format(media_color[0], media_color[1]))
                    print ("Scann")
                    print ("MENOR TERMOOOOOOO: ",minimo)
                    if minimo>0.4:
                        x_objeto = media_color[0]
                        if (x_objeto < centro[0] - tolerancia):
                            # Vira à esquerda
                            vel = Twist(Vector3(0,0,0), Vector3(0,0,math.pi/12.0))
                            #print("ESQUERDA")
                        elif (x_objeto > centro[0] + tolerancia):
                            # Vira à direita
                            vel = Twist(Vector3(0,0,0), Vector3(0,0,-math.pi/12.0))
                            #print("DIREITA")
                        elif (centro[0]- tolerancia < x_objeto < centro[0] + tolerancia): # Gosto de usar a < b < c do Python. Não seria necessário neste caso
                            # Segue em frente
                            vel = Twist(Vector3(0.4,0,0), Vector3(0,0,0))
                            #print("FRENTE")
                    # PEGOU_COLOR SE TORNARA TRUE (FEITO FORA DO IF (LINHA 305:306))
                    if minimo<=0.4:
                        vel = Twist(Vector3(0.5,0,0),Vector3(0,0,-0.00005))
                        if minimo < 0.2: #Valor mínimo do scan (Saber se o robo bateu no verde)
                            pegou_color = True
                # SE NÃO ENCONTROU A COR, CONTINUA NO TRAJETO
                elif len(media_amarelo) != 0 and len(centro) != 0:
                    print ("NÃO PEGOU VERDEEE, NA TRILHA")
                    vel = Twist(Vector3(0,0,0), Vector3(0,0,0))
                    tolerancia = 50
                    if (media_amarelo[0] < centro[0] - tolerancia):
                        # Vira à esquerda
                        vel = Twist(Vector3(0,0,0), Vector3(0,0,+math.pi/12.0))
                        #print("ESQUERDA")
                    elif (media_amarelo[0] > centro[0] + tolerancia):
                        # Vira à direita
                        vel = Twist(Vector3(0,0,0), Vector3(0,0,-math.pi/12.0))
                        #print("DIREITA")
                    elif (centro[0]- tolerancia < media_amarelo[0] < centro[0] + tolerancia): # Gosto de usar a < b < c do Python. Não seria necessário neste caso
                        # Segue em frente
                        vel = Twist(Vector3(0.2,0,0), Vector3(0,0,0))
                        #print("FRENTE")
                # CASO JA TENHA ESBARRADO NO VERDE, COMEÇA A PROCURAR AMARELO
            else:
                if len(media_amarelo) != 0 and len(centro) != 0:
                    print ("PEGOU VERDEEE")
                    vel = Twist(Vector3(0,0,0), Vector3(0,0,0))
                    tolerancia = 50
                    if (media_amarelo[0] < centro[0] - tolerancia):
                        # Vira à esquerda
                        vel = Twist(Vector3(0,0,0), Vector3(0,0,+math.pi/12))
                        print("ESQUERDA")
                    elif (media_amarelo[0] > centro[0] + tolerancia):
                        # Vira à direita
                        vel = Twist(Vector3(0,0,0), Vector3(0,0,-math.pi/12))
                        print("DIREITA")
                    elif (centro[0]- tolerancia < media_amarelo[0] < centro[0] + tolerancia): # Gosto de usar a < b < c do Python. Não seria necessário neste caso
                        # Segue em frente
                        vel = Twist(Vector3(0.2,0,0), Vector3(0,0,0))
                        print("FRENTE")
                else:
                    # SE NÃO TIVER AMARELO NA TELA, RODA ATE ACHAR UM
                    print("PROCURANDO AMARELOO")
                    vel = Twist(Vector3(-0.7,0,0), Vector3(0,0,-1))
                    #TEMPO DO ROBO RODAR
                    rospy.sleep(0.5)

            velocidade_saida.publish(vel)
            rospy.sleep(0.1)
    except rospy.ROSInterruptException:
        print("Ocorreu uma exceção com o rospy")


from graph import GraphLIGA, trigrams
import argparse
import pickle
import pandas as pd
import runtime
import re

parser = argparse.ArgumentParser(description="Build a LIGA graph from given data.")

parser.add_argument('-f', '--filenames', help='filenames in data/raw folder to build graph on', required=True, nargs='+')
parser.add_argument('-s', '--chunksize', help='chunksize for reading text file, default 250', default=250)
parser.add_argument('-l', '--load_pretrained', help='preload pretrained model to build upon', action='count', default=0)
parser.add_argument('-b', '--breakafter', help='break after certain number of chunks', type=int, default=0)
parser.add_argument("-v", "--verbose", help="verbosity", action='count', default=0)
parser.add_argument('-r', '--resolve', help='resolve the graph at the end of build and save', default=0, action='count')

args = parser.parse_args()

convert_nums = lambda x: re.sub(r'[0-9]+[,][0-9]+|[0-9]+', '0', x)

if __name__ == '__main__':

    names=['True', 'Text']

    if args.load_pretrained:
        try:
            with open('graph_unresolved.p', 'rb') as f:
                G = pickle.load(f)
        except FileNotFoundError as e:
            print(e, 'No pretrained graph exists. Creating a new graph instead...')
            G = GraphLIGA()
    else:
        G = GraphLIGA()

    for filename in args.filenames:
        if 'event' in filename:
            lang = filename.replace('_train.csv','')[-2:]
            names = ['Text']

        chunks = pd.read_csv("../data/raw/{}".format(filename), sep="\t", header=None, names=names, chunksize=args.chunksize)

        for i, chunk in enumerate(chunks):

            if args.breakafter != 0 and i > args.breakafter:
                break
            if args.verbose:
                if (i+1)%10==0:
                    print(i)
                else:
                    print(i, end="\t")

            if 'event' in filename:
                chunk['True'] = lang

            #chunk = chunk.drop_duplicates().dropna(subset=['Text'])
            #chunk = chunk[chunk['True'].str.contains('zh')==False]
            #chunk = chunk[chunk['True'].str.contains('ko')==False]
            #chunk = chunk[chunk['True'].str.contains('ja')==False]
            #chunk = chunk[chunk['True'].str.contains('ar')==False]
            #chunk = chunk[chunk['True'].str.contains('hi')==False]
            #chunk = chunk[chunk['True'].str.contains('th')==False]

            # convert all numbers to single character representation, '0', and convert the text again into trigrams.
            chunk['Text'] = chunk['Text'].str.lower().apply(convert_nums).apply(trigrams)

            G.build_partial(chunk)
            print('Built partial graph for {}'.format(filename))

    with open('identifier/graph_unresolved.p', 'wb') as f:
        pickle.dump(G, f)
        print('Pickled unresolved graph as graph_unresolved_e.p')

    if args.resolve:
        G.resolve()
        with open('identifier/graph_resolved.p', 'wb') as g:
            pickle.dump(G, g)
            print('Pickled resolved graph as graph_resolved_e.p')


#!/usr/bin/env python
# Created at 2020/3/22

import click

import tensorflow as tf

from Algorithms.tf2.DoubleDQN.double_dqn import DoubleDQN

@click.command()
@click.option("--env_id", type=str, default="MountainCar-v0", help="Environment Id")
@click.option("--render", type=bool, default=False, help="Render environment or not")
@click.option("--num_process", type=int, default=1, help="Number of process to run environment")
@click.option("--lr", type=float, default=1e-3, help="Learning rate for Policy Net")
@click.option("--gamma", type=float, default=0.99, help="Discount factor")
@click.option("--polyak", type=float, default=0.995,
              help="Interpolation factor in polyak averaging for target networks")
@click.option("--epsilon", type=float, default=0.90, help="Probability controls greedy action")
@click.option("--explore_size", type=int, default=5000, help="Explore steps before execute deterministic policy")
@click.option("--memory_size", type=int, default=100000, help="Size of replay memory")
@click.option("--step_per_iter", type=int, default=1000, help="Number of steps of interaction in each iteration")
@click.option("--batch_size", type=int, default=128, help="Batch size")
@click.option("--min_update_step", type=int, default=1000, help="Minimum interacts for updating")
@click.option("--update_target_gap", type=int, default=50, help="Steps between updating target q net")
@click.option("--max_iter", type=int, default=500, help="Maximum iterations to run")
@click.option("--eval_iter", type=int, default=50, help="Iterations to evaluate the model")
@click.option("--save_iter", type=int, default=50, help="Iterations to save the model")
@click.option("--model_path", type=str, default="trained_models", help="Directory to store model")
@click.option("--log_path", type=str, default="../log/", help="Directory to save logs")
@click.option("--seed", type=int, default=1, help="Seed for reproducing")
def main(env_id, render, num_process, lr, gamma, polyak, epsilon, explore_size, memory_size, step_per_iter, batch_size,
         min_update_step, update_target_gap, max_iter, eval_iter, save_iter, model_path, log_path, seed):
    base_dir = log_path + env_id + "/DoubleDQN_exp{}".format(seed)
    writer = tf.summary.create_file_writer(base_dir)
    ddqn = DoubleDQN(env_id,
                     render=render,
                     num_process=num_process,
                     memory_size=memory_size,
                     lr_q=lr,
                     gamma=gamma,
                     polyak=polyak,
                     epsilon=epsilon,
                     explore_size=explore_size,
                     step_per_iter=step_per_iter,
                     batch_size=batch_size,
                     min_update_step=min_update_step,
                     update_target_gap=update_target_gap,
                     seed=seed)

    for i_iter in range(1, max_iter + 1):
        ddqn.learn(writer, i_iter)

        if i_iter % eval_iter == 0:
            ddqn.eval(i_iter, render=render)

        if i_iter % save_iter == 0:
            ddqn.save(model_path)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
CommerzBank spider created on the top of ATSSpider

scrapy crawl commerzbank -a iteration=1 -a mining_job_id=9999 -a extract=1 -a url="https://www.bechtle-jobs.com/index.php?ac=search_result"

Seed URL:
    https://jobs.commerzbank.com/index.php?ac=search_result
    https://jobs.mhp.com/index.php?ac=search_result
    https://www.bechtle-jobs.com/index.php?ac=search_result
"""

from re import compile
from scrapy.http import Request
from scrapy.selector import Selector
from urlparse import urljoin, urlparse
from urllib import urlencode

from brightcorp.base.atsspiders import ATSSpider
from brightcorp.items import BrightcorpItemLoader
from brightcorp.processors import ConvertDateString, Prefix

pattern = {
    'company': compile(r'(.*?)\.jobs$'),
    'ref_number': compile(r'id=(\d+)'),
}

class CommerzBank(ATSSpider):

    name = 'commerzbank'
    company = ''
    # Taking table header and map to english headers
    mapping_table_headers = {
        'title': 'title',
        'physical_location': 'location',
        'division': 'company',
        'activity_level': 'jobcategory',
        'date': 'date',
        'country': 'country',
    }

    page_params = {
        'ac': 'search_result',
        'ac': 'search_result',
        'hitsperpage': '10',
        'search_criterion_activity_level': '',
        'search_criterion_country': '',
        'search_criterion_entry_level': '',
        'search_criterion_keyword': '',
        'search_criterion_physical_location': '',
        'search_parameter_order_criterion': 'date',
        'search_parameter_order_direction': 'DESC',
        'search_parameter_page_current': '',
    }

    def __init__(self, *args, **kwargs):
        super(CommerzBank, self).__init__(*args, **kwargs)
        if 'url' in kwargs:
            self.company = self.get_company_name(kwargs['url'])

    def parse(self, response):
        """
        Getting the table header and build the meta xpaths.
        Parse SERP page and call GET method to each job urls.
        Pass meta information via GET method.
        """
        sel = Selector(response)
        # set expected job count
        if not self.expected_job_count_set:
            expected_count = sel.xpath(
                '//div[@class="searchoptions"]/h1/span[1]/text() |'
                '//div[@class="searchoptions"]/h1/text() |'
                '//div[@class="searchoptions"]/h2/strong[1]/text()'
            ).extract()
            if expected_count:
                self.expected_job_count = expected_count

        table_header = sel.xpath(
            '//div[@id="search-result"]/table/thead/tr/th/@class'
        ).extract()
        meta_xpaths = {}
        for th in table_header:
            if th in self.mapping_table_headers:
                meta_xpaths[
                    self.mapping_table_headers[th]
                ] = './td[%s]//text()' % str(table_header.index(th) + 1)
        for tr in sel.xpath(
            '//div[@id="search-result"]/table/tbody/tr'
        ):
            href = tr.xpath('./td[1]//a/@href').extract()
            if href:
                meta_data = {}
                for key, value in meta_xpaths.iteritems():
                    meta_data[key] = tr.xpath(value).extract()
                yield Request(
                    callback=self.parse_job_callback(),
                    meta=meta_data,
                    url=urljoin(response.url, href[0])
                )

        # pagination
        next_page = sel.xpath(
            '//ul[contains(@class, "pagination")]/li[@class="active"]/following-sibling::li[1]/a/text()'
        ).extract()
        if next_page:
            self.page_params.update({
                'search_parameter_page_current': str(next_page[0])
            })
            yield Request(
                callback=self.parse,
                url=urljoin(
                    response.url,
                    '/index.php?%s' % urlencode(self.page_params)
                )
            )

    def parse_job(self, response):
        sel = Selector(response)
        description_xpaths = [
            '//div/div/div/div/span[contains(text(), "AUFGABEN")]/../..',
            '//div/div/div/div/span[contains(text(), "PROFIL")]/../..',
            '//div/div/div/span[contains(text(), "FUNCTIEOMSCHRIJVING")]/../..',
            '//div/div/span[contains(text(), "TAKEN")]/../..',
            '//div/div/span[contains(text(), "PROFIL DU POSTE")]/../..',
            '//div/div/span[contains(text(), "TAREAS")]/../..',
            '//div/div/span[contains(text(), "Aufgaben")]/../..',
            '//div/div/span[contains(text(), "Tasks")]/../..',
            '//div/div/h2/a[contains(text(), "Your task")]/../../..',
            '//div/div/h2/a[contains(text(), "Ihre Aufgabe")]/../../..',
            '//div/div/h2/a[contains(text(), "Su tarea")]/../../..',
            '//div[@id="jobad"]/div[@id="job_content"]',
            '//div[@id="task-profile"]/div[@id="task"]',
        ]

        requirements_xpaths = [
            '//div/div/div/div/span[contains(text(), "JOB PROFIL")]/../..',
            '//div/div/div/span[contains(text(), "FUNCTIE-EISEN")]/../..',
            '//div/div/span[contains(text(), "PROFIEL")]/../..',
            '//div/div/span[contains(text(), "PERFIL")]/../..',
            '//div/div/div/div/span[contains(text(), "PROFIL")]/../..',
            '//div/div/span[contains(text(), "Qualifikationen")]/../..',
            '//div/div/span[contains(text(), "Qualifications")]/../..',
            '//div/div/span[contains(text(), "VOS COMPETENCES")]/../..',
            '//div/div/h2/a[contains(text(), "Your Profile")]/../../..',
            '//div/div/h2/a[contains(text(), "Ihr Profil")]/../../..',
            '//div/div/h2/a[contains(text(), "Su perfil")]/../../..',
            '//div[@id="task-profile"]/div[@id="profile"]',
        ]
        company_xpaths = [
            '//div/p[contains(text(), "Company:")]/following-sibling::p[1]/text()',
            '//div/p[contains(text(), "Gesellschaft:")]/following-sibling::p[1]/text()',
            '//div/p[contains(text(), "%s")]/following-sibling::p[1]/text()' % unicode('Compañía:', 'utf-8'),
        ]
        company_description_xpaths = [
            '//div/div[@id="about-us"] |'
            '//div/div/span[contains(text(), "Unternehmensprofil")]/../..',
            '//div/div/span[contains(text(), "At a Glance")]/../..',
            '//div/div/h2/a[contains(text(), "Sobre nosotros")]/../../following-sibling::div[1]',
            '//div/div/h2/a[contains(text(), "%s")]/../../following-sibling::div[1]' % unicode('Über uns', 'utf-8'),
        ]
        location_xpaths = [
            '//div/p[contains(text(), "Locations:")]/following-sibling::p[1]/text()',
            '//div/p[contains(text(), "Standorte:")]/following-sibling::p[1]/text()',
            '//div/p[contains(text(), "Ubicaciones:")]/following-sibling::p[1]/text()',
        ]
        jobtype_xpaths = [
            '//div/p[contains(text(), "Employment type:")]/following-sibling::p[1]/text()',
            '//div/p[contains(text(), "Vertragsart:")]/following-sibling::p[1]/text()',
            '//div/p[contains(text(), "Tipo de contrato:")]/following-sibling::p[1]/text()',
        ]
        experiencerequirements_xpaths = [
            '//div/p[contains(text(), "Experience Required:")]/following-sibling::p[1]/text()',
            '//div/p[contains(text(), "Berufserfahrung:")]/following-sibling::p[1]/text()',
            '//div/p[contains(text(), "Experiencia requerida:")]/following-sibling::p[1]/text()',
        ]

        location = response.meta.get('location')
        country = response.meta.get('country')
        if location and country:
            location = ', '.join([location[0], country[0]])
        loader = BrightcorpItemLoader(selector=sel)

        loader.add_value(
            'title', response.meta.get('title')
        )
        if not loader.get_output_value('title'):
            loader.add_xpath(
                'title',
                '//div[contains(@class, "facts-inner")]/div/h1/text()'
            )
        for description_xpath in description_xpaths:
            loader.add_xpath(
                'description', description_xpath
            )
            if loader.get_output_value('description'):
                break
        loader.add_value(
            'company', response.meta.get('company')
        )
        if not loader.get_output_value('company'):
            loader.add_xpath(
                'company', company_xpaths
            )
        loader.add_xpath(
            'company_description', company_description_xpaths
        )
        loader.add_xpath(
            'referencenumber',
            '//p/label[contains(text(), "%s")]/../text()' % unicode('Kennziffer:', 'utf-8'),
            Prefix('%s-%s-' % (self.name, self.company))
        )
        if not loader.get_output_value('referencenumber'):
            loader.add_value(
                'referencenumber',
                response.url,
                Prefix('%s-%s-' % (self.name, self.company)),
                re=pattern['ref_number']
            )
        loader.add_value('location', location)
        if not loader.get_output_value('location'):
            loader.add_xpath(
                'location', location_xpaths
            )
        loader.add_value(
            'date',
            response.meta.get('date'),
            ConvertDateString('%d.%m.%Y')
        )
        loader.add_value(
            'jobcategory', response.meta.get('jobcategory')
        )
        loader.add_xpath(
            'jobtype', jobtype_xpaths
        )
        loader.add_xpath(
            'expiration_date',
            '//div/p[contains(text(), "Application Deadline:")]/following-sibling::p[1]/text()',
            ConvertDateString('%Y-%m-%d')
        )
        loader.add_xpath(
            'experiencerequirements', experiencerequirements_xpaths
        )
        loader.add_value('url', response.url)
        loader.add_value('apply_url', response.url)
        for requirements_xpath in requirements_xpaths:
            loader.add_xpath(
                'requirements', requirements_xpath
            )
            if loader.get_output_value('requirements'):
                break

        yield loader.load_item()

    def get_company_name(self, url):
        result = ''
        url_parsed = urlparse(url).netloc
        match = pattern['company'].search(url_parsed)
        if match:
            result = match.group(1)
        else:
            url_parts = url_parsed.split('.')
            if len(url_parts) == 3:
                result = url_parts[1]

        return result


import json
from django import template
from django.core.urlresolvers import reverse
from pomco.views import _league
from pomco.models import Team
from pomco.clubs import CLUBS
import pomco

register = template.Library()

@register.filter
def one_error(form):
    mssg = ""
    for field in form:
        if not field.errors:
            continue
        for error in field.errors:
            mssg += field.label + ": " + error
            break
        if mssg:
            break;
    return mssg

@register.simple_tag
def pomco_name():
    return "Mon Beau Tableau"

@register.simple_tag
def disabled_if_not_logged_in(user):
    if user.is_authenticated():
        return ""
    return "disabled=True"

@register.simple_tag
def ligue_landing(stats, team):
    if not team:
        return reverse('ligueoverview', kwargs={'stats':stats})
    if type(team)==Team:
        if team.league == _league(stats):
            return reverse('team', kwargs={'stats':stats, 'team_id':team.url_id()})
        return reverse('ligueoverview', kwargs={'stats':stats})
    return reverse('team', kwargs={'stats':stats, 'team_id':team['url_id']})

@register.inclusion_tag('pomco/clubs_selectoptions.html')
def clubsoptions(stats):
    clubs = []
    context = {
        'clubs':clubs
    }
    l = _league(stats)
    for club in CLUBS[l]:
        clubs.append({
            'value':club,
            'name':CLUBS[l][club],
        })
    if l == pomco.PL:
        for club in clubs:
            club['name']=club['value']
    return context

@register.simple_tag
def datateams(teams):
    dtos = [{key: t.__dict__[key] for key in ["members", "team_name", "league", "id"]} for t in teams]
#    return dtos
    return json.dumps(dtos)


# importar modulos de pygame
# esto es una plantilla
#suppor: utf-8
import pygame, sys

from pygame.locals import *
#antes de esto no se mete nada mas si no da casque.
pygame.init()  #esto lo inicia
ventana = pygame.display.set_mode((700, 700))  #tamano ventana
pygame.display.set_caption("The Wizar of CodeBastards")  #aqui va lo que pone en la cabecera

#cargamos un objeto
Mi_imagen = pygame.image.load("Imagenes/wizardcode.png")
posX, posY = 130, 70
ventana.blit(Mi_imagen,(posX, posY))

while True:
    for evento in pygame.event.get():
        if evento.type == QUIT:
            pygame.quit()
            sys.exit()

    pygame.display.update()  #refresca la ventanica


from typing import List
# Using backtracking
class Solution:
    def permute(self, nums: List[int]) -> List[List[int]]:

        # curr = current elements in the iteration.
        # elements = remaining elements to be considered.
        # added = list of indicaters whether element added to curr or not.
        def permute_helper(curr, added):
            # base condition
            if len(curr) == l:
                res.append(curr[:])
            for i in range(l):
                if not added[i]:
                    # We add the element to curr
                    curr.append(nums[i])
                    added[i] = True
                    permute_helper(curr, added)
                    # Now we backtrack to remove that element to add another element like [1,2] to [1] and then move to [1,3]
                    curr.pop()
                    added[i] = False

        res = []
        l = len(nums)
        added = [False]*len(nums)
        permute_helper([], added)
        return res

    # Without using backtracking
    # import itertools
    # def permute(self, nums: List[int]) -> List[List[int]]:
    #     res = list(itertools.permutations(nums))
    #     return res

soln = Solution()
print(soln.permute([1,2,3]))


# -*- coding: utf-8 -*-
from django.conf.urls import include, url
from .views import FileListView, BCKSHomeView, TodosUpdate
from lockdown.decorators import lockdown

urlpatterns = [
    url(r'^$', lockdown()(BCKSHomeView.as_view()), name='home_bcks'),
    url(r'^update_database(?:/(?P<slug>.*))?/$', 'bcks_files.views.update_database', name='update_database'),
    url(r'^delete_deletables(?:/(?P<slug>.*))?/$', 'bcks_files.views.delete_deletables', name='delete_deletables'),
    url(r'^update_todos/$', lockdown()(TodosUpdate.as_view()), name='update_todos'),
    # url(r'^(?P<slug>[-_\w]+)/$', FileListView.as_view(), name='files'),
    # url(r'^(?P<slug>[-_\w]+)/$', FileTypeCreateView.as_view(), name='files'),
    url(r'^(?P<slug>[-_\w]+)/$', 'bcks_files.views.manage_files', name='files'),
]


import webapp2
from google.appengine.ext import ndb
from google.appengine.api import users
from userModel import UserModel

class UserAuth(webapp2.RequestHandler):
    def __init__(self, requesturi):
        self.requesturi = requesturi

    def userTemplateVals(self):
        url = ''
        url_string = ''
        username = ''
        user = users.get_current_user()
        if user:
            url = users.create_logout_url('/')
            url_string = 'Logout'
            userEmail = "%s" % (user)
            get_user = UserModel.query(UserModel.email==userEmail).fetch()
            if get_user and len(get_user)>0:
                userInfo = get_user[0]
                if userInfo.user_name:
                    username = userInfo.user_name
                else:
                    username = ''
            else:
                username = ''
        else:
            url = users.create_login_url(self.requesturi)
            url_string = 'Login'

        template_values = {
            'url': url,
            'url_string': url_string,
            'user': user,
            'username': username
        }
        return template_values


#!/usr/bin/env python3

import os
import shlex
import subprocess
import sys
import tempfile

def run(command, *args, **kwargs):
    return subprocess.call(shlex.split(command), *args, **kwargs)

container, file_path = sys.argv[1].split(':')
file_name = file_path.split('/')[-1]

with tempfile.TemporaryDirectory() as tmpdir:
    run('docker cp {container}:{file_path} {tmpdir}/{file_name}'.format(
        container=container, file_path=file_path, tmpdir=tmpdir,
        file_name=file_name,
    ))
    run('{editor} {tmpdir}/{file_name}'.format(
        editor=os.environ['EDITOR'], tmpdir=tmpdir, file_name=file_name,
    ))
    run('docker cp {tmpdir}/{file_name} {container}:{file_path}'.format(
        container=container, file_path=file_path, tmpdir=tmpdir,
        file_name=file_name,
    ))


from collections import namedtuple, defaultdict
from datetime import datetime

file_name = 'nyc_parking_tickets_extract-1.csv'

#Goal 1
def cast(data_type, value):
    '''This function returns the value after casting it to the expected data type
        Input parameters:
            1. Data Type: Type to be converted to
            2. Value: Value which needs to be converted
    '''
    if data_type == int:
        return int(value)
    elif data_type == datetime:
        return datetime.strptime(value, '%m/%d/%Y')
    else:
        return str(value)

def cast_row(data_types, data_row):
    '''This function returns a list of values after casting each of the list elements
        to the corresponding expected data type
        1. Data Types: An array of data types
        2. Data Row: An array of values
    '''
    return [cast(data_type, data_value) for data_type, data_value in list(zip(data_types, data_row))]

#To extract column names and data types
with open(file_name, encoding='latin-1') as f:
    headers = f.readline().strip('\n').replace(' ', '_').split(',')
    datarow = f.readline().strip('\n').split(',')
    data_types = []
    for column in datarow:
        if column.isdigit():
            data_types.append(int)
        else:
            try:
                datetime.strptime(column, '%m/%d/%Y')
                data_types.append(datetime)
            except:
                data_types.append(str)
    Ticket = namedtuple('Ticket', headers)

#To extract data
def get_tickets():
    '''This function iterates through all the data rows in the csv
        and yields a named tuple after converting each item in the
        tuple to its corresponding data type.
        Note: 1 row had a "," within the data which was removed.
    '''
    with open(file_name, encoding='latin-1') as f:
        #Skip header row
        next(iter(f))
        for line in f:
            data = cast_row(data_types, line.strip('\n').split(','))
            yield Ticket(*data)

get_ticket = iter(get_tickets())

print(next(get_ticket))
print('\n')
print(next(get_ticket))
print('\n')
print(next(get_ticket))

#Goal 2
violations = defaultdict(int)
for ticket in get_tickets():
    violations[ticket.Vehicle_Make] += 1
print('\n')
print(violations)


#!/usr/bin/env python
#-*- coding: utf-8 -*-

import os
from setuptools import setup, find_packages

if os.name in ["posix", "os2"]: #GNU/Linux and MacOS
    os_kwargs = {"data_files": [
                                ("/usr/share/applications", ["pinguino12.desktop"]),
                                ("/usr/share/icons/hicolor/256x256", ["pinguino/qtgui/resources/art/pinguino_logo-256x256.png"]),
                                ("/usr/share/icons/hicolor/256x256", ["pinguino/qtgui/resources/art/pinguino_logo_background_blue-256x256.png"]),
                                ],
                }

# elif os.name == "os2":  #Mac OS X
    # os_kwargs = {}

elif os.name == "nt":  #Windows
    os_kwargs = {"data_files": [
                                # ("C:\pinguino", ["pinguino12.desktop"]),
                                ("C:\pinguino\icons", ["pinguino/qtgui/resources/art/pinguino_logo-256x256.ico"]),
                                ("C:\pinguino\icons", ["pinguino/qtgui/resources/art/pinguino_logo_background_blue-256x256.ico"]),
                                ],
                }

from pinguino import version

setup(name = "pinguino",
      version = "{}.{}.{}".format(version.MAJOR, version.MINOR, version.PATCH),
      description = "Open Hardware Electronics Prototyping Platform, Open Source Integrated Development Environment (IDE)",
      description_file = "README.rst",

      author = "",
      author_email = "",
      maintainer = "Yeison Cardona",
      maintainer_email = "yeisoneng@gmail.com",

      url = "http://www.pinguino.cc/",
      download_url = "",

      license = "GPLv2",
      keywords = "microchip, electronic, prototyping, IDE",

      classifiers=[#list of classifiers in https://pypi.python.org/pypi?:action=list_classifiers
                   "Environment :: X11 Applications :: Qt",
                   "Intended Audience :: Developers",
                   "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
                   "Operating System :: POSIX :: Linux",
                   "Operating System :: Microsoft :: Windows",
                   "Operating System :: MacOS",
                   "Programming Language :: Python",
                   "Topic :: Software Development",
                   "Topic :: Software Development :: Code Generators",
                   "Topic :: Software Development :: Compilers",
                   "Topic :: Software Development :: Debuggers",
                   ],

      packages = find_packages(),
      include_package_data = True,

      install_requires = [
                          #"gitpython",
                          #"hgapi",
                          "pyusb==1.0.0b2",
                          #"setuptools",
                          #"wheel"
                          #"pyside",
                          ],

      extras_require = {
          "gui":  ["pyside", 'requests']
      },

      scripts = [
                 "cmd/pinguinoide",			# pinguino ide launcher
                 "cmd/pinguinoide.pyw",		# pinguino ide launcher with extension for windows
                 "cmd/pinguino-cmd",		# pinguino command line
                 "cmd/pinguino-cmd.py",		# pinguino command line with extension for windows
                 "cmd/pinguino-reset",
                 "cmd/pinguino-reset.py",
                 "cmd/pinguino-test",
                 "cmd/pinguino-test.py",
                 ],

      zip_safe = False,

      **os_kwargs
      )


# Copyright (c) 2015 Akanda, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import mock
from astara.api import keystone

from astara.test.unit import base

from oslo_config import cfg

class KeystoneTest(base.RugTestBase):
    def setUp(self):
        super(KeystoneTest, self).setUp()
        self.config(auth_region='foo_regin')

    @mock.patch('keystoneclient.session.Session')
    @mock.patch('keystoneclient.auth.load_from_conf_options')
    def test_session(self, mock_load_auth, mock_session):
        fake_auth = mock.Mock()
        mock_load_auth.return_value = fake_auth
        fake_session = mock.Mock()
        mock_session.return_value = fake_session
        ks_session = keystone.KeystoneSession().session
        mock_load_auth.assert_called_with(cfg.CONF, 'keystone_authtoken')
        mock_session.assert_called_with(auth=fake_auth)
        self.assertEqual(ks_session, fake_session)


from __future__ import unicode_literals

from django.conf.urls import include, url, static
from django.conf.urls.i18n import i18n_patterns
from django.contrib import admin
from django.conf import settings

from . import views

admin.autodiscover()

urlpatterns = i18n_patterns(
    # Change the admin prefix here to use an alternate URL for the
    # admin interface, which would be marginally more secure.
    url("^admin/", include(admin.site.urls)),
)

urlpatterns += [

    url("^$", views.index, name="home"),

    url(r'^(?i)categories/(?P<id>[0-9]+)/posts/recent-5/$', views.RecentFiveList.as_view(), name='recent_5'),

    # Take category id as kwarg
    url(r'^(?i)categories/(?P<id>[0-9]+)/posts/$', views.BlogCategoryPosts.as_view(), name="category_posts"),

    url('categories/', views.BlogCategoryList.as_view(), name="category_list"),

    # Take id as kwarg
    # url(r'^posts/(?P<id>[0-9]+)/$', views.BlogPostDetail.as_view(), name="post_detail"),

    # Take slug as kwarg
    url(r'^posts/(?P<slug>[\w\-]+)/$', views.BlogPostDetail.as_view(), name="post_detail"),

    url("posts/", views.BlogPostList.as_view(), name="posts_list"),

    url(r'^.*$', views.index, name="home"),

    url("^", include("mezzanine.urls")),

] + static.static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) # For AWS S3


#!env/bin/python
import argparse
import flask

app = flask.Flask(__name__)

some_list = ['Name', 'Team', 'Position', 'Opponent', 'Our Predictions', 'Value']

value_column_index = 5;
position_column_index = 2;
team_column_index= 1;
name_column_index =0;

nbai = [
    ['LeBron James', 'CLE', 'SF', 'BOS', 28, 'Overvalued'],
    ['Kevin Durant', 'GSW', 'SF', 'HOU', 26, 'Overvalued'],
    ['Kevin Love', 'CLE', 'C', 'BOS', 22, 'Overvalued'],
    ['Stephen Curry', 'GSW', 'PG', 'HOU', 22, 'Undervalued'],
    ['James Harden', 'HOU', 'SG', 'GSW', 21, 'Undervalued'],
    ['Al Horford', 'BOS', 'SF', 'CLE', 21, 'Overvalued'],
    ['David West', 'GSW', 'PF', 'HOU', 19, 'Overvalued'],
    ['Deron Williams', 'CLE', 'PG', 'BOS', 19, 'Overvalued'],
    ['Kyrie Irving', 'BOS', 'PG', 'CLE', 18, 'Undervalued'],
    ['Andre Iguodala', 'GSW', 'SF', 'HOU', 17, 'Undervalued'],
    ['Isaiah Thomas', 'CLE', 'PG', 'BOS', 15, 'Undervalued'],
    ['Klay Thompson', 'GSW', 'SG', 'HOU', 14, 'Undervalued'],
    ['Eric Gordon', 'HOU', 'PF', 'GSW', 14, 'Overvalued'],
    ['Trevor Ariza', 'HOU', 'PF', 'GSW', 14, 'Overvalued'],
    ['Tristan Thompson', 'CLE', 'PF', 'BOS', 13, 'Undervalued'],
    ['Ryan Anderson', 'HOU', 'PF', 'GSW', 13, 'Overvalued'],
    ['Lou Williams', 'HOU', 'SG', 'GSW', 12, 'Overvalued'],
    ['Amir Johnson', 'BOS', 'SF', 'CLE', 12, 'Undervalued'],
    ['Matt Barnes', 'GSW', 'PG', 'HOU', 11, 'Undervalued'],
    ['Tyler Zeller', 'BOS', 'PG', 'CLE', 11, 'Overvalued'],
    ['Draymond Green', 'GSW', 'C', 'HOU', 11, 'Overvalued'],
    ['JaVale McGee', 'GSW', 'C', 'HOU', 11, 'Overvalued'],
    ['JR Smith', 'CLE', 'SG', 'BOS', 11, 'Undervalued'],
    ['Patrick Beverley', 'HOU', 'PF', 'GSW', 11, 'Undervalued'],
    ['Richard Jefferson', 'CLE', 'PF', 'BOS', 11, 'Overvalued'],
    ['Kelly Olynyk', 'BOS', 'SG', 'CLE', 11, 'Undervalued'],
    ['Channing Frye', 'CLE', 'SF', 'BOS', 10, 'Overvalued'],
    ['Marcus Smart', 'BOS', 'PF', 'CLE', 10, 'Undervalued'],
    ['Kyle Korver', 'CLE', 'SG', 'BOS', 10, 'Overvalued'],
]
#    ['Derrick Williams', 'CLE', 'PG', 'BOS', 9, 'Overvalued'],
#    ['Zaza Pachulia', 'GSW', 'PF', 'HOU', 9, 'Overvalued'],
#    ['Jonas Jerebko', 'BOS', 'C', 'CLE', 9, 'Overvalued'],
#    ['Shaun Livingston', 'GSW', 'DF', 'HOU', 8, 'Overvalued']

teamlist = ['CLE', 'GSW', 'HOU', 'BOS']

@app.route('/', defaults={'path': ''})
@app.route('/<path:path>')
def home_page(path):
    return flask.render_template(
        'index.html',
        header_list=some_list,
        website_table=nbai,
        position_index=position_column_index,
        team_index=team_column_index,
        name_index=name_column_index,
        value_index=value_column_index,
        team_list = teamlist
    )

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--host', default='0.0.0.0')
    parser.add_argument('--port', default=5000, type=int, choices=xrange(1, 65536))
    return parser.parse_args()

if __name__ == '__main__':
    args = parse_args()
    app.run(host=args.host, port=args.port)


import os
import random

import chainer
import numpy as np
import librosa
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

class FFT(object):
    def __init__(self, sr=8000, n_fft=1024, hop_length=None, window='hann',
                 loop=100):
        self.sr = sr
        self.n_fft = n_fft
        if hop_length is None:
            self.hop_length = n_fft // 2
        else:
            self.hop_length = hop_length
        self.window = window
        self.loop = loop

    def read_file(self, file):
        y, sr = librosa.core.load(file, self.sr, res_type='kaiser_fast')

        return y

    def fft(self, y):
        # y is file
        if isinstance(y, str):
            y = self.read_file(y)
        D = librosa.core.stft(y, self.n_fft, self.hop_length,
                              window=self.window)
        return D

    def ifft(self, D):
        y = librosa.core.istft(D, self.hop_length,
                               window=self.window)
        return y

    def spectrogram(self, D):
        # D is file
        if isinstance(D, str):
            D = self.fft(D)
        # D is wave
        elif len(D.shape) == 1:
            D = self.fft(D)
        # D is amplitude spectrogram
        S = librosa.core.amplitude_to_db(D, ref=np.max)
        return S

    def GriffinLim(self, S, loop=None, spec_type='db'):
        if loop is None:
            loop = self.loop
        if spec_type == 'db':
            D = librosa.core.db_to_amplitude(S)
        else:
            D = S
        p = np.angle(D)

        for i in range(loop):
            y = self.ifft(D * np.exp(1j * p))
            p = np.angle(self.fft(y))
        return y

    def preprocess(self, S):
        S += 40
        S /= 40
        return np.expand_dims(S, 1)

    def ipreprocess(self, S):
        S *= 40
        S -= 40
        return np.squeeze(S)

    def save_spec_as_wav(self, path, S, preprocessed=False, spec_type='db'):
        if preprocessed:
            S = self.ipreprocess(S)
        y = self.GriffinLim(S, spec_type=spec_type)
        librosa.output.write_wav(path, y, self.sr)

class preprocess(object):
    def __init__(self, fft, length):
        self.fft = fft
        self.length = length

    def __call__(self, *files):
        specs = []
        lengths = []
        for file in files[0]:
            npy = file.replace('.m4a', '_{}_{}_{}_{}.npy'.format(
                self.fft.sr, self.fft.n_fft, self.fft.hop_length,
                self.fft.window))
            wav = file.replace('.m4a', '_{}.wav'.format(self.fft.sr))
            if os.path.exists(npy):
                spec = np.load(npy)
            else:
                if os.path.exists(wav):
                    file = wav
                y = self.fft.read_file(file)
                spec = self.fft.spectrogram(y)
                np.save(npy, spec)
            specs += [spec]
            lengths += [spec.shape[1] - self.length]
        length_start = random.randint(0, min(lengths))
        x = ()
        for i, spec in enumerate(specs):
            spec = spec[:, length_start:length_start + self.length]
            spec = self.fft.preprocess(spec)
            x += (spec,)
        return x

def generate_vocal(encoder, decoder, generator, fft, iterator, device):
    trigger = (500, 'iteration')

    @chainer.training.make_extension(trigger)
    def generate(trainer):
        if trigger[1] == 'epoch':
            n = trainer.updater.epoch
        else:
            n = trainer.updater.iteration

        iterator.reset()
        converter = chainer.dataset.convert.concat_examples
        out = os.path.join(trainer.out,
                           '{}{}'.format(trigger[1], n))
        try:
            os.makedirs(out)
        except OSError:
            pass

        for i, batch in enumerate(iterator):
            with chainer.using_config('enable_backprop', False):
                with chainer.using_config('train', False):
                    acc, vocal = converter(batch, device)
                    # generate
                    h_acc = encoder(acc)
                    h_gen = generator(h_acc)
                    generated = chainer.cuda.to_cpu(decoder(h_gen).data)

                    # ipreprocess
                    acc = fft.ipreprocess(chainer.cuda.to_cpu(acc))[::-1]
                    vocal = fft.ipreprocess(chainer.cuda.to_cpu(vocal))[::-1]
                    generated = fft.ipreprocess(generated)[::-1]

                    # save audio
                    fft.save_spec_as_wav(
                        os.path.join(out, '{}.wav'.format(i)), generated)

                    # save spectrogram
                    spectrograms = np.vstack((acc, vocal, generated))
                    plt.figure()
                    plt.imshow(spectrograms)
                    plt.savefig(os.path.join(out, '{}.png'.format(i)))
                    plt.close()

    return generate

def set_opt(model, optimizer, *hooks):
    optimizer.setup(model)
    for hook in hooks:
        optimizer.add_hook(hook)
    return optimizer


#网址:https://m.w23us.com/
from bs4 import BeautifulSoup
import requests
import time
import sys
import re

class DingDian:

    def GetBook(self, name, author = None, agent = None): #获取小说链接url name:书名,author:作者,agent:代理
        url = 'https://m.w23us.com/home/search'
        Data = {'action':'search', 'q':name}
        html = requests.post(url, data=Data)
        HTML = BeautifulSoup(html.text, 'lxml')
        intro = HTML.find_all(class_="intro")
        return (html.text, name, author, str(intro)[55:-7])

    def GetHtml(self, url): #获取章节信息
        html = BeautifulSoup(url, 'lxml')
        a = html.find_all(id="chapterlist")
        HTML = re.findall('href="(\S+)"', str(a))
        NAME = re.findall('>(.+)</a>', str(a))
        if len(HTML) != len(NAME): #获取的链接数量与章节数量不匹配
            return None
        b = list()
        for i in range(len(HTML)):
            c = ('https://m.w23us.com' + HTML[i], NAME[i])
            b.append(c)
        return b

    def GetText(self, url): #获取小说内容
        isok = True
        text = str()
        while isok:
            html = requests.get(url)
            a = BeautifulSoup(html.text, 'lxml')
            b = a.find_all(id="BookText")
            c = BeautifulSoup(str(b), 'lxml')
            d = c.div.text.replace('\xa0', '').strip() #获取文本内容并删除多余空白字符
            text += d
            next = re.findall('href="(\S+)"', str(c))
            if len(next) == 0:
                isok = False
            else:
                url = 'https://m.w23us.com' + next[0]
        #print(text)
        return text


import pdfplumber
import pyttsx3

pdf_file = ''## Add pdf file path here ##
engine = pyttsx3.init()

def pdf_to_voice(file):
#Open pdf
    with pdfplumber.open(file) as pdf:
        book = ''
        #Get total pages
        total_pages = len(pdf.pages)
        #loop through pages
        for i in range(0, total_pages-1):
            #Extract text
            page = pdf.pages[i].extract_text()
            book = book + page
        #Save to file
        engine.save_to_file(book, 'book.mp3')
        engine.runAndWait()

pdf_to_voice(pdf_file)


# -*- coding: utf-8 -*-
"""
Created on Sat Feb  6 18:15:50 2016

@author: Lingyu
"""

import numpy as np
import csv
import glob
import cv2
import os
from skimage import feature

# obtain the training_set
X=[]
y1=[]
y=[]
name=[]

with open('train.csv') as csvfile:
    reader=csv.DictReader(csvfile)
    for row in reader:
        y1.append(int(row['Prediction']))

# get train_set
file_lists=glob.glob('data/train/*')

for image in file_lists:
    c=os.path.basename(image).split(".")[0]
    name.append(c)
    d=int(c.split("_")[1])
    y.append(y1[d])
    img=cv2.imread(image)
    gray=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    lbp_image=feature.local_binary_pattern(gray,8,3)
    b=lbp_image.max()+1
    #histogram=[np.histogram(iim[...,channel])[0] for channel in [0,1,2]]
    histogram=np.histogram(lbp_image,normed=True,bins=b,range=(0,b))[0]
    X.append(histogram)

#load the test data
name1=[]
name_value=[]
y_pred=[]
Test_set=[]

file_lists_test=glob.glob('data/test/*')
for image in file_lists_test:
    c=os.path.basename(image)
    name1.append(c)
    c=os.path.basename(image).split('.')[0]
    d=int(c.split("_")[1])
    name_value.append(d)
    img=cv2.imread(image)
    gray=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    lbp_image=feature.local_binary_pattern(gray,8,3)
    b=lbp_image.max()+1
    histogram=np.histogram(lbp_image,normed=True,bins=b,range=(0,b))[0]
    Test_set.append(histogram)

indexes=[]
Test_sort=[]
name_sort=[]
indexes=sorted(name_value,key=lambda k: name_value[k])
for n in indexes:
    Test_sort.append(Test_set[n])
    name_sort.append(name1[n])

np.savetxt("test_set.txt",Test_sort)
np.savetxt("test_set_name.txt",name_sort,fmt='%s')
np.savetxt("train_set.txt",X)
np.savetxt("train_label.txt",y,fmt='%u')


from pyecharts import Bar,Line
import random

# bar = Bar("主标题","副标题")
# month = ['{}月'.format(i) for i in range(1,13)]
# v1 = [random.randint(100,200) for i in range(1,13)]
# v2 = [random.randint(100,200) for i in range(1,13)]
# bar.use_theme("dark")
# bar.add('蒸发量',month,v1,mark_line=['average'],mark_point=['max','min'],is_convert=True)
# bar.add('降水量',month,v2,mark_line=['average'],mark_point=['max','min'],is_convert=True)

attr = ["{}天".format(i) for i in range(30)]
v1 = [random.randint(1, 30) for _ in range(30)]
bar = Bar("Bar - datazoom - slider 示例")
bar.add("", attr, v1,is_label_show=True,is_datazoom_show=True,
        datazoom_type='both',datazoom_range=[50,100],yaxis_rotate=30,yaxis_min=5)
bar.render('html/months.html')

# xaxis_rotate=30 x轴坐标值旋转30度
# yaxis_rotate=30 y轴坐标值旋转30度
# yaxis_min=5 表示y轴最小值为5(数值轴有效)
# xaxis_min=5 表示x轴最小值为5(数值轴有效)


from pathlib import Path

projectDirectory = Path().resolve()
labeledDataPath = projectDirectory / 'labeledData'
trainingDataPath = projectDirectory / 'trainingData'
modelPath = projectDirectory / 'models'
modelPathTrain = modelPath / 'train'
modelPathTest = modelPath / 'test'

shuffledMat = 'shuffledMat.mat'
shuffledPickle = 'shuffledPickle.pickle'

scorer ='parts'
trainingFraction = [0.8]
pose_cfg = projectDirectory / 'pose_cfg.yaml'
bodyparts = ['L_F_Paw',
              'L_F_Knee',
              'L_F_Elbow',
              'L_B_Paw',
              'L_B_Knee',
              'L_B_Elbow',
              'R_F_Paw',
              'R_F_Knee',
              'R_F_Elbow',
              'R_B_Paw',
              'R_B_Knee',
              'R_B_Elbow',
              'TailBase',
              'L_EarBase',
              'R_EarBase',
              'Nose',
              'L_Eye',
              'R_Eye',
              'Withers',
              'Throat']

bones = [
    [['L_F_Paw', 'L_F_Knee'], ['L_F_Knee', 'L_F_Elbow'], ['L_F_Elbow', 'R_F_Elbow'], ['R_F_Elbow', 'R_F_Knee'], ['R_F_Paw', 'R_F_Knee']],
    [['R_F_Elbow', 'Withers'], ['L_F_Elbow', 'Withers']],
    [['L_B_Paw', 'L_B_Knee'],['L_B_Knee', 'L_B_Elbow'], ['L_B_Elbow', 'R_B_Elbow'], ['R_B_Elbow', 'R_B_Knee'], ['R_B_Paw', 'R_B_Knee']],
    [['R_B_Elbow', 'TailBase'], ['L_B_Elbow', 'TailBase'], ['TailBase', 'Withers'], ['Withers', 'Throat'], ['Throat', 'Nose'], ['L_EarBase', 'R_EarBase'], ['L_EarBase', 'Nose'], ['L_EarBase', 'L_Eye'], ['R_EarBase', 'R_Eye']]
]


from typing import Tuple, List, Union

from itertools import repeat, permutations
from random import shuffle

from sweetpea.primitives import factor, simple_level, derived_level, get_external_level_name, Factor
from sweetpea.constraints import at_most_k_in_a_row
from sweetpea.internal import get_all_levels
from sweetpea.derivation_processor import DerivationProcessor

def __assert_atmostkinarow_pair(k: int, level: Tuple[factor, Union[simple_level, derived_level]], experiments: List[dict]) -> None:
    sublist = list(repeat(level[1], k + 1))
    for e in experiments:
        assert sublist not in [e[level[0]][i:i+k+1] for i in range(len(e[level[0]]) - (k + 1))]

def __assert_atmostkinarow_factor(k: int, f: factor, experiments: List[dict]) -> None:
    factor_name = f.factor_name
    for level in f.levels:
        level_name = get_external_level_name(level)
        sublist = list(repeat(level_name, k + 1))
        for e in experiments:
            assert sublist not in [e[factor_name][i:i + k + 1] for i in range(len(e[factor_name]) - (k + 1))]

def assert_atmostkinarow(c: at_most_k_in_a_row, experiments: List[dict]) -> None:
    if isinstance(c.level, Factor):
        __assert_atmostkinarow_factor(c.k, c.level, experiments)
    else:
        level_tuples = get_all_levels([c.level])
        for t in level_tuples:
            __assert_atmostkinarow_pair(c.k, t, experiments)

def assert_no_repetition(experiments: List[dict]) -> None:
    for seq in experiments:
        levels_lists = [levels for f, levels in seq.items()]
        transposed = list(map(list, zip(*levels_lists)))
        for t in transposed:
            assert transposed.count(t) == 1, "{} repeats in this trial sequence!".format(t)

def shuffled_design_sample(input, num):
    perms = list(permutations(input))
    shuffle(perms)
    return perms[:num]


import numpy as np
import networkx as nx
import scipy.sparse.linalg as sparse_sp
import itertools
from sklearn.cluster import KMeans

def indicator_vector(node_list,size):
    g = np.zeros(size)
    for node in node_list:
        g[node] = 1.0
    return g

def generate_binary(r):
    return np.array(map(list, itertools.product([0, 1], repeat=r))).T

def best_G(X,C,rank):
    G = np.zeros((rank,X.shape[1]))
    for gene_id, Eg in enumerate(X.T):
        #Gg = max(generate_binary(rank).T, key = lambda v:likelihood(v,Eg,C,P) )
        Gg = min(binary_candidates, key = lambda v:np.linalg.norm(Eg - np.dot(C,v))**2 )
        G[:,gene_id] = Gg
    return G

def E_step(X,initial_G,rank,max_iter=100):
    G = initial_G
    RMSE = np.inf
    for i in range(max_iter):
        C = np.linalg.lstsq(G.T, X.T)[0].T
        G_new = best_G(X,C,rank)
        RMSE_new = np.linalg.norm(np.dot(C,G_new) - X) / np.sqrt(1.0 * X.shape[0] * X.shape[1])
        if RMSE - RMSE_new < (0.001*RMSE) or np.array_equal(G_new,G):
            break
        else:
            G = G_new
            RMSE = RMSE_new
    return C,G

def CG_alternating_koller(X,rank,init):
    if init == 'kmeans+1':
        kmeans = KMeans(n_clusters=rank+1, init='k-means++', n_init=10)
        kmeans.fit(X.T)
        predicted_labels = kmeans.labels_
        centers = kmeans.cluster_centers_
        G = np.zeros((rank,X.shape[1]))
        predicted_labels = np.array(predicted_labels)
        valid_labels = np.argsort(np.linalg.norm(centers , axis = 1))[1:]
        for r,label in enumerate(valid_labels):
            subnetwork_indices = np.where(predicted_labels== label)[0]
            G[r,:] = indicator_vector(subnetwork_indices,X.shape[1])
    elif init == 'kmeans':
        kmeans = KMeans(n_clusters=rank,init='k-means++',n_init=40,n_jobs=10)
        kmeans.fit(X.T)
        predicted_labels = kmeans.labels_
        centers = kmeans.cluster_centers_
        G = np.zeros((rank,X.shape[1]))
        predicted_labels = np.array(predicted_labels)
        valid_labels = range(len(centers))
        for r,label in enumerate(valid_labels):
            subnetwork_indices = np.where(predicted_labels== label)[0]
            G[r,:] = indicator_vector(subnetwork_indices,X.shape[1])
    else:
        G = np.random.choice([0, 1], (rank,X.shape[1]), p=[0.75, 0.25])

    global binary_candidates
    binary_candidates = generate_binary(rank).T
    C,G = E_step(X,G,rank)

    Error = np.linalg.norm(np.dot(C,G)-X)
    RMSE  = Error/np.sqrt(1.0 * X.shape[0] * X.shape[1])
    return {'C':C,'G':G,'Error':Error,'RMSE':RMSE}


from django.conf.urls import patterns,  url

from inventory import views

urlpatterns = patterns(
    '',

    # Basic views
    url(r'^$', views.index, name='index'),
    url(r'^asset/(?P<serial>\w+)/$', views.asset, name='asset'),
    url(r'^view/(?P<collection>\w+)/$', views.view, name='view'),

    # Forms
    url(r'^manageAsset/$', views.manage_asset, name='manage_asset'),  # New asset
    url(r'^manageAsset/(?P<asset_id>\w*)/$', views.manage_asset, name='manage_asset'),  # Update asset

    # Ajax loaded content
    url(r'^view_asset_list/$', views.view_asset_list, name='view_asset_list'),
    url(r'^view_category_list/(?P<collection>\w+)/$', views.view_category_list, name='view_category_list'),

    # Mutators
    url(r'^get_categories/(?P<selected>\w+)/$', views.get_categories, name='get_categories'),
    url(r'^get_models/(?P<category_id>\w+)/$', views.get_models, name='get_models'),  # return models
    url(r'^get_models/(?P<category_id>\w+)/asset_id/(?P<asset_id>\w+)/$', views.get_models, name='get_models'),  # return models with selected item  as serial's model
    url(r'^get_models/(?P<category_id>\w+)/(?P<selected>.+)/$', views.get_models, name='get_models'),  # return models where selected is passed
    url(r'^modify/$', views.modify, name='modify'),
)


from .shop_bot import bot
from telebot.apihelper import ApiException
from ..models.shop_models import User
import time
from threading import Thread

class Sender:
    def __init__(self, users, **message_data):
        self.message_data = message_data
        self._users = users

    def send_message(self):
        users = self._users.filter(is_block=True)
        blocked_ids = []
        for u in users:
            try:
                bot.send_message(
                    u.telegram_id,
                    **self.message_data
                )
            except ApiException as e:
                if e.error_code == 403:
                    blocked_ids.append(u.telegram_id)
                else:
                    raise e
            time.sleep(0.1)

        if not blocked_ids:
            blocked_ids = 0

        User.objects(telegram_id=blocked_ids).update(is_block=True)

def cron_unlock_users():
    while True:
        User.objects(is_block=True).update(is_block=False)
        minute = 60
        hour = 60 * minute
        day = 24 * hour
        time.sleep(2 * day)

# Thread(target=cron_unlock_users()).start()