import os
from typing import List

from dokidokimd.models import MangaSite
from dokidokimd.tools.ddmd_logger import get_logger
from dokidokimd.tools.translator import translate as _

logger = get_logger(__name__)

class MangaStorage(object):
    def __init__(self):
        self.site_extension = 'ddmd'  # type: str

    def store_sites(self, sites_location, sites_to_store) -> bool:
        try:
            if not os.path.isdir(sites_location):
                os.makedirs(sites_location, exist_ok=True)
        except Exception as e:
            logger.critical(_(F'Could not make or access directory {sites_location}\nError message: {e}'))
            return False

        for manga_site in sites_to_store:
            data = manga_site.dump()
            path_to_file = os.path.join(sites_location, F'{manga_site.site_name}.{self.site_extension}')

            try:
                # create new file
                with open(path_to_file, 'wb') as the_file:
                    the_file.write(data)
            except Exception as e:
                logger.critical(
                    _(F'Could not save {manga_site.site_name} site to a file{path_to_file}\nError message: {e}'))
        return True

    def load_sites(self, sites_location) -> List[MangaSite]:
        restored_manga_sites = []
        if not os.path.isdir(sites_location):
            os.makedirs(sites_location, exist_ok=True)
            logger.info(_('No saved state. Creating dir for fresh DB'))
        for file_name in os.listdir(sites_location):
            if file_name.endswith(self.site_extension):
                if os.path.isfile(os.path.join(sites_location, file_name)):
                    logger.info(_(F'Loading last state for {file_name}'))
                    try:
                        with open(os.path.join(sites_location, file_name), 'rb') as the_file:
                            data = the_file.read()
                            manga_site = MangaSite.load_dumped_site(data)
                            restored_manga_sites.append(manga_site)
                    except Exception as e1:
                        logger.warning(_(F'Could not load last state. Error message: {e1}'))
        return restored_manga_sites


#!/usr/bin/python3
# creates and distributes an archive to your web servers

from fabric.api import *
import os.path
import datetime

env.user = 'ubuntu'
env.hosts = ['34.138.47.31', '54.226.1.15']

def do_pack():
    """ .tgz archive """
    local("mkdir -p versions")
    t = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
    archive = local("tar -czvf versions/web_static_{}\
.tgz web_static".format(t))
    if archive.succeeded:
        return ("versions/web_static_{}.tgz".format(t))
    else:
        return None

def do_deploy(archive_path):
    """Fabric script that distrubutes an archive to server"""
    if os.path.exists(archive_path):
        new_path = archive_path.replace('versions/', '')
        file_name = new_path[:-4]
        arc_folder = "/data/web_static/releases/{}".format(new_path)
        put(archive_path, '/tmp/')
        run('mkdir -p /data/web_static/releases/{}'.format(file_name))
        run('tar -xzf /tmp/{} -C {}'.format(new_path, arc_folder[:-4]))
        run('rm /tmp/{}'.format(new_path))
        run('mv {}/web_static/* {}/'.format(arc_folder[:-4], arc_folder[:-4]))
        run('rm -rf {}/web_static'.format(arc_folder[:-4]))
        run('rm -rf /data/web_static/current')
        run('ln -s {} /data/web_static/current'.format(arc_folder[:-4]))
        return True
    else:
        return False

def deploy():
    """Creates and distributes and archive to a server"""
    archive_path = do_pack()
    if archive_path is None:
        return False
    return do_deploy(archive_path)


import os
import json
import pygit2
from fnmatch import fnmatch

from pygit2 import GIT_FILEMODE_BLOB
from pygit2 import GIT_RESET_HARD
from pygit2 import init_repository
from pygit2 import IndexEntry
from pygit2 import Signature
from plural.models.meta.edges import edge_has_index
from plural.models.meta.vertices import vertex_has_index
from plural.models.meta.edges import is_edge_subclass
from plural.models.meta.vertices import is_vertex_subclass
from plural.models.edges import Edge
from plural.models.vertices import Vertex
from plural.models.edges import resolve_edge_name
from plural.models.vertices import resolve_vertex_name
from plural.util import generate_uuid
from plural.util import serialize_commit
from plural.util import AutoCodec

class PluralStore(object):
    """Data store that manipulates a single git repository

    :param path: ``bytes`` - the path to the git repository ``string``
    :param bare: ``bool`` - manipulate git objects only
    :param author_name: ``unicode`` - the name of the default git author.
    :param author_email: ``unicode`` - the email of the default git author.
    """
    def __init__(self, path=None, bare=False,
                 author_name='hexastore',
                 author_email='hexastore@git',
                 default_branch='refs/heads/master',
                 repository=None):
        path = path or self.__class__.__name__.lower()
        self.path = path
        self.bare = bare
        self.repository = repository or init_repository(path, bare=bare)
        self.author = Signature(author_name, author_email)
        self.commiter = Signature(author_name, author_email)
        self.queries = []
        self.default_branch = default_branch

    def add_remote(self, name, url):
        """adds a remote repository

        :param name: ``bytes``
        :param url: ``bytes``
        """
        self.repository.remotes.create(name, url)

    def serialize(self, obj):
        """serializes an object, meant for internal use only.

        :param obj: a hashable object
        :returns: ``string``
        """
        default = AutoCodec().encode
        return json.dumps(obj, sort_keys=True, default=default)

    def deserialize(self, string):
        """deserialize a string into an object, meant for internal use only.

        :param string: a hashable object
        :returns: ``a hashable object``
        """
        return json.loads(string)

    def iter_versions(self, branch='master'):
        """iterates over all the commits of the repo
        :param branch: the branch name
        :returns: an iterator of dictionaries with commit metadata
        """
        for commit in self.repository.walk(self.repository.head.target, pygit2.GIT_SORT_TOPOLOGICAL):
            yield serialize_commit(commit)

    def get_versions(self, branch='master'):
        """same :py:meth:`as iter_versions` but returns a list
        :returns: a list
        """
        results = list(self.iter_versions(branch))
        return results

    def add_spo(self, edge, predicate, data):
        """creates a staged entry of edge, predicate and object

        :param edge:
        :param predicate:
        :param data:
        :returns: ``bytes`` - the blob id
        """
        edge = resolve_edge_name(edge)
        blob_id = self.repository.create_blob(data)
        entry = IndexEntry(os.path.join(edge, predicate), blob_id, GIT_FILEMODE_BLOB)
        self.repository.index.add(entry)
        return blob_id

    def create(self, element, **obj):
        if is_edge_subclass(element):
            new = self.create_edge(element, **obj)
            self.queries.append(
                ' '.join(map(bytes, ['CREATE EDGE', repr(new)]))
            )
            return new
        elif is_vertex_subclass(element):
            new = self.create_vertex(element, **obj)
            self.queries.append(
                ' '.join(map(bytes, ['CREATE VERTEX', repr(new)]))
            )
            return new

    def create_vertex(self, vertex, **obj):
        """creates a staged vertex entry including its indexed fields.

        :param vertex: a string or a :py:class:`Vertex` subclass reference
        :param ``**kw``: the field values
        :returns: an instance of the given vertex

        """
        vertex = resolve_vertex_name(vertex)
        predicate_ids = []

        vertex_uuid = obj.pop('uuid', generate_uuid())
        obj['uuid'] = vertex_uuid

        vertex_data = self.serialize(obj)
        object_hash = bytes(pygit2.hash(vertex_data))
        object_path = os.path.join(vertex, 'objects')

        id_path = os.path.join(vertex, '_ids')
        uuid_path = os.path.join(vertex, '_uuids')

        original_obj = obj.copy()
        origin = obj.pop('origin')
        target = obj.pop('target')

        indexes = {}
        for key in obj.keys():
            value = obj.get(key, None)
            if vertex_has_index(vertex, key):
                indexes[key] = value

            predicate_path = os.path.join(vertex, 'indexes', key)
            predicate_ids.append(self.add_spo(predicate_path, object_hash, value))

        self.add_spo(id_path, vertex_uuid, object_hash)
        self.add_spo(uuid_path, object_hash, vertex_uuid)

        origin_name = resolve_edge_name(origin)
        target_name = resolve_edge_name(target)
        RelationhipModel = Vertex.definition(vertex)

        label = RelationhipModel.label
        # call('Car/incoming/bought_by/Person', 'chuck-uuid', 'car-uuid'),
        # call('___vertices___/Car/bought_by/Person', 'chuck-uuid', 'car-uuid'),
        path_templates = {
            'incoming': '{to}/incoming/{label}/{from}',
            'outgoing': '{from}/outgoing/{label}/{to}',
            'indirect': '{}/indirect/{label}/{}',
        }
        vertex_path_template = path_templates[RelationhipModel.direction]

        ctx = {
            'label': label
        }
        direction = RelationhipModel.direction
        # self.add_spo(object_path, object_hash, vertex_data)

        if direction == 'incoming':
            from_uuid = origin.uuid
            ctx['from'] = origin_name
            to_uuid = target.uuid
            ctx['to'] = target_name
            vertex_path = vertex_path_template.format(**ctx)
            self.add_spo(vertex_path, from_uuid, to_uuid)

        elif direction == 'outgoing':
            from_uuid = target.uuid
            ctx['from'] = target_name
            to_uuid = origin.uuid
            ctx['to'] = origin_name
            vertex_path = vertex_path_template.format(**ctx)
            self.add_spo(vertex_path, from_uuid, to_uuid)

        elif direction == 'indirect':
            from_uuid = target.uuid
            to_uuid = origin.uuid

            path = vertex_path_template.format(target_name, origin_name, **ctx)
            self.add_spo(path, from_uuid, to_uuid)

            path = vertex_path_template.format(origin_name, target_name, **ctx)
            self.add_spo(path, to_uuid, from_uuid)

        return RelationhipModel.from_data(vertex, **original_obj)

    def create_edge(self, edge, **obj):
        """creates a staged edge entry including its indexed fields.

        :param edge: a string or a :py:class:`Edge` subclass reference
        :param ``**kw``: the field values
        :returns: an instance of the given edge

        """
        predicate_ids = []

        edge = resolve_edge_name(edge)
        edge_uuid = obj.pop('uuid', generate_uuid())
        obj['uuid'] = edge_uuid

        edge_data = self.serialize(obj)
        object_hash = bytes(pygit2.hash(edge_data))
        object_path = os.path.join(edge, 'objects')

        id_path = os.path.join(edge, '_ids')
        uuid_path = os.path.join(edge, '_uuids')

        indexes = {}
        for key in obj.keys():
            value = obj.get(key, None)
            if edge_has_index(edge, key):
                indexes[key] = value

            predicate_path = os.path.join(edge, 'indexes', key)
            predicate_ids.append(self.add_spo(predicate_path, object_hash, value))

        self.add_spo(object_path, object_hash, edge_data)
        self.add_spo(id_path, edge_uuid, object_hash)
        self.add_spo(uuid_path, object_hash, edge_uuid)

        return Edge.from_data(edge, **obj)

    def save_nodes(self, *nodes):
        """creates staged entries for all the given edge nodes, regardless of type

        :param ``*nodes``: a list of edge instances
        """

        result = []
        for node in nodes:
            n = self.create_edge(node.__class__.__name__, **node.to_dict())
            result.append(n)

        return result

    def merge(self, *nodes, **kw):
        """saves and commits all given nodes

        :param ``*nodes``: a list of edge instances
        :param auto_commit: ``bool`` - default: ``True``
        :returns: a list with the created nodes
        """
        auto_commit = kw.pop('auto_commit', True)
        result = self.save_nodes(*nodes)
        if auto_commit:
            self.commit()
        return result

    def trace_path(self, entries):
        return map(lambda entry: (entry.path, bytes(entry.oid)), entries)

    def glob(self, pattern, treeish=None):
        treeish = treeish or self.repository.index
        return filter(lambda entry: fnmatch(entry.path, pattern), treeish)

    def scan_all(self, edge_name=None, treeish=None, pattern='*'):
        """scans all nodes

        :returns: a generator that produces :py:class:`Edge` instances with the scanned data
        """
        edge_name = resolve_edge_name(edge_name)
        treeish = treeish or self.repository.index
        pattern = os.path.join(edge_name, 'objects', pattern)
        for entry in self.glob(pattern):
            blob = self.repository[entry.oid]
            data = self.deserialize(blob.data)
            yield Edge.from_data(edge_name, **data)

    def delete(self, *nodes, **kw):
        """deletes and (optionally) commits all given nodes

        :param ``*nodes``: a list of edge instances to be deleted
        :param auto_commit: ``bool`` - default: ``False``
        :returns: a list with the deleted nodes
        """
        auto_commit = kw.pop('auto_commit', False)
        deleted = []
        for node in nodes:
            for path, oid in self.trace_path(self.glob('*{}*'.format(node.uuid))):
                map(self.repository.index.remove, node.get_related_blob_paths(self.repository))
                self.queries.append(
                    ' '.join(map(bytes, ['DELETE', node]))
                )
            deleted.append(node)

        if auto_commit:
            self.commit()

        return nodes

    def get_edge_by_uuid(self, edge_name, uuid):
        """retrieves a edge by id

        :param edge_name: the edge name or type
        :param uuid: the uuid value
        """
        edge_name = resolve_edge_name(edge_name)
        pattern = os.sep.join([edge_name, '_ids', uuid])
        for entry in self.glob(pattern):
            edge_blob_id = self.repository[entry.oid].data
            blob = self.repository[edge_blob_id]
            return Edge.from_data(edge_name, **self.deserialize(blob.data))

    def get_vertex_by_uuid(self, vertex_name, uuid):
        """retrieves a vertex by id

        :param vertex_name: the vertex name or type
        :param uuid: the uuid value
        """
        vertex_name = resolve_vertex_name(vertex_name)
        pattern = os.sep.join([vertex_name, '_ids', uuid])
        for entry in self.glob(pattern):
            vertex_blob_id = self.repository[entry.oid].data
            blob = self.repository[vertex_blob_id]
            return Vertex.from_data(vertex_name, **self.deserialize(blob.data))

    def match_edges_by_index(self, edge_name, field_name, match_callback):
        """retrieves multiple edges by indexed field

        :param edge_name: the edge name or type
        :param uuid: the uuid value
        """
        edge_name = resolve_edge_name(edge_name)
        pattern = os.sep.join([edge_name or '*', 'indexes', '*'])
        # for index, oid in filter(lambda (index, oid): field_name in index, self.glob(pattern)):
        for index, oid in self.trace_path(self.glob(pattern)):
            path, blob_id = os.path.split(index)
            edge_name = path.split(os.sep)[0]
            value = self.repository[oid].data
            if match_callback(value):
                blob = self.repository[blob_id]
                data = self.deserialize(blob.data)
                Definition = Edge.definition(edge_name)
                yield Definition(**data)

    def match_vertices_by_index(self, vertex_name, field_name, match_callback):
        """retrieves multiple vertices by indexed field

        :param vertex_name: the vertex name or type
        :param uuid: the uuid value
        """
        vertex_name = resolve_vertex_name(vertex_name)
        pattern = os.sep.join([vertex_name or '*', 'indexes', '*'])
        # for index, oid in filter(lambda (index, oid): field_name in index, self.glob(pattern)):
        for index, oid in self.trace_path(self.glob(pattern)):
            path, blob_id = os.path.split(index)
            vertex_name = path.split(os.sep)[0]
            value = self.repository[oid].data
            if match_callback(value):
                blob = self.repository[blob_id]
                data = self.deserialize(blob.data)
                Definition = Vertex.definition(vertex_name)
                yield Definition(**data)

    def commit(self, query=None):
        """creates a commit with the staged objects"""
        author = None

        if not query:
            query = '\n'.join(sorted(set(self.queries)))

        # commit
        tree_id = self.repository.index.write_tree()
        parent_commits = []
        if self.repository.references.objects:
            parent_commits = [self.repository.head.target]

        commit_id = self.repository.create_commit(
            self.default_branch,
            author or self.author,
            self.commiter,
            query,
            tree_id,
            parent_commits,
        )
        self.queries = []
        self.repository.head.set_target(commit_id)
        if not self.bare:
            self.repository.reset(commit_id, GIT_RESET_HARD)
        return commit_id


from lib.binary import pad_zeroes, create_encoded_text_parts
from lib.utils import create_str_parts_array, join
from math import sqrt, ceil
from PIL import Image
from sys import argv

FILE = 'ocean.jpg'
STEG_FILE = 'steganographic.png'
START_INDICATION_PIXELS = 4  # number of pixels used for indicating text size
ENCODING_ERROR = 'Can\'t encode {} characters text on {} pixels rgb image'
ARGUMENTS_INFO = """
Please provide arguments:
  (*) --encode, -e [ text ]
  (*) --decode, -d
  (*) --diff, -c
""".strip()

# determines how many pixels are needed to encode text when changing the 2 least significant bits
def get_required_pixels_for_text_encoding(text_size):
    return ceil(text_size * 4 / 3)

def is_encodable(text, image_data):
    image_data_size = len(image_data)
    required_image_data_size = get_required_pixels_for_text_encoding(len(text))
    return START_INDICATION_PIXELS + required_image_data_size <= image_data_size

def convert_image_data(image_data, interceptors=[], skip=0, iterations=None):
    image_size = len(image_data)
    iterations = image_size if iterations is None else iterations
    to = min(skip + iterations, image_size)
    for i in range(skip, to):
        for interceptor in interceptors:
            image_data[i] = tuple(map(interceptor, image_data[i]))

    return image_data

def get_encode_conversion_interceptors(parts):
    return [
        lambda x: pad_zeroes(bin(x), 8),
        lambda x: (x[:6] + parts.pop()) if len(parts) > 0 else x,
        lambda x: int(x, 2),
    ]

def get_decode_conversion_interceptors():
    return [lambda x: pad_zeroes(bin(x), 2)]

def encode_image(text, file=FILE, steganographic_file=STEG_FILE):
    with Image.open(file) as im:
        if not im.mode == 'RGB':
            im = im.convert('RGB')

        image_data = list(im.getdata())
        if not is_encodable(text, image_data):
            raise Exception(ENCODING_ERROR.format(len(text), len(image_data)))

        text_size = len(text)

        text_size_binary = pad_zeroes(bin(text_size), 24)
        encoded_text_size_parts = create_str_parts_array(text_size_binary, is_reversed=True)
        convert_image_data(
            image_data,
            get_encode_conversion_interceptors(encoded_text_size_parts),
            iterations=START_INDICATION_PIXELS,
        )

        text_parts = create_encoded_text_parts(text)
        convert_image_data(
            image_data,
            get_encode_conversion_interceptors(text_parts),
            skip=START_INDICATION_PIXELS,
            iterations=get_required_pixels_for_text_encoding(text_size),
        )

        im.putdata(image_data)
        im.save(steganographic_file)

def decode_image(file=STEG_FILE):
    with Image.open(file) as im:
        if not im.mode == 'RGB':
            im = im.convert('RGB')

        image_data = list(im.getdata())
        decode_interceptors = get_decode_conversion_interceptors()

        convert_image_data(
            image_data,
            decode_interceptors,
            iterations=START_INDICATION_PIXELS,
        )

        start_indicating_bit_string = join([join(image_data[i]) for i in range(0, START_INDICATION_PIXELS)])
        text_size = int(start_indicating_bit_string, 2)

        required_pixels_for_text_encoding = get_required_pixels_for_text_encoding(text_size)
        text_bits_count = text_size * 8  # number of bits that represent the text
        convert_image_data(
            image_data,
            decode_interceptors,
            skip=START_INDICATION_PIXELS,
            iterations=required_pixels_for_text_encoding,
        )

        start, end = START_INDICATION_PIXELS, required_pixels_for_text_encoding + START_INDICATION_PIXELS
        bit_string = join([join(image_data[i]) for i in range(start, end)])
        bit_string = bit_string[:text_bits_count]  # normalize bit string to text bit size
        bits_array = create_str_parts_array(bit_string, 8)
        decoded_text_array = list(map(lambda x: chr(int(x, 2)), bits_array))

        return join(decoded_text_array)

def calc_square_distance(rgb_1, rgb_2):
    r1, g1, b1 = rgb_1
    r2, g2, b2 = rgb_2
    return pow(r1 - r2, 2) + pow(g1 - g2, 2) + pow(b1 - b2, 2)

def get_diff(f1=FILE, f2=STEG_FILE):
    with Image.open(f1) as im1:
        with Image.open(f2) as im2:
            image1_data, image2_data = list(map(lambda im: im.getdata(), [im1, im2]))

            diff_factor = 0
            for i in range(0, len(image1_data)):
                diff_factor += calc_square_distance(image1_data[i], image2_data[i])

    return {'pixels': len(image1_data), 'diff': ceil(sqrt(diff_factor))}

def main():
    if len(argv) > 1:
        arguments, args_dict = argv[1:], {}

        args_pairs_list = create_str_parts_array(arguments)
        for args_pair in args_pairs_list:
            args_pair_size = len(args_pair)

            if args_pair_size == 1:
                args_dict[args_pair[0]] = True

            elif args_pair_size == 2:
                arg, value = args_pair
                args_dict[arg] = value

        file, steg_file = FILE, STEG_FILE
        dict_keys = args_dict.keys()

        if '-f' in dict_keys or '--file' in dict_keys:
            file = args_dict.get('-f') or args_dict.get('--file')

        if '-sf' in dict_keys or '--steg-file' in dict_keys:
            steg_file = args_dict.get('-sf') or args_dict.get('--steg-file')

        if '-c' in dict_keys or '--diff' in dict_keys:
            print(get_diff(file, steg_file))

        elif '-d' in dict_keys or '--decode' in dict_keys:
            print(decode_image(steg_file))

        elif '-e' in dict_keys or '--encode' in dict_keys:
            encode_text = args_dict.get('-e') or args_dict.get('--encode')
            encode_image(encode_text, file, steg_file)
    else:
        print(ARGUMENTS_INFO)

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jan  3 09:55:02 2020

@author: qiuwenjie
"""

'''
cs224w homework 0
Q1 & Q2
'''
import snap as sp
import numpy as np
import matplotlib.pyplot as plt
#load graph
G = sp.LoadEdgeList(sp.PNGraph, "Wiki-Vote.txt",0,1)

#------------------------------------------------------------------------------
#Q1
GNodesNum= G.GetNodes()
print("Q1.1 The number of nodes in the network: ",GNodesNum)

SelfLoopNum = 0
for EI in G.Edges():
    if EI.GetSrcNId() == EI.GetDstNId():
        SelfLoopNum += 1
print("Q1.2 The number of nodes with a self-edge: ",SelfLoopNum)

DirecEdgeNum = 0
for EI in G.Edges():
    if EI.GetSrcNId() != EI.GetDstNId():
        DirecEdgeNum += 1
print("Q1.3 The number of directed edges in the network: ",DirecEdgeNum)

UDirecEdgeNum = 0
for EI in G.Edges():
    if EI.GetSrcNId() != EI.GetDstNId():
        if G.IsEdge(EI.GetDstNId(),EI.GetSrcNId()) == False:
            UDirecEdgeNum += 1
        else:
            UDirecEdgeNum +=0.5
print("Q1.4 The number of undirected edges in the network: ",UDirecEdgeNum)

RecEdgeNum = 0
for EI in G.Edges():
    if EI.GetSrcNId() != EI.GetDstNId():
        if G.IsEdge(EI.GetDstNId(),EI.GetSrcNId()):
            RecEdgeNum += 0.5
print("Q1.5 The number of reciprocated edges in the network: ",RecEdgeNum)

ZOutNum = 0
for NI in G.Nodes():
    if NI.GetOutDeg() == 0:
        ZOutNum +=1
print("Q1.6 The number of nodes of zero-out edge: ",ZOutNum)

ZInNum = 0
for NI in G.Nodes():
    if NI.GetInDeg() == 0:
        ZInNum +=1
print("Q1.7 The number of nodes of zero-in edge: ",ZInNum)

OutG10Num = 0
for NI in G.Nodes():
    if NI.GetOutDeg() > 10:
        OutG10Num +=1
print("Q1.8 The number of nodes with more than 10 outgoing edge: ",OutG10Num)

InF10Num = 0
for NI in G.Nodes():
    if NI.GetInDeg() < 10:
        InF10Num +=1
print("Q1.9 The number of nodes with fewer than 10 incoming edge: ",InF10Num)
#Q1 end
#------------------------------------------------------------------------------

#------------------------------------------------------------------------------
#Q2
OutDegDict = {}
for NI in G.Nodes():
    OutDeg = NI.GetOutDeg()
    if OutDeg > 0:
        if OutDeg in OutDegDict.keys():
            OutDegDict[OutDeg] += 1
        else:
            OutDegDict[OutDeg] = 1
OutDegArr = np.array(sorted(OutDegDict))
OutDegArrNum = np.zeros(OutDegArr.shape[0])
for i in range(len(OutDegArrNum)):
    OutDegArrNum[i] = OutDegDict[OutDegArr[i]]
print('Q2')
ax = plt.gca()
ax.set_xscale("log")
ax.set_yscale("log")
ax.set_adjustable("datalim")
ax.set_xlabel('Out-Degree')
ax.set_ylabel('Count')
ax.set_title('The distribution of out-degrees of nodes in the network')
plt.plot(OutDegArr, OutDegArrNum,'-',label='Out-Degree Distribution')
LogOutDegArr = np.log10(OutDegArr)
LogOutDegArrNum = np.log10(OutDegArrNum)
reg=np.polyfit(LogOutDegArr,LogOutDegArrNum,1)
LogOutDegArrNumEst=np.power(10,reg[1])*np.power(OutDegArr,reg[0])
plt.plot(OutDegArr,LogOutDegArrNumEst,'-',label='Least-Square Regression')
ax.set_xlim(OutDegArr[0], OutDegArr[-1])
#ax.set_ylim(min(OutDegArrNum), max(OutDegArrNum))
ax.grid()
plt.legend()


#Asynchronous file I/O and database operation

import asyncio
import aiofiles

async def read_file(file_path):
    async with aiofiles.open(file_path, mode='r') as file:
        content = await file.read()
        print(f"Read from {file_path}: {content}")

async def write_file(file_path, content):
    async with aiofiles.open(file_path, mode='w') as file:
        await file.write(content)
        print(f"Wrote to {file_path}")

async def main():
    file_path = 'example.txt'
    content_to_write = 'Hello, Async I/O!'

    await write_file(file_path, content_to_write)

    await read_file(file_path)

# Run the asynchronous event loop
asyncio.run(main())


import argparse
import numpy as np
from tqdm import tqdm
from glob import glob
from PIL import Image
from multiprocessing import Pool

from metric import metric as module_metric

parser = argparse.ArgumentParser(description='Image Inpainting')
parser.add_argument('--real_dir', required=True, type=str)
parser.add_argument('--fake_dir', required=True, type=str)
parser.add_argument("--metric", type=str, nargs="+")
args = parser.parse_args()

def read_img(name_pair):
    rname, fname = name_pair
    rimg = Image.open(rname)
    fimg = Image.open(fname)
    return np.array(rimg), np.array(fimg)

def main(num_worker=8):

    real_names = sorted(list(glob(f'{args.real_dir}/*.png')))
    fake_names = sorted(list(glob(f'{args.fake_dir}/*.png')))
    print(f'real images: {len(real_names)}, fake images: {len(fake_names)}')
    real_images = []
    fake_images = []
    pool = Pool(num_worker)
    for rimg, fimg in tqdm(pool.imap_unordered(read_img, zip(real_names, fake_names)), total=len(real_names), desc='loading images'):
        real_images.append(rimg)
        fake_images.append(fimg)

    # metrics prepare for image assesments
    metrics = {met: getattr(module_metric, met) for met in args.metric}
    evaluation_scores = {key: 0 for key,val in metrics.items()}
    for key, val in metrics.items():
        evaluation_scores[key] = val(real_images, fake_images, num_worker=num_worker)
    print(' '.join(['{}: {:6f},'.format(key, val) for key,val in evaluation_scores.items()]))

if __name__ == '__main__':
    main()


from sklearn import preprocessing  # import StandardScaler
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
import numpy as np
from sklearn import linear_model

# (a)
X = [4.0,5.0,5.6,6.8,7.0,7.2,8.0,0.8,1.0,1.2,2.5,2.6,3.0,4.3]
X = np.asarray(X).transpose()
t = [1,1,1,1,1,1,1,0,0,0,0,0,0,0]
t = np.asarray(t).transpose().reshape(len(t),1)

standardized_X = preprocessing.scale(X)

plt.scatter(standardized_X,t,color='green')

# (b)
xtest = np.arange(-1.5,1.5,0.1)
xtest_matrix = xtest.reshape(len(xtest),1)

# ridge regression
ridgeModel = linear_model.Ridge(alpha=0.1)
ridgeModel.fit(standardized_X.reshape(len(X),1),t)
predRidge = ridgeModel.predict(xtest_matrix)

# logistic regression L2 regurlization

lr = LogisticRegression(C=10)
lrModel = lr.fit(standardized_X.reshape(len(X),1),np.ravel(t))

predLog = lrModel.predict(xtest_matrix)
plt.plot(xtest,predRidge,'r')
plt.plot(xtest,predLog,'y')
plt.title('problem2 part(b) plot')
plt.show()

print('w1 for logit is: ',lrModel.coef_)
print('w0 for logit is: ',lrModel.intercept_)
print('w1 for ridge is: ',ridgeModel.coef_)
print('w0 for ridge is: ',ridgeModel.intercept_)

# (c)
new_a = np.append(X,12)
new_a = preprocessing.scale(new_a)
x_new = np.asarray(new_a).transpose()

new_b = np.append(t,1)
t_new = np.asarray(new_b).transpose()

### new ridge regression
ridgeModel2 = linear_model.Ridge(alpha=0.1)
ridgeModel2.fit(x_new.reshape(len(X)+1,1),np.ravel(t_new))

predRidge2 = ridgeModel2.predict(xtest_matrix)
plt.plot(xtest,predRidge2,'b')

# logistic regression L2 regurlization

lr2 = LogisticRegression(C=10)
lrModel2 = lr2.fit(x_new.reshape(len(X)+1,1),t_new)

predLog2 = lrModel2.predict(xtest_matrix)
plt.scatter(x_new,t_new,color='green')
plt.plot(xtest,predLog2,'g')
plt.title('problem2 part(c) plot')
plt.show()

print('w1 for logit is: ',lrModel2.coef_)
print('w0 for logit is: ',lrModel2.intercept_)
print('w1 for ridge is: ',ridgeModel2.coef_)
print('w0 for ridge is: ',ridgeModel2.intercept_)


#!/usr/bin/env python
import rospy
import cv2
from cv_bridge import CvBridge
from ir_marker_extractor.msg import ir_markers
from geometry_msgs.msg import Point
from sensor_msgs.msg import Image
from pyclustering.cluster.bsas import bsas
from dynamic_reconfigure.server import Server
from ir_marker_extractor.cfg import ir_marker_extractor_cfgsConfig
import numpy as np

def recongigureator_callback(config,level):
    #Image Thresholding Parameters
    node_processor.marker_extractor.high_threshold=config.high_threshold
    node_processor.marker_extractor.low_threshold=config.low_threshold
    #BSAS Parameters
    node_processor.marker_extractor.bsas_enable=config.bsas_enable
    node_processor.marker_extractor.max_clusters=config.max_clusters
    node_processor.marker_extractor.maxWhitePixels=config.max_white_pixels
    node_processor.marker_extractor.threshold=config.clustering_threshold
    #Blub Detection Parameters
    node_processor.marker_extractor.blubParams.minThreshold=config.blub_min_threshold
    node_processor.marker_extractor.blubParams.maxThreshold=config.blub_max_threshold
    node_processor.marker_extractor.blubParams.filterByArea=config.blub_filterByArea
    node_processor.marker_extractor.blubParams.minArea=config.blub_minArea
    node_processor.marker_extractor.blubParams.filterByCircularity=config.blub_filterByCircularity
    node_processor.marker_extractor.blubParams.minCircularity=config.blub_minCircularity
    node_processor.marker_extractor.blubParams.filterByConvexity=config.blub_filterByConvexity
    node_processor.marker_extractor.blubParams.minConvexity=config.blub_minConvexity
    node_processor.marker_extractor.blubParams.filterByInertia=config.blub_filterByInertia
    node_processor.marker_extractor.blubParams.minInertiaRatio=config.blub_minInertiaRatio
    node_processor.marker_extractor.blubParams.blobColor=config.blub_color
    node_processor.marker_extractor.updateParameters()
    return config
class markerExteractor():
    def __init__(self):
        self.bsas_enable=False
        self.maxWhitePixels=300
        self.low_threshold=100
        self.high_threshold=255
        self.max_clusters = 8
        self.threshold = 20
        self.blubParams = cv2.SimpleBlobDetector_Params()
        self.blubParams.minThreshold = 50;
        self.blubParams.maxThreshold = 255;
        self.blubParams.filterByArea = True
        self.blubParams.minArea = 0
        self.blubParams.filterByCircularity = True
        self.blubParams.minCircularity = 0.3
        self.blubParams.filterByConvexity = True
        self.blubParams.minConvexity = 0.7
        self.blubParams.filterByInertia = True
        self.blubParams.minInertiaRatio = 0.1
        self.blubParams.blobColor = 255

        ver = (cv2.__version__).split('.')
        if int(ver[0]) < 3 :
            self.blubDetector = cv2.SimpleBlobDetector(self.blubParams)
        else :
            self.blubDetector = cv2.SimpleBlobDetector_create(self.blubParams)
    def updateParameters(self):
        ver = (cv2.__version__).split('.')
        if int(ver[0]) < 3 :
            self.blubDetector = cv2.SimpleBlobDetector(self.blubParams)
        else :
            self.blubDetector = cv2.SimpleBlobDetector_create(self.blubParams)
    def detect(self,frame):
        self.cms=[]
        self.image_ROIs=[]
        self.keypoints=[]
        if len(frame.shape)==2:
            img_gray=frame
        else:
            img_gray=cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        ret,img_thresh = cv2.threshold(img_gray,
                                       self.low_threshold,self.high_threshold,cv2.THRESH_TOZERO)
        #Find the clusters
        self.nonzro_samples = cv2.findNonZero(img_thresh)
        if self.nonzro_samples is None:
            return -1,None  #No markers in the image
        else:
            self.nonzro_samples=self.nonzro_samples.reshape(-1, 2).astype('float32')
        if self.nonzro_samples.shape[0]<self.maxWhitePixels:
            if self.bsas_enable:
                result=self.process_bsas(img_thresh)
            else:
                result=self.process_no_bsas(img_thresh)
            return result
        else:
            return -2,None
    def process_bsas(self,img_thresh):
            bsas_instance = bsas(self.nonzro_samples, self.max_clusters, self.threshold)
            bsas_instance.process()
            clusters = bsas_instance.get_clusters()
            #Calculate the center of the clusters and the Regions of Interests
            self.ROIs=np.zeros((len(clusters),4))
            for i,cluster in enumerate(clusters):
                current_batch=self.nonzro_samples[cluster]
                self.cms.append(np.sum(current_batch,axis=0)/current_batch.shape[0])
                row_max=np.max(current_batch[:,1],axis=0)+6
                row_min=np.min(current_batch[:,1],axis=0)-6
                col_max=np.max(current_batch[:,0],axis=0)+6
                col_min=np.min(current_batch[:,0],axis=0)-6
                self.ROIs[i,:]=[row_min,row_max,col_min,col_max]
            for roi in self.ROIs.astype('int32'):
                self.image_ROIs.append(img_thresh.copy()[roi[0]:roi[1],roi[2]:roi[3]])
            #Return The Results
            marker_points=[]
            for i,roi in enumerate(self.image_ROIs):
                keys_in_roi=self.blubDetector.detect(roi)
                for key in keys_in_roi:
                    #Calculate the global coordinate of marker points.
                    #The points are returned in (X(Col),Y(Row)) coordinate.
                    marker_points.append([key.pt[0]+self.ROIs.astype('float32')\
                    [i,2],key.pt[1]+self.ROIs.astype('float32')[i,0]])
            return 0,np.array(marker_points)
    def process_no_bsas(self,img_thresh):
        blubs=self.blubDetector.detect(img_thresh)
        marker_points=[]
        for key in blubs:
            marker_points.append([key.pt[0] ,key.pt[1]])
        return 0,np.array(marker_points)

class process():
    def __init__(self,debug_image):
        self.debug_image=debug_image
        self.marker_extractor=markerExteractor()
        self.bridge=CvBridge()
    def process(self,image):
        cv_image=self.bridge.imgmsg_to_cv2(image,desired_encoding='mono8')
        #print(cv_image.shape)
        status,markers=self.marker_extractor.detect(cv_image)
        if self.debug_image:
            img_color=cv2.cvtColor(cv_image,cv2.COLOR_GRAY2BGR)
            if markers is not None:
                for marker in markers:
                    cv2.circle(img_color,(int(round(marker[0])), int(round(marker[1]))), 3,
                                                                           (255,0,255),-1)
            debug_image=self.bridge.cv2_to_imgmsg(img_color)
        else:
            debug_image=None
        return status,markers,debug_image
def image_callback(image):
    markers_msg=ir_markers()
    status,markers,debug_image=node_processor.process(image)
    if debug_image is not None:
        debug_image_pub.publish(debug_image)
    if status==-2: #too many white pixels in the image print
                    #('too many white pixels dtected in the image')
        markers_msg.status=status
        markers_msg.header.stamp=image.header.stamp
        pub.publish(markers_msg)
        return
    if status==-1 or len(markers)==0: #no markers in the image
        #print('no markers in the image')
        markers_msg.status=status
        markers_msg.header.stamp=image.header.stamp
        pub.publish(markers_msg)
        return
    marker_list=[]
    for marker in markers:
        marker_list.append(Point(marker[0],marker[1],0))
    markers_msg.status=status
    markers_msg.header.stamp=image.header.stamp
    markers_msg.markers=marker_list
    #[Point(np.random.randn(),2,3),Point(4,5,6)]#
    pub.publish(markers_msg)

if __name__ =='__main__':
    rospy.init_node('ir_marker_extractor_node')
    if rospy.has_param('~debug_image'):
        debug_image=rospy.get_param('~debug_image')
        debug_image_pub=rospy.Publisher('/ir_marker_extractor/debug_image',Image,queue_size=1)
    else:
        debug_image=False
    node_processor=process(debug_image)
    pub=rospy.Publisher('/ir_marker_extractor/ir_markers',ir_markers,queue_size=1)
    rospy.Subscriber('/camera/image_raw',Image,image_callback)
    srv=Server(ir_marker_extractor_cfgsConfig,recongigureator_callback)
    rospy.spin()


# --------------
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# Load the dataframe
df = pd.read_csv(path)

#Code starts here
#Task 1
p_a=df[df['fico']>700].shape[0]/df.shape[0]
print("Probability of fico credit score greater than 700 is ",p_a)

p_b=df[df['purpose']=='debt_consolidation'].shape[0]/df.shape[0]
print("Probability of purpose equal to debt_consolidation is ",p_b)

df1=df[df['purpose']=='debt_consolidation']
# df1.head()
p_a_b=df1[df1['fico']>700].shape[0]/df1.shape[0]
print("probablityp(B|A)",p_a_b)

result= (p_a_b==p_a)
print(result)

#Task 2
prob_lp=df[df['paid.back.loan']=='Yes'].shape[0]/df.shape[0]
print(prob_lp)

prob_cs=df[df['credit.policy']=='Yes'].shape[0]/df.shape[0]
print(prob_cs)

new_df=df[df['paid.back.loan']=='Yes']
# new_df.head()

prob_pd_cs=new_df[new_df['credit.policy']=='Yes'].shape[0]/new_df.shape[0]
print(prob_pd_cs)

bayes=prob_pd_cs*prob_lp/prob_cs
print("value of bayes is",round(bayes,4))

#Task 3
pur=df['purpose'].value_counts(normalize=True)
pur.plot(kind='bar')
plt.xlabel("purpose")
plt.ylabel("P(purl)")
plt.show()

df1=df[df['paid.back.loan'] == 'No']
# df1.head()

purpl=df1['purpose'].value_counts(normalize=True)
purpl.plot(kind='bar')
plt.xlabel("purpose where paid.back.loan is No")
plt.ylabel("p(purpl)")
plt.show()

#Task 4
inst_median=df['installment'].median()
print(inst_median)

inst_mean=df['installment'].mean()
print(inst_mean)

df['installment'].hist(normed=True,bins=50)
plt.axvline(x=inst_median,color='r')
plt.axvline(x=inst_mean,color='g')
plt.xlabel("installment")
plt.ylabel("Probability")
plt.show()

df['log.annual.inc'].hist(normed=True,bins=50)
plt.xlabel("log.annual.inc")
plt.ylabel("Probability")
plt.show()


#! /usr/bin/env python
import os, sys
import shutil
import subprocess
import json, glob

class NotebookFinder(object):
    """
    Given the path to a notebook.json file, locate the notebooks
    given the supplied command-line options (projects and suites).
    """

    def __init__(self, spec_path, projects=[], suites=[]):
        spec = json.load(open(spec_path, 'r'))
        # Assuming file in doc/nbpublisher
        root = os.path.abspath(os.path.join(__file__, '..', '..', '..'))
        # Get current project name from spec
        project = [k for k in spec.keys() if k != 'links'][0]
        # If not projects specified, test current project
        projects = [project] if not projects else projects
        # Dictionary of root directories by project
        root_dirs = self.root_directories(root, spec, projects)

        # Relative to top level of each project
        self.spec = self.expand_spec(spec, root_dirs)
        self.files = self.expand_paths(self.spec, root_dirs, suites)

    def root_directories(self, root, spec, projects):
        """
        Return a dictionary of root directories for the required
        projects.
        """
        root_dirs = {}
        for project in projects:
            relpath =  spec['links'].get(project, False)
            if relpath:
                link_path = os.path.abspath(os.path.join(root, relpath))
                path = link_path
            elif project in spec:
                path = root
            elif relpath is False:
                continue

            if not os.path.isdir(path):
                raise Exception("Invalid link path: %s" % link_path)
            root_dirs[project] = path

        return root_dirs

    def expand_spec(self, spec, root_dirs, location=['doc', 'notebook.json']):
        """
        Follow the links to other notebook.json files to create an
        expanded specification dictionary with all the projects to
        test.
        """
        spec = dict((k,v) for (k,v) in spec.items() if k != 'links')
        for name, root in root_dirs.items():
            specpath = os.path.join(root,  *location)
            if (name not in spec) and not os.path.isfile(specpath):
                print("Could not find %r in %s" % (os.path.sep.join(location), root))
            elif (name not in spec):
                spec[name] = json.load(open(specpath, 'r'))[name]
        return spec

    def expand_paths(self, spec, root_dirs, suites=[]):
        """
        Using the expanded spec, find all the specified notebooks,
        filtering by the specified suites (if specified).
        """
        files = []
        for project, group in spec.items():
            selection = [s for s in suites if s in group]
            selection = selection if suites else group.keys()
            for suite in selection:
                key = (str(project), str(suite))
                group_list = []
                for relpath in group[suite]:
                    pattern = os.path.join(root_dirs[project], relpath)
                    for match in glob.glob(pattern):
                        group_list.append(str(match))
                files.append((key, group_list))
        return files

def switch_reference_branch(ref_dir, project):
    # Not yet implemented...
    pass

def run_notebook_test(notebook, project, suite, ref_dir, test_dir, regen=False):
    path, filename = os.path.split(__file__)
    test_script = os.path.abspath(os.path.join(path, 'nbtest.py'))

    notebook_name =  os.path.split(notebook)[1]
    if not notebook_name.endswith('.ipynb'):
        print("Not an IPython notebook (%s)...skipping." % notebook_name)
        return 0

    py_version =  "_py%d" % sys.version_info[0]
    ref_dir = os.path.join(ref_dir, project + '_' + notebook_name[:-6] + py_version)
    test_dir = os.path.join(test_dir, project + '_' + notebook_name[:-6] + py_version)
    if regen and os.path.isdir(ref_dir):  shutil.rmtree(ref_dir)

    cmds = ['ipython', test_script, notebook, ref_dir, test_dir, str(regen)]
    proc = subprocess.Popen(cmds,
                            stderr=subprocess.PIPE,
                            cwd=os.path.split(notebook)[0])
    _,stderr = proc.communicate()
    print(str(stderr.decode()))
    for idx, line in enumerate(stderr.splitlines()[-10:]):
        error = 'FAILED (errors=' in str(line)
        failure = 'FAILED (failures=' in str(line)
        if error or failure:
            return 1
    return proc.returncode

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument('projects', nargs='*', default=[])
    parser.add_argument('-s', '--suites', nargs='*', default=[])
    parser.add_argument('-p', '--paths', nargs='*')
    parser.add_argument('-r', '--regen', action="store_true")
    args = parser.parse_args()

    if args.paths is not None:
        assert len(args.projects) == 1
        file_list = [((args.projects[0], "custom"), args.paths)]
    else:
        file_list = NotebookFinder('../notebook.json', projects=args.projects,
                                   suites=args.suites).files

    ref_dir = os.path.abspath(os.path.join(__file__, '..', '..','reference_data'))
    test_dir = os.path.abspath(os.path.join(__file__, '..', '..','test_data'))

    if not os.path.isdir(ref_dir):
        raise Exception("No reference directory: %s" % ref_dir)
    if not os.path.isdir(test_dir):
        raise Exception("No test directory: %s" % test_dir)

    retcode = 0
    for (project, suite), paths in sorted(file_list):
        for path in paths:
            switch_reference_branch(ref_dir, project)
            retcode |= run_notebook_test(path, project, suite, ref_dir, test_dir, regen=args.regen)
    sys.exit(retcode)


""" Wrapper around TRF to annotate a VCF """
import sys
import logging
import argparse
import tempfile
from collections import defaultdict

import pysam
import pyfaidx
import truvari
from acebinf import cmd_exe, setup_logging

# Start with just insertions and that sequence
# Eventually you can intersect with known tandem repeat regions as well
# and use those regions (with the edit) to better predict a 'difference' in number of copies
# Or you can just annotate the absolute number of the individual... probably bot

# Also, running single threaded isn't great, but also I don't care at this point...
# It'll be a heavy step, but I'm not judging the software on speed for a bit...

class TRFAnno():
    """ Class for trf annotation """
    TRNAME = tempfile.NamedTemporaryFile().name # Need to remove this
    FANAME = tempfile.NamedTemporaryFile().name # Need to remove this
    TRFCOLS = [("TRF_starts", int),
               ("TRF_ends", int),
               ("TRF_periods", int),
               ("TRF_copies", float),
               ("TRF_consizes", int),
               ("TRF_pctmats", int),
               ("TRF_pctindels", int),
               ("TRF_scores", int),
               ("TRF_As", int),
               ("TRF_Cs", int),
               ("TRF_Gs", int),
               ("TRF_Ts",  int),
               ("TRF_entropies", float),
               ("TRF_repeats", str),
               ("unk1", None),
               ("unk2", None),
               ("unk3", None)]

    def __init__(self, in_vcf, out_vcf="/dev/stdout", executable="trf409.linux64",
                 min_length=50, threshold=0.8, full=False,
                 trf_params="2 7 7 80 10 50 500 -m -f -h -d -ngs",
                 refanno=None, ref=None, pctovl=0.2):
        """ The setup """
        self.in_vcf = in_vcf
        self.out_vcf = out_vcf
        self.executable = executable
        self.min_length = min_length
        self.threshold = threshold
        self.full = full

        if "-ngs" not in trf_params:
            trf_params = trf_params + " -ngs "
        self.trf_params = trf_params

        if refanno and not ref:
            logging.error("-R requires -r")
            exit(1)

        self.refanno = truvari.make_bedanno_tree(refanno) if refanno else None
        self.ref = pyfaidx.Fasta(ref) if ref else None
        self.pctovl = pctovl

        self.n_header = self.edit_header()
        self.cmd = f"{self.executable} {TRFAnno.FANAME} {self.trf_params} > {TRFAnno.TRNAME}"

    def edit_header(self):
        """
        New VCF INFO fields
        """
        header = None
        with pysam.VariantFile(self.in_vcf, 'r') as fh:
            header = fh.header.copy()
        # if intersect_only: Do I want to sometimes turn this off?
        # Probably, actully
        header.add_line(('##INFO=<ID=TRF_repeats,Number=.,Type=String,'
                         'Description="TRF repeat sequences">'))
        header.add_line(('##INFO=<ID=TRF_periods,Number=.,Type=Integer,'
                         'Description="TRF periods">'))
        header.add_line(('##INFO=<ID=TRF_copies,Number=.,Type=Float,'
                         'Description="TRF repeat copies">'))
        header.add_line(('##INFO=<ID=TRF_scores,Number=.,Type=Integer,'
                         'Description="TRF repeat scores">'))

        # Take this out
        if self.refanno and self.ref:
            header.add_line(('##INFO=<ID=TRF_diffs,Number=.,Type=Float,'
                             'Description="Alternate allele copy number difference from '
                             'its reference SREP_repeat copy number">'))
        if self.refanno:
            for headinfo in self.refanno[1]:
                header.add_line(headinfo)

        if self.full:
            header.add_line(('##INFO=<ID=TRF_starts,Number=.,Type=Integer,'
                             'Description="TRF starts">'))
            header.add_line(('##INFO=<ID=TRF_ends,Number=.,Type=Integer,'
                             'Description="TRF ends">'))
            header.add_line(('##INFO=<ID=TRF_periods,Number=.,Type=Integer,'
                             'Description="TRF periods">'))
            header.add_line(('##INFO=<ID=TRF_consizes,Number=.,Type=Integer,'
                             'Description="TRF consizes">'))
            header.add_line(('##INFO=<ID=TRF_pctmats,Number=.,Type=Integer,'
                             'Description="TRF pctmats">'))
            header.add_line(('##INFO=<ID=TRF_pctindels,Number=.,Type=Integer,'
                             'Description="TRF pctindels">'))
            header.add_line(('##INFO=<ID=TRF_As,Number=.,Type=Integer,'
                             'Description="TRF As">'))
            header.add_line(('##INFO=<ID=TRF_Cs,Number=.,Type=Integer,'
                             'Description="TRF Cs">'))
            header.add_line(('##INFO=<ID=TRF_Gs,Number=.,Type=Integer,'
                             'Description="TRF Gs">'))
            header.add_line(('##INFO=<ID=TRF_Ts,Number=.,Type=Integer,'
                             'Description="TRF Ts">'))
            header.add_line(('##INFO=<ID=TRF_entropies,Number=.,Type=Float,'
                             'Description="TRF entropies">'))
            # TRF HIT - if refanno, just report if we hit a reference TRF region
            # Might want some pct/ovl -- this is handled... nowhere
            pass
        # TODO: Need to put a source line that says this thing was run with whatever parameters
        return header

    def run_trf(self, seq):
        """
        Runs trf on the ref/alt sequences
        returns list of hits
        """
        def parse_output():
            """
            Parse the outputs from trf, turn to a dictionary
            """
            hits = []
            with open(TRFAnno.TRNAME, 'r') as fh:
                name = fh.readline()
                if name == "": # no hits
                    return hits
                name = name.strip()[1:]
                while True:
                    # If there are multiple, need to parameters for 'take best' or take top N or something
                    # Will need name now that there's ref/alt seq
                    data = fh.readline()
                    if data == "":
                        break
                    if data.startswith("@"):
                        name = data.strip()[1:]
                        continue
                    data = data.strip().split(' ')
                    data = {x[0]: x[1](y) for x, y in zip(TRFAnno.TRFCOLS, data) if not x[0].startswith("unk")}
                    # don't really need until parallel
                    data["TRF_scores"] = int(data["TRF_scores"])
                    hits.append(data)
            return hits

        with open(TRFAnno.FANAME, 'w') as fout:
            #for seq in seqs:
            fout.write(">a\n%s\n" % (seq))

        ret = cmd_exe(self.cmd)
        if ret.ret_code != 0:
            logging.error("Couldn't run trf")
            logging.error(str(ret))
            exit(ret.ret_code)
        return parse_output()

    def __old_srep(self):
        """
        legacy code
        """
        srep_hits = None
        # Make srep_hits if we have the bed
        if self.refanno:
            srep_hits = self.refanno[0][entry.chrom].overlap(entry.start, entry.stop)

    def annotate(self, entry, altseq):
        """
        Annotates an Insertion, returns the annotation information
        """
        trf_hits = self.run_trf(altseq)
        best_hit = None
        entry_size = truvari.entry_size(entry)
        for hit in trf_hits:
            size_trf = abs(hit["TRF_ends"] - hit["TRF_starts"]) + 1
            pct = size_trf / entry_size  # The TR that covers the most of the insertion
            if pct < self.threshold:
                continue
            if best_hit is None:
                best_hit = hit
                continue
            if best_hit is None \
               or (hit["TRF_periods"] < best_hit["TRF_periods"] \
               and hit["TRF_copies"] > best_hit["TRF_copies"]):
                best_hit = hit
        return best_hit

    def annotate_entry(self, entry, altseq):
        """
        Annotates an insertion. Insertions are assumed to have a
        refspan of a single anchor base and full ALT sequence
        Returns the edited entry
        """
        return self.edit_entry(entry, [self.annotate(entry, altseq)])

    def edit_entry(self, entry, trf_annos):
        """
        puts the annos in vcf entry
        return the edited entry
        """
        if not trf_annos:
            return entry

        # Let's assume we're only doing the alt allele
        srep_hits = None
        try:
            entry = truvari.copy_entry(entry, self.n_header)
        except TypeError:
            return entry
        n_dat = defaultdict(list)
        for i in trf_annos:
            if i is None:
                continue
            if self.full:
                for key, cnvt in TRFAnno.TRFCOLS:
                    if cnvt:
                        n_dat[key].append(cnvt(i[key]))
            else:
                n_dat["TRF_repeats"].append(i["TRF_repeats"])
                n_dat["TRF_periods"].append(int(i["TRF_periods"]))
                n_dat["TRF_copies"].append(float(i["TRF_copies"]))
                n_dat["TRF_scores"].append(int(i["TRF_scores"]))

        # Adding srep hits to n_dat also
        # this is getting kinda choppy with the converters
        # may have over engineered it early
        if srep_hits is not None:
            for i in srep_hits:
                for k,v in i.data.items():
                    n_dat[k].extend(v)

        # Can calculate the diffs
        if self.refanno and self.ref and srep_hits and alt_annos:
            lookup = {}
            diffs = []
            # make a lookup of the trf_annos from the reference
            for i in srep_hits:
                i = i.data
                lookup[i["SREP_repeats"][0]] = i["SREP_copies"][0]

            has_diff = False
            # Subtract each alt_repeat from its corresponding ref_repeat
            for alt_repeat, alt_copy in zip(n_dat["TRF_repeats"], n_dat["TRF_copies"]):
                if alt_repeat in lookup:
                    has_diff = True
                    diffs.append(alt_copy - lookup[alt_repeat])
                else:
                    diffs.append(None)

            if has_diff:
                n_dat["TRF_diffs"] = diffs

        for key in n_dat:
            entry.info[key] = n_dat[key]

        return entry

    def run(self):
        """
        The work
        """
        logging.info("Annotating VCF")
        # should probably 'batch' it instead of running individually
        with pysam.VariantFile(self.in_vcf) as vcf, \
                pysam.VariantFile(self.out_vcf, 'w', header=self.n_header) as output:
            for entry in vcf:
                sz = truvari.entry_size(entry)
                if sz >= self.min_length \
                   and truvari.entry_variant_type(entry) == "INS":
                    entry = self.annotate_entry(entry, entry.alts[0])
                output.write(entry)

def parse_args(args):
    """
    Pull the command line parameters
    """
    def restricted_float(x):
        x = float(x)
        if x < 0.0 or x > 1.0:
            raise argparse.ArgumentTypeError("%r not in range [0.0, 1.0]" % (x,))
        return x
    parser = argparse.ArgumentParser(prog="trf", description=__doc__,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("-i", "--input", type=str, default="/dev/stdin",
                        help="VCF to annotate (stdin)")
    parser.add_argument("-o", "--output", type=str, default="/dev/stdout",
                        help="Output filename (stdout)")
    parser.add_argument("-e", "--executable", type=str, default="trf409.linux64",
                        help="Path to tandem repeat finder (%(default)s)")
    parser.add_argument("-f", "--full", action="store_true",
                        help="Write full trf output to entries")
    parser.add_argument("-m", "--min-length", type=int, default=50,
                        help="Minimum size of entry to annotate (%(default)s)")
    parser.add_argument("-t", "--threshold", type=restricted_float, default=.8,
                        help="Threshold for pct of allele covered (%(default)s)")
    parser.add_argument("-T", "--trf-params", type=str, default="2 7 7 80 10 50 500 -m -f -h -d -ngs",
                        help="Default parameters to send to trf (%(default)s)")
    parser.add_argument("-R", "--ref-bed", type=str, default=None,
                        help="Reference bed of simple repeats")
    parser.add_argument("-r", "--ref", type=str, default=None,
                        help="Reference fasta file (use with -R)")
    parser.add_argument("--debug", action="store_true",
                        help="Verbose logging")
    args = parser.parse_args(args)
    setup_logging(args.debug)
    return args

def trf_main(cmdargs):
    """ Main """
    args = parse_args(cmdargs)
    anno = TRFAnno(in_vcf=args.input,
                   out_vcf=args.output,
                   executable=args.executable,
                   min_length=args.min_length,
                   threshold=args.threshold,
                   full=args.full,
                   trf_params=args.trf_params,
                   refanno=args.ref_bed,
                   ref=args.ref)
    anno.run()
    logging.info("Finished trf")

if __name__ == '__main__':
    trf_main(sys.argv[1:])

"""
1) I can't guarantee that TRF alt seq hits are going to happend
    But I'm returning nulls - not good. need to remove I think
    |

So 1- you can give up on the reference, totally un-needed unti you get to 'denovo mode'
Which at this point you should just abandon until it beocmes a feature request
"""


__author__ = 'alexcomu'

## Steps
# 1 - Find every pair of movies that were watched by tha same person
# 2 - Measure the similarity of their rating across all users who watched both
# 3 - Sort by movie, then by similarity strength

## MapReduce Problem

## Mapper: Extract User -> (Movie, Rating)
## Reducer: Groups all movie, rating pairs by user

## Mapper: Output all movies viewed by the same user (movie1, movie2) -> (rating1, rating2)
## Reducer: Compute rating-based similarity between each movie pair (movie1, movie2) -> (similarity, number of users who saw both)

## Mapper: Make movie name, similarity score the sorting key
## Reducer: Display the final sorted output

## -> Movie sorted by Name with a list of movie ordered by similaity

from mrjob.job import MRJob
from mrjob.step import MRStep
from math import sqrt
from itertools import combinations # permutations

class MovieSimilarities(MRJob):

    def configure_options(self):
        super(MovieSimilarities, self).configure_options()
        self.add_file_option('--items', help='Insert the path to u.item')

    def load_movie_names(self):
        # Load database of movie names to print pretty name instead ID.
        self.movieNames = {}

        with open("u.item") as f:
            for line in f:
                fields = line.split('|')
                self.movieNames[int(fields[0])] = fields[1]

    def steps(self):
        # 3 steps process
        return [
            MRStep(mapper=self.mapper_parse_input,
                    reducer=self.reducer_ratings_by_user),
            MRStep(mapper=self.mapper_create_item_pairs,
                    reducer=self.reducer_compute_similarity),
            MRStep(mapper=self.mapper_sort_similarities,
                    mapper_init=self.load_movie_names,
                    reducer=self.reducer_output_similarities)]

    def mapper_parse_input(self, key, line):
        # Outputs userID => (movieID, rating)
        (userID, movieID, rating, timestamp) = line.split('\t')
        yield userID, (movieID, float(rating))

    def reducer_ratings_by_user(self, user_id, itemRatings):
        #Group (item, rating) pairs by userID

        ratings = []
        for movieID, rating in itemRatings:
            ratings.append((movieID, rating))

        yield user_id, ratings

    def mapper_create_item_pairs(self, user_id, itemRatings):
        # Find every pair of movies each user has seen, and emit
        # each pair with its associated ratings

        # "combinations" finds every possible pair from the list of movies
        # this user viewed.
        for itemRating1, itemRating2 in combinations(itemRatings, 2):
            movieID1 = itemRating1[0]
            rating1 = itemRating1[1]
            movieID2 = itemRating2[0]
            rating2 = itemRating2[1]

            # Produce both orders so sims are bi-directional
            yield (movieID1, movieID2), (rating1, rating2)
            yield (movieID2, movieID1), (rating2, rating1)

    def cosine_similarity(self, ratingPairs):
        # Computes the cosine similarity metric between two
        # rating vectors.
        numPairs = 0
        sum_xx = sum_yy = sum_xy = 0
        for ratingX, ratingY in ratingPairs:
            sum_xx += ratingX * ratingX
            sum_yy += ratingY * ratingY
            sum_xy += ratingX * ratingY
            numPairs += 1

        numerator = sum_xy
        denominator = sqrt(sum_xx) * sqrt(sum_yy)

        score = 0
        if (denominator):
            score = (numerator / (float(denominator)))

        return (score, numPairs)

    def reducer_compute_similarity(self, moviePair, ratingPairs):
        # Compute the similarity score between the ratings vectors
        # for each movie pair viewed by multiple people

        # Output movie pair => score, number of co-ratings

        score, numPairs = self.cosine_similarity(ratingPairs)

        # Enforce a minimum score and minimum number of co-ratings
        # to ensure quality
        if (numPairs > 10 and score > 0.95):
            yield moviePair, (score, numPairs)

    def mapper_sort_similarities(self, moviePair, scores):
        # Shuffle things around so the key is (movie1, score)
        # so we have meaningfully sorted results.
        score, n = scores
        movie1, movie2 = moviePair

        yield (self.movieNames[int(movie1)], score), \
            (self.movieNames[int(movie2)], n)

    def reducer_output_similarities(self, movieScore, similarN):
        # Output the results.
        # Movie => Similar Movie, score, number of co-ratings
        movie1, score = movieScore
        for movie2, n in similarN:
            yield movie1, (movie2, score, n)

if __name__ == '__main__':
    MovieSimilarities.run()


import requests
import bs4
result = requests.get('https://en.wikipedia.org/wiki/Machine_learning')
soup = bs4.BeautifulSoup(result.text, 'lxml')
items = soup.select('.toctext')
for item in items:
    print(item.text)

count = 0
for image in soup.select('.thumbimage'):
    image= requests.get('http:'+image['src'])
    f=open('./assets/images/image_'+str(count)+'.jpg','wb')
    f.write(image.content)
    count +=1


import boto3
from decimal import Decimal, getcontext

def calculate_cost():

    client = boto3.client('ce')
    result = client.get_cost_and_usage(
        TimePeriod = {
            'Start': '2020-08-06',
            'End': '2020-08-07'
        },
        Granularity = 'DAILY',
        Filter =
                {
                    "Dimensions": {
                        "Key": "SERVICE",
                        "Values": ['AWS Lambda']
                    }
                },
        Metrics = ["BlendedCost"],
        GroupBy = [
            {
                'Type': 'TAG',
                'Key': 'app'
            }
        ]
    )
    resultsByTime = result.get('ResultsByTime')
    for item in resultsByTime:
        if item.get('Groups'):
            #print(item)
            oneDayCost = item.get('Groups')[0].get('Metrics').get('BlendedCost').get('Amount')
            #print(f"{oneDayCost}")
            return oneDayCost

def push_cost_metric_to_cloudwatch(oneDayCost):
    """
    Sends custom cost metric to cloudwatch AWS/Lambda namespace.
    :param lambdas_data: List of Lambda functions data.
    :return: None.
    """
    cloudwatch = boto3.client('cloudwatch', 'us-west-2')

    response = cloudwatch.put_metric_data(
        MetricData = [
            {
                'MetricName': 'LastOneDayCost',
                'Dimensions': [
                    {
                        'Name': 'By Function Name',
                        'Value': 'LambdaLogging'
                    }
                ],
                'Unit': 'None',
                'Value': oneDayCost
            },
        ],
        Namespace='Lambda'
    )
    print(f"{response}")

if __name__ == '__main__':
   oneDayCost = calculate_cost()
   getcontext().prec = 20
   #oneDayCost = Decimal(oneDayCost)
   oneDayCost = '{0:1f}'.format(Decimal(oneDayCost))
   print(f"{float(oneDayCost)}")
   #push_cost_metric_to_cloudwatch(oneDayCost)


from typing import Tuple

import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import tensorflow_datasets as tf_ds
from PIL import Image
from tensorflow.image import rgb_to_hsv
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow_io.python.experimental.color_ops import rgb_to_lab
from tqdm import tqdm

from autoencoders.ColoringAutoEncoder import AutoEncoderFormat

def load_image(path, format: AutoEncoderFormat, size: Tuple[int, int]) -> Tuple[np.ndarray, np.ndarray]:
    img = Image.open(path)
    img = img.resize(size)
    gray_scale = np.mean(np.asarray(img, dtype='float32') / 255.0, axis=2)
    gray_scale = np.reshape(gray_scale, (gray_scale.shape[0], gray_scale.shape[1], 1))
    img = np.asarray(img, dtype='float32') / 255.0
    if format == AutoEncoderFormat.HSV:
        img = rgb_to_hsv(img)
    elif format == AutoEncoderFormat.LAB:
        img = rgb_to_lab(img)
        img = (img + np.array([0, 128, 128])) / np.array([100, 255, 255])
    return img, gray_scale

def load_data(name, nb_samples, size=100):
    size = (size, size)
    train = tf_ds.load(name=name, split=tf_ds.Split.TEST)
    train = train.shuffle(1024).prefetch(tf.data.experimental.AUTOTUNE)

    train_x = prepare_data(train, "Train dataset", nb_samples, size)

    val = tf_ds.load(name=name, split=tf_ds.Split.TRAIN)
    val = val.shuffle(1024).prefetch(tf.data.experimental.AUTOTUNE)

    val_x = prepare_data(val, "Validation dataset", nb_samples // 5, size)
    return train_x, val_x

def prepare_data(dataset, name, n, size):
    samples = list(dataset)[:n]
    data_x = []
    first = True
    for element in tqdm(samples, desc=name):
        img = Image.fromarray(np.array(element["image"]))
        if size is not None:
            img = img.resize(size)
        x = np.asarray(img, dtype="float32") / 255.0
        if first:
            plt.imshow(x)
            plt.show()
            first = False
        data_x.append(x)
    data_x = np.array(data_x, dtype='float32')
    return data_x

class AutoEncoderImageDataGenerator(tf.keras.utils.Sequence):

    def __init__(self, x_train, batch_size: int, shuffle: bool, format: AutoEncoderFormat, data_ag: bool):
        super(AutoEncoderImageDataGenerator, self).__init__()
        self.__image_data_generator = ImageDataGenerator(
            rotation_range=30,
            width_shift_range=1.0,
            height_shift_range=1.0,
            horizontal_flip=True,
            vertical_flip=True,
        )
        self.__x_train = x_train
        self.__batch_size: int = batch_size
        self.__shuffle: bool = shuffle
        self.__format: AutoEncoderFormat = format
        self.__data_ag: bool = data_ag

    def __getitem__(self, index):
        batch = self.__x_train[index * self.__batch_size: (index + 1) * self.__batch_size]
        return self.__prepare_data(batch)

    def __len__(self):
        return len(self.__x_train) // self.__batch_size

    def on_epoch_end(self):
        if self.__shuffle:
            self.__x_train = np.random.shuffle(self.__x_train)

    def __prepare_data(self, batch):
        x_data = []
        y_data = []
        for i in batch:
            if self.__data_ag:
                x = self.__image_data_generator.random_transform(i)
            else:
                x = i
            x_data.append(tf.math.reduce_mean(x, axis=2, keepdims=True))
            if self.__format == AutoEncoderFormat.RGB:
                y = x
            elif self.__format == AutoEncoderFormat.HSV:
                y = rgb_to_hsv(x)
            elif self.__format == AutoEncoderFormat.LAB:
                y = rgb_to_lab(x)
                y = (y + np.array([0, 128, 128])) / np.array([100, 255, 255])
            else:
                raise ValueError(f"No format: {self.__format}")
            y_data.append(y)
        x_data = np.array(x_data)
        y_data = np.array(y_data)
        return x_data, y_data


import numpy as np

import cv2
import torch
from torchvision import transforms as T
from app.features.utils import ModelUtils
from app.models.model import load_model, device

def predict(filename):
    print("Loading PyTorch model")
    model = load_model()

    data = {"success": False}
    classes = ['_', 'apple', 'orange', 'banana']

    # get image
    img = cv2.imread(filename)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32)
    # img = cv2.resize(img, (480, 480), cv2.INTER_AREA)
    img /= 255.0

    # Define a transform to convert the image to tensor
    transform = T.transforms.ToTensor()

    # Convert the image to PyTorch tensor
    img2 = transform(img)

    utils = ModelUtils()

    with torch.no_grad():
        prediction = model([img2.to(device())])[0]
        prediction = utils.nms(prediction, threshold=0.3)
    source_img = cv2.imread(filename)
    source_img = cv2.cvtColor(source_img, cv2.COLOR_BGR2RGB)

    utils.plot_box(filename, source_img, prediction, classes)

    data["prediction"] = classes[prediction['labels'][0]]
    data["success"] = True
    data["filename"] = filename
    # os.remove(f.filename)
    return data


import cv2, os
import random
import numpy as np

import torch
from torch.utils.data import Dataset
import torchvision.transforms as tfms

class VOC_Dataset(Dataset):
    def __init__(self, image_dir, label_txt, image_size=448, grid_size=7, num_bboxes=2, num_classes=20, train=True, debug=False):

        self._train = train
        self._debug = debug
        self.image_size = image_size

        self.S, self.B, self.C = grid_size, num_bboxes, num_classes
        self.mean_rgb = np.array([122.6791434, 116.66876762, 104.00698793], dtype=np.float32)
        self.to_tensor = tfms.Compose([
            tfms.ToTensor(),
            # tfms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
            tfms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
        ])

        self.paths, self.bboxes, self.labels = [], [], []
        with open(label_txt) as f:
            lines = f.readlines()

        for line in lines:
            attrs = line.strip().split()

            img_name = attrs[0]
            path = os.path.join(image_dir, img_name)
            self.paths.append(path)

            num_bboxes = (len(attrs)-1)//5
            bbox, label = [], []

            for n in range(num_bboxes):
                x1         = float(attrs[5*n + 1])
                y1         = float(attrs[5*n + 2])
                x2         = float(attrs[5*n + 3])
                y2         = float(attrs[5*n + 4])
                confidence = int(attrs[5*n + 5])
                bbox.append([x1, y1, x2, y2])
                label.append(confidence)
            self.bboxes.append(torch.Tensor(bbox))
            self.labels.append(torch.LongTensor(label))

        self.length = len(self.paths)

    def __getitem__(self, index):
        path = self.paths[index]
        img = cv2.imread(path)
        if self._debug:
            original_img = img.copy()

        boxes = self.bboxes[index].clone()
        labels = self.labels[index].clone()

        if self._train:
            img, boxes = self.lr_flip(img, boxes)
            img, boxes = self.ud_flip(img, boxes)
            img, boxes = self.width_scale(img, boxes)
            img, boxes = self.height_scale(img, boxes)
            img = self.blur(img)
            img = self.brightness(img)
            img = self.hue(img)
            img = self.saturation(img)
            img, boxes, labels = self.shift(img, boxes, labels)

        if self._debug:
            debug_dir = './debug'
            os.makedirs(debug_dir, exist_ok=True)

            img_show = img.copy()
            box_show = boxes.numpy().reshape(-1)
            n = len(box_show) // 4

            for b in range(n):
                pt1 = (int(box_show[4*b + 0]), int(box_show[4*b + 1]))
                pt2 = (int(box_show[4*b + 2]), int(box_show[4*b + 3]))
                cv2.rectangle(img_show, pt1=pt1, pt2=pt2, color=(0,255,0), thickness=2)

                cv2.imwrite(os.path.join(debug_dir, './train_valid/debug_{}.jpg'.format(index+1)), img_show)
                # cv2.imwrite(os.path.join(debug_dir, './train_valid/original_{}.jpg'.format(index+1)), original_img)

        h, w, _ = img.shape
        boxes /= torch.Tensor([[w, h, w, h]]).expand_as(boxes) # normalize (x1, y1, x2, y2) w.r.t. image width/height.
        target = self.to_target(boxes, labels) # [S, S, 5 x B + C]

        img = cv2.resize(img, dsize=(self.image_size, self.image_size), interpolation=cv2.INTER_LINEAR)
        # img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        img = self.to_tensor(img)
        return img, target

    def __len__(self):
            return self.length

    def to_target(self, bboxes, labels):
        S, B, C = self.S, self.B, self.C
        N = 5 * B + C

        target = torch.zeros(S, S, N)
        cell_size = 1.0 / float(S)
        bboxes_wh = bboxes[:, 2:] - bboxes[:, :2] # width and height for each box, [n, 2]
        bboxes_ct = (bboxes[:, 2:] + bboxes[:, :2]) / 2.0 # center x & y for each box, [n, 2]

        for b in range(bboxes.size(0)):
            ct, wh, label = bboxes_ct[b], bboxes_wh[b], int(labels[b])

            ij = (ct / cell_size).ceil() - 1.0
            i, j = int(ij[0]), int(ij[1])
            base_xy = ij * cell_size # x & y of the cell left-top corner.
            offset_xy = (ct - base_xy) / cell_size # x & y of the box on the cell, normalized from 0.0 to 1.0.

            # TBM, remove redundant dimensions from target tensor.
            # To remove these, loss implementation also has to be modified.
            for k in range(B):
                s = 5 * k
                target[j, i, s  :s+2] = offset_xy
                target[j, i, s+2:s+4] = wh
                target[j, i, s+4    ] = 1.0
            target[j, i, 5*B + label] = 1.0

        return target

    def lr_flip(self, img, bboxes):
        if random.random() < 0.5:
            return img, bboxes

        h, w, _ = img.shape
        img = np.fliplr(img)

        x1, x2 = bboxes[:, 0], bboxes[:, 2]
        new_x1 = w - x2
        new_x2 = w - x1
        bboxes[:, 0], bboxes[:, 2] = new_x1, new_x2

        return img, bboxes

    def ud_flip(self, img, bboxes):
        if random.random() < 0.5:
            return img, bboxes

        h, w, _ = img.shape
        img = np.flipud(img)

        y1, y2 = bboxes[:, 1], bboxes[:, 3]
        new_y1 = h - y2
        new_y2 = h - y1
        bboxes[:, 1], bboxes[:, 3] = new_y1, new_y2

        return img, bboxes

    def width_scale(self, img, bboxes):
        if random.random() < 0.5:
            return img, bboxes

        scale = random.uniform(0.8, 1.2)
        h, w, _ = img.shape
        img = cv2.resize(img, dsize=(int(w*scale), h), interpolation=cv2.INTER_LINEAR)

        scale_tensor = torch.FloatTensor([[scale, 1.0, scale, 1.0]]).expand_as(bboxes)
        bboxes *= scale_tensor

        return img, bboxes

    def height_scale(self, img, bboxes):
        if random.random() < 0.5:
            return img, bboxes

        scale = random.uniform(0.8, 1.2)
        h, w, _ = img.shape
        img = cv2.resize(img, dsize=(w, int(h*scale)), interpolation=cv2.INTER_LINEAR)

        scale_tensor = torch.FloatTensor([[1.0, scale, 1.0, scale]]).expand_as(bboxes)
        bboxes *= scale_tensor

        return img, bboxes

    def blur(self, img):
        if random.random() < 0.5:
            return img

        ksize = random.choice([2, 3, 4, 5])
        img = cv2.blur(img, (ksize, ksize))
        return img

    def brightness(self, img):
        if random.random() < 0.5:
            return img

        hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
        h, s, v = cv2.split(img)
        adjust = random.uniform(0.6, 1.0)
        v = v * adjust
        v = np.clip(v, 0, 255).astype(hsv.dtype)
        hsv = cv2.merge((h, s, v))
        img = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)

        return img

    def hue(self, img):
        if random.random() < 0.5:
            return img

        hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
        h, s, v = cv2.split(hsv)
        adjust = random.uniform(1.0, 1.5)
        h = h * adjust
        h = np.clip(h, 0, 255).astype(hsv.dtype)
        hsv = cv2.merge((h, s, v))
        img = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)

        return img

    def saturation(self, img):
        if random.random() < 0.5:
            return img

        hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
        h, s, v = cv2.split(hsv)
        adjust = random.uniform(1.0, 1.5)
        s = s * adjust
        s = np.clip(s, 0, 255).astype(hsv.dtype)
        hsv = cv2.merge((h, s, v))
        img = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)

        return img

    def shift(self, img, boxes, labels):
        if random.random() < 0.5:
            return img, boxes, labels

        centre = (boxes[:, 2:] + boxes[:, :2]) / 2.0

        h, w, c = img.shape
        img_out = np.zeros((h, w, c), dtype=img.dtype)
        # mean_bgr = self.mean_rgb[::-1]
        # img_out[:, :] = mean_bgr

        dx = random.uniform(-w*0.2, w*0.2)
        dy = random.uniform(-h*0.2, h*0.2)
        dx, dy = int(dx), int(dy)

        if dx >= 0 and dy >= 0:
            img_out[dy:, dx:] = img[:h-dy, :w-dx]
        elif dx >= 0 and dy < 0:
            img_out[:h+dy, dx:] = img[-dy:, :w-dx]
        elif dx < 0 and dy >= 0:
            img_out[dy:, :w+dx] = img[:h-dy, -dx:]
        elif dx < 0 and dy < 0:
            img_out[:h+dy, :w+dx] = img[-dy:, -dx:]

        centre = centre + torch.FloatTensor([[dx, dy]]).expand_as(centre) # [n, 2]
        mask_x = (centre[:, 0] >= 0) & (centre[:, 0] < w) # [n,]
        mask_y = (centre[:, 1] >= 0) & (centre[:, 1] < h) # [n,]
        mask = (mask_x & mask_y).view(-1, 1) # [n, 1], mask for the boxes within the image after shift.

        boxes_out = boxes[mask.expand_as(boxes)].view(-1, 4) # [m, 4]
        if len(boxes_out) == 0:
            return img, boxes, labels
        shift = torch.FloatTensor([[dx, dy, dx, dy]]).expand_as(boxes_out) # [m, 4]

        boxes_out = boxes_out + shift
        boxes_out[:, 0] = boxes_out[:, 0].clamp_(min=0, max=w)
        boxes_out[:, 2] = boxes_out[:, 2].clamp_(min=0, max=w)
        boxes_out[:, 1] = boxes_out[:, 1].clamp_(min=0, max=h)
        boxes_out[:, 3] = boxes_out[:, 3].clamp_(min=0, max=h)

        labels_out = labels[mask.view(-1)]

        return img_out, boxes_out, labels_out

def test():
    from torch.utils.data import DataLoader
    from torch.utils.data import random_split

    image_dir = './data/VOC2012'
    label_txt = './data/voc2012.txt'

    dataset = VOC_Dataset(image_dir, label_txt, debug=True)

    valid_ratio = .2
    dataset_size = len(dataset)
    valid_size = int(valid_ratio * dataset_size)

    train_set, valid_set = random_split(dataset, [dataset_size-valid_size, valid_size])
    valid_set._train = False

    train_loader = DataLoader(train_set, batch_size=1, shuffle=False, num_workers=0)
    valid_loader = DataLoader(valid_set, batch_size=1, num_workers=0)

    data_iter = iter(train_loader)
    for i in range(5):
        img, target = next(data_iter)
        print(img.size(), target.size())

    data_iter = iter(valid_loader)
    for i in range(5):
        img, target = next(data_iter)
        print(img.size(), target.size())

if __name__ == "__main__":
    test()


from collect_training_data import CollectImageData
from train_model import TrainingModel
from utils import *
import cv2

if(yes_or_no("Do you want to collect training data?")):
    fc = CollectImageData()

    fc.ExtractAndSaveFacesFromCamera('dataset')

if(yes_or_no('Do you want to retrain the model?')):
    tm = TrainingModel()

    tm.TrainingModel('dataset')

if(yes_or_no('Is it ready to recognize faces?')):
    recognizer = cv2.face.LBPHFaceRecognizer_create()
    recognizer.read('Trained_Models/face_trainer.yml')

    cascadePath = 'DATA/haarcascades/haarcascade_frontalface_default.xml'
    faceCascade = cv2.CascadeClassifier(cascadePath)
    font = cv2.FONT_HERSHEY_SIMPLEX

    names = ['unkown','Jin']

    cam = cv2.VideoCapture(0+cv2.CAP_DSHOW)
    cam.set(3, 640)
    cam.set(4, 480)

    minW = 0.1*cam.get(3)
    minH = 0.1*cam.get(4)

    while True:
        ret, img = cam.read()

        gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

        faces = faceCascade.detectMultiScale(gray_img,
                                            scaleFactor=1.2,
                                            minNeighbors=15,
                                            minSize=(int(minW), int(minH)))
        for (x, y, w, h) in faces:
            cv2.rectangle(img, (x, y), (x+w, y+h), (0, 255, 0), 2)

            id, confidence = recognizer.predict(gray_img[y:y+h, x:x+w])

            if(confidence < 50):
                id = names[id]
                confidence = " {0}%".format(round(100.0-confidence))
            else:
                id = 'unknown'
                confidence = " {0}%".format(round(100.0-confidence))
            cv2.putText(
                img,
                str(id),
                (x+5, y-5),
                font,
                1,
                (255, 255, 255),
                2,
            )

            cv2.putText(
                img,
                str(confidence),
                (x+5, y+h-5),
                font,
                1,
                (255, 255, 0),
                1,
            )
        cv2.imshow('camera', img)
        k = cv2.waitKey(1) & 0Xff
        if k == 27:
            break
    cam.release()
    cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('core', '0004_review'),
    ]

    operations = [
        migrations.AddField(
            model_name='location',
            name='seating',
            field=models.IntegerField(blank=True, null=True, choices=[(0, b'None'), (1, b'Minimal'), (2, b'Some'), (3, b'Ample')]),
        ),
        migrations.AddField(
            model_name='location',
            name='wifi',
            field=models.IntegerField(blank=True, null=True, choices=[(0, b'None'), (1, b'Spotty'), (2, b'Strong')]),
        ),
    ]


from trusthouse.models.incident import Incident
from ..extensions import db, app
from datetime import datetime

def new_incident(category, description, address):
    """
    Creates a new incident entry, saving it into the Incident table.
    Each entry is linked to both Address & Maps tables.

    Returns the Incident object back to the user.
    """
    with app.app_context():
        new_incident_entry = Incident(
            category=category.lower(),
            description=description.lower(),
            date=datetime.now(),
            area=address,
        )
        db.session.add(new_incident_entry)
        db.session.commit()
    return new_incident_entry


''' Item Price Rule url'''
from django.urls import path
from .views import ItemPriceRuleViews, ItemPriceRuleUpdateViews, ItemPriceRuleMultipleStatusUpdate

urlpatterns = [
    path('multiple/', ItemPriceRuleMultipleStatusUpdate.as_view(),
         name='item_price_rule_multiple_status_update_delete'),
    path('<itemPriceRuleID>/', ItemPriceRuleUpdateViews.as_view(),
         name='item_price_rule_updtae'),
    path('', ItemPriceRuleViews.as_view(),
         name='item_price_rule_get_create'),

]


from django.urls import path
from . import views

urlpatterns = [
    path('', views.shop, name='shop-home'),
    path('about', views.about, name='shop-about'),
    path('pricing', views.pricing, name='shop-pricing'),
    path('contact', views.contact, name='shop-contact'),

]


#!/usr/bin/python3
from pyspark.sql import SparkSession
from pyspark.sql.functions import *
from pyspark.sql.types import *
from pyspark.sql.window import Window
from pyspark.sql import Row

def triggerPeriodGroup(rdd):
    temp = rdd.asDict()
    global i
    if temp["trigger_period"] > 120:
        i+=1
    temp["trigger_period_group"]=i
    return Row(**temp)

if __name__ == "__main__":
    # Create SparkSession
    spark = SparkSession \
        .builder \
        .getOrCreate()

    # Create a DataFrame from a JSON file
    event_df = spark.read.json("hdfs://localhost/user/cloudera/SparkSQLBehaviorsAnalysis4Linechatbot/data/eventLog")

    # Explode all elements in the column (event_df["events"]) -> all record represent every behavior event
    allbehaviors_df = event_df.select(explode(event_df["events"]).alias("events"))

    # Select columns which will be analyzed later
    clear_df = allbehaviors_df.select(allbehaviors_df["events"]["timestamp"].alias("timestamp"), \
                                      from_unixtime(allbehaviors_df["events"]["timestamp"]/1000, "yyyy-MM-dd HH:mm:ss").alias("event_time"), \
                                      allbehaviors_df["events"]["source"]["userId"].alias("userid"), \
                                      allbehaviors_df["events"]["type"].alias("type"), \
                                      allbehaviors_df["events"]["message"]["text"].alias("messageEvent"), \
                                      allbehaviors_df["events"]["postback"].alias("postbackEvent"))

    # analysis(staying time)
    # trigger_period in clear_df
    window_clause1 = Window.partitionBy(clear_df["userid"]).orderBy(clear_df["timestamp"])

    window_specification1 = window_clause1.rowsBetween(-1, 0)

    clear_df = clear_df.withColumn("trigger_period", ((clear_df["timestamp"] - first(clear_df["timestamp"]).over(window_specification1))/1000))

    # triggerPeriodGroup_df
    i=0; triggerPeriodGroup_df = clear_df.rdd.map(triggerPeriodGroup).toDF(sampleRatio=0.2)

    # once_staying_time in triggerPeriodGroup_df
    window_clause2 = Window.partitionBy([triggerPeriodGroup_df["userid"], triggerPeriodGroup_df["trigger_period_group"]]) \
                           .orderBy(triggerPeriodGroup_df["trigger_period_group"])

    window_specification2 = window_clause2.rowsBetween(Window.unboundedPreceding, Window.unboundedFollowing)

    triggerPeriodGroup_df = triggerPeriodGroup_df.withColumn("once_staying_time", \
                            (sum(triggerPeriodGroup_df["trigger_period"]).over(window_specification2) - \
                             max(triggerPeriodGroup_df["trigger_period"]).over(window_specification2))) \
                             .orderBy(triggerPeriodGroup_df["userid"])

    # staying_time
    staying_time_df = triggerPeriodGroup_df.groupBy( [triggerPeriodGroup_df["userId"], triggerPeriodGroup_df["trigger_period_group"]]) \
                                           .agg({"once_staying_time":"avg"}) \
                                           .withColumnRenamed("avg(once_staying_time)", "once_staying_time") \
                                           .orderBy([triggerPeriodGroup_df["userId"], triggerPeriodGroup_df["trigger_period_group"]])

    staying_time_df = staying_time_df.select(staying_time_df["userId"], \
                                             staying_time_df["once_staying_time"])

    # save to json file
    staying_time_df.write.json("hdfs://localhost/user/cloudera/SparkSQLBehaviorsAnalysis4Linechatbot/analysisResults/staying_time_analysis")

    # Create a DataFrame from a csv file
    memberProfile_df = spark.read.csv("hdfs://localhost/user/cloudera/SparkSQLBehaviorsAnalysis4Linechatbot/data/memberProfile4MySQL", header=True)

    staying_time_with_memberProfile_df = staying_time_df.join(memberProfile_df, staying_time_df["userId"] == memberProfile_df["line_id"], 'left_outer') \
                                                        .select(staying_time_df["userId"], \
                                                                memberProfile_df["username"], \
                                                                memberProfile_df["sex"], \
                                                                staying_time_df["once_staying_time"]) \
                                                        .fillna({"username":"unknown", "sex":"unknown"})

    # save to json file
    staying_time_with_memberProfile_df.write.json("hdfs://localhost/user/cloudera/SparkSQLBehaviorsAnalysis4Linechatbot/analysisResults/staying_time_with_memberProfile_df")

#========================= (Launch Spark by 「Interactive Development Environment」)
# pyspark --master spark://172.21.0.2:7077

#========================= (Launch Spark by 「Shell Script」)
# spark-submit --master spark://172.21.0.2:7077 message_event_analysis.py


import datetime

import requests
import forecastio

import keys
import config

class OutsideData():
    def __init__(self):
        self.cached_temp = 70
        self.time_of_last_request = None
        self.minutes_5 = datetime.timedelta(minutes=config.temp_cache_frequency_minutes)

    def get_outside_temp(self, lat, lon):
        current_time = datetime.datetime.utcnow()
        if self.time_of_last_request is not None:
            time_from_last_request = current_time - self.time_of_last_request
            if time_from_last_request < self.minutes_5:
                return self.cached_temp, None

        try:
            temp = self.get_forecast_io_temp(lat, lon)
        except:
            error = 'failure of weather api this will probably resolve itself using last known outside temp'
            return self.cached_temp, error

        self.cached_temp = temp
        self.time_of_last_request = current_time
        return temp, None

    def get_forecast_io_temp(self, lat, lon):
        forecast = forecastio.load_forecast(keys.dark_sky_key, lat, lon)
        temp = forecast.currently().temperature
        return temp

    def get_open_weather_temp(lat, lon):
        uri = 'http://api.openweathermap.org/data/2.5/weather'
        params = {'lat': lat, 'lon': lon, 'APPID':keys.open_weather_key}
        result = requests.get(uri, params=params)
        if result.status_code != 200:
            print(result.status_code)
        else:
            results = result.json()
            temp_k = result.json()['main']['temp']
            temp_f = (temp_k - 273.15) * 1.8 + 32.0
        return temp_f

outside_data = OutsideData()


from flask import render_template, session, request, Markup, redirect
from app import app, LINK

from requests import post
from json import loads

@app.route('/sys_study')
@app.route('/sys_study/')
def sys_study():
	ip = request.remote_addr

	if 'token' in session:
		req = loads(post(LINK, json={
			'method': 'study.start',
			'token': session['token'],
			'ip': ip,
			'ladder': int(request.args.get('ladder')),
			'step': int(request.args.get('step')),
			'teacher': int(request.args.get('user')),
		}).text)

		if req['error']:
			return render_template('message.html', cont=req['message'])

		src = LINK + 'space/' + str(req['id'])
		return redirect(src)
	else:
		return redirect(LINK + 'login?url=' + url)