import numpy as np
import argparse
import socket
import time
import os
import scipy.misc
import sys
import glob
import matplotlib.pyplot as plt
from sklearn import datasets, decomposition, manifold, preprocessing

import os,sys,inspect
currentdir = "/usr/local/featureExtractionParty/external/pointnet_spine_ae"
sys.path.insert(0,currentdir)
import importlib

import pandas as pd
from sklearn.cluster import KMeans

import meshparty
import time
from meshparty import trimesh_io, trimesh_vtk, trimesh_repair
import trimesh
from trimesh.primitives import Sphere
import os
import h5py
from meshparty import skeleton, skeleton_io
import json
import math
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from scipy import sparse
from meshparty import mesh_filters
from meshparty import utils
import numpy as np
import pickle
import pandas
import cloudvolume
from meshparty import skeletonize
from trimesh.ray import ray_pyembree
from multiprocessing import Pool
from functools import partial
from scipy import sparse
from meshparty import skeleton_io
from annotationframeworkclient import infoservice
import trimesh
from meshparty import skeletonize
from meshparty import mesh_skel_utils
from scipy.sparse import csgraph
print("Finished imports")

# Import full classifications dataframe
classes_file = 'synapses_PSS.pkl'
classes_df = pd.read_pickle(classes_file)

# Import dually innervated spines list
spines_file = 'dually-innervated_reindexed.pkl'
spines_df = pd.read_pickle(spines_file)

# Choose which pair of meshes to work with (i corresponds to index in list of pairs)
i = 39344

# Load meshes and create mesh actors for each mesh
spine1_file = r'Meshes/' + str(spines_df['cell id'][i]) + '/' + str(i) + '/spine_' + str(spines_df['spine 1 index'][i]) + '.off'
spinemesh1 = trimesh.exchange.load.load_mesh(spine1_file)
spinemesh1_mp = meshparty.trimesh_io.Mesh(vertices=spinemesh1.vertices, faces=spinemesh1.faces)
spinemesh1_actor = trimesh_vtk.mesh_actor(spinemesh1, color=(0,0,1), opacity=0.2)
spine2_file = r'Meshes/' + str(spines_df['cell id'][i]) + '/' + str(i) + '/spine_' + str(spines_df['spine 2 index'][i]) + '.off'
spinemesh2 = trimesh.exchange.load.load_mesh(spine2_file)
spinemesh2_mp = meshparty.trimesh_io.Mesh(vertices=spinemesh2.vertices, faces=spinemesh2.faces)
spinemesh2_actor = trimesh_vtk.mesh_actor(spinemesh2, color=(1,0,0), opacity=0.2)

# Find synapse location and make a point cloud actor for the synapse of each mesh
synapse1_loc = [spines_df['spine 1 synapse'][i][0]*4,spines_df['spine 1 synapse'][i][1]*4,spines_df['spine 1 synapse'][i][2]*40]
synapse2_loc = [spines_df['spine 2 synapse'][i][0]*4,spines_df['spine 2 synapse'][i][1]*4,spines_df['spine 2 synapse'][i][2]*40]
synapse1_actor = trimesh_vtk.point_cloud_actor([synapse1_loc],size=100,color=(0,0,1),opacity=1)
synapse2_actor = trimesh_vtk.point_cloud_actor([synapse2_loc],size=100,color=(1,0,0),opacity=1)

# Skeletonize and create a skeleton actor for each mesh
skeleton1 = meshparty.skeletonize.skeletonize_mesh(spinemesh1_mp,compute_radius=False,cc_vertex_thresh=0)
skeleton1_actor = meshparty.trimesh_vtk.skeleton_actor(skeleton1,color=(0,0,1),opacity=1)
skeleton2 = meshparty.skeletonize.skeletonize_mesh(spinemesh2_mp,compute_radius=False,cc_vertex_thresh=0)
skeleton2_actor = meshparty.trimesh_vtk.skeleton_actor(skeleton2,color=(1,0,0),opacity=1)

# Find opening and point on skeleton closest to opening for each mesh
spine1broken = trimesh.repair.broken_faces(spinemesh1_mp)
spine1openings = spinemesh1_mp.submesh([spine1broken], append=True)
spine1openings = meshparty.trimesh_io.Mesh(vertices=spine1openings.vertices, faces=spine1openings.faces)
print(spine1openings)
if spine1openings.body_count == 1:
    path_opening1_a = meshparty.mesh_skel_utils.point_to_skel_meshpath(spinemesh1_mp,skeleton1,spine1openings.vertices[0])
    opening_ind1_a = np.where(skeleton1.vertices == spinemesh1.vertices[path_opening1_a[-1]])
    opening_ind1_a = opening_ind1_a[0][0]
    opening_ind1_b = -1
elif spine1openings.body_count == 2:
    mask1 = mesh_filters.filter_largest_component(spine1openings)
    spine1openings_a = spine1openings.apply_mask(mask1)
    spine1openings_b_vertices = []
    for i in spine1openings.vertices:
        if i not in spine1openings_a.vertices:
            spine1openings_b_vertices.append(i.tolist())
    path_opening1_a = meshparty.mesh_skel_utils.point_to_skel_meshpath(spinemesh1_mp,skeleton1,spine1openings.vertices[0])
    path_opening1_b = meshparty.mesh_skel_utils.point_to_skel_meshpath(spinemesh1_mp,skeleton1,spine1openings_b_vertices[0])
    opening_ind1_a = np.where(skeleton1.vertices == spinemesh1.vertices[path_opening1_a[-1]])
    opening_ind1_a = opening_ind1_a[0][0]
    opening_ind1_b = np.where(skeleton1.vertices == spinemesh1.vertices[path_opening1_b[-1]])
    opening_ind1_b = opening_ind1_b[0][0]
else:
    mask1 = mesh_filters.filter_largest_component(spine1openings)
    spine1openings = spine1openings.apply_mask(mask1)
    path_opening1_a = meshparty.mesh_skel_utils.point_to_skel_meshpath(spinemesh1_mp,skeleton1,spine1openings.vertices[0])
    opening_ind1_a = np.where(skeleton1.vertices == spinemesh1.vertices[path_opening1_a[-1]])
    opening_ind1_a = opening_ind1_a[0][0]
    opening_ind1_b = -1

spine2broken = trimesh.repair.broken_faces(spinemesh2_mp)
spine2openings = spinemesh2_mp.submesh([spine2broken], append=True)
spine2openings = meshparty.trimesh_io.Mesh(vertices=spine2openings.vertices, faces=spine2openings.faces)
print(spine2openings)
if spine2openings.body_count == 1:
    path_opening2_a = meshparty.mesh_skel_utils.point_to_skel_meshpath(spinemesh2_mp,skeleton2,spine2openings.vertices[0])
    opening_ind2_a = np.where(skeleton2.vertices == spinemesh2.vertices[path_opening2_a[-1]])
    opening_ind2_a = opening_ind2_a[0][0]
    opening_ind2_b = -1
elif spine2openings.body_count == 2:
    mask2 = mesh_filters.filter_largest_component(spine2openings)
    spine2openings_a = spine2openings.apply_mask(mask2)
    spine2openings_b_vertices = []
    for i in spine2openings.vertices:
        if i not in spine2openings_a.vertices:
            spine2openings_b_vertices.append(i.tolist())
    path_opening2_a = meshparty.mesh_skel_utils.point_to_skel_meshpath(spinemesh2_mp,skeleton2,spine2openings.vertices[0])
    path_opening2_b = meshparty.mesh_skel_utils.point_to_skel_meshpath(spinemesh2_mp,skeleton2,spine2openings_b_vertices[0])
    opening_ind2_a = np.where(skeleton2.vertices == spinemesh2.vertices[path_opening2_a[-1]])
    opening_ind2_a = opening_ind2_a[0][0]
    opening_ind2_b = np.where(skeleton2.vertices == spinemesh2.vertices[path_opening2_b[-1]])
    opening_ind2_b = opening_ind2_b[0][0]
else:
    mask2 = mesh_filters.filter_largest_component(spine2openings)
    spine2openings = spine2openings.apply_mask(mask2)
    path_opening2_a = meshparty.mesh_skel_utils.point_to_skel_meshpath(spinemesh2_mp,skeleton2,spine2openings.vertices[0])
    opening_ind2_a = np.where(skeleton2.vertices == spinemesh2.vertices[path_opening2_a[-1]])
    opening_ind2_a = opening_ind2_a[0][0]
    opening_ind2_b = -1

# Create a path from the synapse to the opening for each mesh/synapse
# The path consists of two parts: (1) a path from the synapse to the nearest point on the skeleton, and
#                                 (2) a path along the skeleton to the opening
path1_1 = meshparty.mesh_skel_utils.point_to_skel_meshpath(spinemesh1_mp,skeleton1,synapse1_loc)
meshverts1 = spinemesh1.vertices.tolist()
skelverts1 = skeleton1.vertices.tolist()
inds1_a = path1_1
inds1_b = path1_1[1:]
index = np.where(skeleton1.vertices == spinemesh1.vertices[path1_1[-1]])
index = index[0][0]
if opening_ind1_b == -1:
    opening_ind1 = opening_ind1_a
    skeleton1.reroot(opening_ind1_a)
    path1_2 = skeleton1.path_to_root(index)
    for i in path1_2[1:]:
        mesh_ind = np.where(spinemesh1.vertices == skeleton1.vertices[i])
        mesh_ind = mesh_ind[0][0]
        inds1_a.append(mesh_ind)
        inds1_b.append(mesh_ind)
    inds1_b = np.append(inds1_b,inds1_b[-1])
    line1 = meshparty.trimesh_vtk.linked_point_actor(np.array(meshverts1),np.array(meshverts1),inds_a=inds1_a,inds_b=inds1_b,opacity=1,line_width=5,color=(0,0,1))
else:
    skeleton1_a = skeleton1
    skeleton1_a.reroot(opening_ind1_a)
    path1_2a = skeleton1_a.path_to_root(index)
    skeleton1_b = skeleton1
    skeleton1_b.reroot(opening_ind1_b)
    path1_2b = skeleton1_b.path_to_root(index)
    if len(path1_2a) > len(path1_2b) and len(path1_2b) > 1:
        path1_2 = path1_2b
        opening_ind1 = opening_ind1_b
    else:
        path1_2 = path1_2a
        opening_ind1 = opening_ind1_a
    for i in path1_2[1:]:
        mesh_ind = np.where(spinemesh1.vertices == skeleton1.vertices[i])
        mesh_ind = mesh_ind[0][0]
        inds1_a.append(mesh_ind)
        inds1_b.append(mesh_ind)
    inds1_b = np.append(inds1_b,inds1_b[-1])
    line1 = meshparty.trimesh_vtk.linked_point_actor(np.array(meshverts1),np.array(meshverts1),inds_a=inds1_a,inds_b=inds1_b,opacity=1,line_width=5,color=(0,0,1))

path2_1 = meshparty.mesh_skel_utils.point_to_skel_meshpath(spinemesh2_mp,skeleton2,synapse2_loc)
meshverts2 = spinemesh2.vertices.tolist()
skelverts2 = skeleton2.vertices.tolist()
inds2_a = path2_1
inds2_b = path2_1[1:]
index = np.where(skeleton2.vertices == spinemesh2.vertices[path2_1[-1]])
index = index[0][0]
if opening_ind2_b == -1:
    opening_ind2 = opening_ind2_a
    skeleton2.reroot(opening_ind2_a)
    path2_2 = skeleton2.path_to_root(index)
    for i in path2_2[1:]:
        mesh_ind = np.where(spinemesh2.vertices == skeleton2.vertices[i])
        mesh_ind = mesh_ind[0][0]
        inds2_a.append(mesh_ind)
        inds2_b.append(mesh_ind)
    inds2_b = np.append(inds2_b,inds2_b[-1])
    line2 = meshparty.trimesh_vtk.linked_point_actor(np.array(meshverts2),np.array(meshverts2),inds_a=inds2_a,inds_b=inds2_b,opacity=1,line_width=5,color=(1,0,0))
else:
    skeleton2_a = skeleton2
    skeleton2_a.reroot(opening_ind2_a)
    path2_2a = skeleton2_a.path_to_root(index)
    skeleton2_b = skeleton2
    skeleton2_b.reroot(opening_ind2_b)
    path2_2b = skeleton2_b.path_to_root(index)
    if len(path2_2a) > len(path2_2b) and len(path2_2b) > 1:
        path2_2 = path2_2b
        opening_ind2 = opening_ind2_b
    else:
        path2_2 = path2_2a
        opening_ind2 = opening_ind2_a
    for i in path2_2[1:]:
        mesh_ind = np.where(spinemesh2.vertices == skeleton2.vertices[i])
        mesh_ind = mesh_ind[0][0]
        inds2_a.append(mesh_ind)
        inds2_b.append(mesh_ind)
    inds2_b = np.append(inds2_b,inds2_b[-1])
    line2 = meshparty.trimesh_vtk.linked_point_actor(np.array(meshverts2),np.array(meshverts2),inds_a=inds2_a,inds_b=inds2_b,opacity=1,line_width=5,color=(1,0,0))

end_actor1 = trimesh_vtk.point_cloud_actor([skelverts1[opening_ind1]],size=100,color=(0.7,0.9,1),opacity=1)
end_actor2 = trimesh_vtk.point_cloud_actor([skelverts2[opening_ind2]],size=100,color=(0.9,0.7,0.6),opacity=1)

# Compare the two lines
line1points = []
verts_ind1 = 0
for i in meshverts1:
    if verts_ind1 in inds1_a:
        line1points.append(i)
    verts_ind1 += 1
line2points = []
verts_ind2 = 0
for i in meshverts2:
    if verts_ind2 in inds2_a:
        line2points.append(i)
    verts_ind2 += 1
class Linemesh:
    def __init__(self, vertices):
        self.vertices = vertices
line1mesh = Linemesh(line1points)
#cloud = trimesh.PointCloud(line1points)
#line1mesh = cloud.convex_hull
compare = meshparty.mesh_filters.filter_spatial_distance_from_points(line1mesh,line2points,200)
print(compare)
numTrue = 0
for i in compare:
    if i == True:
        numTrue += 1
if numTrue > len(compare)/2:
    print("one head")
else:
    print("two heads")

#print(synapse1_loc)
#print(synapse2_loc)
syn1loc_array = np.array(synapse1_loc)
syn2loc_array = np.array(synapse2_loc)
squared_dist = np.sum((syn1loc_array-syn2loc_array)**2,axis=0)
syn_dist = np.sqrt(squared_dist)
print(syn_dist)

# Visualize the meshes, synapses, skeletons, and lines
renderer1 = trimesh_vtk.render_actors(actors=[spinemesh1_actor,spinemesh2_actor,synapse1_actor,synapse2_actor,skeleton1_actor,skeleton2_actor,end_actor1,end_actor2,line1,line2],do_save=False)

# Visualize the meshes, synapses, and lines (no skeletons)
renderer2 = trimesh_vtk.render_actors(actors=[spinemesh1_actor,spinemesh2_actor,synapse1_actor,synapse2_actor,line1,line2],do_save=False)


#
#           ShoeanahScrapy scraper.py | 2021 (c) Mrmagicpie
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
#
import scrapy
from pprint import PrettyPrinter
#
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
#
class projectItem(scrapy.item.Item):
    """
    A fancy scrapy project dictionary.
    """
    package            = scrapy.Field()
    version            = scrapy.Field()
    author             = scrapy.Field()
    project_url        = scrapy.Field()
    linkTo             = scrapy.Field()
    description        = scrapy.Field()
    release            = scrapy.Field()
    pip                = scrapy.Field()
    rss                = scrapy.Field()
    short_description  = scrapy.Field()
#
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
#
class PyPiScraper(scrapy.Spider):
    """
    The main Scrapy spider class.
    """

    # PPrint because cool.
    pp = PrettyPrinter(indent=4)

    # All scrapy spider names must be unique or it gets confused and might run the wrong one.
    name = "PyPi"

    # PyPi Search value. Not scrapy related.
    search = "discord"

    # start_urls must be a list of urls you want to scrape.
    start_urls = ["https://pypi.org/search/?q=" + search,]

    # Not sure what this does but we have it :KEK:
    sourceIndex = ['Python Package Index',]

    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

    # Try to always start your scraper using a "parse" or "start_request" function
    def parse(self, response):
        """
        Start the request sequence by loading the search page, gathering how many pages in total and then starting the request(s).
        :param response: Basic scrapy request response.
        :return: Returns a sequence of requests.
        """

        # Find number of search results.
        page_numbers = response.xpath('//a[@class="button button-group__button"][3]/text()').extract_first()
        # Turn the str into an int.
        page_numbers = int(page_numbers)
        # Not sure how to do it better at the moment so this keeps track of what page to request.
        number = 0
        # Loop through all the page numbers.
        for page in range(page_numbers):
            number = number + 1
            # Specify the URL to scrape to be used in our request.
            url = "https://pypi.org/search/?q=discord&page=" + f"{number}"
            # Specify the scrapy request, url is our url with page number above, and callback is what function we want
            # to call when we make this request.
            request = scrapy.Request(url, callback=self.parse_two)
            # Yield the request(make the request)
            yield request

    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

    def parse_two(self, response):
        """
        Callback function from parse to deal with each requested page.
        :param response: Basic scrapy request response.
        :return: Returns the first part of our package dictionary.
        """
        # Put all HTML contents into an html file named "response.html".
        # with open("response.html", "w") as f:
        #     f.write(response.text)
        # Using our fancy dictionary class.
        project = projectItem()
        # Find all packages listed on the page.
        package_table = response.xpath("//ul[@aria-label='Search results']/li")
        # Loop through each one to pull information.
        for packages in package_table:
            # Find the package name.
            project['package'] = packages.xpath(".//span[@class='package-snippet__name']/text()").extract_first()
            if self.search in project['package'].lower():
                # Get the current package version from the package block.
                project['version'] = packages.xpath(".//span[@class='package-snippet__version']/text()").extract_first()
                # Find the href link from the block.
                project['linkTo'] = packages.xpath(".//a[@class='package-snippet']/@href").extract_first()
                # Turn it into a usable link for scrapy to use in followLinkTo.
                project['linkTo'] = "https://pypi.org" + project['linkTo']
                # Find the release date of the package.
                project['release'] = packages.xpath(".//time/text()").extract_first().replace("\n", "").strip()
                # Get the short description shown on the search page.
                project['short_description'] = packages.xpath('.//p[@class="package-snippet__description"]/text()').extract_first()
                # self.pp.pprint(project)
                # Form our request using the packages linkTo, specify we want to use followLinkTo for the callback and
                # and send a copy of the current information so we can use it in followLinkTo.
                request = scrapy.Request(project['linkTo'], callback=self.followLinkTo, meta={'item': project.copy()})
                # Yield the request(make the request)
                yield request
            # If the package title does not include your search pass so we don't gather non accurate information.
            else:
                pass

    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

    def followLinkTo(self, response):
        """
        Callback function to sort through the HTML from each packages individual page.
        :param response: Basic scrapy request response.
        :return: Returns the final package dictionary.
        """
        # Redefine our project dictionary using the "parse" callback's version.
        project = response.meta['item']
        # Scrape the package's description.
        project['description'] = response.xpath("//div[@class='project-description']").extract_first()
        # Scrape the package's pip install command.
        project['pip'] = response.xpath("//span[@id='pip-command']/text()").extract_first()
        # Scrape the package's home URL(ex. GitHub or Docs website).
        project['project_url'] = response.xpath("//a[@class='vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--condensed']/@href").extract_first()
        # Scrape the package's RSS feed href.
        project['rss'] = response.xpath('//span[@class="reset-text margin-top"]/a[2]/@href').extract_first()
        # Turn the package's scraped RSS feed into a usable URL.
        project['rss'] = "https://pypi.org" + project['rss']
        # Define where to find the Authors.
        sidebar_author = response.xpath("//div[@class='sidebar-section'][5]/span")
        # Predefine the Authors list to append authors to.
        project['author'] = []
        # Loop over each author in the selection.
        for author in sidebar_author:
            # Find the author's name.
            oop = author.xpath(".//span[@class='sidebar-section__maintainer']/a/span[2]/text()").extract_first()
            # Strip/replace common characters if the author is not None.
            if oop is not None:
                oop = oop.replace("\n", "") \
                    .replace(" ", "")
            # Add the author to the authors list.
            project['author'].append(oop)
        # Print the package to the terminal.
        self.pp.pprint(project)
        # Yield the final request(to be handled by a pipeline or other output)
        yield project

#
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
#
# Don't mind this, blame: https://github.com/ShuanaOnGitHub
# def check_for_big_pp(self, ppcheck):
#     if ppcheck >= 8: # inches
#         print("Uwu, nice pp bro")
#     else:
#         print("no smol pp :W")

#
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
#


import typing

from graphql import (
    GraphQLBoolean,
    GraphQLFloat,
    GraphQLID,
    GraphQLInt,
    GraphQLString,
    GraphQLType,
)

from .scalars import ID

TYPE_HINTS_REGISTRY: typing.Dict[str, typing.Type] = {}
ANNOTATION_REGISTRY: typing.Dict[typing.Type, GraphQLType] = {
    str: GraphQLString,
    int: GraphQLInt,
    float: GraphQLFloat,
    bool: GraphQLBoolean,
    ID: GraphQLID,
}

GRAPHQL_TYPE_TO_STRAWBERRY_TYPE: typing.Dict[GraphQLType, typing.Type] = {}

def register_type(
    cls: typing.Type, graphql_type: GraphQLType, *, store_type_information: bool = True
) -> None:
    """Register a Strawberry type alongside its GraphQL type"""

    # register the annotation so we can use when getting the graphql type for
    # an annotation in type_converter
    ANNOTATION_REGISTRY[cls] = graphql_type
    GRAPHQL_TYPE_TO_STRAWBERRY_TYPE[graphql_type] = cls

    # register names to classes so that we can pass them to get_type_hints
    TYPE_HINTS_REGISTRY[cls.__name__] = cls

    # we currently have to store the "graphql core" type onto the
    # class, but we can't do that for all types (like builtins),
    # so we have an escape hatch for that

    if store_type_information:
        cls.graphql_type = graphql_type

def get_strawberry_type_for_graphql_type(
    graphql_type: GraphQLType,
) -> typing.Optional[typing.Type]:
    return GRAPHQL_TYPE_TO_STRAWBERRY_TYPE.get(graphql_type)

def get_type_for_annotation(annotation: typing.Type) -> typing.Optional[GraphQLType]:
    return ANNOTATION_REGISTRY.get(annotation)

def get_registered_types() -> typing.Dict[str, typing.Type]:
    return TYPE_HINTS_REGISTRY


from django.urls import path
from .views import SuburbTweets,QueryAllTweets,HomePage,HashTag,UserMention,SuburbRank,Suburb,LoadSuburbHashTag,SuburbHashTag,SuburbUserMention

urlpatterns = [
    path('', HomePage.as_view(), name='HomePage'),
    path('hashTag', HashTag.as_view(), name='hashTag'),
    path('userMention',UserMention.as_view(),name='userMention'),
    path('suburbRank',SuburbRank.as_view(),name='suburbRank'),
    path('suburb',Suburb.as_view(),name='suburb'),
    path('uploadSuburbHashTag',LoadSuburbHashTag.as_view(),name='uploadSuburbHashTag'),
    path('suburbHashTag/',SuburbHashTag.as_view(),name='suburbHashTag'),
    path('suburbUserMention/',SuburbUserMention.as_view(),name='suburbUserMention'),
    path('queryAllTweets/',QueryAllTweets.as_view(),name='queryAllTweets'),
    path('suburbTweets/',SuburbTweets.as_view(),name='suburbTweets')
]


from collections import OrderedDict
from products import product_to_business, short_envs

class Pattern:

    # class level list of all supported patterns & fields
    pattern_fields = {
        'Solo' : OrderedDict([
                ('Queue Name', (suggest_queue_name, 'Q')),
                ('Queue Manager', '{qmgr}')
            ]),
        'Alias' : OrderedDict([
                ('Alias Name', (suggest_queue_name, 'A')),
                ('Queue Name', (suggest_queue_name, 'Q')),
                ('Queue Manager', '{qmgr}'),
                ('Cluster', '')
            ]),
        'Point to Point' : OrderedDict([
                ('Remote Queue Name', (suggest_queue_name, 'R')),
                ('Remote Queue Manager', '{qmgr}'),
                ('Local Queue Name', (suggest_queue_name, 'Q')),
                ('Local Queue Manager', '{qmgr}')
            ]),
        'Router' : OrderedDict([
                ('Source Remote Queue Name', (suggest_queue_name, 'R')),
                ('Source Queue Manager', '{qmgr}'),
                ('Destination Remote Queue Name', (suggest_queue_name, 'R')),
                ('Destination Queue Manager', '{qmgr}')
            ]),
        'Server to Client' : OrderedDict([
                ('Queue Manager', '{qmgr}'),
                ('Channel Name', (suggest_channel_name, 'TO')),
                ('SSL CipherSpec', ''),
                ('SSL Peername', '')
        ])
    }

    # eg P.FOO.BAR.{id}.Q
    def suggest_queue_name(product, env, suffix):
        return ('%s.%s.%s.{id}.%s' % (
                    env,
                    product_to_business[product],
                    product,
                    suffix
                ))

    # eg P.FOO.BAR.TO.{id}
    def suggest_channel_name(product, env, direction):
        return ('%s.%s.%s.%s.{id}' % (
                    env,
                    product_to_business[product],
                    product,
                    direction
                ))

    def __init__(self, name, product, env):
        self.name = name
        self.product = product
        self.env = env
        self.pattern_fields = Pattern.pattern_fields[name]

        # go through the fields and invoke suggestion functions
        for k,v in self.pattern_fields.items():
            try:
                f,s = v
                self.pattern_fields[k] = f(product, env, s);
            except ValueError: # not a tuple
                pass
            except TypeError: # f is not a function?
                pass

    # factory method in case we need to specialize things later
    def Get(name, product, env):
            return Pattern(name, product, env)


import argparse
import os
import random

def read_fasta(fasta_path):
    contignames = []
    contigs = {}
    with open(fasta_path, 'r') as rf:
        contigname = ''
        contigseq = ''
        for line in rf:
            if line.startswith('>'):
                if contigname != '' and contigseq != '':
                    contigs[contigname] = contigseq
                    contignames.append(contigname)
                contigname = line.strip()[1:].split(' ')[0]
                contigseq = ''
            else:
                contigseq += line.strip().upper()
        contigs[contigname] = contigseq
        contignames.append(contigname)
    rf.close()
    return contignames, contigs

def get_classifier_name(contigname):
    main_class = ''
    sub_class = ''
    parts = contigname.split("#")
    class_name = parts[1].split(" ")[0]
    c_parts = class_name.split("/")
    if len(c_parts) > 0:
        main_class = c_parts[0]
        if len(c_parts) > 1:
            sub_class = c_parts[1]
    return parts[0], main_class, sub_class

def getReverseSequence(sequence):
    base_map = {'A': 'T', 'T': 'A', 'C': 'G', 'G': 'C'}
    res = ''
    length = len(sequence)
    i = length - 1
    while i >= 0:
        base = sequence[i]
        if base not in base_map.keys():
            base = 'N'
        else:
            base = base_map[base]
        res += base
        i -= 1
    return res

if __name__ == '__main__':
    # w_fasta_path = '/public/home/hpc194701009/Liao_Results/classifier_multi/repeatscout_ant_r_3.fa'
    # fasta_path = '/public/home/hpc194701009/Liao_Results/classifier_multi/repeatscout_ant.fa'
    # contignames, contigs = read_fasta(fasta_path)

    # w_contignames, w_contigs = read_fasta(w_fasta_path)
    #
    # if len(w_contignames) != len(contignames):
    #     print("error")
    #
    # for name in contignames:
    #     if contigs[name] != w_contigs[name]:
    #         print("error")

    # count = 0
    # added_contignames = []
    # total_len = len(contignames)
    # with open(w_fasta_path, 'w') as f_save:
    #     while count < total_len:
    #         remain_names = list(set(contignames).difference(set(added_contignames)))
    #         remain_len = len(remain_names)
    #         index = random.randint(0, remain_len-1)
    #         #print(remain_len, index)
    #         header = '>' + remain_names[index]
    #         f_save.write(header + '\n')
    #         f_save.write(contigs[remain_names[index]] + '\n')
    #
    #         count += 1
    #         added_contignames.append(remain_names[index])

    # 1.parse args
    parser = argparse.ArgumentParser(description='run reference coverage calculation...')
    parser.add_argument('-o', metavar='origin repeat classified path',
                        help='input origin repeat classified path')
    parser.add_argument('-s', metavar='spark repeat classified path',
                        help='input spark repeat classified path')
    args = parser.parse_args()

    origin_classified_path = args.o
    spark_classified_path = args.s

    o_map = {}
    o_contignames, o_contigs = read_fasta(origin_classified_path)
    s_map = {}
    s_contignames, s_contigs = read_fasta(spark_classified_path)
    # r_o_map = {}
    # r_s_map = {}
    # r_o_contigs = {}
    # r_s_contigs = {}
    # for name in o_contigs.keys():
    #     contig = o_contigs[name]
    #     r_contig = getReverseSequence(contig)
    #     contig = contig if contig < r_contig else r_contig
    #     r_o_contigs[contig] = name
    #
    # for name in s_contigs.keys():
    #     contig = s_contigs[name]
    #     r_contig = getReverseSequence(contig)
    #     contig = contig if contig < r_contig else r_contig
    #     r_s_contigs[contig] = name
    #
    # for contig in r_o_contigs.keys():
    #     contig_name = r_o_contigs[contig]
    #     repeat_id, main_class, sub_class = get_classifier_name(contig_name)
    #     r_o_map[contig] = (main_class, sub_class)
    #
    # for contig in r_s_contigs.keys():
    #     contig_name = r_s_contigs[contig]
    #     repeat_id, main_class, sub_class = get_classifier_name(contig_name)
    #     r_s_map[contig] = (main_class, sub_class)
    #
    #
    # total_same_num = 0
    # total_diff_num = 0
    # sub_class_diff_num = 0
    #
    # if len(o_map.keys()) != len(s_map.keys()):
    #     print("origin file length is not equal to spark file")
    # else:
    #     total_diff_id = []
    #     for contig in r_o_map.keys():
    #         (main_class1, sub_class1) = r_o_map[contig]
    #         (main_class2, sub_class2) = r_s_map[contig]
    #         if main_class1 != main_class2:
    #             total_diff_num += 1
    #             total_diff_id.append(contig)
    #         elif sub_class1 != sub_class2:
    #             sub_class_diff_num += 1
    #         else:
    #             total_same_num += 1
    #
    #     print("total_same_num=%d, total_diff_num=%d, sub_class_diff_num=%d, sum=%d" %(total_same_num, total_diff_num, sub_class_diff_num, total_same_num + total_diff_num + sub_class_diff_num))
    #     print("all sequence number=%d" % len(r_o_map.keys()))
    #     print(total_diff_id[0:10])

    for name in o_contignames:
        repeat_id, main_class, sub_class = get_classifier_name(name)
        o_map[repeat_id] = (main_class, sub_class)

    for name in s_contignames:
        repeat_id, main_class, sub_class = get_classifier_name(name)
        s_map[repeat_id] = (main_class, sub_class)

    total_same_num = 0
    total_diff_num = 0
    sub_class_diff_num = 0

    if len(o_map.keys()) != len(s_map.keys()):
        print("origin file length is not equal to spark file")
    else:
        total_diff_id = []
        for repeat_id in o_map.keys():
            (main_class1, sub_class1) = o_map[repeat_id]
            (main_class2, sub_class2) = s_map[repeat_id]
            if main_class1 != main_class2:
                total_diff_num += 1
                total_diff_id.append(repeat_id)
            elif sub_class1 != sub_class2:
                sub_class_diff_num += 1
            else:
                total_same_num += 1

        print("total_same_num=%d, total_diff_num=%d, sub_class_diff_num=%d, sum=%d" %(total_same_num, total_diff_num, sub_class_diff_num, total_same_num + total_diff_num + sub_class_diff_num))
        print("all sequence number=%d" %len(o_map.keys()))
        #print(total_diff_id[0:10])

        # last_repeat_ids = []
        # spark_sub_dir = '/public/home/hpc194701009/Liao_Results/classifier/'
        # sub_ids = []
        # for i in range(0, 345):
        #     p = spark_sub_dir + str(i) + '/RepeatLib_dmel.fa.tmp.classified'
        #     tmp_contignames, tmp_contigs = read_fasta(p)
        #     last_contig_name = tmp_contignames[len(tmp_contignames)-1]
        #     last_repeat_id, main_class, sub_class = get_classifier_name(last_contig_name)
        #     last_repeat_ids.append(last_repeat_id)
        #
        # # total_diff_id = sorted(total_diff_id, key=lambda x: x.split('_')[1], reverse=False)
        # # last_repeat_ids = sorted(last_repeat_ids, key=lambda x: x.split('_')[1], reverse=False)
        # diff_id_num = 0
        # for diff_id in total_diff_id:
        #     if not diff_id in last_repeat_ids:
        #         print("diff id=%s not in last_repeat_ids" %diff_id)
        #         diff_id_num += 1
        # print("number of diff id not in last_repeat_ids = %d" %diff_id_num)

        # print(total_diff_id)
        # print(last_repeat_ids)

        # last_sequence_unknow_num = 0
        # for item in last_repeat_ids:
        #     last_repeat_id = item[0]
        #     main_class = item[1]
        #     sub_class = item[0]
        #     if main_class == 'Unknown':
        #         last_sequence_unknow_num += 1
        #
        #     if not last_repeat_id in total_diff_id:
        #         print("last_repeat_id=%s not in total_diff_ids" %last_repeat_id)
        #
        # print("last_repeat_ids_num=%d, last_sequence_unknow_num=%d" %(len(last_repeat_ids), last_sequence_unknow_num))


def sendRainState():
        #Get Data
        url = "https://api.forecast.io/forecast/" + key + "/" + locLat + "," + locLong
        response = urllib.urlopen(url).read()

        #Interpret Data
        posRain = response.find('"precipType":"rain"')
        if(posRain != -1):
                posComma2 = response.rfind(",",0,posRain)
                posComma1 = response.rfind(",",0,posComma2)
                rainChance = float(response[posComma1+21:posComma2])
                print(rainChance)

                if (rainChance>=0.5):
                        ser.write('2&') #it will rain
                else :
                        ser.write('3&') #it will not rain

def getValuesAndPublsih():
        ser.write('1&')

        #Arduino sends messages

        myTrashString = ser.readline()
        #recTemp = int(ser.readline())
        #print(recTemp)
        #recHumAir = int(ser.readline())
        #print(recHumAir)
        recHumEarth = int(ser.readline())
        print(recHumEarth)
        recLight = int(ser.readline())
        print(recLight)

        #publish

        sendString =  " ; Humidity Earth: " + str(recHumEarth) + " ; Light: " + str(recLight)
        #""" "Temperature: " + str(recTemp) + " ; Humidity Air: " + str(recHumAir) +"""
        publish.single("/SmartGarden/Station1", sendString, hostname=publisherAdress)

	#Console Output

import serial,time,urllib
from datetime import datetime
import paho.mqtt.publish as publish

ser = serial.Serial('/dev/ttyACM0',9600)

#User defined Data:
getDataIntervall = 19 #in minutes, minimum 2
sendWeatherIntervall = 19 #in hours, minimum 2
key = "" #use your own key from https://developer.forecast.io/
locLat = "47.2267" #position rapperswil -> gets zurich
locLong = "08.8167"
publisherAdress = "localhost"

#System Variables, don't change these:
gotValues = False
sentWeather = False

print("setup complete")

while(1):
        #Get Data from Arduino
        if(datetime.now().second%getDataIntervall==0):
                if(gotValues == False):
                        print("getValues")
                        getValuesAndPublsih()
                        gotValues = True
        else:
                gotValues = False

        #Get and process weather data
        if(datetime.now().second%sendWeatherIntervall==10):
                if(sentWeather == False):
                        print("getRainData")
                        sendRainState()
                        sentWeather = True
        else:
                sentWeather = False


# -*- coding: utf-8 -*-

"""Project Stubs Module."""

import sys
from pathlib import Path
from typing import List, Sequence, Union

from boltons import setutils

from micropy.project.modules import ProjectModule
from micropy.stubs import StubManager
from micropy.stubs.stubs import DeviceStub

class StubsModule(ProjectModule):
    """Project module for handling Stubs.

    Args:
        stub_manager (StubManager): StubManager instance.
        stubs (List[Type[Stub]], optional): Initial Stubs to use.

    """
    PRIORITY: int = 9

    def __init__(self, stub_manager: StubManager,
                 stubs: Sequence[DeviceStub] = None, **kwargs):
        super().__init__(**kwargs)
        self.stub_manager: StubManager = stub_manager
        self._stubs: Sequence[DeviceStub] = stubs or []

    @property
    def context(self):
        """Component stub context."""
        paths = setutils.IndexedSet()
        if self.stubs:
            frozen = [s.frozen for s in self.stubs]
            fware_mods = [s.firmware.frozen
                          for s in self.stubs if s.firmware is not None]
            stub_paths = [s.stubs for s in self.stubs]
            paths.update(*frozen, *fware_mods, *stub_paths)

        return {
            "stubs": set(self.stubs),
            "paths": paths,
            "datadir": self.parent.data_path,
        }

    @property
    def config(self) -> dict:
        """Component specific config values.

        Returns:
            dict: Current config.

        """
        stubs = {s.name: s.stub_version for s in self._stubs}
        return {
            'stubs': stubs
        }

    @property
    @ProjectModule.hook()
    def stubs(self) -> Union[StubManager, Sequence[DeviceStub]]:
        """Component stubs.

        Returns:
            List[micropy.stubs.Stub]: List of stubs used in project.

        """
        return self._resolve_subresource(self._stubs)

    @stubs.setter
    def stubs(self, val):
        """Sets project stubs.

        Args:
            val (List[micropy.stubs.Stub]): List of stubs to set.

        """
        self._stubs = val

    def _resolve_subresource(self,
                             stubs: List[DeviceStub]) -> Union[StubManager, Sequence[DeviceStub]]:
        """Resolves stub resource.

        Args:
            stubs (stubs): Stubs Passed to Manager

        """
        if not hasattr(self, "_parent"):
            return self._stubs
        if not self.parent.exists:
            return self._stubs
        try:
            resource = set(
                self.stub_manager.resolve_subresource(stubs,
                                                      self.parent.data_path))
        except OSError as e:
            self.log.error("Failed to Create Stub Links!", exception=e)
            sys.exit(1)
        else:
            return resource

    def _load_stub_data(self, stub_data=None, **kwargs):
        """Loads Serialized Stub Data.

        Args:
            stub_data (dict): Dict of Stubs

        """
        _data = self.config.get('stubs')
        data = {**stub_data, **_data}
        for name, location in data.items():
            _path = Path(location).absolute()
            if Path(_path).exists():
                yield self.stub_manager.add(_path)
            yield self.stub_manager.add(name)

    def load(self, **kwargs):
        """Loads stubs from info file.

        Args:
            stub_list (dict): Dict of Stubs

        """
        stub_data = self.parent.config.get('stubs', default={})
        stubs = list(self._load_stub_data(stub_data=stub_data))
        stubs.extend(self.stubs)
        self.stubs = self._resolve_subresource(stubs)
        return self.update()

    def create(self):
        """Create stub project files."""
        self.parent.context.merge(self.context)
        self.log.info(
            f"Stubs: $[{' '.join(str(s) for s in self.stubs)}]")
        return self.load()

    def update(self):
        """Update current project stubs."""
        self.parent.context.set('stubs', self.stubs)
        self.parent.context.set('paths', self.context.get('paths'))
        self.parent.config.set('stubs', {s.name: s.stub_version for s in self.stubs})
        return self.stubs

    @ProjectModule.hook()
    def add_stub(self, stub, **kwargs):
        """Add stub to project.

        Args:
            stub (Stub): Stub object to add

        Returns:
            [Stubs]: Project Stubs

        """
        loaded = self.stubs or []
        stubs = [*loaded, stub]
        self.log.info("Loading project...")
        self.stubs = self._resolve_subresource(stubs)
        self.log.info("Updating Project Info...")
        self.parent.update()
        self.log.info(
            f"Project Stubs: $[{' '.join(str(s) for s in self.stubs)}]")
        self.log.success("\nProject Updated!")
        return self.stubs


import webapp2
import jinja2
import os

jinja_environment = jinja2.Environment(
    extensions=['jinja2.ext.autoescape'],
    loader=jinja2.FileSystemLoader(
        os.path.dirname(__file__) + "/../templates"),
    autoescape=True)

class BaseHandler(webapp2.RequestHandler):       #Handlers are roughly equivalent to controllers in MVC

    def __init__(self, request=None, response=None):
        super(BaseHandler, self).__init__(request, response)
        # your own code goes here...

    def render(self, template_name, template_values):
        template = jinja_environment.get_template(template_name)
        self.response.out.write(template.render(template_values))


from django.shortcuts import redirect, render
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.forms import modelformset_factory
from django.core import serializers

from apps.subjects.models import Subject

from apps.users.decorators import city_required

@city_required
@login_required
def objects_form(request, Model, template='abs/objects_form.html'):
    """Универсальная вьюха для работы с тарифами, адресами, образованием, документами и прочими связанными сущностями"""

    extra = 0 if Model.objects.filter(user=request.user).exists() else 1

    formset = modelformset_factory(Model, can_delete=True, extra=extra, **Model.form)(data=request.POST or None, files=request.FILES or None, queryset=Model.objects.filter(user=request.user))

    formset = Model.modify_formset(request, formset)

    if request.POST and formset.is_valid():

        for form in formset.save(commit=False):
            form.user = request.user
            form.save()

        for obj in formset.deleted_objects:
            obj.delete()

        formset.save_m2m()

        messages.add_message(request, messages.SUCCESS, 'Сохранено')
        return redirect(request.META.get('HTTP_REFERER'))

    subjects_and_additions = serializers.serialize('json', Subject.objects.all())

    return render(request, template, {
        'formset': formset,
        'Model': Model,
        'subjects_and_additions' : subjects_and_additions,
    })


import torch
import torch.optim as optim
import torch.utils.data
import torchvision
import torchvision.transforms as transforms
import torchvision.utils
import torchvision.datasets
import torch.backends.cudnn as cudnn

import numpy as np
import os
import argparse

from datasets import *
from log import *
from models import *
from utils import *

import warnings
warnings.filterwarnings("ignore")

# python main.py --result results
parser = argparse.ArgumentParser(description='PyTorch CIFAR10 Training')
parser.add_argument('--dataset', required=True, help='cifar10 | cifar100 | tinyimagenet | imagenet')
parser.add_argument('--root', required=False, help='path to dataset')
parser.add_argument('--result', default='results', help='output folder')

parser.add_argument('--epoch', type=int, default=90, help='epoch')
parser.add_argument('--batch-size', type=int, default=256, help='batch size')
parser.add_argument('--lr', default=0.1, type=float, help='learning rate')
parser.add_argument('--image-size', type=int, default=64, help='the size of the input image to network')
parser.add_argument('--padding', type=int, default=8, help='padding')
parser.add_argument('--num-classes', type=int, default=10, help='number of classes')

parser.add_argument('--ratio', type=float, default=0, help='epoch')
parser.add_argument('--key', default=None, help='key')
parser.add_argument('--random-key', action='store_true', default=False, help='random keys and random transformation matrices')
parser.add_argument('--user-num', default=1, type=int, help='total number of user')
parser.add_argument('--var', default=0.0, type=float, help='random noise var')
parser.add_argument('--cut-size', default=0.0, type=float, help='cut out size')
parser.add_argument('--adv', action='store_true', default=False, help='enable adversarial training')
parser.add_argument('--epsilon', type=float, default=0.1, help='adversarial training')
parser.add_argument('--smooth', action='store_true', default=False, help='smooth label')
parser.add_argument('--smoothing-coef', type=float, default=0.1, help='smoothing coefficient')
parser.add_argument('--tau', type=int, default=15, help='tau')
# parser.add_argument('--indices', default=None, help='indices path')
args = parser.parse_args()

presult = os.getcwd() + "/" + args.result
if not os.path.exists(presult):
    os.mkdir(presult)

log = Log(presult+"/results.txt")
log.info(str(args))

device = 'cuda' if torch.cuda.is_available() else 'cpu'
best_acc = 0  # best test accuracy
start_epoch = 0  # start from epoch 0 or last checkpoint epoch

normalize = transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
# normalize = transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))

transform_train = transforms.Compose([
    transforms.RandomCrop(args.image_size, padding=args.padding),
    #transforms.RandomResizedCrop(args.image_size),
    #transforms.RandomAffine(20, shear=(-15,15)),
    #transforms.ColorJitter(brightness=0.1, contrast=0.2, saturation=0.2, hue=0.2),
    transforms.RandomHorizontalFlip(),
    # AddGaussianNoise(0, args.var),
    # Cutout(length=args.cut_size, val=0.5),
    #transforms.RandomVerticalFlip(),
    normalize
])

transform_test = transforms.Compose([
    normalize
])

key = args.key
M = None

if args.random_key:
    keys = random.choices(np.arange(args.tau*2,360,args.tau*2), k=args.user_num)
    matrices = []
    for i in range(args.user_num):
        m = torch.randn(3,3).uniform_(-1, 1)
        m = m/m.sum()
        matrices.append(m)

    key = keys
    M = matrices
    print("random key")
    torch.save(torch.Tensor(keys), presult + "/keys.t7")
    torch.save(matrices, presult + "/matrices.t7")

if args.dataset == 'cifar10':
    print("cifar10")
    trainset = hue_Cifar10(train=True, transform=transform_train, key=key, ratio=args.ratio, M=M)
    testset = hue_Cifar10(train=False, transform=transform_test)
elif args.dataset == 'cifar100':
    print("cifar100")
    trainset = hue_Cifar100(train=True, transform=transform_train, key=key, ratio=args.ratio, M=M)
    testset = hue_Cifar100(train=False, transform=transform_test)
elif args.dataset == 'tinyimagenet':
    print("tinyimagenet")
    trainset = hue_TinyImageNet(transform=transform_train, key=key, ratio=args.ratio, train=True, M=M)
    testset = hue_TinyImageNet(transform=transform_test, train=False)
elif args.dataset == 'imagenet':
    trainset = hue_ImageNet(transform=transform_train, key=key, ratio=args.ratio, train=True)
    testset = hue_ImageNet(transform=transform_test, train=False)

torch.save(trainset.getIndices(), presult + "/indices.t7")
trainloader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size, shuffle=True, num_workers=8)
testloader = torch.utils.data.DataLoader(testset, batch_size=args.batch_size, shuffle=False, num_workers=8)

# Model
print('==> Building model..')
print(args.num_classes)
net = torchvision.models.resnet50(num_classes=args.num_classes) if args.dataset == 'tinyimagenet'  else ResNet50(num_classes=args.num_classes)
#net = torchvision.models.vgg19_bn(num_classes=args.num_classes)
#net = torchvision.models.mobilenet_v2(num_classes=args.num_classes)
#net = torchvision.models.shufflenet_v2_x1_0(num_classes=args.num_classes)
#net = torchvision.models.inception_v3(num_classes=args.num_classes)
#net = torchvision.models.densenet161(num_classes=args.num_classes)
#net = torchvision.models.googlenet(num_classes=args.num_classes)
#net = VGG('VGG16')
#net = LeNet()
#net = resnet18()
# net = wide_resnet50_2()
# net = PreActResNet18()
# net = GoogLeNet()
# net = DenseNet121()
# net = ResNeXt29_2x64d()
# net = MobileNet()
# net = MobileNetV2()
# net = DPN92()
# net = ShuffleNetG2()
# net = SENet18()
# net = ShuffleNetV2(1)
# net = EfficientNetB0()
net = net.to(device)

log.info(net)

if device == 'cuda':
    net = torch.nn.DataParallel(net)
    cudnn.benchmark = True

criterion = nn.CrossEntropyLoss()
if args.smooth:
    criterion = LabelSmoothingLoss(args.num_classes, args.smoothing_coef)
optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=5e-4)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 30, gamma=0.1)

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def save_model(net, d, filename, acc=0):
    if not os.path.isdir(d):
        os.mkdir(d)

    state = {
        'net': net.module.state_dict(),
        'acc': acc
    }
    extension = '.t7'
    torch.save(state, d + '/' + filename + extension )

def train(epoch):
    net.train()
    train_loss = 0
    correct = 0
    total = 0
    for batch_idx, (inputs, targets) in enumerate(trainloader):
        inputs, targets = inputs.to(device), targets.to(device)

        if args.adv:
            optimizer.zero_grad()

            x_adv = fgsm_attack(net, inputs, targets, args.epsilon, -1, 1)
            outputs = net(x_adv)
            loss = criterion(outputs, targets)
            loss.backward()

            optimizer.step()

        optimizer.zero_grad()
        outputs = net(inputs)
        loss = criterion(outputs, targets)
        loss.backward()

        optimizer.step()

        train_loss += loss.item()
        _, predicted = outputs.max(1)
        total += targets.size(0)
        correct += predicted.eq(targets).sum().item()

    return train_loss/len(trainloader), 100.*correct/total

def test(epoch):
    global best_acc
    net.eval()
    test_loss = 0
    correct = 0
    total = 0
    with torch.no_grad():
        for batch_idx, (inputs, targets) in enumerate(testloader):
            inputs, targets = inputs.to(device), targets.to(device)
            outputs = net(inputs)
            loss = criterion(outputs, targets)

            test_loss += loss.item()
            _, predicted = outputs.max(1)
            total += targets.size(0)
            correct += predicted.eq(targets).sum().item()

    # Save checkpoint.
    acc = 100.*correct/total
    if epoch %10==0:
        save_model(net, presult + '/checkpoint', 'ep_{:02d}_model'.format(epoch), acc)

    if acc > best_acc:
        save_model(net, presult + '/checkpoint', 'best_model', acc)
        best_acc = acc

    return acc

for epoch in range(start_epoch, start_epoch+args.epoch):
    train_loss, train_acc = train(epoch)
    test_acc = test(epoch)

    scheduler.step()
    log.info("Epoch {:4d}, Err {:.4f}, Train Acc {:.4f}, Test Acc {:.4f}, lr {:.4f}, best {:.4f}".format(
        epoch, train_loss, train_acc, test_acc, optimizer.param_groups[0]['lr'], best_acc))


# -*- coding: utf-8 -*-
"""
"""
from __future__ import unicode_literals
from __future__ import print_function

import logging

import zmq

from p2p0mq.constants import (
    SPEED_MEDIUM, SPEED_FAST, SPEED_SLOW,
    LOOP_CONTINUE, TRACE, ISOLATE, HEART_BEAT_INTERVAL, TRACE_PACKETS, TRACE_NET)
from p2p0mq.errors import ValidationError, MissingCertificateError
from p2p0mq.message import Message
from p2p0mq.message_queue.fast import FastMessageQueue
from p2p0mq.message_queue.medium import RegularMessageQueue
from p2p0mq.message_queue.slow import SlowMessageQueue
from p2p0mq.peer import Peer
from p2p0mq.utils.thread.netthread import KoNetThread

logger = logging.getLogger('p2p0mq.app.c')

class Sender(KoNetThread):
    """
    A thread that sends requests and information
    to peers.

    These are "clients" according to the
    [zmq_curve](http://api.zeromq.org/4-1:zmq-curve) which states that:
    > A socket using CURVE can be either client or server,
    > at any moment, but not both.
    > The role is independent of bind/connect direction.

    To become a CURVE client, the local peer sets the ZMQ_CURVE_SERVERKEY
    option with the long-term public key of the server it intends to
    connect to, or accept connections from, next. The local peer then
    sets the ZMQ_CURVE_PUBLICKEY and ZMQ_CURVE_SECRETKEY
    options with its client long-term key pair.

    The sender provides three queues:
    - fast (sends all available messages every loop);
    - medium (sends oldest message of each peer every loop);
    - slow (sends oldest overall message;one message per loop).
    """
    def __init__(self, *args, **kwargs):
        """ Constructor. """
        super(Sender, self).__init__(*args, **kwargs)
        self.name = 'p2p0mq.C.th' if self.app is None or self.app.uuid is None \
            else ('%s-p2p0mq.C.th' % self.app.uuid[-4:-1].decode("utf-8"))
        self.slow_queue = SlowMessageQueue()
        self.medium_queue = RegularMessageQueue()
        self.fast_queue = FastMessageQueue()
        self.connection_queue = SlowMessageQueue()

    def create(self):
        """ Called at thread start to initialize the state. """
        logger.debug("Sender is being created...")
        super().create()

        # The ROUTER socket type talks to a set of peers, using explicit
        # addressing so that each outgoing message is sent to a specific
        # peer connection. ROUTER works as an asynchronous replacement
        # for REP, and is often used as the basis for servers that talk
        # to DEALER clients.
        # https://rfc.zeromq.org/spec:28/REQREP/#the-router-socket-type
        new_socket = self.context.socket(zmq.ROUTER)

        # Do not keep messages in memory that were not send yet when
        # we attempt to close the socket.
        # http://api.zeromq.org/2-1:zmq-setsockopt#toc15
        new_socket.setsockopt(zmq.LINGER, 0)

        # The ZMQ_IDENTITY option shall set the identity of
        # the specified socket. Socket identity determines if
        # existing ØMQ infrastructure (message queues,
        # forwarding devices) shall be identified with a
        # specific application and persist across multiple
        # runs of the local peer.
        #
        # If the socket has no identity, each run of an
        # application is completely separate from other runs.
        # However, with identity set the socket shall re-use
        # any existing ØMQ infrastructure configured by the
        # previous run(s). Thus the local peer may receive
        # messages that were sent in the meantime, message
        # queue limits shall be shared with previous run(s)
        # and so on.

        # http://api.zeromq.org/2-1:zmq-setsockopt#toc6
        new_socket.setsockopt(zmq.IDENTITY, self.app.uuid)
        new_socket.setsockopt(zmq.ROUTER_MANDATORY, 1)

        if not self.app.no_encryption:
            logger.debug(
                "Application %r loads certificates for its Sender from %s",
                self.app.uuid, self.app.private_file)
            client_public, client_secret = zmq.auth.load_certificate(
                self.app.private_file)

            # http://api.zeromq.org/4-1:zmq-curve
            # To become a CURVE client, the local peer sets the
            # ZMQ_CURVE_SERVERKEY option with the long-term public key
            # of the server it intends to connect to, or accept
            # connections from, next.
            # new_socket.curve_serverkey = \
            #     str(peer_data["serverkey"]).encode('ascii')

            # The local peer then sets the ZMQ_CURVE_PUBLICKEY and
            # ZMQ_CURVE_SECRETKEY options with its client long-term key pair.
            # Set long term secret key (for server)
            new_socket.curve_publickey = client_public
            new_socket.curve_secretkey = client_secret

        # Connect to the address where we listen for incoming data
        # and send our requests.
        # address = self.address
        # new_socket.bind(address)
        # logger.debug("Sender bound to %s", address)

        self.socket = new_socket
        logger.debug("Sender has been created")
        return new_socket

    def terminate(self):
        """ Called at thread end to free resources. """
        super().terminate()
        assert self.socket is None
        logger.debug("sender (client) thread was terminated on app %r",
                     self.app.uuid)

    def send_message(self, message):
        """
        Encode and send a single message.

        If the socket throws an error:
        - we check if the time to live has expired, in which case we drop
        the message and inform the handler.
        - we tell the handler (Concern class) that we could not send it (
        by default the handler will give us back the same message to
        put it back in the queue until the message expires).

        Returns:
            The message to be enqueued or None. This is not a
            `(PRIORITY, message)` format.
        """
        try:
            logger.log(TRACE_NET, "encoding message for wire: %r", message)
            assert message.valid_for_send(self.app), "%r is invalid" % message
            encoded = message.encode(self.app.uuid)
            logger.log(TRACE_PACKETS, ">>>>>>>>>>>>>>>  %r", encoded)
            self.socket.send_multipart(encoded)
            message.handler.message_sent(message)
            return None

        except (zmq.error.ZMQError, KeyError, TypeError) as exc:

            logger.log(TRACE, "Sender failed to send message",
                       exc_info=True)

            if message.time_to_live < self.app.tick:
                # Inform the handler that the message is being dropped.
                message.handler.message_dropped(message)
                return None
            else:
                # Inform the handler that the message could not be send.
                return message.handler.send_failed(message, exc)

    def execute_queue(self, queue):
        """
        Called during the execution step to send messages from a queue.

        The method takes one or more messages from a queue, encodes
        the message and sends it to the socket.
        """
        logger.log(TRACE, "sending queue %s", queue)

        to_send = queue.dequeue()
        for message in to_send:
            re_message = self.send_message(message)
            if re_message:
                logger.log(TRACE, "Failed message is being re-queued")
                queue.enqueue(re_message)
            else:
                logger.log(TRACE, "Message is being dropped")

    def connect_peers(self):
        """
        Called during the execution step to connect to peers.
        """
        while not self.connection_queue.empty():
            result = self.connection_queue.dequeue()
            assert len(result) == 1
            result = result[0]
            assert len(result) == 1
            peer = list(result)[0]
            message = result[peer]
            logger.debug("Connecting peer %r...", peer.uuid)
            try:
                if not hasattr(peer, '_first_connect'):
                    if not self.app.no_encryption:
                        # https://grokbase.com/t/zeromq/zeromq-dev/151j3cp0x0/about-curve-and-router-sockets
                        server_key = self.app.cert_key_by_uuid(
                            uuid=peer.uuid, public=True)
                        logger.debug("First connect to peer %r with key %r",
                                     peer.uuid, server_key)
                        if server_key is None:
                            message.handler.send_failed(
                                message, MissingCertificateError())
                            return
                        self.socket.curve_serverkey = server_key

                    setattr(peer, '_first_connect', peer.uuid)
                    self.socket.setsockopt(
                        zmq.CONNECT_RID, peer.uuid)

                self.socket.connect(peer.address)
                self.fast_queue.enqueue(message)
            except zmq.error.ZMQError as exc:
                message.handler.send_failed(message, exc)

    def execute(self):
        """
        Called to execute the main part of the thread.

        We're simply looping through our queues and ask each one
        to give us some messages that we then try to send.
        """
        self.connect_peers()
        for queue in (self.fast_queue, self.medium_queue, self.slow_queue):
            self.execute_queue(queue)
        return LOOP_CONTINUE

    def enqueue(self, message, priority=SPEED_MEDIUM):
        """ Adds one or more messages to internal queue
        to be send later. """
        assert Message.validate_messages_for_send(message, self.app)
        if priority == SPEED_MEDIUM:
            self.medium_queue.enqueue(message)
        elif priority == SPEED_FAST:
            self.fast_queue.enqueue(message)
        elif priority == SPEED_SLOW:
            self.slow_queue.enqueue(message)
        else:
            raise ValidationError("Unknown priority: %r", priority)
        self.sleep.set()

    def enqueue_all(self, requests=None, replies=None, routed=None):
        """ Enqueues all kinds of messages. """
        fast = []
        medium = []
        slow = []

        if requests is not None:
            if SPEED_FAST in requests:
                fast = requests[SPEED_FAST]
            if SPEED_MEDIUM in requests:
                medium = requests[SPEED_MEDIUM]
            if SPEED_SLOW in requests:
                slow = requests[SPEED_SLOW]

        if replies is not None:
            if SPEED_FAST in replies:
                fast = fast + replies[SPEED_FAST]
            if SPEED_MEDIUM in replies:
                medium = medium + replies[SPEED_MEDIUM]
            if SPEED_SLOW in replies:
                slow = slow + replies[SPEED_SLOW]

        if routed is not None:
            fast = fast + routed

        total = len(fast) + len(medium) + len(slow)

        if len(fast) > 0:
            assert Message.validate_messages_for_send(fast, self.app)
            self.fast_queue.enqueue(fast)
        if len(medium) > 0:
            assert Message.validate_messages_for_send(medium, self.app)
            self.medium_queue.enqueue(medium)
        if len(slow) > 0:
            assert Message.validate_messages_for_send(slow, self.app)
            self.slow_queue.enqueue(slow)

        if total > 0:
            self.sleep.set()

    def enqueue_fast(self, message):
        """ Adds one or more messages to internal fast queue
        to be send later. """
        assert Message.validate_messages_for_send(message, self.app)
        self.fast_queue.enqueue(message)
        self.sleep.set()

    def enqueue_slow(self, message):
        """ Adds one or more messages to internal slow queue
        to be send later. """
        assert Message.validate_messages_for_send(message, self.app)
        self.slow_queue.enqueue(message)
        self.sleep.set()

    def enqueue_medium(self, message):
        """ Adds one or more messages to internal medium queue
        to be send later. """
        assert Message.validate_messages_for_send(message, self.app)
        self.medium_queue.enqueue(message)
        self.sleep.set()


from conans import ConanFile, AutoToolsBuildEnvironment, MSBuild, tools
import os
import shutil

class IrrlichtConan(ConanFile):
    name = "irrlicht"
    version = "1.8.4"
    license = "http://irrlicht.sourceforge.net/?page_id=294"
    url = "https://github.com/mpusz/conan-irrlicht"
    description = "An open source high performance realtime 3D engine written in C++"
    exports = "LICENSE.md"
    exports_sources = ["*.patch"]
    settings = "os", "compiler", "build_type", "arch"
    options = {"shared": [True, False], "fPIC": [True, False]}
    default_options = {"shared": False, "fPIC": True}

    @property
    def _subfolder(self):
        return "irrlicht-%s" % self.version

    def config_options(self):
        if self.settings.os == 'Windows':
            del self.options.fPIC

    def source(self):
        zip_name = "irrlicht-%s.zip" % self.version
        tools.get("http://downloads.sourceforge.net/irrlicht/%s" % zip_name, sha1="38bf0223fe868d243d6a39d0dc191c8df6e03b3b")

    def system_requirements(self):
#        if self.settings.os == "Macos":
#            self.run("brew cask install xquartz")

        if self.settings.os == "Linux" and tools.os_info.is_linux:
            installer = tools.SystemPackageTool()
            if tools.os_info.with_apt:
                if self.settings.arch == "x86":
                    arch_suffix = ':i386'
                elif self.settings.arch == "x86_64":
                    arch_suffix = ':amd64'
                packages = ['libgl1-mesa-dev%s' % arch_suffix]
                packages.append('libxcursor-dev%s' % arch_suffix)

            if tools.os_info.with_yum:
                if self.settings.arch == "x86":
                    arch_suffix = '.i686'
                elif self.settings.arch == 'x86_64':
                    arch_suffix = '.x86_64'
                packages = ['mesa-libGL-devel%s' % arch_suffix]
                packages.append('libXcursor-devel%s' % arch_suffix)

            for package in packages:
                installer.install(package)

    def _patch_add_shared_lib_links(self):
        # Irrlicht does that only for install step and without the links create Conan does not link correctly
        tools.replace_in_file("Makefile", "cp $(SHARED_FULLNAME) $(LIB_PATH)", """cp $(SHARED_FULLNAME) $(LIB_PATH)
	cd $(LIB_PATH) && ln -s -f $(SHARED_FULLNAME) $(SONAME)
	cd $(LIB_PATH) && ln -s -f $(SONAME) $(SHARED_LIB)""")

    def _patch_mingw(self):
        # patch library name
        tools.replace_in_file("Makefile", "-ld3dx9d", "-ld3dx9")

    def _patch_macos(self):
        # fix compilation
        shutil.move("Irrlicht.cpp", "Irrlicht.mm")
        shutil.move("COpenGLDriver.cpp", "COpenGLDriver.mm")
        # comment unsupported options
        tools.replace_in_file("Makefile", "staticlib_osx sharedlib_osx: LDFLAGS += --no-export-all-symbols --add-stdcall-alias", "#staticlib_osx sharedlib_osx: LDFLAGS += --no-export-all-symbols --add-stdcall-alias")
        # uncomment Macosx linker flags
        tools.replace_in_file("Makefile", "#sharedlib_osx: LDFLAGS", "sharedlib_osx: LDFLAGS")
        # add Macosx specific sources
        tools.replace_in_file("Makefile", "Irrlicht.o os.o", "Irrlicht.o os.o MacOSX/CIrrDeviceMacOSX.o MacOSX/OSXClipboard.o MacOSX/AppDelegate.o")
        # fix window creation
        tools.patch(patch_file=os.path.join(self.source_folder, "osx-window-creation.patch"), strip=2)
        # fix shared libraries
        self._patch_add_shared_lib_links()

    def _patch_linux(self):
        # fix shared libraries
        self._patch_add_shared_lib_links()

    def build(self):
        if self.settings.compiler == "Visual Studio":
            msbuild = MSBuild(self)
            msbuild.build("../source/%s/source/Irrlicht/Irrlicht11.0.sln" % self._subfolder)
            # TBD....
        else:
            with tools.chdir(os.path.join(self._subfolder, "source", "Irrlicht")):
                autotools = AutoToolsBuildEnvironment(self, win_bash=tools.os_info.is_windows)
                if self.settings.os != 'Windows':
                    autotools.fpic = self.options.fPIC

                if tools.os_info.is_windows:
                    self._patch_mingw()
                    make_target = "sharedlib_win32" if self.options.shared else "staticlib_win32"
                elif tools.os_info.is_macos:
                    self._patch_macos()
                    autotools.include_paths.append(os.getcwd())
                    make_target = "sharedlib_osx" if self.options.shared else "staticlib_osx"
                else:
                    self._patch_linux()
                    make_target = "sharedlib" if self.options.shared else "staticlib"

                autotools.make(target=make_target)

    def package(self):
        self.copy(pattern="*license*", dst="licenses", ignore_case=True, keep_path=False)

        include_folder = os.path.join(self._subfolder, "include")
        self.copy(pattern="*", dst="include", src=include_folder)

        media_folder = os.path.join(self._subfolder, "media")
        self.copy(pattern="*", dst="media", src=media_folder)

        if tools.os_info.is_windows:
            folder = "Win32-gcc"
        elif tools.os_info.is_macos:
            folder = "MacOSX"
        else:
            folder = "Linux"

        lib_folder = os.path.join(self._subfolder, "lib", folder)
        bin_folder = os.path.join(self._subfolder, "bin", folder)

        self.copy(pattern="*.dll", src=bin_folder, dst="bin", keep_path=False)
        self.copy(pattern="*.lib", src=lib_folder, dst="lib", keep_path=False)
        self.copy(pattern="*.a", src=lib_folder, dst="lib", keep_path=False)
        self.copy(pattern="*.so*", src=lib_folder, dst="lib", keep_path=False)
        self.copy(pattern="*.dylib", src=lib_folder, dst="lib", keep_path=False)

    def package_info(self):
        self.cpp_info.libs = tools.collect_libs(self)
        if tools.os_info.is_windows:
            if not self.options.shared:
                self.cpp_info.defines.extend(['_IRR_STATIC_LIB_'])
                self.cpp_info.libs.extend(['opengl32', 'm', 'winmm'])
        elif tools.os_info.is_macos:
            frameworks = ['Cocoa', 'Carbon', 'OpenGL', 'IOKit']
            for framework in frameworks:
                self.cpp_info.exelinkflags.append("-framework %s" % framework)
            self.cpp_info.sharedlinkflags = self.cpp_info.exelinkflags
        else:
            if not self.options.shared:
                self.cpp_info.libs.extend(['GL', 'Xxf86vm', 'Xext', 'X11', 'Xcursor'])

        self.output.info(self.cpp_info.libs)


import xarray as xr
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

"""Extraindo para uma posicao (Umuarama-PR) a serie historica diaria
da temperatura maxima (Tmax), calculando a sua media
mensal (01/1980-12/2010) e exportando dados diarios em arquivo cvs:

Tmax_daily_UT_Brazil_v2.1_19800101_19891231.nc
Tmax_daily_UT_Brazil_v2.1_19900101_19991231.nc
Tmax_daily_UT_Brazil_v2.1_20000101_20061231.nc
Tmax_daily_UT_Brazil_v2.1_20070101_20131231.nc
Tmax_daily_UT_Brazil_v2.1_20140101_20170731.nc
"""

# set correct path of the variables
path_var = 'D:/Dropbox/ParaUbuntu/netcdfgrid3/'
ds = xr.open_mfdataset(path_var + 'Tmax_daily_UT_Brazil_v2*.nc')

# pegando a variavel Tmax entre 01/01/1980 a 31/12/2010
Tmax_data = ds.Tmax.sel(time=slice('1980-01-01','2010-12-31'))

# pegando os dados para o posicao de Umuarama/Parana
Tmax_data_temporal = Tmax_data.sel(latitude=-23.76,longitude=-53.30,
                                   method='nearest')
# plotando dados diarios
plt.subplot(121), Tmax_data_temporal.plot()

# plotando a media mensal
Tmax_mean_month = Tmax_data_temporal.groupby('time.month').mean('time')
plt.subplot(122), Tmax_mean_month.plot()
plt.show()

# exportando dados diarios em cvs: nome do arquivo 'Tmax.cvs'
fileName = 'Tmax.csv'
days = np.array(Tmax_data_temporal.time)
data_dataframe = pd.Series(np.array(Tmax_data_temporal), index=days)
data_dataframe.to_csv(fileName)


from macros.libs import wap
from macros.settings import local_settings as app_settings
from xml.etree.ElementTree import parse, fromstring

class WolframProcessor():

    def __init__(self, wolfram_alpha_engine_name):
        self.wolfram_alpha_engine_name = wolfram_alpha_engine_name

    def parse_query(self, query_string, response_text=None):

        if response_text is not None:
            root = fromstring(response_text.encode("ascii", "ignore"))
            pods = response_text
        else:
            pods, is_error = self.query_api(query_string)
            root = fromstring(pods)
        macros = [["Calories", "total calories", "mean value"], \
                  ["Protein and amino acids", "protein", "mean value"], \
                  ["Carbohydrates", "total carbohydrates", "mean value"], \
                  ["Fats and fatty acids", "total fat", "mean value"] \
            ]
        data = {}
        for macro in macros:
            # Return a string from the XML
            plaintext = self.extract_plaintext_from_pods(root, macro[0])
            # Convert the string to an array
            matrix = self.plaintext_to_matrix(plaintext)
            # convert the array to a dictionary
            dictionary = self.matrix_to_dict(matrix)
            value = self.value_string_to_number(dictionary[macro[1]][macro[2]])
            data[macro[1]] = value
        return data, pods

    def query_api(self, queryString):
        """
        Query wolframalpha with a string
        Return the pods
        """

        server = app_settings.WOLFRAM_SERVER
        app_id = app_settings.WOLFRAM_APPID
        waeo = self.wolfram_alpha_engine_name(app_id, server)
        query = waeo.CreateQuery(queryString)
        query_result = waeo.PerformQuery(query)
        waeqr = wap.WolframAlphaQueryResult(query_result)
        is_error = waeqr.IsError()
        xml_result = waeqr.XmlResult
        pods = xml_result
        return pods, is_error

    def extract_plaintext_from_pods(self, root, pod_name):
        """
        Take pods and extract the plaintext strings for given pod names
        """
        try:
            found_element = root.findall("./pod[@title='%s']/subpod/plaintext" % pod_name)
            plaintext = found_element[0].text.replace("rangeprotein", r"range\nprotein")
            return plaintext
        except:
            raise Exception('Could not get data from Wolfram Alpha')

    def plaintext_to_matrix(self, plaintext):
        """
        Take the plaintext table and convert it to a dict
        """

        matrix = list()

        table_rows = plaintext.split("\n")

        for row in table_rows:
            matrix.append(row.split(" | "))

        return matrix

    def matrix_to_dict(self, matrix):

        result = dict()

        for x in range(1, len(matrix)):
            zipped = dict(zip(matrix[0], matrix[x]))
            result[zipped[""]] = zipped
            # Remove the empty cell in the upper-left corner of table
            del result[zipped[""]][""]

        return result

    def value_string_to_number(self, value_string):
        if value_string[-2:] == " g":
            return float(value_string[:-2])
        elif value_string[-2:] == 'mg':
            return 0
        elif value_string[-3:] == 'Cal':
            return float(value_string[:-3])
        else:
            return 0


from django.core.management.base import BaseCommand

from tweet_commune.submission_actions import SubmissionActions
from ...logger.tweet_logger import TweetLogger
from ...models.submission import Submission
from ...raffle import Raffle

class Command(BaseCommand):
    """
    Management command to post Submission to Twitter
    """

    help = "Post the top submission"

    def handle(self, *args, **options):
        try:
            raffle = Raffle(multiplier=5)
            submission = raffle.choices(Submission.queue.all())[0]
            if submission:
                actions = SubmissionActions(submission)
                actions.post()
            else:
                TweetLogger.empty_queue()
        except Exception as e:
            TweetLogger.critical(e)


import tweepy
import threading
import time

class ApiObject:

    def __init__(self, id, auth):
        self.id = id
        self.tweepyapi = tweepy.API(auth)
        self.active = True

    def __worker(self):
        time.sleep(905) # sleep 15:05 min
        self.active = True
        print('Api {} is now Active.'.format(self.id))

    def start_reactivation_timer(self):
        print('Api {} starts reactivation timer.'.format(self.id))
        reactivation_thread = threading.Thread(target=self.__worker)
        reactivation_thread.start()


import sqlite3

conn = sqlite3.connect('allgdinfo.db')
c = conn.cursor()

t = ('赵伟',)
c.execute('SELECT * FROM gdinfo WHERE name=?', t)
all_com = c.fetchall()
for one in all_com:
    print(one)
    #print('\n')

t = ('红杉资本',)
c.execute('SELECT * FROM gdinfo WHERE name=?', t)
all_com = c.fetchall()
for one in all_com:
    print(one)

# We can also close the connection if we are done with it.
# Just be sure any changes have been committed or they will be lost.
conn.close()


import logging

from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.db.models.signals import m2m_changed
from django.dispatch import receiver

from rdmo.questions.models import Catalog
from rdmo.projects.models import Project, Membership
from rdmo.views.models import View

logger = logging.getLogger(__name__)

@receiver(m2m_changed, sender=View.catalogs.through)
def m2m_changed_view_catalog_signal(sender, instance, **kwargs):
    catalogs = instance.catalogs.all()

    if catalogs:
        catalog_candidates = Catalog.objects.exclude(id__in=[catalog.id for catalog in catalogs])

        # Remove catalog candidates for all sites
        projects = Project.objects.filter(catalog__in=catalog_candidates, views=instance)
        for proj in projects:
            proj.views.remove(instance)

@receiver(m2m_changed, sender=View.sites.through)
def m2m_changed_view_sites_signal(sender, instance, **kwargs):
    sites = instance.sites.all()
    catalogs = instance.catalogs.all()

    if sites:
        site_candidates = Site.objects.exclude(id__in=[site.id for site in sites])
        if not catalogs:
            # if no catalogs are selected, update all
            catalogs = Catalog.objects.all()

        # Restrict chosen catalogs for chosen sites
        projects = Project.objects.filter(site__in=site_candidates, catalog__in=catalogs, views=instance)
        for project in projects:
            project.views.remove(instance)

@receiver(m2m_changed, sender=View.groups.through)
def m2m_changed_view_groups_signal(sender, instance, **kwargs):
    groups = instance.groups.all()
    catalogs = instance.catalogs.all()

    if groups:
        users = User.objects.exclude(groups__in=groups)
        memberships = [membership.id for membership in Membership.objects.filter(role='owner', user__in=users)]
        if not catalogs:
            # if no catalogs are selected, update all
            catalogs = Catalog.objects.all()

        # Restrict chosen catalogs for chosen groups
        projects = Project.objects.filter(memberships__in=list(memberships), catalog__in=catalogs, views=instance)
        for project in projects:
            project.views.remove(instance)


import requests
from bs4 import BeautifulSoup
from sites.utils import grab_advt, USER_AGENT

def parse_monster_advt(page):
    print('Parsing monster advt')
    soup_obj = BeautifulSoup(page, 'lxml')

    if len(soup_obj) == 0:
        soup_obj = BeautifulSoup(page, 'html5lib')

    desc = get_description(soup_obj)
    title = get_title(soup_obj)
    salary = get_salary(soup_obj)
    date_posted = get_date_posted(soup_obj)
    experience = get_experience_requirements(soup_obj)
    organization = get_hiring_organization(soup_obj)

    if desc and title:
        return {
            "title": title,
            "description": desc,
            "salary": salary,
            "experience": experience,
            "datePosted": date_posted,
            "hiringOrganization": organization
        }
    return {}

def get_date_posted(soup_obj):
    date_posted = soup_obj.find("meta", itemprop="datePosted")
    if date_posted:
        return date_posted['content'].strip()
    return None

def get_experience_requirements(soup_obj):
    experience = soup_obj.find("span", itemprop="experienceRequirements")
    if experience:
        return experience.text.strip()
    return None

def get_hiring_organization(soup_obj):
    org = soup_obj.find("li", itemprop="hiringOrganization")
    if not org:
        org = soup_obj.find("dd", itemprop="hiringOrganization")
    if org:
        name = org.find("span", itemprop="name")
        if name:
            return name.text.strip()
        return org.text.strip()
    return None

def get_salary(soup_obj):
    salary = soup_obj.find('div', itemprop="baseSalary")
    if not salary:
        salary = soup_obj.find('span', itemprop="baseSalary")
    if salary:
        return salary.text.strip()
    return None

def get_title(soup_obj):
    title = soup_obj.find('h1', itemprop="title")
    if not title:
        title = soup_obj.find('div', itemprop="title")
    if not title:
        title = soup_obj.find('h3')
    if title:
        return title.text.strip()
    return None

def get_description(soup_obj):
    desc = soup_obj.find('div', itemprop="description")
    if not desc:
        desc = soup_obj.find('div', {"class": "jobview-section"})
    if not desc:
        desc = soup_obj.find('div', {"id": "jobBodyContent"})
    if not desc:
        desc = soup_obj.find('div', {"id": "CJT_jobBodyContent"})
    if desc:
        return desc.text.strip()
    return None

def parse_monster_search_results(results):
    print('Parsing monster search results')
    soup_obj = BeautifulSoup(results, 'lxml')

    if len(soup_obj) == 0:
        soup_obj = BeautifulSoup(results, 'html5lib')

    jobs = soup_obj.find_all('article', {"itemtype": "https://schema.org/JobPosting"})

    if jobs:
        results = []
        for job in jobs:
            m = job.find("meta", itemprop="url")
            if m:
                results.append(m['content'])
        return results
    return []

def search_monster(keywords="semantic web", location="united kingdom"):
    print("Searching monster for keywords: {} in location: {}".format(keywords, location))
    url = "http://www.monster.co.uk/jobs/search/"

    headers = {
        "user-agent": USER_AGENT
    }

    params = {
        'q': keywords,
        'where': location,
    }

    r = requests.get(url, headers=headers, params=params, allow_redirects=True)
    if r.status_code == 200:
        print("Search successful.")
        return r.text
    else:
        print(r.status_code)
        print("Search was borked.")
        return ""

def monster_pipeline(query, location):
    search_results = search_monster(query, location)
    job_links = parse_monster_search_results(search_results)
    for link in job_links:
        raw_advt = grab_advt(link)
        description = parse_monster_advt(raw_advt)
        description.update({"link": link})
        yield description

if __name__ == '__main__':
    d = monster_pipeline("sparql", "london")


import urllib3
import requests
import random
import os
import time
import sys
import json
from cookies import cookie_data
def parseProxy(proxyUrl):
  return "://".join(proxyUrl.split("#")[::-1])

def getProxies():
  proxies=['lum-customer-c_b6481680-zone-tiktok_api_res_zone1-gip-us_46375_ca_berkeley_0:gy6bek012jez@zproxy.lum-superproxy.io:22225','lum-customer-c_b6481680-zone-tiktok_api_res_zone1-gip-us_46375_ca_berkeley_3:gy6bek012jez@zproxy.lum-superproxy.io:22225','lum-customer-c_b6481680-zone-tiktok_api_res_zone1-gip-us_7018_ca_berkeley_2:gy6bek012jez@zproxy.lum-superproxy.io:22225','lum-customer-c_b6481680-zone-tiktok_api_res_zone1-gip-us_7922_ca_berkeley_1:gy6bek012jez@zproxy.lum-superproxy.io:22225']
  return proxies
class TiktokRequest:

  def __init__(self, uniqueId):
    self.uniqueId = uniqueId
    self.request = None
    self.enableProxies = True
    self.proxies = {}
    if self.enableProxies:
      self.proxy = random.choice(getProxies())
      self.proxies = {"http": 'http://' + self.proxy, "https": 'http://' + self.proxy}
  def userRequest(self, retry):
    url = f"https://m.tiktok.com/api/user/detail/?uniqueId={self.uniqueId}&language=en&verifyFp="
    if retry:
      self.signature = self.requestSignature(url)
    self.request = self.getRequest(url + '&_signature=' + self.signature)

    return self.request

  def itemListRequest(self, user, cursor, retry):
    url=f'https://m.tiktok.com/api/post/item_list/?aid=1988&count=30&cursor={cursor}&secUid={user.secUid}'
    self.signature = self.requestSignature(url)
    self.request = self.getRequest(self.signature)
    return self.request

  def commentsListRequest(self, post, cursor, retry):
    print(post.id)
    url=f"https://www.tiktok.com/api/comment/list/?aweme_id={post.id}&cursor={cursor}&count=20&aid=1988&app_language=en&device_platform=web_pc&current_region=NL&fromWeb=1&channel_id=3"
    self.signature = self.requestSignature(url)
    self.request = self.getRequest(self.signature, True)
    return self.request

  def resetSigner(self):
    requests.get('http://localhost:8080/reset')
    print('reset', flush=True)

  def getRequest(self, url, sendCokies=False):
    payload={}
    headers={
              'authority': 'www.tiktok.com',
              'accept': 'application/json, text/plain, /',
              'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',
              'referer': f'https://www.tiktok.com//@{self.uniqueId}?lang=en',
              'accept-language': 'en-US,en;q=0.9',
            }
    if sendCokies:
      headers['cookie'] = cookie_data
    while True:
      try:
        request = requests.get(url, headers=headers, proxies=self.proxies, verify=(not self.enableProxies),data=payload)
      except (ConnectionResetError, requests.exceptions.ProxyError, requests.exceptions.ConnectionError) as e:
        print("Proxy Connnection error, trying new proxy", flush=True)
        self.proxy = random.choice(getProxies())
        self.proxies = {"http": 'http://' + self.proxy, "https": 'http://' + self.proxy}
        time.sleep(1)
      else:
        break
    return request

  def requestSignature(self, url):
    while True:
      try:
        """
        Here We sign our base url with the help of node server
        """
        req_sign = requests.post('http://localhost/signature', data=url).json()
        req_sign=req_sign["data"]["signed_url"]
      except (ConnectionResetError, requests.exceptions.ProxyError, requests.exceptions.ConnectionError) as e:
        print("Signer Connnection error, trying new signer", flush=True)
        self.proxy = random.choice(getProxies())
        self.proxies = {"http": 'http://' + self.proxy, "https": 'http://' + self.proxy}
        time.sleep(1)
      else:
        break
    return req_sign


# Authors: Valentino Constantinou <vc@valentino.io>
# License: Apache 2.0

from PyNomaly import loop

import numpy as np
from numpy.testing import assert_array_equal

import pandas as pd
import pytest

from sklearn.metrics import roc_auc_score

from sklearn.utils import check_random_state
from sklearn.utils.testing import assert_greater
from sklearn.utils.testing import assert_true
from sklearn.utils.testing import assert_equal
from sklearn.utils.testing import assert_warns

from sklearn.datasets import load_iris

import warnings

# load the iris dataset
# and randomly permute it
rng = check_random_state(0)
iris = load_iris()
perm = rng.permutation(iris.target.size)
iris.data = iris.data[perm]
iris.target = iris.target[perm]

def test_loop():
    # Toy sample (the last two samples are outliers):
    X = np.array([[-2, -1], [-1, -1], [-1, -2], [1, 2], [1, 2], [2, 1], [5, 3],
                  [-4, 2]])

    # Test LocalOutlierProbability:
    clf = loop.LocalOutlierProbability(X, n_neighbors=5)
    score = clf.fit().local_outlier_probabilities
    share_outlier = 2. / 8.
    predictions = [-1 if s > share_outlier else 1 for s in score]
    assert_array_equal(predictions, 6 * [1] + 2 * [-1])

    # Assert smallest outlier score is greater than largest inlier score:
    assert_greater(np.min(score[-2:]), np.max(score[:-2]))

def test_loop_performance():
    # Generate train/test data
    rng = check_random_state(2)
    X = 0.3 * rng.randn(120, 2)

    # Generate some abnormal novel observations
    X_outliers = rng.uniform(low=-4, high=4, size=(20, 2))
    X_test = np.r_[X, X_outliers]
    X_labels = np.r_[
        np.repeat(1, X.shape[0]), np.repeat(-1, X_outliers.shape[0])]

    # fit the model
    clf = loop.LocalOutlierProbability(X_test, n_neighbors=X_test.shape[0] - 1)

    # predict scores (the lower, the more normal)
    score = clf.fit().local_outlier_probabilities
    share_outlier = X_outliers.shape[0] / X_test.shape[0]
    X_pred = [-1 if s > share_outlier else 1 for s in score]

    # check that roc_auc is good
    assert_greater(roc_auc_score(X_pred, X_labels), .985)

def test_lambda_values():
    # Generate train/test data
    rng = check_random_state(2)
    X = 0.3 * rng.randn(120, 2)

    # Generate some abnormal novel observations
    X_outliers = rng.uniform(low=-4, high=4, size=(20, 2))
    X_test = np.r_[X, X_outliers]

    # Fit the model with different extent (lambda) values
    clf1 = loop.LocalOutlierProbability(X_test, extent=1)
    clf2 = loop.LocalOutlierProbability(X_test, extent=2)
    clf3 = loop.LocalOutlierProbability(X_test, extent=3)

    # predict scores (the lower, the more normal)
    score1 = clf1.fit().local_outlier_probabilities
    score2 = clf2.fit().local_outlier_probabilities
    score3 = clf3.fit().local_outlier_probabilities

    # Get the mean of all the scores
    score_mean1 = np.mean(score1)
    score_mean2 = np.mean(score2)
    score_mean3 = np.mean(score3)

    # check that expected the means align with expectation
    assert_greater(score_mean1, score_mean2)
    assert_greater(score_mean2, score_mean3)

def test_parameters():
    # Generate train/test data
    rng = check_random_state(2)
    X = 0.3 * rng.randn(120, 2)

    # fit the model
    clf = loop.LocalOutlierProbability(X).fit()

    # check that the model has attributes post fit
    assert_true(hasattr(clf, 'n_neighbors') and
                clf.n_neighbors is not None)
    assert_true(hasattr(clf, 'extent') and
                clf.extent is not None)
    assert_true(hasattr(clf, 'cluster_labels') and
                clf._cluster_labels() is not None)
    assert_true(hasattr(clf, 'prob_distances') and
                clf.prob_distances is not None)
    assert_true(hasattr(clf, 'prob_distances_ev') and
                clf.prob_distances_ev is not None)
    assert_true(hasattr(clf, 'norm_prob_local_outlier_factor') and
                clf.norm_prob_local_outlier_factor is not None)
    assert_true(hasattr(clf, 'local_outlier_probabilities') and
                clf.local_outlier_probabilities is not None)

def test_n_neighbors():
    X = iris.data
    clf = loop.LocalOutlierProbability(X, n_neighbors=500).fit()
    assert_equal(clf.n_neighbors, X.shape[0] - 1)

    clf = loop.LocalOutlierProbability(X, n_neighbors=500)
    assert_warns(UserWarning, clf.fit)
    assert_equal(clf.n_neighbors, X.shape[0] - 1)

def test_extent():
    X = np.array([[1, 1], [1, 0]])
    clf = loop.LocalOutlierProbability(X, n_neighbors=2, extent=4)
    assert_warns(UserWarning, clf.fit)

def test_data_format():
    X = [1.3, 1.1, 0.9, 1.4, 1.5, 3.2]
    clf = loop.LocalOutlierProbability(X, n_neighbors=3)
    assert_warns(UserWarning, clf.fit)

def test_missing_values():
    X = np.array([1.3, 1.1, 0.9, 1.4, 1.5, np.nan, 3.2])
    clf = loop.LocalOutlierProbability(X, n_neighbors=3)

    with pytest.raises(SystemExit) as record:
        clf.fit()

    assert record.type == SystemExit

def test_small_cluster_size():
    # Generate train/test data
    rng = check_random_state(2)
    X = 0.3 * rng.randn(120, 2)

    # Generate some abnormal novel observations
    X_outliers = rng.uniform(low=-4, high=4, size=(20, 2))
    X = np.r_[X, X_outliers]
    # Generate cluster labels
    a = [0] * 120
    b = [1] * 18
    cluster_labels = a + b

    with pytest.warns(UserWarning) as record:
        warnings.warn(
            "Number of neighbors specified larger than smallest cluster. Specify a number of neighbors smaller than the smallest cluster size (observations in smallest cluster minus one).",
            UserWarning)

    loop.LocalOutlierProbability(X, n_neighbors=50, cluster_labels=cluster_labels)

    # check that only one warning was raised
    assert len(record) == 1
    # check that the message matches
    assert record[0].message.args[
               0] == "Number of neighbors specified larger than smallest cluster. Specify a number of neighbors smaller than the smallest cluster size (observations in smallest cluster minus one)."

def test_stream_fit():
    # Generate train/test data
    rng = check_random_state(2)
    X = 0.3 * rng.randn(120, 2)

    # Generate some abnormal novel observations
    X_outliers = rng.uniform(low=-4, high=4, size=(20, 2))
    X = np.r_[X, X_outliers]

    # Fit the model
    X_train = X[0:138]
    X_test = X[139]
    clf = loop.LocalOutlierProbability(X_train)

    with pytest.raises(SystemExit) as record:
        clf.stream(X_test)

    assert record.type == SystemExit

def test_stream_cluster():
    # Generate train/test data
    rng = check_random_state(2)
    X = 0.3 * rng.randn(120, 2)

    # Generate some abnormal novel observations
    X_outliers = rng.uniform(low=-4, high=4, size=(20, 2))
    X = np.r_[X, X_outliers]

    # Generate cluster labels
    a = [0] * 120
    b = [1] * 18
    cluster_labels = a + b

    # Fit the model
    X_train = X[0:138]
    X_test = X[139]
    clf = loop.LocalOutlierProbability(X_train,
                                       cluster_labels=cluster_labels).fit()

    with pytest.warns(UserWarning) as record:
        warnings.warn(
            "Stream approach does not support clustered data. Automatically refit using single cluster of points.",
            UserWarning)

    clf.stream(X_test)

    # check that only one warning was raised
    assert len(record) == 1
    # check that the message matches
    assert record[0].message.args[
               0] == "Stream approach does not support clustered data. Automatically refit using single cluster of points."

def test_stream_performance():
    # Generate train/test data
    rng = check_random_state(2)
    X = 0.3 * rng.randn(120, 2)

    # Generate some abnormal novel observations
    X_outliers = rng.uniform(low=-4, high=4, size=(20, 2))
    X = np.r_[X, X_outliers]

    X_train = X[0:100]
    X_test = X[100:140]

    # Fit the models in standard and stream form
    m = loop.LocalOutlierProbability(X).fit()
    scores_noclust = m.local_outlier_probabilities

    m_train = loop.LocalOutlierProbability(X_train)
    m_train.fit()
    X_train_scores = m_train.local_outlier_probabilities

    X_test_scores = []
    for idx in range(X_test.shape[0]):
        X_test_scores.append(m_train.stream(X_test[idx]))
    X_test_scores = np.array(X_test_scores)

    stream_scores = np.hstack((X_train_scores, X_test_scores))

    # calculate the rmse and ensure score is below threshold
    rmse = np.sqrt(((scores_noclust - stream_scores) ** 2).mean(axis=None))
    assert_greater(0.35, rmse)