"""
Created on: 10/11/17
Author: Nikolaos Apostolakos
"""

from __future__ import division, print_function

import os
import pytest
import numpy as np

from ..util_fixtures import temp_dir_fixture

##############################################################################

@pytest.fixture()
def sed_list_fixture():
    """Returns a list of SEDs to be used for testing.

    The SEDs are the following:
      File   ID  Data
     ------ ---- --------------
       1     1   [(1., 100.), (1.5, 168.3), (1.7,96.12)]
       1     3   [(167.7,4.2), (456.49,7.8)]
       1    12   [(1.,1.), (2.,2.), (3.,3.), (4.,4.),(5.,5.)]
       1     7   [(56.,78.1)]
       2    34   [(6.,6.), (7.,7.)]
       2    56   [(8.,8.), (9.,9.)]
    """
    return {
        1:[
            [1, np.asarray([(1., 100.), (1.5, 168.3), (1.7,96.12)], dtype=np.float32)],
            [3, np.asarray([(167.7,4.2), (456.49,7.8)], dtype=np.float32)],
            [12, np.asarray([(1.,1.), (2.,2.), (3.,3.), (4.,4.),(5.,5.)], dtype=np.float32)],
            [7, np.asarray([(56.,78.1)], dtype=np.float32)]
        ],
        2:[
            [34, np.asarray([(6.,6.), (7.,7.)], dtype=np.float32)],
            [56, np.asarray([(8.,8.), (9.,9.)], dtype=np.float32)]
        ]
    }

##############################################################################

@pytest.fixture()
def sed_data_files_fixture(temp_dir_fixture, sed_list_fixture):
    """Creates SED data files to be used for testing.

    Returns: A map with keys the file indices and values the paths of the files

    The created files contains the SEDs of the sed_list_fixture.
    """

    result = {}
    for file_index in sed_list_fixture:
        filename = os.path.join(temp_dir_fixture, 'sed_data_{}.bin'.format(file_index))
        with open(filename, 'wb') as out:
            for i, data in sed_list_fixture[file_index]:
                # Write the index
                np.asarray([i], dtype='int64').tofile(out)
                # Write the length
                np.asarray([len(data)], dtype='uint32').tofile(out)
                # Write the data
                data_arr = np.asarray(data, dtype='float32')
                data_arr.flatten().tofile(out)
        result[file_index] = filename

    return result

##############################################################################

@pytest.fixture()
def redshift_bins_fixture():
    """Returns an array with the redsift bins of the PDZs to be used for testing"""
    return np.asarray([1,2,5,6,8,9], dtype=np.float32)

##############################################################################

@pytest.fixture()
def pdz_list_fixture():
    """Returns a list of PDZs to be used for testing."""
    return {
        1: [
            [1, np.asarray([1,2,3,4,5,6], dtype=np.float32)],
            [3, np.asarray([2,3,4,5,6,7], dtype=np.float32)],
            [12, np.asarray([3,4,5,6,7,8], dtype=np.float32)],
            [7, np.asarray([4,5,6,7,8,9], dtype=np.float32)]
        ],
        2: [
            [34, np.asarray([11,12,13,14,15,16], dtype=np.float32)],
            [56, np.asarray([12,13,14,15,16,17], dtype=np.float32)]
        ]
    }

##############################################################################

@pytest.fixture()
def pdz_data_files_fixture(temp_dir_fixture, redshift_bins_fixture, pdz_list_fixture):
    """Creates PDZ data files to be used for testing.

    Returns: A map with keys the file indices and values he path to the newly created files

    The created files contain the PDZs of the pdz_list_fixture and the redshift
    bins of the redshift_bins_fixture.
    """

    result = {}
    for file_index in pdz_list_fixture:
        filename = os.path.join(temp_dir_fixture, 'pdz_data_{}.bin'.format(file_index))
        with open(filename, 'wb') as out:
            # Write the length of each PDZ
            np.asarray([len(redshift_bins_fixture)], dtype=np.uint32).tofile(out)
            # Write the redshift bins
            np.asarray(redshift_bins_fixture, dtype=np.float32).tofile(out)
            #Write all the PDZs
            for i, data in pdz_list_fixture[file_index]:
                np.asarray([i], dtype=np.int64).tofile(out)
                np.asarray(data, dtype=np.float32).tofile(out)
        result[file_index] = filename

    return result

##############################################################################

@pytest.fixture()
def reference_sample_dir_fixture(temp_dir_fixture, sed_data_files_fixture,
                                 pdz_data_files_fixture, sed_list_fixture,
                                 redshift_bins_fixture):
    """Returns a directory which contains a reference sample"""

    # Create the index file
    pdz_length = len(redshift_bins_fixture)
    pdz_offset = 4 + 4 * pdz_length
    with open(os.path.join(temp_dir_fixture, 'index.bin'), 'wb') as f:

        for file_index in sed_list_fixture:
            sed_pos = 0
            pdz_pos = pdz_offset
            for obj_id, sed_data in sed_list_fixture[file_index]:
                np.asarray([obj_id], dtype=np.int64).tofile(f)
                np.asarray([file_index], dtype=np.uint16).tofile(f)
                np.asarray([sed_pos], dtype=np.int64).tofile(f)
                np.asarray([file_index], dtype=np.uint16).tofile(f)
                np.asarray([pdz_pos], dtype=np.int64).tofile(f)
                sed_pos += 8 + 4 + 4 * sed_data.size
                pdz_pos += 8 + 4 * pdz_length

        # Add two more objects without data
        np.asarray([100], dtype=np.int64).tofile(f)
        np.asarray([0], dtype=np.uint16).tofile(f)
        np.asarray([-1], dtype=np.int64).tofile(f)
        np.asarray([0], dtype=np.uint16).tofile(f)
        np.asarray([-1], dtype=np.int64).tofile(f)
        np.asarray([101], dtype=np.int64).tofile(f)
        np.asarray([0], dtype=np.uint16).tofile(f)
        np.asarray([-1], dtype=np.int64).tofile(f)
        np.asarray([0], dtype=np.uint16).tofile(f)
        np.asarray([-1], dtype=np.int64).tofile(f)

    return temp_dir_fixture

##############################################################################


import pandas as pd
import random
from ETE_scaling import scale_encode_combination
from pprint import pprint as pp
from sklearn import preprocessing
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import make_scorer, accuracy_score

import pickle, joblib
import time

# DATA PREPROCESSING STEP 1=====================================================
# 1-1. Load data
dirty_data = pd.read_csv('dirty_tracks.csv')
dataset = dirty_data

# 1-2. Checking data
print("dataset information")
#print(dataset.info())
#print(dataset.describe())
#print(dataset.shape)

# 1.3. Fill missing data
print("fill missing data")
#print(dataset.isnull().sum())
dataset = dataset.fillna(dataset.median())
dataset = dataset.apply(lambda x: x.fillna(x.value_counts().index[0]))
#print(dataset.isnull().sum())

# 1.4.1. Drop not use column (unique data)
print("drop unique data")
dataset = dataset.drop(columns=['id', 'name', 'artists', 'id_artists'])
print("drop dummy data")
dataset = dataset.drop(columns=['mode', 'explicit'])

# 1.4.2. Drop duplicated data
dataset.duplicated().sum()
dataset = dataset[~dataset.duplicated()]
print("drop duplicate")

# 1.4.3. Drop popularity zero data
# => A song with zero popularity is an outlier that does not reflect proper information.
dataset = dataset[dataset.popularity > 0]
print("drop popularity == 0")

# 1.4.4. Drop Speechiness song (only instrumental)
dataset = dataset[dataset.speechiness > 0]
print("drop only instrumental")

# 1.4.5. Drop Liveness song
# => If the concert is live, it is highly likely to be duplicated because existing music sources may exist.
dataset = dataset[dataset.liveness < 0.9]
print("drop Liveness")

# 1.4.6. Drop Duration_ms short data
# => Less than a minute of data is outlier data, which is likely to be advertised music.
dataset = dataset[dataset.duration_ms > 60000] # 1min
print("drop Duration_ms")

print(dataset.shape)
print(dataset['release_date'].isna().sum())

# 1.5 Modity release date (YYYY -> YYY (first 3 chat))
# ref : https://stackoverflow.com/questions/36505847/substring-of-an-entire-column-in-pandas-dataframe/36506041
dataset['release_date'] = dataset['release_date'].str[0:3]
#class에 값이 1개 있는 190은 drop
dataset = dataset[dataset.release_date != '190']

feature = dataset.drop('release_date',axis=1)
target = dataset['release_date']

# DATA PREPROCESSING STEP 2=====================================================
# 2.1 Set numerical column and categorical column
numerical_feature_list = ['duration_ms', 'danceability', 'energy', 'loudness', 'speechiness', 'acousticness', 'instrumentalness', 'liveness', 'valence', 'tempo']
categorical_feature_list = ['time_signature', 'key']

# 2.2 Run scaling & encoding using "scale_encode_combination" function (return type : dictionary)
print("Scaling and Encoding feature")
combination_dataset = scale_encode_combination(feature, numerical_feature_list, categorical_feature_list)
#pp(combination_dataset)

# MODEL TRAINING STEP=====================================================
for key, mydataset in combination_dataset.items():

    print("Set traom and test")
    print(feature.shape)
    print(target.shape)

    # 3.2 Split train and test
    train_X, test_X, train_Y, test_Y = train_test_split(feature, target.ravel(), test_size = .3, shuffle=True, stratify=target)
    print(train_X)
    print(train_Y)
    print(train_X.shape)
    print(train_Y.shape)

    train_X, val_X, train_Y, val_Y= train_test_split(train_X, train_Y, test_size=0.3, random_state=12)

    # 3.3 Set model
    print("Set model")
    model = RandomForestClassifier()

    # for XGBRegressor
    param_grid = {
    'n_estimators': [100, 200, 300],
    'max_features': ['sqrt', 'log2'],
    'max_depth' : [3, 5, 7],
    'criterion' :['gini', 'entropy']
    }

    # 3.5 Define an evaluation metric as root mean squared error in the scoring parameter
    # 3.6 Set and RUN GridSearchCV
    start_time = time.time()
    model_test = GridSearchCV(model, param_grid, cv=5, n_jobs=4, verbose=2)
    model_test.fit(train_X, train_Y)
    print("The time that this function finish :", time.time() - start_time)

    # 3.6 Show Result & Evaluation
    print('model 2 best estimator: ', model_test.best_estimator_)
    print('model 2 best parameters: ', model_test.best_params_)

    best_model = model_test.best_estimator_
    predict_test = best_model.predict(test_X)
    print('Best model 2 score : ', best_model.score(test_X, test_Y))
    print("Accuracy for Random Forest on CV data: ", accuracy_score(test_Y, predict_test))

    #파일 이름 구성 -> combination type + classification인지 regresison인지 + dirty data/originaldata + 몇번쨰 시도인지 + 그외 저장해야 하는 정보
    with open(key + '_rf' + '_model2_classification_dirtydata.pkl', 'wb') as f:
        pickle.dump(model_test, f)


import os
import json
import pandas as pd
from collections import namedtuple
import requests
import yaml

from allensdk.core.swc import read_swc
from vtkplotter import *
from vtk.util.numpy_support import numpy_to_vtk, vtk_to_numpy
import vtk

def connected_to_internet(url='http://www.google.com/', timeout=5):
	try:
		_ = requests.get(url, timeout=timeout)
		return True
	except requests.ConnectionError:
		print("No internet connection available.")
	return False

def send_query(query_string, clean=False):
	response = requests.get(query_string)
	if response.ok:
		if not clean:
			return response.json()['msg']
		else:
			return response.json()
	else:
		raise ValueError("Invalide query string: {}".format(query_string))

def listdir(fld):
	if not os.path.isdir(fld):
		raise FileNotFoundError("Could not find directory: {}".format(fld))

	return [os.path.join(fld, f) for f in os.listdir(fld)]

def load_json(filepath):
	if not os.path.isfile(filepath) or not ".json" in filepath.lower(): raise ValueError("unrecognized file path: {}".format(filepath))
	with open(filepath) as f:
		data = json.load(f)
	return data

def load_yaml(filepath):
	if filepath is None or not os.path.isfile(filepath): raise ValueError("unrecognized file path: {}".format(filepath))
	if not "yml" in filepath and not "yaml" in filepath: raise ValueError("unrecognized file path: {}".format(filepath))
	return yaml.load(open(filepath), Loader=yaml.FullLoader)

"""
import allensdk.core.swc as swc

# if you ran the examples above, you will have a reconstruction here
file_name = 'cell_types/specimen_485909730/reconstruction.swc'
morphology = swc.read_swc(file_name)

# subsample the morphology 3x. root, soma, junctions, and the first child of the root are preserved.
sparse_morphology = morphology.sparsify(3)

# compartments in the order that they were specified in the file
compartment_list = sparse_morphology.compartment_list

# a dictionary of compartments indexed by compartment id
compartments_by_id = sparse_morphology.compartment_index

# the root soma compartment
soma = morphology.soma

# all compartments are dictionaries of compartment properties
# compartments also keep track of ids of their children
for child in morphology.children_of(soma):
	print(child['x'], child['y'], child['z'], child['radius'])

"""

def load_volume_file(filepath, **kwargs):
	if not os.path.isfile(filepath): raise FileNotFoundError(filepath)

	if ".x3d" in filepath.lower(): raise ValueError("BrainRender cannot use .x3d data as they are not supported by vtkplotter")

	elif "nii" in filepath.lower() or ".label" in filepath.lower():
		import nibabel as nb
		data = nb.load(filepath)
		d = data.get_fdata()

		act = Volume(d, **kwargs)

	else:
		act = load(filepath, **kwargs)
		if act is None:
			raise ValueError("Could not load {}".format(filepath))

	return act

if __name__ == "__main__":
	# testing load_neuron_swc
	NEURONS_FILE = "Examples/example_files/AA0007.swc"
	load_neuron_swc(NEURONS_FILE)


import GradientDescent as gd
import Plotter as pl
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
df = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data', header=None)

y = df.iloc[0:100,4].values
y = np.where(y == 'Iris-setosa' , -1 , 1)

X = df.iloc[0:100 , [0,2]].values

X_std = np.copy(X)
X_std[:,0] = (X[:,0] - X[:,0].mean())/X[:,0].std()
X_std[:,1] = (X[:,1] - X[:,1].mean())/X[:,1].std()

ada = gd.AdalineSGD(n_iter = 15, eta = 0.01)
ada.fit(X_std,y)

pl.plot_decision_regions(X_std,y,classifier=ada)
plt.title('Gradient Descent')
plt.xlabel('sepal length')
plt.ylabel('petal length')
plt.legend(loc = 'upper left')

plt.show()
plt.plot(range(1, len(ada.cost_)+1), ada.cost_, marker='o')
plt.xlabel('iterations')
plt.ylabel('Squared Error Sum')
plt.show()


# Copyright (c) 2019, NVIDIA CORPORATION.

import warnings

import pyarrow.parquet as pq

import cudf
import cudf._lib as libcudf
from cudf.utils import ioutils

@ioutils.doc_read_parquet_metadata()
def read_parquet_metadata(path):
    """{docstring}"""

    pq_file = pq.ParquetFile(path)

    num_rows = pq_file.metadata.num_rows
    num_row_groups = pq_file.num_row_groups
    col_names = pq_file.schema.names

    return num_rows, num_row_groups, col_names

@ioutils.doc_read_parquet()
def read_parquet(
    filepath_or_buffer,
    engine="cudf",
    columns=None,
    row_group=None,
    skip_rows=None,
    num_rows=None,
    strings_to_categorical=False,
    use_pandas_metadata=True,
    *args,
    **kwargs,
):
    """{docstring}"""

    filepath_or_buffer, compression = ioutils.get_filepath_or_buffer(
        filepath_or_buffer, None, **kwargs
    )
    if compression is not None:
        ValueError("URL content-encoding decompression is not supported")

    if engine == "cudf":
        df = libcudf.parquet.read_parquet(
            filepath_or_buffer,
            columns,
            row_group,
            skip_rows,
            num_rows,
            strings_to_categorical,
            use_pandas_metadata,
        )
    else:
        warnings.warn("Using CPU via PyArrow to read Parquet dataset.")
        pa_table = pq.read_pandas(
            filepath_or_buffer, columns=columns, *args, **kwargs
        )
        df = cudf.DataFrame.from_arrow(pa_table)

    return df

@ioutils.doc_to_parquet()
def to_parquet(df, path, *args, **kwargs):
    """{docstring}"""
    warnings.warn(
        "Using CPU via PyArrow to write Parquet dataset, this will "
        "be GPU accelerated in the future"
    )
    pa_table = df.to_arrow()
    pq.write_to_dataset(pa_table, path, *args, **kwargs)


"""
multiprocessing_demo1.py - multiprocessing demo from Chapter 8

Creates a Process instance directly.
"""

from multiprocessing import Process
from zipfile import ZipFile, ZIP_DEFLATED

def zip_it(infile, outfile):
    with ZipFile(outfile, 'w', ZIP_DEFLATED) as f:
        f.write(infile)
    print('Finished', infile)

# On Windows the subprocesses will import (i.e. execute) the main module
# when they start. You need to protect the main code to avoid creating
# subprocesses recursively.
if __name__ == "__main__":

    # create a Process instance that will call zip_it, passing args tuple
    child_process = Process(target=zip_it,
                            args=('inventory.csv', 'inventory.zip'))

    child_process.start()  # start the Process executing

    print('Main process continues to run in foreground')

    child_process.join()  # Wait for the child process to finish

    print('Main process waited until child_process was done.')


# コンフィグデータ参照
import config, json

# 標準モジュール参照
import sys
import urllib
import webbrowser
import unicodedata

# OAunth認証用モジュール参照
import oauth2

# 外部モジュール参照
import tweepy
from requests_oauthlib import OAuth1Session

# PyQt5モジュール参照
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *

# 認証用UIクラス指定
from authorization import Ui_Pin_Dialog as auth_ui

# 独自モジュール
import MJT_Definitions

class TWAuthController(QDialog):
    TW_User_name = "[UserName]"
    TW_USER_id   = "[UserID"

    RESOURCE_URL	= "https://api.twitter.com/1.1/statuses/update.json"
    USERLOOKUP_URL	= "https://api.twitter.com/1.1/users/lookup.json"
    REQUEST_TOKEN_URL = "https://api.twitter.com/oauth/request_token"
    ACCESS_TOKEN_URL  = "https://api.twitter.com/oauth/access_token"
    AUTHENTICATE_URL  = "https://api.twitter.com/oauth/authenticate"

    CACHE_PATH = "__pycache__/"

    flag = False

    def __init__(self, parent = None):
        super(TWAuthController, self).__init__(parent)
        self.CONSUMER_KEY    = config.consumer_key
        self.CONSUMER_SECRET = config.consumer_secret
        self.app             = oauth2.Consumer(self.CONSUMER_KEY, self.CONSUMER_SECRET)

        self.authui = auth_ui()
        self.authui.setupUi(self)
        self.connect_signal_slot()

        print("TWAuthController : CONSTRUCTOR PROCESS COMPLETE")

    def __del__(self):
        print("TWAuthController : DESTRUCTOR PROCESS COMPLETE")

    def get_saved_token(self, fileref):
        list = [string.strip() for string in fileref.readlines()]
        self.ACCESS_TOKEN = list[0]
        self.ACCESS_TOKEN_SECRET = list[1]

	# アプリケーション認証URL取得
    def get_auth_url(self):
        self.set_request_token_content()
        request_token = self.request_token_content["oauth_token"][0]
        query = urllib.parse.urlencode({"oauth_token": request_token})
        return self.AUTHENTICATE_URL + "?" + query

	# トークン情報を持つ辞書(dict)生成
    def get_access_token_dict(self, PIN):
        oauth_token = self.request_token_content["oauth_token"][0]
        oauth_token_secret = self.request_token_content["oauth_token_secret"][0]
        token = oauth2.Token(oauth_token, oauth_token_secret)
        client = oauth2.Client(self.app, token)
        body = urllib.parse.urlencode({"oauth_verifier": PIN})
        response, content = client.request(self.ACCESS_TOKEN_URL, "POST", body = body)
        return urllib.parse.parse_qs(content.decode())

	# トークン要求のための情報生成
    def set_request_token_content(self):
        client = oauth2.Client(self.app)
        response, content = client.request(self.REQUEST_TOKEN_URL, "GET")
        self.request_token_content = urllib.parse.parse_qs(content.decode())

	# OAuthによるユーザ認証
    def authorize_user(self):
        self.path = '.savedata'
        try:
            with open(self.path) as fileref:
                self.get_saved_token(fileref)
        except FileNotFoundError:
            # self.ACCESS_TOKEN 		= config.access_token
            # self.ACCESS_TOKEN_SECRET = config.access_token_secret
            self.auth_url = self.get_auth_url()
            webbrowser.open(self.auth_url)

            print("PINコードを入力してください。\n", end = "")
            # PIN = int(input(">> "))
            PIN = self.check_pin_code()

            self.access_token_content = self.get_access_token_dict(PIN)
            self.ACCESS_TOKEN = self.access_token_content["oauth_token"][0]
            self.ACCESS_TOKEN_SECRET = self.access_token_content["oauth_token_secret"][0]

            print("Authorization Completed.------------------------------------------------")
            print("ACCESS TOKEN        = " + self.ACCESS_TOKEN)
            print("ACCESS TOKEN SECRET = " + self.ACCESS_TOKEN_SECRET)
            print("------------------------------------------------------------------------")

            try:
                with open(self.path, mode="w") as fileref:
                    string = self.ACCESS_TOKEN + "\n" + self.ACCESS_TOKEN_SECRET
                    fileref.write(string)
            except FileNotFoundError as e:
                print("An error occured!")
                print(e)
                sys.exit()
            else:
                print("Saved ")

        # OAuthによる認証処理試行
        try:
            # self.TWITTER_OAUTH = OAuth1Session(self.CONSUMER_KEY, self.CONSUMER_SECRET, self.ACCESS_TOKEN, self.ACCESS_TOKEN_SECRET)
            self.OAUTH 		= tweepy.OAuthHandler(self.CONSUMER_KEY, self.CONSUMER_SECRET)
            self.OAUTH.set_access_token(self.ACCESS_TOKEN, self.ACCESS_TOKEN_SECRET)
            self.APIInst 	= tweepy.API(self.OAUTH)
            self.UserObject = self.APIInst.me()
            self.UserName	= self.UserObject.name
            self.UserID		= self.UserObject.screen_name

        # すべての例外をキャッチ
        except Exception as e:
            print(e)
            return False
        else :
            print("Authorization Completed.")
            return True

    def connect_signal_slot(self):
        self.authui.buttonBox.accepted.connect(self.pin_accepted)
        self.authui.buttonBox.rejected.connect(self.pin_rejected)

    def pin_rejected(self):
        print("Authorization Rejected.")
        self.flag = False
        self.close()

    def pin_accepted(self):
        print("Authorization Accepted.")
        self.flag = True
        self.close()

    def check_pin_code(self):
        self.show()
        if (self.flag == False):
            sys.exit()
        return self.authui.textEdit.toPlainText()

    def show(self):
        self.exec_()

if __name__ == '__main__':
    app = QApplication(sys.argv)
    tmp = TWAuthController()
    tmp.check_pin_code(None)
    sys.exit(app.exec())


#!/usr/bin/env python3

import logging
import time

import matplotlib.pyplot as plt
import numpy as np
from tensorboardX import SummaryWriter
import torch

from nets import MappingNet, DiscriminatorNet
from toy_data import generate_data

logging.basicConfig()
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

writer = SummaryWriter()

def training_step(
    net: str,
    discriminator: DiscriminatorNet,
    optimizer: torch.optim.Optimizer,
    loss_criterion: torch.nn.modules.loss._Loss,
    point: torch.Tensor,
    source: torch.Tensor,
):
    """
    Performs the training step on either the mapper or the discriminator.

    :param net: 'mapper' or 'discriminator'
    :param discriminator: discriminator neural network
    :param optimizer: optimizer for the neural network being trained
    :param loss_criterion: loss function
    :param point: data vector of shape (1, 2)
    :param source: vector of shape (1, 2) describing the source of
        the point, either a mapped point or a true target point
    """

    # zero grad the optimizer
    optimizer.zero_grad()

    # get discriminator output for the given point
    discrim_output = discriminator(point)

    # calculate loss, the mapper tries to make the output incorrect and the
    # discriminator tries to make the output match the source
    if net == "mapper":
        loss = loss_criterion(discrim_output, 1 - source)
    elif net == "discriminator":
        loss = loss_criterion(discrim_output, source)
    else:
        raise Exception(f'Invalid parameter for "net": {net}')

    # backward propagate
    loss.backward(retain_graph=True)

    # optimizer step
    optimizer.step()

    return loss.data.item()

def main():
    # Use CPU for this simple model
    device = torch.device("cpu")

    # Generate a toy training dataset
    points, points_rotated = generate_data(sample_size=10000)
    data = np.hstack((points, points_rotated))
    data_tensor = torch.tensor(data=data, dtype=torch.float, device=device)

    # Generate a toy test dataset
    test_origin, test_target = generate_data(sample_size=1000)
    test_tensor = torch.tensor(
        data=test_origin, dtype=torch.float32, device=device
    )

    # create neural nets, define optimizers and loss criterion
    map_net = MappingNet().to(device)
    discrim_net = DiscriminatorNet().to(device)
    map_optimizer = torch.optim.Adam(params=map_net.parameters(), lr=1e-4)
    discrim_optimizer = torch.optim.Adam(
        params=discrim_net.parameters(), lr=1e-4
    )
    criterion = torch.nn.BCELoss()

    # begin training loops
    logger.info("Training has begun!")
    logger.info("Access TensorBoard to view results.")

    t0 = time.time()
    global_step = 0
    for epoch in range(25):

        for data_pair in data_tensor:
            global_step += 1

            # get input and target
            origin = data_pair[:2].view(1, 2)
            target = data_pair[2:].view(1, 2)

            # define source tensors
            origin_source = torch.tensor(data=[[1.0]], device=device)
            target_source = torch.tensor(data=[[0.0]], device=device)

            # train discriminator on the true target point
            discrim_loss = training_step(
                net="discriminator",
                discriminator=discrim_net,
                optimizer=discrim_optimizer,
                loss_criterion=criterion,
                point=target,
                source=target_source,
            )

            # train each net on a point mapped from the origin to the target
            mapped_point = map_net(origin)
            map_loss = training_step(
                net="mapper",
                discriminator=discrim_net,
                optimizer=map_optimizer,
                loss_criterion=criterion,
                point=mapped_point,
                source=origin_source,
            )
            discrim_loss = training_step(
                net="discriminator",
                discriminator=discrim_net,
                optimizer=discrim_optimizer,
                loss_criterion=criterion,
                point=mapped_point,
                source=origin_source,
            )

            # Logging in tensorboard
            writer.add_scalar(
                tag="loss/mapper",
                scalar_value=map_loss,
                global_step=global_step,
            )
            writer.add_scalar(
                tag="loss/discriminator",
                scalar_value=discrim_loss,
                global_step=global_step,
            )

            if not global_step % 500:
                logger.info(f"Global Step : {global_step}")

        # predict on test data
        test_pred = map_net(test_tensor).data.numpy()

        # plot test data, its transformation, and the true target space
        plt.clf()
        plt.scatter(
            x=test_origin[:, 0],
            y=test_origin[:, 1],
            c="red",
            label="origin space",
        )
        plt.scatter(
            x=test_target[:, 0],
            y=test_target[:, 1],
            c="blue",
            label="target space",
        )
        plt.scatter(
            x=test_pred[:, 0],
            y=test_pred[:, 1],
            c="yellow",
            label="transformed data",
        )
        plt.axis([-5, 5, -5, 5])
        plt.legend(loc="upper left")

        fig = plt.gcf()

        writer.add_figure(tag="plot", figure=fig, global_step=global_step)

    # Print information from the training
    t1 = time.time()
    logger.info(f"Time taken to finish: {t1 - t0}")

if __name__ == "__main__":
    main()


import requests
import ubicacion
import database

class obtener_Locacion():
    r = requests.get('https://rickandmortyapi.com/api/location/')
    texto = r.json()
    paginas = texto['info']['pages']
    db = database.basedatos()
    db.crear_tabla_locaciones()

    for i in range(1,paginas+1):
        r = requests.get('https://rickandmortyapi.com/api/location/?page={}'.format(i))
        texto = r.json()
        location = texto['results']
        for locacion in location:
            residentes = []
            res = locacion['residents']
            for j in range(0,len(res)):
                if len(res) != 0:
                    cadena = res[j]

                    parte = ''
                    parte += cadena[42]
                    if len(cadena) >= 44:
                        parte += cadena[43]
                    if len(cadena) == 45:
                        parte += cadena[44]
                    residentes.append(parte)
                else:
                    parte = 'Sin Residentes'
                    residentes.append(parte)
            loc = ubicacion.ubicaciones(locacion['id'],locacion['name'],locacion['type'],locacion['dimension'],locacion['residents'],residentes,locacion['url'])
            db.insertar_locaciones(loc)

if __name__ == '__main__':
    obtener_Locacion()


# -*- coding: utf-8 -*-
'''
    >>> MON == 'Понедельник'
    True
    >>> MON == 'Вторник'
    False
    >>> days[1] == '2'
    True
    >>> days[3] == 'THU'
    True
    >>> 'sat' == SAT
    True

    >>> SUN
    Weekday(7)
'''

import datetime

class Weekday(object):
    def __init__(self, names):
        self._names = tuple(names)

    def __eq__(self, other):
        if isinstance(other, str):
            return other.strip().lower() in self._names or other == repr(self)
        return self._names == other._names

    def __repr__(self):
        return 'Weekday({0})'.format(self._names[0])

    def __hash__(self):
        return hash(repr(self))

MON = Weekday(['mon', '1', 'monday', 'понедельник', 'пн'])
TUE = Weekday(['tue', '2', 'tuesday', 'вторник', 'вт'])
WED = Weekday(['wed', '3', 'wednesday', 'среда', 'ср'])
THU = Weekday(['thu', '4', 'thursday', 'четверг', 'чт'])
FRI = Weekday(['fri', '5', 'friday', 'пятница', 'пт'])
SAT = Weekday(['sat', '6', 'saturday', 'суббота', 'сб'])
SUN = Weekday(['sun', '7', 'sunday', 'воскресенье', 'вс'])

days = [
    MON,
    TUE,
    WED,
    THU,
    FRI,
    SAT,
    SUN,
]

def today():
    return days[datetime.datetime.now().weekday()]

if __name__ == '__main__':
    import doctest
    doctest.testmod()


from math import floor
from random import randint
import numpy as np

class Simulation:

    DURATION = 20

    def __init__(self, agent, policy, market):
        self.agent = agent
        self.policy = policy
        self.market = market
        self.init_cash = self.agent.cash
        self.start_day = randint(0, floor(market.duration/2))
        self.end_day = self.start_day + self.DURATION

    def run_transaction(self, action, stock_price):
        try:
            self.agent.transaction(action, stock_price)
            cost = 0.002
        except ValueError:
            cost = 0.05

        transaction_cost = cost*stock_price
        assets = self.agent.cash + self.agent.stock*self.market.stock_price[self.end_day-1]
        assets -= transaction_cost
        reward = ((assets - self.init_cash) / self.init_cash)

        return reward, assets

    def compute_cost_function(self, y, action_prob, reward):
        return (y - action_prob) * reward

    def run(self):

        for i in range(self.start_day, self.end_day):

            state = np.concatenate((self.market.indices[i], self.agent.state), axis=0)
            action_prob = self.policy.decide(state)

            if np.random.uniform() < action_prob:
                action = "buy"
                y = 1
            else:
                action = "sell"
                y = 0

            reward, assets = self.run_transaction(action, self.market.stock_price[i])
            y_hat = self.compute_cost_function(y, action_prob, reward)

            self.policy.train(state, y_hat, i)

        return self.policy, assets


import torch
from torch import nn

from models.UNet import setup_unet
from models.perception_loss import vgg16_feat, perceptual_loss
from .base_model import BaseModel
from . import networks
from parse_config import ConfigParser
import parse_config

class AsyndganModel(BaseModel):
    """ This class implements the pix2pix model, for learning a mapping from input images to output images given paired data.

    The model training requires '--dataset_mode aligned' dataset.
    By default, it uses a '--netG unet256' U-Net generator,
    a '--netD basic' discriminator (PatchGAN),
    and a '--gan_mode' vanilla GAN loss (the cross-entropy objective used in the orignal GAN paper).

    pix2pix paper: https://arxiv.org/pdf/1611.07004.pdf
    """
    @staticmethod
    def modify_commandline_options(parser, is_train=True):
        """Add new dataset-specific options, and rewrite default values for existing options.

        Parameters:
            parser          -- original option parser
            is_train (bool) -- whether training phase or test phase. You can use this flag to add training-specific or test-specific options.

        Returns:
            the modified parser.

        For pix2pix, we do not use image buffer
        The training objective is: GAN Loss + lambda_L1 * ||G(A)-B||_1
        By default, we use vanilla GAN loss, UNet with batchnorm, and aligned datasets.
        """
        # changing the default values to match the pix2pix paper (https://phillipi.github.io/pix2pix/)
        parser.set_defaults(norm='batch', netG='unet_256', dataset_mode='aligned')
        if is_train:
            parser.set_defaults(pool_size=0, gan_mode='vanilla')
            parser.add_argument('--lambda_L1', type=float, default=100.0, help='weight for L1 loss')
            parser.add_argument('--delta_perceptual', type=float, default=1.0, help='weight for perceptual loss')

            parser.add_argument('--lambda_G', type=float, default=0.1, help='weight for asyndgan G ')
            parser.add_argument('--lambda_D', type=float, default=0.05, help='weight for asyndgan D')

        return parser

    def __init__(self, opt):
        """Initialize the pix2pix class.

        Parameters:
            opt (Option class)-- stores all the experiment flags; needs to be a subclass of BaseOptions
        """
        BaseModel.__init__(self, opt)
        # specify the training losses you want to print out. The training/test scripts will call <BaseModel.get_current_losses>
        # self.loss_names = ['G_GAN', 'G_L1', 'G_Seg', 'D_real', 'D_fake']
        self.loss_names = ['G_GAN_all', 'G_L1_all', 'G_perceptual_all', 'D_real_all', 'D_fake_all']
        # specify the images you want to save/display. The training/test scripts will call <BaseModel.get_current_visuals>
        self.visual_names = ['real_A_2', 'fake_B_2', 'real_B_2','real_A_7', 'fake_B_7', 'real_B_7']
        # specify the models you want to save to the disk. The training/test scripts will call <BaseModel.save_networks> and <BaseModel.load_networks>
        if self.isTrain:
            self.model_names = ['G', 'D']
        else:  # during test time, only load G
            self.model_names = ['G']
        # define networks (both generator and discriminator)
        self.netG = networks.define_G(opt.input_nc, opt.output_nc, opt.ngf, opt.netG, opt.norm,
                                      not opt.no_dropout, opt.init_type, opt.init_gain, self.gpu_ids)

        if self.isTrain:  # define a discriminator; conditional GANs need to take both input and output images; Therefore, #channels for D is input_nc + output_nc
            self.netD = []
            # for i in range(10):
            for i in range(4):
                self.netD.append(networks.define_D(opt.input_nc + opt.output_nc, opt.ndf, opt.netD,
                                          opt.n_layers_D, opt.norm, opt.init_type, opt.init_gain, self.gpu_ids))
            # self.netD = networks.define_D(opt.input_nc + opt.output_nc, opt.ndf, opt.netD,
            #                               opt.n_layers_D, opt.norm, opt.init_type, opt.init_gain, self.gpu_ids)

        if self.isTrain:
            # define loss functions
            self.criterionGAN = networks.GANLoss(opt.gan_mode).to(self.device)
            self.criterionL1 = torch.nn.L1Loss()
            # self.criterionSeg = nn.BCEWithLogitsLoss()
            # initialize optimizers; schedulers will be automatically created by function <BaseModel.setup>.
            self.optimizer_G = torch.optim.Adam(self.netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999))
            self.optimizer_D = []
            for i in self.netD:
                opt_D = torch.optim.Adam(i.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999))
                self.optimizer_D.append(opt_D)
                self.optimizers.append(opt_D)
            # self.optimizer_D = torch.optim.Adam(self.netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999))
            self.optimizers.append(self.optimizer_G)
            # self.optimizers.append(self.optimizer_D)

            self.vgg_model = vgg16_feat().cuda()
            self.criterion_perceptual = perceptual_loss()
            # self.unet = setup_unet().cuda()

    def set_input(self, input):
        """Unpack input data from the dataloader and perform necessary pre-processing steps.

        Parameters:
            input (dict): include the data itself and its metadata information.

        The option 'direction' can be used to swap images in domain A and domain B.
        """
        AtoB = self.opt.direction == 'AtoB'
        self.real_A = []
        self.real_B = []
        self.image_paths = []
        # for i in range(10):
        for i in range(4):
            self.real_A.append(input['A_' + str(i)].to(self.device))
            self.real_B.append(input['B_' + str(i)].to(self.device))
            self.image_paths.append(input['A_paths_' + str(i)])

        self.real_A_2 = self.real_A[0]
        self.real_B_2 = self.real_B[0]
        self.real_A_7 = self.real_A[1]
        self.real_B_7 = self.real_B[1]

    def forward(self):
        """Run forward pass; called by both functions <optimize_parameters> and <test>."""
        self.fake_B = []
        for i in range(len(self.real_A)):
            self.fake_B.append(self.netG(self.real_A[i]))# G(A)
        # self.fake_B = self.netG(self.real_A)  # G(A)
        self.fake_B_2 = self.fake_B[0]
        self.fake_B_7 = self.fake_B[1]

    def backward_D(self):
        """Calculate GAN loss for the discriminator"""

        self.loss_D_fake = []
        self.loss_D_real = []
        for i in range(len(self.real_A)):
            # Fake; stop backprop to the generator by detaching fake_B
            fake_AB = torch.cat((self.real_A[i], self.fake_B[i]),1)  # we use conditional GANs; we need to feed both input and output to the discriminator
            pred_fake = self.netD[i](fake_AB.detach())
            self.loss_D_fake.append(self.criterionGAN(pred_fake, False))
            # Real
            real_AB = torch.cat((self.real_A[i], self.real_B[i]), 1)
            pred_real = self.netD[i](real_AB)
            self.loss_D_real.append(self.criterionGAN(pred_real, True))

        self.loss_D_fake_all = None
        self.loss_D_real_all = None
        for i in range(len(self.loss_D_fake)):
            if self.loss_D_fake_all is None:
                self.loss_D_fake_all = self.loss_D_fake[i]
                self.loss_D_real_all = self.loss_D_real[i]
            else:
                self.loss_D_fake_all += self.loss_D_fake[i]
                self.loss_D_real_all += self.loss_D_real[i]

        # combine loss and calculate gradients
        self.loss_D = (self.loss_D_fake_all + self.loss_D_real_all)*self.opt.lambda_D #0.05
        self.loss_D.backward()

        # # Fake; stop backprop to the generator by detaching fake_B
        # fake_AB = torch.cat((self.real_A, self.fake_B), 1)  # we use conditional GANs; we need to feed both input and output to the discriminator
        # pred_fake = self.netD(fake_AB.detach())
        # self.loss_D_fake = self.criterionGAN(pred_fake, False)
        # # Real
        # real_AB = torch.cat((self.real_A, self.real_B), 1)
        # pred_real = self.netD(real_AB)
        # self.loss_D_real = self.criterionGAN(pred_real, True)
        # # combine loss and calculate gradients
        # self.loss_D = (self.loss_D_fake + self.loss_D_real) * 0.5
        # self.loss_D.backward()

    def backward_G(self):
        """Calculate GAN and L1 loss for the generator"""
        self.loss_G_GAN = []
        self.loss_G_L1 = []
        self.loss_G_perceptual = []

        # for i in range(10):
        for i in range(4):
            # First, G(A) should fake the discriminator
            fake_AB = torch.cat((self.real_A[i], self.fake_B[i]), 1)
            pred_fake = self.netD[i](fake_AB)
            self.loss_G_GAN.append(self.criterionGAN(pred_fake, True))
            self.loss_G_L1.append(self.criterionL1(self.fake_B[i], self.real_B[i]) * self.opt.lambda_L1)

            pred_feat = self.vgg_model(self.fake_B[i])
            target_feat = self.vgg_model(self.real_B[i])
            self.loss_G_perceptual.append(self.criterion_perceptual(pred_feat, target_feat) * self.opt.delta_perceptual)

        self.loss_G_GAN_all = None
        self.loss_G_L1_all = None
        self.loss_G_perceptual_all = None
        for i in range(len(self.loss_G_GAN)):
            if self.loss_G_GAN_all is None:
                self.loss_G_GAN_all = self.loss_G_GAN[i]
                self.loss_G_L1_all = self.loss_G_L1[i]
                self.loss_G_perceptual_all = self.loss_G_perceptual[i]
            else:
                self.loss_G_GAN_all += self.loss_G_GAN[i]
                self.loss_G_L1_all += self.loss_G_L1[i]
                self.loss_G_perceptual_all += self.loss_G_perceptual[i]

        self.loss_G = (self.loss_G_GAN_all + self.loss_G_L1_all + self.loss_G_perceptual_all)*self.opt.lambda_G #0.1
        self.loss_G.backward()

        # # First, G(A) should fake the discriminator
        # fake_AB = torch.cat((self.real_A, self.fake_B), 1)
        # pred_fake = self.netD(fake_AB)
        # self.loss_G_GAN = self.criterionGAN(pred_fake, True)
        # # Second, G(A) = B
        # self.loss_G_L1 = self.criterionL1(self.fake_B, self.real_B) * self.opt.lambda_L1
        #
        # pred_feat = self.vgg_model(self.fake_B)
        # target_feat = self.vgg_model(self.real_B)
        # self.loss_G_perceptual = self.criterion_perceptual(pred_feat, target_feat) * self.opt.delta_perceptual

        # pred_seg = self.unet(self.fake_B)
        #
        # pred_seg = torch.sigmoid(pred_seg)
        #
        # self.loss_G_Seg = self.criterionSeg(pred_seg, self.seg)

        # combine loss and calculate gradients
        # self.loss_G = self.loss_G_GAN + self.loss_G_L1 + self.loss_G_Seg
        # self.loss_G = self.loss_G_GAN + self.loss_G_L1 + self.loss_G_perceptual
        # self.loss_G.backward()

    def optimize_parameters(self):
        self.forward()                   # compute fake images: G(A)
        # update D
        self.set_requires_grad(self.netD, True)  # enable backprop for D
        for opt in self.optimizer_D:
            opt.zero_grad()
        self.backward_D()
        for opt in self.optimizer_D:
            opt.step()
        # self.optimizer_D.zero_grad()     # set D's gradients to zero
        # self.backward_D()                # calculate gradients for D
        # self.optimizer_D.step()          # update D's weights
        # update G
        self.set_requires_grad(self.netD, False)  # D requires no gradients when optimizing G
        self.optimizer_G.zero_grad()        # set G's gradients to zero
        self.backward_G()                   # calculate graidents for G
        self.optimizer_G.step()             # udpate G's weights


from xai.brain.wordbase.nouns._contour import _CONTOUR

#calss header
class _CONTOURING(_CONTOUR, ):
	def __init__(self,):
		_CONTOUR.__init__(self)
		self.name = "CONTOURING"
		self.specie = 'nouns'
		self.basic = "contour"
		self.jsondata = {}


"""
This lambda function is used to send message to FIFO
for each pipe delim file created in S3 bucket
"""
import json
import boto3
import ast

sqr = boto3.resource('sqs')
sqs = boto3.client('sqs')

def lambda_handler(event, context):
    """
    Lambda start function, based on event trigger
    If event is empty or incorrect, will try to use fixed file
    The message will be sent to FIFO as soon as pipe delim file created with meta data info
    """
    srcBucket="pipedelimfiles-for-table"
    srcFile="repair_order/repair_order/RO_3612514224.csv"

    #print("event: ", event)
    try:
        srcBucket=event['Records'][0]['s3']['bucket']['name']
        srcFile=event['Records'][0]['s3']['object']['key']
    except:
        print("Will be using default values from Lambda")
    print("fileName:     ", srcFile)
    print("sourceBucket: ", srcBucket)

    if "log_table" in srcFile:
        return {
            'statusCode': 200,
            'body': json.dumps('Log Tables will not be uploaded to Redshift')
        }

    queue = sqr.get_queue_by_name(QueueName='sendToRedShiftUpload.fifo')
    response = queue.send_message(
        MessageBody= (srcBucket+":"+srcFile).strip(),
        MessageGroupId='testGrp1'
    )

    # The response is NOT a resource, but gives you a message ID and MD5
    print(response.get('MessageId'))
    print(response.get('MD5OfMessageBody'))

    return {
        'statusCode': 200,
        'body': json.dumps('Hello from Lambda!')
    }


import h5py
import cv2
import pandas as pd
import numpy as np
from tqdm import tqdm
import argparse
from sklearn.preprocessing import OneHotEncoder
from gluoncv.data import VOCDetection

from ssd import SSD300
from utils import df_iou

MAX_SCALE = 0.9
MIN_SCALE = 0.2
NUM_SCALES = 6
IMG_SIZE = 300
CHANNELS = 3
NUM_VOC_CLASSES = 20
IOU_THRESHOLD = 0.5

def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_root', default=None, help='Path to Pascal VOC dataset root')
    parser.add_argument('--out_path', default='data.h5', help='Path to output')
    parser.add_argument('--train', default=True, type=bool, help='Train or test dataset')
    parser.add_argument('--single_data', default=None, type=int, help='Select single datapoint')
    return parser.parse_args()

def create_class_encoder():
    ohe = OneHotEncoder()
    ohe.fit(np.array(range(NUM_VOC_CLASSES + 1)).reshape(-1, 1))
    return ohe

def get_anchor_boxes():
    model = SSD300(1)
    return model.anchors

def add_corner_coordinates(df):
    df['x0'] = df['center_x'] - df['w'] / 2
    df['y0'] = df['center_y'] - df['h'] / 2
    df['x1'] = df['center_x'] + df['w'] / 2
    df['y1'] = df['center_y'] + df['h'] / 2

def bounding_box_to_df(bounding_boxes, table_type):
    bounding_boxes = pd.DataFrame(bounding_boxes, columns=['center_x', 'center_y', 'w', 'h'])
    bounding_boxes[table_type + '_id'] = range(len(bounding_boxes))
    bounding_boxes['temp'] = True
    add_corner_coordinates(bounding_boxes)
    return bounding_boxes

def match_ground_truths_to_priors(bounding_boxes, class_ids, anchors, iou_threshold=0.5):
    pred_type = 'prediction'
    gt_type = 'gt'

    anchors = bounding_box_to_df(anchors, pred_type)

    ground_truth_boxes = bounding_box_to_df(bounding_boxes, gt_type)
    ground_truth_boxes['class_id'] = class_ids

    associations = pd.merge(anchors, ground_truth_boxes, how='outer', suffixes=('_' + pred_type, '_' + gt_type), on='temp')
    associations = associations.drop(columns=['temp'])
    associations['iou'] = df_iou(associations)
    associations['match'] = (associations['iou'] > iou_threshold)

    best_priors = associations.iloc[associations.groupby('prediction_id').apply(lambda g: g.iou.idxmax())]
    return best_priors[['center_x_gt', 'center_y_gt', 'w_gt', 'h_gt']], best_priors['iou'], best_priors['class_id']

def preprocess_image(image):
    return cv2.resize(image.asnumpy(), (IMG_SIZE, IMG_SIZE)) / 255.0

def convert_to_anchor_coordinates(image, bounding_boxes):
    image_width = image.shape[1]
    image_height = image.shape[0]
    x1 = bounding_boxes[:, 0]
    y1 = bounding_boxes[:, 1]
    x2 = bounding_boxes[:, 2]
    y2 = bounding_boxes[:, 3]
    abs_width = x2 - x1
    abs_height = y2 - y1
    abs_center_x = x1 + abs_width / 2
    abs_center_y = y1 + abs_height / 2
    cx = abs_center_x / image_width
    cy = abs_center_y / image_height
    w = abs_width / image_width
    h = abs_height / image_height
    return np.stack([cx, cy, w, h], axis=1)

def assign_classes(classes, matches):
    # Assign an extra class ('background') to any anchor box, where the highest match is not above IoU threshold
    classes = classes.values.reshape((-1, 1))
    classes[~matches] = NUM_VOC_CLASSES + 1
    return classes

def convert(args):
    if args.train:
        splits = [(2007, 'trainval'), (2012, 'trainval')]
    else:
        splits = [(2007, 'test')]

    gluon_dataset = VOCDetection(root=args.data_root, splits=splits)
    if args.single_data:
        rng = [args.single_data]
        num_data = 1
    else:
        rng = range(len(gluon_dataset))
        num_data = len(gluon_dataset)

    with h5py.File(args.out_path, 'w') as f:
        group = f.create_group('main')

        anchors = np.concatenate(get_anchor_boxes(), axis=0)

        num_total_priors = anchors.shape[0]

        anchors_dataset = group.create_dataset('anchors', anchors.shape, dtype='f')
        anchors_dataset[:] = anchors
        image_dataset = group.create_dataset('image', (num_data, IMG_SIZE, IMG_SIZE, 3), dtype='f')
        bounding_boxes_dataset = group.create_dataset('bounding_box', (num_data, num_total_priors, 4), dtype='f')
        match_dataset = group.create_dataset('match', (num_data, num_total_priors, 1))
        num_match_dataset = group.create_dataset('num_match', (num_data, 1), dtype='i')
        class_dataset = group.create_dataset('class', (num_data, num_total_priors, 1), dtype='i')
        #class_dataset = group.create_dataset('class', (num_data, num_total_priors, NUM_VOC_CLASSES + 1), dtype='i')
        #ohe_encoder = create_class_encoder()

        for i, data_index in tqdm(enumerate(rng)):
            image, label = gluon_dataset[data_index]
            bounding_boxes = label[:, :4]
            class_ids = label[:, 4:5]
            image_dataset[i] = preprocess_image(image)
            bounding_boxes = convert_to_anchor_coordinates(image, bounding_boxes)
            bounding_boxes, ious, classes = match_ground_truths_to_priors(bounding_boxes, class_ids, anchors)
            #class_ids = ohe_encoder.transform(class_ids)
            bounding_boxes_dataset[i] = bounding_boxes.values
            matches = ious.values.reshape((-1, 1)) > IOU_THRESHOLD
            match_dataset[i] = np.where(matches, 1.0, 0.0)
            num_match_dataset[i] = np.sum(matches)
            class_dataset[i] = assign_classes(classes, matches)

if __name__ == "__main__":
    args = get_args()
    convert(args)


#####################
#importing libraires
import cv2
import numpy as np
from matplotlib import pyplot as plt

#path
path = 'C:/A_stuff/Learning/Machine Learning/Udemy/Computer Vision OpenCV/Pictures/'

# Template Image
image = cv2.imread(path+'Shapes.png')
gray = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
cv2.imshow('identifying Image',image)
cv2.waitKey()

# -*- coding: utf-8 -*-
"""
Created on Thu Sep  6 20:07:27 2018

@author: akansal2
"""

#threshold
ret,thresh = cv2.threshold(gray,127,255,1)

#extract contours
_,contours,hierarchy = cv2.findContours(thresh.copy(),cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)

for cnt in contours:

    approx = cv2.approxPolyDP(cnt,0.01*cv2.arcLength(cnt,True),True)

    if len(approx) == 3:
        shape_name = 'Triangle'
        cv2.drawContours(image,[cnt],0,(0,255,0),-1)

        #find contour center to place text at the center
        M= cv2.moments(cnt)
        cx = int(M['m10']/M['m00'])
        cy = int(M['m01']/M['m00'])
        cv2.putText(image,shape_name,(cx-50,cy),cv2.FONT_HERSHEY_COMPLEX,1,(0,0,0),1)

    elif len(approx) == 4:
        x,y,w,h = cv2.boundingRect(cnt)
        M = cv2.moments(cnt)
        cx = int(M['m10']/M['m00'])
        cy = int(M['m01']/M['m00'])

        #check if it is square or rectangle
        if abs(w-h) == 0:
            shape_name = 'Square'
            cv2.drawContours(image,[cnt],0,(0,255,0),-1)
            cv2.putText(image,shape_name,(cx-50,cy),cv2.FONT_HERSHEY_COMPLEX,1,(0,0,0),1)

        else:
            shape_name = 'Rectangle'
            cv2.drawContours(image,[cnt],0,(0,255,0),-1)
            cv2.putText(image,shape_name,(cx-50,cy),cv2.FONT_HERSHEY_COMPLEX,1,(0,0,0),1)

    elif len(approx) == 10:
        shape_name = 'Star'
        cv2.drawContours(image,[cnt],0,(0,255,0),-1)
        M = cv2.moments(cnt)
        cx = int(M['m10']/M['m00'])
        cy = int(M['m01']/M['m00'])
        cv2.putText(image,shape_name,(cx-50,cy),cv2.FONT_HERSHEY_COMPLEX,1,(0,0,0),1)
    elif len(approx) >= 15:
        shape_name = 'Circle'
        cv2.drawContours(image,[cnt],0,(0,255,0),-1)
        M = cv2.moments(cnt)
        cx = int(M['m10']/M['m00'])
        cy = int(M['m01']/M['m00'])
        cv2.putText(image,shape_name,(cx-50,cy),cv2.FONT_HERSHEY_COMPLEX,1,(0,0,0),1)

cv2.imshow('Identified Images',image)
cv2.waitKey()

cv2.destroyAllWindows()


# -*- encoding: utf-8
"""
This test checks that all the internal links in the site (links that point to
other pages on the site) are pointing at working pages.
"""

from http_crawler import crawl

def test_no_links_are_broken(baseurl):
    responses = []
    for rsp in crawl(baseurl, follow_external_links=False):
        responses.append(rsp)

    failed_responses = [rsp for rsp in responses if rsp.status_code != 200]
    failures = [f'{rsp.url} ({rsp.status_code})' for rsp in failed_responses]
    print('\n'.join(failures))
    assert len(failures) == 0


from __future__ import print_function

import os
import sys
import json
import argparse as ap
import subprocess
from os import path
from collections import namedtuple, OrderedDict

import jedi

SOURCE_FILE_BATCH = 10

def log(msg):
    if _verbose:
        sys.stderr.write(msg + '\n')

def error(msg):
    if not _quiet:
        sys.stderr.write(msg + '\n')

def graph_wrapper(dir_, pretty=False, nSourceFilesTrunc=None):
    os.chdir(dir_)          # set working directory to be source directory

    source_files = get_source_files('.')
    if nSourceFilesTrunc is not None:
        source_files = source_files[:nSourceFilesTrunc]

    all_data = {'Defs': [], 'Refs': []}
    for i in range(0, len(source_files), SOURCE_FILE_BATCH):
        log('processing source files %d to %d of %d' % (i, i + SOURCE_FILE_BATCH, len(source_files)))
        batch = source_files[i:i + SOURCE_FILE_BATCH]

        args = ["python", "-m", "grapher.graph", "--dir", "."]
        if _verbose:
            args.append('--verbose')
        if _quiet:
            args.append('--quiet')
        if pretty:
            args.append('--pretty')
        args.append('--files')
        args.extend(batch)
        p = subprocess.Popen(args, stdout=subprocess.PIPE)
        out, err = p.communicate()
        if err is not None:
            sys.stderr.write(err)

        data = json.loads(out.decode('utf-8'))
        all_data['Defs'].extend(order_dict(dct) for dct in data['Defs'])
        all_data['Refs'].extend(order_dict(dct) for dct in data['Refs'])

    json_indent = 2 if pretty else None
    print(json.dumps(all_data, indent=json_indent))

def order_dict(dct):
    return OrderedDict(sorted(dct.items(), key=lambda x: x[0]))

def graph(dir_, source_files, pretty=False):
    os.chdir(dir_)          # set working directory to be source directory

    jedi.cache.never_clear_cache = True  # never clear caches, because running in batch

    modules_and_files = [(filename_to_module_name(f), f) for f in source_files]

    defs = []
    refs = []
    for file in source_files:
        d, r = get_defs_refs(file)
        defs += d
        refs += r

    # Add module/package defs
    for module, filename in modules_and_files:
        defs.insert(0, Def(
            Path=module.replace('.', '/'),
            Kind='module',
            Name=module.split('.')[-1],
            File=filename,
            DefStart=0,
            DefEnd=0,
            Exported=True,
            Docstring='',           # TODO: extract module/package-level doc
            Data=None,
        ))

    # De-duplicate definitions (local variables may be defined in more than one
    # place). Could do something smarter here, but for now, just take the first
    # definition that appears. (References also point to the first definition.)
    unique_defs = []
    unique_def_paths = set()
    for def_ in defs:
        if not def_.Path in unique_def_paths:
            unique_defs.append(def_)
            unique_def_paths.add(def_.Path)

    # Self-references, dedup
    unique_refs = []
    unique_ref_keys = set()
    for def_ in unique_defs:
        ref = Ref(
            DefPath=def_.Path,
            DefFile=path.abspath(def_.File),
            Def=True,
            File=def_.File,
            Start=def_.DefStart,
            End=def_.DefEnd,
            ToBuiltin=False,
        )
        ref_key = (ref.DefPath, ref.DefFile, ref.File, ref.Start, ref.End)
        if ref_key not in unique_ref_keys:
            unique_ref_keys.add(ref_key)
            unique_refs.append(ref)
    for ref in refs:
        ref_key = (ref.DefPath, ref.DefFile, ref.File, ref.Start, ref.End)
        if ref_key not in unique_ref_keys:
            unique_ref_keys.add(ref_key)
            unique_refs.append(ref)

    json_indent = 2 if pretty else None
    # Use OrderedDict to have reproducible text outputs.
    dct = {'Defs': [order_dict(d.__dict__) for d in unique_defs],
           'Refs': [order_dict(r.__dict__) for r in unique_refs]}
    print(json.dumps(order_dict(dct), indent=json_indent))

def get_source_files(dir_):
    source_files = []
    for dirpath, dirnames, filenames in os.walk(dir_):
        rel_dirpath = os.path.relpath(dirpath, dir_)
        for filename in filenames:
            if os.path.splitext(filename)[1] == '.py':
                source_files.append(os.path.normpath(os.path.join(rel_dirpath, filename)))
    return source_files

def get_defs_refs(file_path):
    defs, refs = [], []

    # Get a clean UTF8 source.
    linecoler = LineColToOffConverter(jedi.Script(path=file_path).source)
    names = jedi.names(path=file_path, all_scopes=True, references=True)
    for name in names:
        if name.is_definition():
            def_ = jedi_def_to_def(name, file_path, linecoler)
            defs.append(def_)
        else:
            try:
                full_name = full_name_of_def(name, from_ref=True)
                if full_name == '':
                    raise Exception('full_name is empty')
                start = linecoler.convert(name.line, name.column)
                refs.append(Ref(
                    DefPath=full_name.replace('.', '/'),
                    DefFile=path.relpath(name.module_path),
                    Def=False,
                    File=file_path,
                    Start=start,
                    End=start + len(name.name),
                    ToBuiltin=name.in_builtin_module(),
                ))
            except Exception as e:
                error('failed to convert ref (%s) in source file %s: %s'
                      % (name, file_path, e))

    return defs, refs

def jedi_def_to_def(def_, source_file, linecoler):
    full_name = full_name_of_def(def_)

    start = linecoler.convert(def_.line, def_.column)

    return Def(
        Path=full_name.replace('.', '/'),
        Kind=def_.type,
        Name=def_.name,
        File=path.relpath(def_.module_path),
        DefStart=start,
        DefEnd=start + len(def_.name),
        Exported=True,          # TODO: not all vars are exported
        Docstring=def_.docstring(),
        Data=None,
    )

def full_name_of_def(def_, from_ref=False):
    # TODO: This function
    # - currently fails for tuple assignments (e.g., 'x, y = 1, 3')
    # - doesn't distinguish between m(module).n(submodule) and m(module).n(contained-variable)

    if def_.in_builtin_module():
        return def_.name

    if def_.type == 'statement':
        # kludge for self.* definitions
        if def_.parent().type == 'function' and def_.name == 'self':
            parent = def_.parent()
            while parent.type != 'class':
                parent = parent.parent()
            full_name = ('%s.%s' % (parent.full_name, def_.name))
        else:
            full_name = ('%s.%s' % (def_.full_name, def_.name))
    elif def_.type == 'param':
        full_name = ('%s.%s' % (def_.full_name, def_.name))
    else:
        full_name = def_.full_name

    module_path = os.path.relpath(def_.module_path)
    if from_ref:
        module_path = abs_module_path_to_relative_module_path(module_path)

    supermodule = supermodule_path(module_path).replace('/', '.')

    # definition definitions' full_name property contains only the promixal module, so we need to add back the parent
    # module components. Luckily, the module_path is relative in this case.
    return path.join(supermodule, full_name)

def supermodule_path(module_path):
    if path.basename(module_path) == '__init__.py':
        return path.dirname(path.dirname(module_path))
    return path.dirname(module_path)

def abs_module_path_to_relative_module_path(module_path):
    relpath = path.relpath(module_path)  # relative from pwd (which is set in main)
    if not relpath.startswith('..'):
        return relpath
    components = module_path.split(os.sep)
    pIdx = -1
    for i, cmpt in enumerate(components):
        if cmpt in ['site-packages', 'dist-packages']:
            pIdx = i
            break
    if pIdx != -1:
        return path.join(*components[i + 1:])

    for i, cmpt in enumerate(components):
        if cmpt.startswith('python'):
            pIdx = i
            break
    if pIdx != -1:
        return path.join(*components[i + 1:])
    raise Exception("could not convert absolute module path %s to relative module path" % module_path)

Def = namedtuple('Def', ['Path', 'Kind', 'Name', 'File', 'DefStart', 'DefEnd', 'Exported', 'Docstring', 'Data'])
Ref = namedtuple('Ref', ['DefPath', 'DefFile', 'Def', 'File', 'Start', 'End', "ToBuiltin"])

def filename_to_module_name(filename):
    if path.basename(filename) == '__init__.py':
        return path.dirname(filename).replace('/', '.')
    return path.splitext(filename)[0].replace('/', '.')

class LineColToOffConverter(object):
    def __init__(self, source):
        source_lines = source.split('\n')
        cumulative_off = [0]
        for line in source_lines:
            cumulative_off.append(cumulative_off[-1] + len(line) + 1)
        self._cumulative_off = cumulative_off

    # Converts from (line, col) position to byte offset. line is 1-indexed, col is 0-indexed
    def convert(self, line, column):
        line = line - 1  # convert line to 0-indexed
        if line >= len(self._cumulative_off):
            return None, 'requested line out of bounds %d > %d' % (line + 1, len(self._cumulative_off) - 1)
        return self._cumulative_off[line] + column

if __name__ == '__main__':
    argser = ap.ArgumentParser(description='graph.py is a command that dumps all Python definitions and references found in code rooted at a directory')
    argser.add_argument('--dir', help='path to root directory of code')
    argser.add_argument('--files', help='path code files', nargs='+')
    argser.add_argument('--pretty', help='pretty print JSON output', action='store_true', default=False)
    argser.add_argument('--verbose', help='verbose', action='store_true', default=False)
    argser.add_argument('--quiet', help='quiet', action='store_true', default=False)
    argser.add_argument('--maxfiles', help='maximum number of files to process', default=None, type=int)
    args = argser.parse_args()

    _verbose, _quiet = args.verbose, args.quiet
    if args.files is not None and len(args.files) > 0:
        graph(args.dir, args.files, pretty=args.pretty)
    elif args.dir is not None and args.dir != '':
        graph_wrapper(args.dir, pretty=args.pretty, nSourceFilesTrunc=args.maxfiles)
    else:
        error('target directory must not be empty')
        sys.exit(1)


def summarize_card_data(cards):
    cmc_data = [0 for x in range(9)]
    color_data = [0 for x in range(6)]
    total_cards_data = 0
    for card in cards:
        if card.type_land:
            # skip lands because they skew the data
            continue
        total_cards_data += 1
        if card.color_white + card.color_blue + card.color_black + card.color_red + card.color_green == 0:
            color_data[5] += 1
        else:
            color_data[0] += card.color_white
            color_data[1] += card.color_blue
            color_data[2] += card.color_black
            color_data[3] += card.color_red
            color_data[4] += card.color_green
        if card.cmc < 8:
            cmc_data[card.cmc] += 1
        else:
            cmc_data[8] += 1
    cmc_data_formatted = ",".join([str(x) for x in cmc_data])
    color_data_formatted = ",".join([str(x) for x in color_data])
    return {'cmc_data_formatted': cmc_data_formatted,
            'color_data_formatted': color_data_formatted,
            'total_cards_data': total_cards_data,}

def display_stats(wins, losses, draws, percentage):
        display_percentage = "{0:.0f}%".format(percentage*100)
        return '{}-{}-{} ({})'.format(wins,
                                                                                losses,
                                                                                draws,
                                                                                display_percentage)

def build_total_ordering_cards(Card):
    cards = Card.objects.all().order_by('type_land','-color_monocolor',
                    'color_numerical','-type_creature',
                    '-type_planeswalker','type_artifact',
                    'type_enchantment','type_sorcery',
                    'type_instant','cmc','name')
    cur = 1
    for card in cards:
        card.total_ordering = cur
        card.save()
        cur += 1


from PyQt5.QtWidgets import QApplication,QWidget
import sys
from RSA_ui import Ui_Form
import random
import math

class RSA(QWidget, Ui_Form):
    def __init__(self):
        super(RSA, self).__init__()
        self.setupUi(self)
        self.setupSignal()
        self.key_length = 1024

    def setupSignal(self):
        self.encry.clicked.connect(self.onEncryptionClicked)
        self.decry.clicked.connect(self.onDecryptionClicked)
        self.clear_1.clicked.connect(self.onClear1Clicked)
        self.clear_2.clicked.connect(self.onClear2Clicked)
        self.pushButton.clicked.connect(self.genKey)

    def onClear1Clicked(self):
        self.textEdit_1.clear()
        self.textBrowser_1.clear()

    def onClear2Clicked(self):
        self.textEdit_2.clear()
        self.textBrowser_2.clear()

    def fast_exp_mode(self, a, b, c):
        res = 1
        a = a % c
        while b != 0:
            if b % 2 == 1:
                res = (res * a) % c
            b >>= 1
            a = (a * a) % c
        return res

    def extend_gcd(self, a, b):
        if b == 0:
            return (1, 0, a)
        x1 = 1
        y1 = 0

        x2 = 0
        y2 = 1
        while b != 0:
            q = a // b
            r = a % b
            a = b
            b = r
            x = x1 - q * x2
            x1 = x2
            x2 = x

            y = y1 - q*y2
            y1 = y2
            y2 = y
        return (x1, y1, a)

    def find_E_D(self, ln, bit):
        while 1:
            e = random.randint(0, 1 << bit)#e随机产生
            x,y,r =  self.extend_gcd(e, ln)#e与ln需要互质
            if r == 1:#e与ln就是互质了
                d = x if x >= 0 else x+ln
                return (e, d)

    def find_prime(self, bit):
        while True:
            n = random.randint(0, 1 << bit)
            if n % 2 != 0:
                found = 1
                for i in range(0, 10):
                    if self.prime_test(n) == 0:
                        found = 0
                        break
                if found:
                    return n

    def prime_test(self, number):
        q = number - 1
        k = 0
        while q % 2 == 0:
            k += 1
            q = q // 2
        a = random.randint(2, number-2)

        if self.fast_exp_mode(a, q, number) == 1:
            return 1

        for j in range(0, k):
            if self.fast_exp_mode(a, (2**j)*q, number) == number-1:
                return 1

        return 0

    def genKey(self):
        p = self.find_prime(int(self.key_length/2))
        q = self.find_prime(int(self.key_length/2))
        self.n = p * q
        ln = (p-1) * (q-1)
        self.e, self.d = self.find_E_D(ln, int(self.key_length/2))
        self.key.setText("公钥：(%x, %x)       私钥：(%x, %x)"%(self.n, self.e, self.n, self.d))

    def string2int(self, m):
        result = []
        res = 0
        l = self.key_length // 8
        i = 0
        for c in m:
            res <<= 8
            res |= ord(c)
            i += 1
            if i % l == 0:
                result.append(res)
                res = 0
        if i%l != 0:
            result.append(res)
        return result

    def int2string(self, ints):
        result = []
        while ints:
            ch_int = ints & (0xFF)
            result.insert(0, chr(ch_int))
            ints >>= 8
        return ''.join(result)

    def onEncryptionClicked(self):
        m = self.textEdit_1.toPlainText()
        if m:
            if len(m) > self.key_length:
                m = m[:self.key_length]
            int_number = self.string2int(m)[0]
            result = []
            encry_text = self.fast_exp_mode(int_number, self.e, self.n)
            hex_text = "%x"%encry_text
            result.append(hex_text)
            self.textBrowser_1.setText(''.join(result))

    def onDecryptionClicked(self):
        m = self.textEdit_2.toPlainText()
        if m:
            ints = int(m, 16)
            ints = self.fast_exp_mode(ints, self.d, self.n)
            text = self.int2string(ints)
            self.textBrowser_2.setText(text)

if __name__ == "__main__":
    app = QApplication(sys.argv)
    rsa = RSA()
    rsa.show()
    sys.exit(app.exec_())

非对称加密算法实现/非对称加密---RSA/RSA.py


import pandas as pd
import numpy as np

if __name__ == "__main__":
    data_path = "../data/"
    categories = pd.read_json(data_path+"category_2018.json", lines=True, chunksize=1000)
    big = pd.read_csv(data_path+"category_big_2018.csv", names=['cb_id', 'category'])
    pre_df = pd.DataFrame(columns=['category', 'cb_id'])

    info = dict()
    for idx, row in big.iterrows():
        info[row['category']] = row['cb_id']
    print(info)
    for idx, category in enumerate(categories):
        print(f"{idx} done")
        for idx, row in category.iterrows():
            if len(row['category']) <= 2:
                pre_df = pre_df.append(pd.Series(['Extra',  int(info[row['category'][1]])], index=pre_df.columns), ignore_index=True)
            else :
                pre_df = pre_df.append(pd.Series([row['category'][2], int(info[row['category'][1]])], index=pre_df.columns), ignore_index=True)
            pre_df.drop_duplicates(inplace=True)
    pre_df.index = np.arange(1, len(pre_df)+1)
    pre_df.to_csv("../data/category_mid_2018.csv")


import sys
# inFile = open(sys.stdin, 'r', encoding='utf8')
# outFile = open('output.txt', 'w', encoding='utf8')
p = []
c = []
a = set()
b = set()
# f = sys.stdin.read()
# for line in sys.stdin:
#     print(line.rstrip())
for line in sys.stdin:
        c = line.split()
        for i in range(len(c)):
            a.add(c[i].strip())
# print(a)
print(len(a))


import gensim
# Operating System
import os
# Regular Expression
import re
# nltk packages
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.stem.porter import *
from nltk.stem.snowball import SnowballStemmer
# Basic Packages
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
# PCA Package
from sklearn.decomposition import PCA
stemmer = SnowballStemmer("english")
from nltk.tokenize import word_tokenize, sent_tokenize
import requests #for making HTTP requests in Python

# !pip install sentence-transformers
# !pip install huggingface-hub==0.0.12
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity

# download the stopword for the english
stopWords = pd.read_csv('stopwords.txt').values

# define the function to get the text from the url
def crawler(url):
    webpage = requests.get(url)
    webpagetext = BeautifulSoup(webpage.text, "html.parser")
    all_p = webpagetext.find_all('p')
    text = ""
    for item in all_p:
        text = text + item.get_text()
    return text
# wiki_text = crawler(url_wiki)

class MySentences(object):
    def __init__(self, fnamelist):
        self.fnamelist = fnamelist
        # Creating a set of vocabulary
        self.vocabulary = set([])

    def __iter__(self):
        for fname in self.fnamelist:
            for line in open(fname, encoding='latin1'):
                # Find all the words that has letters from 2 - 15. If the words are longer than that ignore.
                words = re.findall(r'(\b[A-Za-z][a-z]{2,15}\b)', line)
                # Stemming a word.
                words = [ stemmer.stem(word.lower()) for word in words if not word.lower() in stopWords]
                for word in words:
                    self.vocabulary.add(word)
                yield words

def get_reward(url, wiki_url):

	# web crawling, get the text for the both link
	url_text = crawler(url)
	wiki_text = crawler(wiki_url)

	# tokenized the text of  both websites
	tokenized_url = sent_tokenize(url_text)
	tokenized_wiki = sent_tokenize(wiki_text)

	sentences_url = MySentences(tokenized_url) # a memory-friendly iterator
	sentences_wiki = MySentences(tokenized_wiki) # a memory-friendly iterator
	model = SentenceTransformer('bert-base-nli-mean-tokens')

	sentences = [sentences_url, sentences_wiki]
	sentence_embeddings = model.encode(sentences)
	similarity = cosine_similarity( [sentence_embeddings[0]], sentence_embeddings[1:])

	return similarity

# sentences_nvd = MySentences(text_nvd) # a memory-friendly iterator
# sentences_wiki = MySentences(text_wiki) # a memory-friendly iterator

# text_nvd = crawler(url_nvd)    # input nvd url for the keyword e.g., Injection, return the text from the nvd website
# text_wiki = crawler(url_wiki)  # input wiki url for the keyword e.g., Injection, return the text from the wiki website

# tokenized_nvd = sent_tokenize(text_nvd)
# tokenized_wiki = sent_tokenize(text_wiki)

# sentences_nvd = MySentences(tokenized_nvd) # a memory-friendly iterator
# sentences_wiki = MySentences(tokenized_wiki) # a memory-friendly iterator

# # # Initialize our model:
# # from sentence_transformers import SentenceTransformer
# # from sklearn.metrics.pairwise import cosine_similarity

# model = SentenceTransformer('bert-base-nli-mean-tokens')

# sentences = [sentences_nvd, sentences_wiki]
# sentence_embeddings = model.encode(sentences)
# # sentence_embeddings.shape ##

# # calculate the similarity for the text
# # Let's calculate cosine similarity for sentence 0:
# similarity = cosine_similarity(
#     [sentence_embeddings[0]],
#     sentence_embeddings[1:])
# similarity


# Python routines useful for handling ikg's LEGO robot data.
# Author: Claus Brenner, 28.10.2012
from math import sin, cos, pi

# In previous versions, the S record included the number of scan points.
# If so, set this to true.
s_record_has_count = True

# Class holding log data of our Lego robot.
# The logfile understands the following records:
# P reference position (of the robot)
# S scan data
# I indices of poles in the scan data (determined by an external algorithm)
# M motor (ticks from the odometer) data
# F filtered data (robot position, or position and heading angle)
# E (error) standard deviation in position, or position and heading angle
# L landmark (reference landmark, fixed)
# D detected landmark, in the scanner's coordinate system. C is cylinders
# W something to draw in the world coordinate system. C is cylinders.
#
class LegoLogfile(object):
    def __init__(self):
        self.reference_positions = []
        self.scan_data = []
        self.pole_indices = []
        self.motor_ticks = []
        self.filtered_positions = []
        self.filtered_stddev = []
        self.landmarks = []
        self.detected_cylinders = []
        self.world_cylinders = []
        self.last_ticks = None

    def read(self, filename):
        """Reads log data from file. Calling this multiple times with different
           files will result in a merge of the data, i.e. if one file contains
           M and S data, and the other contains M and P data, then LegoLogfile
           will contain S from the first file and M and P from the second file."""
        # If information is read in repeatedly, replace the lists instead of appending,
        # but only replace those lists that are present in the data.
        first_reference_positions = True
        first_scan_data = True
        first_pole_indices = True
        first_motor_ticks = True
        first_filtered_positions = True
        first_filtered_stddev = True
        first_landmarks = True
        first_detected_cylinders = True
        first_world_cylinders = True
        f = open(filename)
        for l in f:
            sp = l.split()
            # P is the reference position.
            # File format: P timestamp[in ms] x[in mm] y[in mm]
            # Stored: A list of tuples [(x, y), ...] in reference_positions.
            if sp[0] == 'P':
                if first_reference_positions:
                    self.reference_positions = []
                    first_reference_positions = False
                self.reference_positions.append( (int(sp[2]), int(sp[3])) )

            # S is the scan data.
            # File format:
            #  S timestamp[in ms] distances[in mm] ...
            # Or, in previous versions (set s_record_has_count to True):
            #  S timestamp[in ms] count distances[in mm] ...
            # Stored: A list of tuples [ [(scan1_distance,... ), (scan2_distance,...) ]
            #   containing all scans, in scan_data.
            elif sp[0] == 'S':
                if first_scan_data:
                    self.scan_data = []
                    first_scan_data = False
                if s_record_has_count:
                    self.scan_data.append(tuple(map(int, sp[3:])))
                else:
                    self.scan_data.append(tuple(map(int, sp[2:])))

            # I is indices of poles in the scan.
            # The indices are given in scan order (counterclockwise).
            # -1 means that the pole could not be clearly detected.
            # File format: I timestamp[in ms] index ...
            # Stored: A list of tuples of indices (including empty tuples):
            #  [(scan1_pole1, scan1_pole2,...), (scan2_pole1,...)...]
            elif sp[0] == 'I':
                if first_pole_indices:
                    self.pole_indices = []
                    first_pole_indices = False
                self.pole_indices.append(tuple(map(int, sp[2:])))

            # M is the motor data.
            # File format: M timestamp[in ms] pos[in ticks] tachoCount[in ticks] acceleration[deg/s^2] rotationSpeed[deg/s] ...
            #   (4 values each for: left motor, right motor, and third motor (not used)).
            # Stored: A list of tuples [ (inc-left, inc-right), ... ] with tick increments, in motor_ticks.
            # Note that the file contains absolute ticks, but motor_ticks contains the increments (differences).
            elif sp[0] == 'M':
                ticks = (int(sp[2]), int(sp[6]))
                if first_motor_ticks:
                    self.motor_ticks = []
                    first_motor_ticks = False
                    self.last_ticks = ticks
                self.motor_ticks.append(
                    tuple([ticks[i]-self.last_ticks[i] for i in range(2)]))
                self.last_ticks = ticks

            # F is filtered trajectory. No time stamp is used.
            # File format: F x[in mm] y[in mm]
            # OR:          F x[in mm] y[in mm] heading[in radians]
            # Stored: A list of tuples, each tuple is (x y) or (x y heading)
            elif sp[0] == 'F':
                if first_filtered_positions:
                    self.filtered_positions = []
                    first_filtered_positions = False
                self.filtered_positions.append( tuple( map(float, sp[1:])) )

            # E is error of filtered trajectory. No time stamp is used.
            # File format: E (angle of main axis)[in radians] std-dev1 std-dev2
            # OR:          The same format but with std-dev-heading appended.
            # Note: std-dev1 is along the main axis, std-dev2 is along the
            # second axis, which is orthogonal to the main axis.
            # Stored: A list of tuples, each tuple is
            # (angle, std-dev1, std-dev2) or
            # (angle, std-dev1, std-dev2, std-dev-heading).
            elif sp[0] == 'E':
                if first_filtered_stddev:
                    self.filtered_stddev = []
                    first_filtered_stddev = False
                self.filtered_stddev.append( tuple( map(float, sp[1:])) )

            # L is landmark. This is actually background information, independent
            # of time.
            # File format: L <type> info...
            # Supported types:
            # Cylinder: L C x y diameter.
            # Stored: List of (<type> info) tuples.
            elif sp[0] == 'L':
                if first_landmarks:
                    self.landmarks = []
                    first_landmarks = False
                if sp[1] == 'C':
                    self.landmarks.append( tuple(['C'] + list(map(float, sp[2:]))) )

            # D is detected landmarks (in each scan).
            # File format: D <type> info...
            # Supported types:
            # Cylinder: D C x y x y ...
            #   Stored: List of lists of (x, y) tuples of the cylinder positions,
            #   one list per scan.
            elif sp[0] == 'D':
                if sp[1] == 'C':
                    if first_detected_cylinders:
                        self.detected_cylinders = []
                        first_detected_cylinders = False
                    cyl = map(float, sp[2:])
                    self.detected_cylinders.append([(cyl[2*i], cyl[2*i+1]) for i in range(len(cyl)/2)])

            # W is information to be plotted in the world (in each scan).
            # File format: W <type> info...
            # Supported types:
            # Cylinder: W C x y x y ...
            #   Stored: List of lists of (x, y) tuples of the cylinder positions,
            #   one list per scan.
            elif sp[0] == 'W':
                if sp[1] == 'C':
                    if first_world_cylinders:
                        self.world_cylinders = []
                        first_world_cylinders = False
                    cyl = list(map(float, sp[2:]))
                    self.world_cylinders.append([(cyl[2*i], cyl[2*i+1]) for i in range(int(len(cyl)/2))])

        f.close()

    def size(self):
        """Return the number of entries. Take the max, since some lists may be empty."""
        return max(len(self.reference_positions), len(self.scan_data),
                   len(self.pole_indices), len(self.motor_ticks),
                   len(self.filtered_positions), len(self.filtered_stddev),
                   len(self.detected_cylinders), len(self.world_cylinders))

    @staticmethod
    def beam_index_to_angle(i, mounting_angle = -0.06981317007977318):
        """Convert a beam index to an angle, in radians."""
        return (i - 330.0) * 0.006135923151543 + mounting_angle

    @staticmethod
    def scanner_to_world(pose, point):
        """Given a robot pose (rx, ry, heading) and a point (x, y) in the
           scanner's coordinate system, return the point's coordinates in the
           world coordinate system."""
        dx = cos(pose[2])
        dy = sin(pose[2])
        x, y = point
        return (x * dx - y * dy + pose[0], x * dy + y * dx + pose[1])

    def info(self, i):
        """Prints reference pos, number of scan points, and motor ticks."""
        s = ""
        if i < len(self.reference_positions):
            s += " | ref-pos: %4d %4d" % self.reference_positions[i]

        if i < len(self.scan_data):
            s += " | scan-points: %d" % len(self.scan_data[i])

        if i < len(self.pole_indices):
            indices = self.pole_indices[i]
            if indices:
                s += " | pole-indices:"
                for idx in indices:
                    s += " %d" % idx
            else:
                s += " | (no pole indices)"

        if i < len(self.motor_ticks):
            s += " | motor: %d %d" % self.motor_ticks[i]

        if i < len(self.filtered_positions):
            f = self.filtered_positions[i]
            s += " | filtered-pos:"
            for j in (0,1):
                s += " %.1f" % f[j]
            if len(f) > 2:
                s += " %.1f" % (f[2] / pi * 180.)

        if i < len(self.filtered_stddev):
            stddev = self.filtered_stddev[i]
            s += " | stddev:"
            # Print stddev in both axes, and theta, if present.
            # Don't print the orientation angle stddev[0].
            for j in (1,2):
                s += " %.1f" % stddev[j]
            if len(stddev) > 3:
                s += " %.1f" % (stddev[3] / pi * 180.)

        return s


# -*- coding: utf-8 -*-
"""
Editor de Spyder

Este es un archivo temporal.
"""

from scrapy.item import Item, Field
from scrapy.spiders import CrawlSpider
from scrapy.spiders import Rule
from scrapy.linkextractors import LinkExtractor
from scrapy.loader.processors import Join
from scrapy.loader.processors import MapCompose
from scrapy.loader import ItemLoader
from bs4 import BeautifulSoup

class AptitusItem(Item):
    empleo = Field()
    empresa = Field()
    ubicacion = Field()
    sueldo = Field()
    descripcion = Field()
    requisitos = Field()

class AptitusCrawler(CrawlSpider):
    name = "aptituscrawler"
    start_urls = ['https://aptitus.com/buscar-trabajo-en-peru']
    allowed_domains = ['aptitus.com']

    rules = (
        Rule(LinkExtractor(allow = r'page='), follow=True),
        Rule(LinkExtractor(allow = r'/ofertas-de-trabajo'), follow=True, callback='parse_items'),
    )

    def parse_items(self, response):
        item = ItemLoader(AptitusItem(), response)
        item.add_xpath('empleo', '/html/body/div[3]/div/div[4]/div[1]/div[2]/div[1]/h1/text()')
        item.add_xpath('empresa', '/html/body/div[3]/div/div[4]/div[1]/div[2]/div[1]/div[3]/div/a/text()')
        item.add_xpath('ubicacion', '/html/body/div[3]/div/div[4]/div[1]/div[2]/div[1]/div[3]/div/div/ul[1]/li[1]/a/text()')
        item.add_xpath('sueldo', '/html/body/div[3]/div/div[4]/div[1]/div[2]/div[1]/div[3]/div/div/ul[1]/li[3]/span[2]/text()')
        item.add_xpath('descripcion', '/html/body/div[3]/div/div[4]/div[1]/div[2]/div[3]/div[1]/div[1]/div[2]/div/p[2]/text()')
        item.add_xpath('requisitos', '/html/body/div[3]/div/div[4]/div[1]/div[2]/div[3]/div[1]/div[2]/div[2]/div[1]/div/text()')

        yield item.load_item()


# context.area: VIEW_3D
import bpy
import random
import sys, os, importlib
if 'damageutils' in sys.modules: importlib.reload(sys.modules['damageutils'])
extra_path = os.environ["BLENDER_UTILITIES"]
if extra_path not in sys.path: sys.path.append(extra_path)
import damageutils

def create(seed_in: int, project=True):
    ob = bpy.context.active_object
    bm = damageutils.initialize_bmesh(ob.data) # creates a BMesh object (enables BVHTree and neighbor algo)
    vcount = len(ob.data.vertices) - 200 # limit for sampling random vertices

    used = damageutils.voronoi_crack(bm.verts[random.randint(0, vcount)], set([]), -0.15, 2, seed_in, 3000, use_x=False, use_y=True, use_z=False, project=False)

    bm.to_mesh(ob.data) # Writes changed data to mesh -- if this isn't called nothing is saved

    # Scene setup
    mat = bpy.data.materials.get("Neutral") # Choices -- Concrete, Deterioration, Bricks
    bpy.data.materials["Neutral"].node_tree.nodes["Principled BSDF"].inputs[0].default_value = (random.random(), random.random(), random.random(), 0.5)
    ob.data.materials.append(mat)
    bpy.ops.object.light_add(type='SUN', radius=1, location=(-4, 4, 4), rotation=(-1, 0, 0.7))
    bpy.context.object.data.energy = 3
    bpy.ops.object.camera_add(enter_editmode=False, align='CURSOR', location=(0, 4.88, 0), rotation=(1.57, 0, 3.14))
    bpy.context.object.data.lens = 23
    bpy.context.scene.camera = bpy.data.objects["Camera"]

    if project: # projects the crack onto a 2d plane -- requires post-processing to be useful
        used_final = []
        [used_final.append(bmv) for bmv in used if bmv not in used_final]
        for v in used_final:
            v.co.y += 0.15
        damageutils.project_crack(bpy.data.objects['Camera'], used_final, '../batch_output/' + str(render_id) + '.csv')

for render_id in range(1, 1000 + 1):
    damageutils.set_resolution(128, 128)
    seed = random.randint(0, 10000)
    random.seed(seed)
    damageutils.clean()
    damageutils.plane(8, 7)
    create(seed)
    #damageutils.write_csv(bpy.data.objects.get('Cylinder'), '../mesh_data/%s.csv' % render_id)
    bpy.context.scene.render.filepath = os.path.dirname(os.getcwd()) + "/batch/" + str(render_id) + ".png"
    bpy.ops.render.render(write_still=True)
    print("Completed full execution %d" % render_id)


#!/usr/bin/python
# -*- coding: utf-8 -*-

# django-pyvows extensions
# https://github.com/rafaelcaricio/django-pyvows

# Licensed under the MIT license:
# http://www.opensource.org/licenses/mit-license
# Copyright (c) 2011 Rafael Caricio rafael@caricio.com

from pyvows import Vows, expect

from lxml.cssselect import CSSSelector
from lxml.etree import fromstring

from django.http import HttpResponse
from django.template.loader import render_to_string

class Url(object):
    def __init__(self, context, path):
        self.context = context
        self.path = path

class Model(object):
    def __init__(self, context, model):
        self.context = context
        self.model = model

class Template(object):
    def __init__(self, template_name, context):
        self.template_name = template_name
        self.context = context
        self.doc = None

    def load(self):
        if self.doc is None:
            self.doc = fromstring(render_to_string(self.template_name, self.context))
        return self.doc

    def select_element(self, selector):
        sel = CSSSelector(selector)
        return sel(self.load())

    def _to_contain(self, selector):
        return len(self.select_element(selector)) > 0

    def get_text(self, selector):
        return "".join((c.text for c in self.select_element(selector)))

    def __unicode__(self):
        return self.template_name

@Vows.assertion
def to_be_mapped(topic):
    verify_url_is_mapped_to_method(topic)

@Vows.assertion
def to_match_view(topic, view):
    verify_url_is_mapped_to_method(topic, view, True)

def verify_url_is_mapped_to_method(topic, method=None, assert_method_as_well=False):
    assert isinstance(topic, Url), "Only django_pyvows.Url items can be verified for mapping"

    from django.conf import settings
    project_urls = settings.ROOT_URLCONF

    urlpatterns = __import__(project_urls).urls.urlpatterns

    found = False
    matches_method = False
    for urlpattern in urlpatterns:
        regex = urlpattern.regex
        pattern = regex.pattern
        actual_method = urlpattern.callback

        if topic.path == pattern:
            found = True
        if method == actual_method:
            matches_method = True

    assert found, "Expected url(%s) to be mapped but it wasn't"
    if assert_method_as_well:
        assert matches_method, "Expected url(%s) to match method(%s), but it didn't"

@Vows.assertion
def to_be_http_response(topic):
    expect(topic).to_be_instance_of(HttpResponse)

@Vows.assertion
def to_have_contents_of(topic, expected):
    expect(topic.content).to_be_like(expected)

@Vows.assertion
def to_contain(topic, selector):
    assert isinstance(topic, Template), "Only django_pyvows.Template items can be verified for mapping"
    assert topic._to_contain(selector), "Expected template(%s) to have an element(%s), but it don't have" % \
            (unicode(topic), selector)

@Vows.assertion
def not_to_contain(topic, selector):
    assert isinstance(topic, Template), "Only django_pyvows.Template items can be verified for mapping"
    assert not topic._to_contain(selector), "Expected template(%s) to not have an element(%s), but it have" % \
            (unicode(topic), selector)

@Vows.assertion
def to_be_cruddable(topic, defaults={}):
    import django.db.models.fields as fields
    instance = __create_or_update_instance(topic, None, defaults)

    assert instance, "An instance could not be created for model %s" % topic.model.__name__

    retrieved = topic.model.objects.get(id=instance.id)
    assert retrieved.id == instance.id, "An instance could not be retrieved for model %s with id %d" % (topic.model.__name__, instance.id)

    for key, value in defaults.iteritems():
        assert value == getattr(retrieved, key), "The default specified value of '%s' should have been set in the '%s' property of the instance but it was not" % (value, key)

    updated = __create_or_update_instance(topic, retrieved, defaults)

    for field, value in topic.model._meta._field_cache:
        if field.__class__ == fields.AutoField:
            continue

        if field.name in defaults:
            continue

        assert getattr(updated, field.name) != getattr(instance, field.name), "The instance should have been updated but the field %s is the same in both the original instance and the updated one (%s)." % (field.name, getattr(updated, field.name))

    instance.delete()
    object_count = topic.model.objects.count()
    assert object_count == 0, "Object should have been deleted, but it wasn't (count: %d)" % object_count

def __create_or_update_instance(topic, instance, defaults):
    import django.db.models.fields as fields
    arguments = {}
    for field, value in topic.model._meta._field_cache:
        if field.__class__ == fields.AutoField:
            continue

        if field.name in defaults:
            arguments[field.name] = defaults[field.name]
            continue

        if field.__class__ == fields.CharField:
            __add_char_value_for(field, instance, arguments)

    if instance:
        for key, value in arguments.iteritems():
            setattr(instance, key, value)

        instance.save()
        return instance

    return topic.model.objects.create(**arguments)

def __add_char_value_for(field, instance, arguments):
    value = "monty python"
    if instance:
        value = getattr(instance, field.name) + '2'
    if field.max_length:
        if instance:
            value = value[:len(value) - 2] + '2'
        value = (value * field.max_length)[:field.max_length]
    arguments[field.name] = value