########################
#file: zbw_snap.py
#Author: zeth willie
#Contact: zeth@catbuks.com, www.williework.blogspot.com
#Date Modified: 01/17/17
#To Use: type in python window  "import zbw_snap; zbw_snap.snap()"
#Notes/Descriptions: use to simply snap one object to another. Option for translate, rotate or both. Two options for multi-select: either snap all to the first selection or snap the last selection to the average of the first selections
########################

import maya.cmds as cmds
from functools import partial

widgets = {}

def snapUI():
    """simple snap UI for snapping"""

    if cmds.window("snapWin", exists=True):
        cmds.deleteUI("snapWin", window=True)
        cmds.windowPref("snapWin", remove=True)

    widgets["win"] = cmds.window("snapWin", t="zbw_snap", w=210, h=100, rtf=True)
    widgets["mainCLO"] = cmds.columnLayout(w=210, h=100)
    cmds.text("Select the target object(s),\nthen the object(s) you want to snap", al="center", w=210)
    cmds.separator(h=5, style="single")
    widgets["cbg"] = cmds.checkBoxGrp(l="Options: ", ncb=2, v1=1, v2=1, l1="Translate", l2="Rotate", cal=[(1,"left"),(2,"left"), (3,"left")], cw=[(1,50),(2,75),(3,75)])
    widgets["avgRBG"] = cmds.radioButtonGrp(nrb=2, l1="Snap all to first", l2="Snap last to avg", cal=[(1,"left"),(2,"left"),(3,"left")], cw=[(1,100),(2,100)],sl=1)
    widgets["rpCB"] = cmds.checkBox(l="Use Rotate Pivot To Query Position?", v=1)
    widgets["snapPivCB"] = cmds.checkBox(l="Snap via pivot? (vs. translate value)", v=1)
    cmds.separator(h=5, style="single")
    widgets["snapButton"] = cmds.button(l="Snap obj(s)!", w=210, h=40, bgc=(.6,.8,.6), c=partial(snapIt, False))
    widgets["snapPivButton"] = cmds.button(l="Snap pivot!", w=210, h=20, bgc=(.8,.6,.6), c=partial(snapIt, True))

    cmds.window(widgets["win"], e=True, w=5, h=5)
    cmds.showWindow(widgets["win"])

def snapIt(pivt=False, *args):
    """
    Does the snapping by xform. Should work with any rotation order, etc

    Args:
        piv (bool): a boolean for whether to snap the actual object or just pivot. True = Pivot only, False = Transform

    """

    mode = cmds.radioButtonGrp(widgets["avgRBG"], q=True, sl=True)

    translate = cmds.checkBoxGrp(widgets["cbg"], q=True, v1=True)
    rotate = cmds.checkBoxGrp(widgets["cbg"], q=True, v2=True)
    pivot = cmds.checkBox(widgets["rpCB"], q=True, v=True)
    moveViaPivot = cmds.checkBox(widgets["snapPivCB"], q=True, v=True)

    sel = cmds.ls(sl=True, fl=True)
    if len(sel) < 2:
        return()
    # get pt components for later
    selPts = cmds.filterExpand(sm=(31, 28, 30, 35, 46, 47, 51), ex=True)
    # remove these from selection (faces and edges)
    selEdgeFace = cmds.filterExpand(sm=(32, 34))

    if mode==1: # all to first

        target = sel[0]
        objects = sel[1:]

        for x in objects: # get rid of faces and edges (return weird positions)
            if selEdgeFace and (x in selEdgeFace):
                objects.remove(x)
            if selEdgeFace and (x in selPts):
                objects.remove(x)

        targetPos = (0,0,0)

        if selPts and (target in selPts):
            targetPos = cmds.pointPosition(target)
        elif pivot:
            targetPos = cmds.xform(target, ws=True, q=True, rp=True)
        else:
            targetPos = cmds.xform(target, ws=True, q=True, t=True)

        tarRot = cmds.xform(target, ws=True, q=True, ro=True)
        tarRO = cmds.xform(target, q=True, roo=True)

        for obj in objects:
            if rotate:
                cmds.select(obj, r=True)
                objRO = cmds.xform(obj, q=True, roo=True)
                # if not pivt:
                cmds.xform(obj, roo=tarRO)
                cmds.xform(obj, ws=True, ro=tarRot)
                cmds.xform(obj, roo=objRO, p=True)

            if translate:
                if pivt:
                    cmds.xform(obj, ws=True, a=True, piv=targetPos, p=True)
                else:
                    if not moveViaPivot:
                        cmds.xform(obj, ws=True, t=targetPos)
                    else:
                        objPiv = cmds.xform(obj, q=True, ws=True, rp=True)
                        objTrans = cmds.xform(obj, q=True, ws=True, t=True)
                        endPos = (objTrans[0]-objPiv[0]+targetPos[0], objTrans[1]-objPiv[1]+targetPos[1], objTrans[2]-objPiv[2]+targetPos[2])
                        cmds.xform(obj, ws=True, t=endPos)

        cmds.select(objects, r=True)

    else:
        if (len(sel)>=2):
            objects = sel[0:-1]
            target = sel[-1:][0]

            for x in objects:
                if selEdgeFace and (x in selEdgeFace):
                    objects.remove(x)

            tarRO = cmds.xform(target, q=True, roo=True)

            txList = []
            tyList = []
            tzList = []
            rxList = []
            ryList = []
            rzList = []
            TX, TY, TZ, RX, RY, RZ = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
            for obj in objects:
                component = False
                if selPts and (obj in selPts):
                     component = True

                if pivot and not component:
                    objPos = cmds.xform(obj, q=True, ws=True, rp=True)
                if (not pivot and not component) or component:
                    objPos = cmds.xform(obj, q=True, ws=True, t=True)

                txList.append(objPos[0])
                tyList.append(objPos[1])
                tzList.append(objPos[2])

                #convert it to the rot order of the target
                objRO = cmds.xform(obj, q=True, roo=True)
                cmds.xform(obj, p=True, roo=objRO)
                #get the rotation
                objRot = cmds.xform(obj, q=True, ws=True, ro=True)
                rxList.append(objRot[0])
                ryList.append(objRot[1])
                rzList.append(objRot[2])
                #convert it back
                cmds.xform(obj, p=True, roo=objRO)

            #now average them all
            for tx in txList:
                TX += tx
            for ty in tyList:
                TY += ty
            for tz in tzList:
                TZ += tz
            for rx in rxList:
                RX += rx
            for ry in ryList:
                RY += ry
            for rz in rzList:
                RZ += rz

            avgTx = TX/len(txList)
            avgTy = TY/len(tyList)
            avgTz = TZ/len(tzList)
            avgRx = RX/len(rxList)
            avgRy = RY/len(ryList)
            avgRz = RZ/len(rzList)

            if rotate:
                cmds.select(target, r=True)
                cmds.xform(target, ws=True, ro=(avgRx, avgRy, avgRz))

            if translate:
                cmds.select(target, r=True)
                if not pivt:
                    cmds.xform(target, ws=True, t=(avgTx, avgTy,avgTz))
                else:
                    cmds.xform(target, ws=True, piv=(avgTx, avgTy, avgTz), p=True)

        else:
            cmds.warning("You need to select two objects or more!")

        cmds.select(target, r=True)

def snap(*args):
    """function to run the script"""
    snapUI()


import datetime
from django.db.models import Q

from django.db import models
from django.contrib import admin
from django import forms
from django.forms import CheckboxSelectMultiple
from tinymce.widgets import AdminTinyMCE

from story.models import Story, ScrapingRule, Buyer
from story.forms import StoryAdminForm

CUSTOM_PUB_STATUS = (
    (-1, 'Rejected'),
    (0, 'Draft'),
    (1, 'Pending Review'),
)

# class SourceExist(SimpleListFilter):
#     # Human-readable title which will be displayed in the
#     # right admin sidebar just above the filter options.
#     title = _('Source')
#
#     # Parameter for the filter that will be used in the URL query.
#     parameter_name = 'Source'
#
#     def lookups(self, request, model_admin):
#         """
#         Returns a list of tuples. The first element in each
#         tuple is the coded value for the option that will
#         appear in the URL query. The second element is the
#         human-readable name for the option that will appear
#         in the right sidebar.
#         """
#         return (
#             ('at', _('AT')),
#             ('others', _('Others')),
#
#         )
#
#     def queryset(self, request, queryset):
#         """
#         Returns the filtered queryset based on the value
#         provided in the query string and retrievable via
#         `self.value()`.
#         """
#         # sorce may be empty or None
#         # to decide how to filter the queryset.
#         if self.value() == 'others':
#             return queryset.filter(Q(source__source__name=None) | Q(source__source__name=""))
#         elif self.value() == 'at':
#             return queryset.exclude(id__in=queryset.filter(Q(source__source__name=None) | Q(source__source__name="")))
#         return queryset

class StoryAdmin(admin.ModelAdmin):
    formfield_overrides = {
        models.ManyToManyField: {'widget': CheckboxSelectMultiple},
    }
    form = StoryAdminForm
    fieldsets = ((None, {'fields': (
                                    ('title', 'slug'), ('rule_box',), ('comments'),
                                    ('body_text','approved_by',),
                                    ('original_text', 'status',
                                     'pub_date', 'created_by',),
                                    ('url',),
                                    )
                         }
                  ),
                 )

    list_display = ('title', 'pub_date', 'created_by', 'created_on',
                    'approved_by', 'approved_on', 'comments', 'status')
    prepopulated_fields = {"slug": ("title",)}
    # list_editable = ()
    # readonly_fields = ('domain', 'source')
    search_fields = ['title', 'body_text', 'url']
    date_hierarchy = 'created_on'
    list_filter = ('status', 'pub_date', 'created_on', 'approved_on',
                   'created_by',
                   'approved_by')
    list_per_page = 50
    #raw_id_fields = ('domain', 'source')
    save_on_top = True

    def formfield_for_dbfield(self, db_field, **kwargs):
        # increase the size of the text box for title and secondary headline
        if db_field.name in ['title']:
            kwargs['widget'] = forms.Textarea(attrs={'rows': 2, 'cols': 100})
        # customize comment box size
        elif db_field.name in ['rule_box']:
            kwargs['widget'] = forms.Textarea(attrs={'rows': 3, 'cols': 100})
        elif db_field.name in ['comments']:
            kwargs['widget'] = forms.Textarea(attrs={'rows': 2, 'cols': 100})
        elif db_field.name in ['body_text', 'original_text']:
            kwargs['widget'] = forms.Textarea(attrs={'rows': 15, 'cols': 100})
            kwargs['widget'] = AdminTinyMCE

        return super(StoryAdmin, self).formfield_for_dbfield(db_field, **kwargs)

    def get_readonly_fields(self, request, obj=None):
        if obj and obj.rule_box:  # In edit mode
            return ('rule_box',) + self.readonly_fields
        return self.readonly_fields

    # def response_add(self, request, obj, post_url_continue=None):
    #     base_path = u"/admin/story/story/"
    #     if '_addanother' in request.POST:
    #         domain_id = request.POST['domain']
    #         source_id = request.POST['source']
    #         body_text = ""
    #         if domain_id and source_id:
    #             body_text = obj.source.get_leadline_and_source()
    #         query_string = u'add/?domain={}&source={}&body_text={}'.format(
    #             domain_id, source_id, body_text
    #         )
    #         path = base_path + query_string
    #         return HttpResponseRedirect(path)
    #     elif '_continue' in request.POST:
    #         path = u'{}{}/change/'.format(base_path, obj.id)
    #         message = u'The story "{}" was changed successfully. ' \
    #                   u'You may edit it again below.'.format(obj.title)
    #         self.message_user(request, message, level=messages.INFO)
    #         return HttpResponseRedirect(path)
    #     else:
    #         return HttpResponseRedirect(base_path)

    # def response_change(self, request, obj, post_url_continue=None):
    #     base_path = u"/admin/story/story/"
    #     if '_addanother' in request.POST:
    #         domain_id = request.POST['domain']
    #         source_id = request.POST['source']
    #         body_text = ""
    #         if domain_id and source_id:
    #             body_text = obj.source.get_leadline_and_source()
    #         query_string = u'add/?domain={}&source={}&body_text={}'.format(
    #             domain_id, source_id, body_text
    #         )
    #         path = base_path + query_string
    #         return HttpResponseRedirect(path)
    #
    #     elif '_continue' in request.POST:
    #         return super(StoryAdmin, self).response_change(request, obj)
    #     else:
    #         return HttpResponseRedirect(base_path)

    def queryset(self, request):
        """
        user having scraper permission should see stroies created by userself.
        """
        qs = super(StoryAdmin, self).queryset(request).select_related()
        if request.user.is_superuser or request.user.has_perm('story.view_all_story'):
            return qs
        elif request.user.has_perm('story.can_publish_own_story'):
            return qs.filter(created_by=request.user)
        else:
            return qs.filter(created_by=request.user)

    def formfield_for_choice_field(self, db_field, request, **kwargs):
        """override status: can only see Rejected, Draft and Pending Review"""
        if db_field.name == "status" and not \
                (request.user.has_perm('story.can_publish_all_story') or
                 request.user.has_perm('story.can_publish_own_story')):
            kwargs["choices"] = CUSTOM_PUB_STATUS
            return db_field.formfield(**kwargs)
        return super(StoryAdmin, self).formfield_for_choice_field(
            db_field, request, **kwargs)

    def save_model(self, request, obj, form, change):
        if obj.status in [-1, 2]:
            obj.approved_on = datetime.datetime.now()
            obj.approved_by = request.user
        if not change:
            obj.created_by = request.user
        obj.updated_by = request.user
        obj.save()

class ScrapingRuleAdmin(admin.ModelAdmin):
    fieldsets = ((None, {'fields': (('short_name'), ('rule', 'is_active')),
                         }
                  ),
                 )
    list_display = ('short_name', 'rule', 'created_by', 'created_on')

    def save_model(self, request, obj, form, change):
        if not change:
            obj.created_by = request.user
        obj.updated_by = request.user
        obj.save()

class BuyerAdmin(admin.ModelAdmin):
    fieldsets = ((None, {'fields': (('name', 'slug',),
                                    ('syndication_id', 'active')),
                         }
                  ),
                 )
    list_display = ('name', 'slug', 'syndication_id', 'active')
    list_filter = ('active',)
    search_fields = ('name',)

admin.site.register(Buyer, BuyerAdmin)
admin.site.register(ScrapingRule, ScrapingRuleAdmin)
admin.site.register(Story, StoryAdmin)


import codecs
import fnmatch
import os
import random
import sys
import re
from collections import *
import nltk
import numpy as np
import tensorflow as tf
from six.moves import cPickle as pickle
import logging

import csv
# for long CSV
maxInt = sys.maxsize
decrement = True
while decrement:
	# decrease the maxInt value by factor 10
	# as long as the OverflowError occurs.
	decrement = False
	try:
		csv.field_size_limit(maxInt)
	except OverflowError:
		maxInt = int(maxInt / 10)
		decrement = True

class MyError(Exception):
  def __init__(self, value):
    self.value = value
  def __str__(self):
    return repr(self.value)

class TextDataHandler:
	_force_read_input = False
	_word_to_id = {}
	_id_to_word = {}
	_words = []
	_unknown_terms = set()
	_unknown_terms_freq = 0

	def __init__(self, all_doc_path, save_dir):

		files = Utilities.recursive_glob(all_doc_path, "*")
		Utilities.create_file_dir(save_dir)
		# base_name = os.path.basename(os.path.normpath(all_doc_path))
		# word_to_id
		word_to_id_pickle = os.path.join(save_dir, 'word_to_id.pkl')
		self._word_to_id = self._maybe_pickling(word_to_id_pickle, self._build_dict_from_a_set_of_file, files)

		# id_to_word
		id_to_word_pickle = os.path.join(save_dir, 'id_to_word.pkl')
		self._id_to_word = self._maybe_pickling(id_to_word_pickle, self._build_reverse_dict, self._word_to_id)

	def _maybe_pickling(self, pickle_name, func, *func_param):
		if os.path.exists(pickle_name) and not self._force_read_input:
			# You may override by setting force=True.
			print('%s already present - Skipping pickling.' % pickle_name)
			try:
				with open(pickle_name, 'rb') as f:
					data = pickle.load(f)
			except Exception as e:
				print('Unable to process data from', pickle_name, ':', e)
				raise
		else:
			data = func(*func_param)
			self.pickling(pickle_name, data)
		return data

	def pickling(self, pickle_file_name, data):
		print('Pickling %s' % pickle_file_name)
		try:
			with open(pickle_file_name, 'wb') as f:
				pickle.dump(data, f, pickle.HIGHEST_PROTOCOL)
		except Exception as e:
			logging.error('Unable to save data to', pickle_file_name, ':', e)

	@staticmethod
	def clean_str(string):
		"""
		string cleanin
		"""
		string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
		string = re.sub(r"\'s", " \'s", string)
		string = re.sub(r"\'ve", " \'ve", string)
		string = re.sub(r"n\'t", " n\'t", string)
		string = re.sub(r"\'re", " \'re", string)
		string = re.sub(r"\'d", " \'d", string)
		string = re.sub(r"\'ll", " \'ll", string)
		string = re.sub(r",", " , ", string)
		string = re.sub(r"!", " ! ", string)
		string = re.sub(r"\(", " \( ", string)
		string = re.sub(r"\)", " \) ", string)
		string = re.sub(r"\?", " \? ", string)
		string = re.sub(r"\s{2,}", " ", string)
		#       string = re.sub(r"[^A-Za-z0-9]", " ", string)
		return string.strip().lower()

	def read_words(self, filename):
		"""
		Tokenization using NLTK
		"""
		with tf.gfile.GFile(filename, "r") as f:
			return nltk.word_tokenize(TextDataHandler.clean_str(f.read()))

	def _build_dict_from_a_set_of_file(self, filenames):
		counter = Counter()
		file_cnt = len(filenames)
		cnt = 0
		for filename in filenames:
			if cnt % 500 == 0:
				print(str(cnt) + " out of " + str(file_cnt) + " has been processed")
			words = self.read_words(filename)
			counter.update(words)
			cnt += 1
		count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
		# if vocab_size == -1:
		# 	vocab_size = len(count_pairs)
		# words, _ = list(zip(*count_pairs[:vocab_size]))
		words, _ = list(zip(*count_pairs))
		word_to_id = dict(zip(words, range(1, len(words) + 1)))
		return word_to_id

	def get_vocab_size(self):
		if not self._word_to_id.values:
			raise MyError('first build the dictionary...')
		return len(self._word_to_id) + 1  # unknown

	def truncate_vocab(self, new_vocab_size):
		if new_vocab_size < 0:
			print("New Vocab Size:", self.get_vocab_size())
			return
		self._word_to_id = OrderedDict(sorted(self._word_to_id.items(), key=lambda item: item[1])[:new_vocab_size - 1])
		self._id_to_word = self._build_reverse_dict(self._word_to_id)
		print("New Vocab Size:", self.get_vocab_size())

	def _build_reverse_dict(self, original_dict):
		if not original_dict:
			raise MyError('first build the dictionary...')
		reverse_dict = dict(zip(original_dict.values(), original_dict.keys()))
		return reverse_dict

	def get_id_of_word(self, word):
		#     return self._word_to_id.get(word, 0) # dictionary['UNK'] = 0
		if word in self._word_to_id:
			return self._word_to_id[word]
		else:  # dictionary['UNK'] = 0
			self._unknown_terms.add(word)
			self._unknown_terms_freq += 1
			return 0

	def get_word_of_id(self, i):
		if i == 0:
			word = 'UNK'
		elif i in self._id_to_word:
			word = self._id_to_word[i]
		else:
			print(i)
			raise MyError('unknown_id')
		return word

	def word_list_to_id_list(self, words):
		return [self.get_id_of_word(word) for word in words]

	def id_list_to_word_list(self, ids):
		return [self.get_word_of_id(i) for i in ids]

	def get_vocab_dict(self):
		return self._word_to_id.copy()

	def get_rev_vocab_dict(self):
		return self._id_to_word.copy()

	def get_vocab(self):
		if not self._words:
			self._words = list(self._word_to_id)
		return self._words

	def get_one_hot_vector(self, wordIds_vec):
		vocab_size = self.get_vocab_size()
		vec = (np.arange(vocab_size) == np.array(wordIds_vec)[:, None]).astype(np.float32)
		return np.array(vec)

	def get_ids_from_one_hot_vector(self, one_hot_Vec):
		#to be implimented
		return None

	def get_binary_vector(self, wordIds_vec):
		vec = np.zeros(self.get_vocab_size(), dtype=int)
		inices = list(set(wordIds_vec))
		vec[inices] = 1
		return np.array(vec)

	def get_ids_from_binary_vector(self, binary_vector):
		return np.nonzero(binary_vector)

	def get_freq_vector(self, wordIds_vec):
		vec = np.zeros(self.get_vocab_size(), dtype=int)
		counter = Counter(wordIds_vec)
		vec[list(counter.keys())] = list(counter.values())
		return np.array(vec)

	def get_ids_from_freq_vector(self, freq_vector):
		# return [[freq_vector[i]] * i for i in np.nonzero(freq_vector)]
		return None

class BatchCreator4RNN:
	def __init__(self, tensor, batch_size, seq_length):
		self.tensor = tensor
		self.batch_size = batch_size
		self.seq_length = seq_length
		self.create_batches()
		self.pointer = 0

	# abcd -> x:abc y:bcd
	def create_batches(self):
		self.num_batches = int(self.tensor.size / (self.batch_size *
																							 self.seq_length))
		if self.num_batches == 0:
			assert False, "Not enough data. Make seq_length and batch_size small."

		self.tensor = self.tensor[:self.num_batches * self.batch_size * self.seq_length]
		xdata = self.tensor
		ydata = np.copy(self.tensor)

		ydata[:-1] = xdata[1:]
		ydata[-1] = xdata[0]
		self.x_batches = np.split(xdata.reshape(self.batch_size, -1), self.num_batches, 1)
		self.y_batches = np.split(ydata.reshape(self.batch_size, -1), self.num_batches, 1)

	def next_batch(self):
		x, y = self.x_batches[self.pointer], self.y_batches[self.pointer]
		self.pointer += 1
		return x, y

	def reset_batch_pointer(self):
		self.pointer = 0

class Retrieval_Data_Util(object):
	doc_query_pairs = []

	def __init__(self, runres, qrel):
		with codecs.open(qrel, "r", encoding='utf-8', errors='ignore') as f:
			csv_reader = csv.reader(f, delimiter=' ', quoting=csv.QUOTE_NONE)
			for row in csv_reader:
				if row[3] == '1':
					self.doc_query_pairs.append((row[2], row[0]))
			f.close()
		self._runRes = runres

	def get_rel_qd(self):
		d = [tup[0] for tup in self.doc_query_pairs]
		q = [tup[1] for tup in self.doc_query_pairs]
		return d, q

	def get_pseudo_rel_qd(self, top_k):
		d = []
		q = []
		with codecs.open(self._runRes, "r", encoding='utf-8', errors='ignore') as f:
			csv_reader = csv.reader(f, delimiter=' ', quoting=csv.QUOTE_NONE)
			for row in csv_reader:
				if int(row[3]) <= top_k:
					d.append(row[2])
					q.append(row[0])
			f.close()
		return d, q

	def get_label(self, d, q):
		return (d, q) in self.doc_query_pairs

class Utilities():
	@staticmethod
	def shufflize(data, label):
		assert len(data) == len(label)
		perm = np.random.permutation(len(data))
		return data[perm], data[perm]

	@staticmethod
	def recursive_glob(treeroot, pattern):
		results = []
		for base, dirs, files in os.walk(treeroot):
			goodfiles = fnmatch.filter(files, pattern)
			results.extend(os.path.join(base, f) for f in goodfiles)
		return results

	@staticmethod
	def create_file_dir(filename):
		if not os.path.exists(os.path.dirname(filename)):
			try:
				os.makedirs(os.path.dirname(filename))
			except OSError as exc:  # Guard against race condition
				if exc.errno != errno.EEXIST:
					raise

	@staticmethod
	def sample_distribution(distribution):
		"""Sample one element from a distribution assumed to be an array of normalized
    probabilities.
    """
		r = random.uniform(0, 1)
		s = 0
		for i in range(len(distribution)):
			s += distribution[i]
			if s >= r:
				return i
		return len(distribution) - 1

	@staticmethod
	def random_distribution(vector_size):
		"""Generate a random column of probabilities."""
		b = np.random.uniform(0.0, 1.0, size=[1, vector_size])
		return b / np.sum(b, 1)[:, None]

	@staticmethod
	def accuracy(predictions, labels):
		return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))
						/ predictions.shape[0])


"""
mechpy configuration, singleton,
ie:
>>> from config import config
>>> config.SCREEN_WIDTH
768
"""

#
# Import python system libraries
#
import os, sys, logging, ConfigParser, shutil

log = logging.getLogger(__name__)

#
# configuration singleton
#
config = None

def init():
    """
    Initialize the singleton once in main
    """
    global config
    config = Config()

#
# Helper functions for creating properties
# TODO move these into class Config
#

def _section_key_split(seckey):
    idx = seckey.index('.')
    section = seckey[0:idx]
    key = seckey[idx+1:]
    return (section, key)

def _str_property(seckey, docstr=None):
    (section, key) = _section_key_split(seckey)
    return property(
        lambda self: self._config.get(section, key),
        lambda self, x: self._config.set(section, key, x),
        None,
        docstr)

def _bool_property(seckey, docstr=None):
    (section, key) = _section_key_split(seckey)
    return property(
        lambda self: self._config.getboolean(section, key),
        lambda self, x: self._config.set(section, key, x),
        None,
        docstr)

def _int_property(seckey, docstr=None):
    (section, key) = _section_key_split(seckey)
    return property(
        lambda self: self._config.getint(section, key),
        lambda self, x: self._config.set(section, key, x),
        None,
        docstr)

def _float_property(seckey, docstr=None):
    (section, key) = _section_key_split(seckey)
    return property(
        lambda self: self._config.getfloat(section, key),
        lambda self, x: self._config.set(section, key, x),
        None,
        docstr)

#
# list of properties attached to the configuration properties file
#
class Config:

    #
    # List of all configuration properties
    #

    SCREEN_WIDTH = _int_property("gfx.screen_width",
        """Width of the display screen""")
    SCREEN_HEIGHT = _int_property("gfx.screen_height",
        """Height of the display screen""")

    def __init__(self):
        """
        Call this function whenever you want the configuration to be
        initialized, should be called in main().
        """
        global _config

        #
        # Load configuration
        #
        src_dir = os.path.split(os.path.abspath(__file__))[0]
        main_dir = os.path.abspath(os.path.join(src_dir, '..'))
        data_dir = os.path.join(main_dir, 'data')
        user_dir = os.getenv("HOME")

        if os.name == 'posix':
            config_dir = os.path.join(user_dir, '.config')
            if not os.path.isdir(config_dir):
                config_dir = os.path.join(user_dir, '.local', 'Share')
                if not os.path.isdir(config_dir):
                    log.info("Can't find ~/.config or ~/.local, not linux?")
                    sys.exit(1)
        else:
            log.info("Not linux?")
            sys.exit(1)

        program_config_dir = os.path.join(config_dir, 'mechpy')
        program_config_file = os.path.join(program_config_dir, 'config.ini')

        if not os.path.exists(program_config_dir):
            log.info("Configuration directory did not exist, creating")
            os.mkdir(program_config_dir)

        if not os.path.exists(program_config_file):
            log.info("Configuration did not exist, copying over default")
            default_config_file = os.path.join(data_dir,
                                               'default_config.ini')
            shutil.copyfile(default_config_file, program_config_file)

        self._config = ConfigParser.SafeConfigParser()
        self._config.read(program_config_file)


from pyspark.sql import SparkSession
from pyspark.sql.functions import col

S3_DATA_SOURCE_PATH = 's3://adbproj1/data-input/houseRent.csv'
S3_DATA_OUTPUT_PATH = 's3://adbproj1/data-output'

def main():

    spark = SparkSession.builder.appName('ADBMSProject').getOrCreate()

    all_data = spark.read.csv(S3_DATA_SOURCE_PATH,header=True)

    print('Total number of records in the source data :%s' % all_data.count())

    selected_data = all_data.where((col('beds') == 2) & (col('sqfeet')>1500))
    print('Total number of beds with 2 and sqfeet > 1500 are :%s'%selected_data.count())

    selected_data = all_data.where((col('beds') == 1) & (col('sqfeet')>1500))
    print('Total number of beds with 1 and sqfeet > 1500 are  :%s',selected_data.count())

    selected_data = all_data.where((col('baths')==1) & (col('type')=='apartment'))
    print('Total number of baths = 1 with type is apartment : %s', selected_data.count())

    selected_data = all_data.where((col('baths')==2) & (col('type')=='house'))
    print('Total number of baths=2 with type is house : %s', selected_data.count())

    selected_data = all_data.where((col('baths')==1) & (col('type')=='townhouse'))
    print('Total number of baths=2 with type is townhouse : %s', selected_data.count())

    selected_data = all_data.where(col('parking_options').isin(['attached garage', 'street parking', 'off-street parking']))
    print('Total number of records with parking options of attached garage, street parking, or off-street parking: %s' % selected_data.count())

     # select data where state = 'CA' and price < 3000
    selected_data = all_data.where((col('state') == 'ca') & (col('price') < 3000))
    print('Total number of records where state = "CA" and price < 3000: %s' % selected_data.count())

     # select data where cats_allowed = true and dogs_allowed = true and smoking_allowed = false
    selected_data = all_data.where((col('cats_allowed') == 1) & (col('dogs_allowed') == 1) & (col('smoking_allowed') == 1))
    print('Total number of records where cats_allowed = 1, dogs_allowed = 1, and smoking_allowed = 1: %s' % selected_data.count())

    selected_data.write.mode('overwrite').parquet(S3_DATA_OUTPUT_PATH)
    print('selected data is successfully  saved to s3 : %s' % S3_DATA_OUTPUT_PATH)

if __name__ == '__main__' :
    main()


import re
from typing import Dict

def select_converter(sql_string: str) -> str:
    functions_to_convert = {
        r"(^|\s)sel ": "select ",
        r"median\(.+?\)": "percentile_cont(placeholder, 0.5) over()",
    }

    return _replace_matches(functions_to_convert, sql_string)

def type_converter(sql_string: str) -> str:
    types_to_convert = {
        r"(\s)character\(.+?\)": " string",
        r"(\s)varchar\(.+?\)": " string",
        r"(\s)char\(.+?\)": " string",
        r"(\s)integer": " int64",
        r"(\s)smallint": " int64",
        r"(\s)byteint": " int64",
        r"(\s)bigint": " int64",
        r"(\s)float": " float64",
    }

    return _replace_matches(types_to_convert, sql_string)

def comparison_converter(sql_string: str) -> str:
    comparison_operator_to_convert = {
        r"(\s)eq(\s)": " = ",
        r"(\s)le(\s)": " <= ",
        r"(\s)lt(\s)": " < ",
        r"(\s)ne(\s)": " != ",
        r"(\s)ge(\s)": " >= ",
        r"(\s)gt(\s)": " > ",
    }

    return _replace_matches(comparison_operator_to_convert, sql_string)

def schema_converter(sql_string: str) -> str:
    schema_to_convert = {}

    return _replace_matches(schema_to_convert, sql_string)

def _replace_matches(patterns: Dict, sql: str) -> str:
    for key, value in patterns.items():
        regex = re.compile(key)
        if regex.search(sql):
            matches = regex.findall(sql)
            for match in matches:
                if re.search("placeholder", value):
                    colname = re.findall(r"\((.+?)\)", match)
                    temp_value = re.sub("placeholder", colname[0], value)
                else:
                    temp_value = value
                sql = regex.sub(temp_value, sql, 1)

    return sql


from lib.action import NapalmBaseAction

class NapalmGetFacts(NapalmBaseAction):
    """Get Facts from a network device via NAPALM
    """

    def run(self, **std_kwargs):

        with self.get_driver(**std_kwargs) as device:
            result = {'raw': device.get_facts()}

            if self.htmlout:
                result['html'] = self.html_out(result['raw'])

        return (True, result)


import cv2
import onnx
import torch
import numpy as np
from albumentations import (Compose, Resize,)
from albumentations.augmentations.transforms import Normalize
from albumentations.pytorch.transforms import ToTensor
from torchvision import models

from config import DETECTION_THRESHOLD, CLASSES, ONNX_FILE_PATH
from model import create_model, convert_to_onnx

import tensorrt as trt
import pycuda.driver as cuda
import pycuda.autoinit

# logger to capture errors, warnings, and other information during the build and inference phases
TRT_LOGGER = trt.Logger()

def preprocess_image(img_path):

    image = cv2.imread(img_path)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB).astype(np.float32)

    # make the pixel range between 0 and 1
    image /= 255.0
    # bring color channels to front
    image = np.transpose(image, (2, 0, 1)).astype(float)
    # convert to tensor
    image = torch.tensor(image, dtype=torch.float)
    # add batch dimension
    image = torch.unsqueeze(image, 0)

    return image

def postprocess(output_data, img_path):
    orig_image = cv2.imread(img_path)
    image_name = img_path.split('/')[-1].split('.')[0]
    outputs = [{k: v.to('cpu') for k, v in t.items()} for t in output_data]

    if len(outputs[0]['boxes'] != 0):
        boxes = outputs[0]['boxes'].data.numpy()
        scores = outputs[0]['scores'].data.numpy()
        # filter out boxes according to `detection_threshold`
        boxes = boxes[scores >= DETECTION_THRESHOLD].astype(np.int32)
        draw_boxes = boxes.copy()
        # get all the predicited class names
        pred_classes = [CLASSES[i] for i in outputs[0]['labels'].cpu().numpy()]

        # draw the bounding boxes and write the class name on top of it
        for j, box in enumerate(draw_boxes):
            cv2.rectangle(orig_image,
                          (int(box[0]), int(box[1])),
                          (int(box[2]), int(box[3])),
                          (0, 0, 255), 2)
            cv2.putText(orig_image, pred_classes[j],
                        (int(box[0]), int(box[1] - 5)),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0),
                        2, lineType=cv2.LINE_AA)
        cv2.imshow('Prediction', orig_image)
        cv2.waitKey(1)
        cv2.imwrite(f"test_predictions/{image_name}.jpg", orig_image, )

def build_engine(onnx_file_path, engine_file_path = ""):

    network_creation_flag = 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_PRECISION)
    network_creation_flag |= 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)

    with trt.Builder(TRT_LOGGER) as builder, builder.create_network(
            network_creation_flag) as network, builder.create_builder_config() as config, trt.OnnxParser(network,
                                                                                TRT_LOGGER) as parser, trt.Runtime(
            TRT_LOGGER) as runtime:
        config.max_workspace_size = 1 << 30
        builder.max_batch_size = 1

        print('Loading ONNX file from path {}...'.format(onnx_file_path))
        with open(onnx_file_path, 'rb') as model:
            print('Beginning ONNX file parsing')
            if not parser.parse(model.read()):
                print('ERROR: Failed to parse the ONNX file.')
                for error in range(parser.num_errors):
                    print(parser.get_error(error))
                return None

        network.get_input(0).shape = [1, 3, 608, 608]
        print('Completed parsing of ONNX file')
        print('Building an engine from file {}; this may take a while...'.format(onnx_file_path))
        plan = builder.build_serialized_network(network, config)
        engine = runtime.deserialize_cuda_engine(plan)
        print("Completed creating Engine")
        with open(engine_file_path, "wb") as f:
            f.write(plan)
        context = engine.create_execution_context()
        print("Completed creating Engine")
        return engine, context

def main():
    # initialize TensorRT engine and parse ONNX model
    engine, context = build_engine(ONNX_FILE_PATH)
    # get sizes of input and output and allocate memory required for input data and for output data
    for binding in engine:
        if engine.binding_is_input(binding):  # we expect only one input
            input_shape = engine.get_binding_shape(binding)
            input_size = trt.volume(input_shape) * engine.max_batch_size * np.dtype(np.float32).itemsize  # in bytes
            device_input = cuda.mem_alloc(input_size)
        else:  # and one output
            output_shape = engine.get_binding_shape(binding)
            # create page-locked memory buffers (i.e. won't be swapped to disk)
            host_output = cuda.pagelocked_empty(trt.volume(output_shape) * engine.max_batch_size, dtype=np.float32)
            device_output = cuda.mem_alloc(host_output.nbytes)

    # Create a stream in which to copy inputs/outputs and run inference.
    stream = cuda.Stream()

    # preprocess input data
    host_input = np.array(preprocess_image("data/test/images/fruit1.png").numpy(), dtype=np.float32, order='C')
    cuda.memcpy_htod_async(device_input, host_input, stream)

    # run inference
    context.execute_async(bindings=[int(device_input), int(device_output)], stream_handle=stream.handle)
    cuda.memcpy_dtoh_async(host_output, device_output, stream)
    stream.synchronize()

    # postprocess results
    output_data = torch.Tensor(host_output).reshape(engine.max_batch_size, output_shape[0])
    postprocess(output_data)

if __name__ == '__main__':
    img_path = "data/test/Images/fruit104.png"
    model_path = "outputs/model100.pth"

    # path = convert_to_onnx(model_path, img_path)

    main()


import lief

from aleph.exceptions import ProcessorRuntimeException
from aleph.models import Processor

class ELF(Processor):

    binary = None
    filetypes = ["application/x-elf"]

    def process(self, sample):

        sample_data = sample["data"]

        try:
            self.binary = lief.ELF.parse(raw=sample_data)
        except Exception as e:
            raise ProcessorRuntimeException("Unable to parse ELF file: %s" % str(e))

        metadata = {
            "entry_point": self.binary.entrypoint,
            "image_base": self.binary.imagebase,
            "virtual_size": self.binary.virtual_size,
            "header": self.get_header(),
            "sections": self.get_sections(),
            "segments": self.get_segments(),
            "static_symbols": self.get_symbols(),
            "dynamic_symbols": self.get_symbols(dynamic=True),
            "is_pie": self.binary.is_pie,
        }

        return metadata

    def get_header(self):

        header = self.binary.header

        return {
            "arm_flags_list": list(header.arm_flags_list),
            "file_type": str(header.file_type).split(".")[1],
            "size": header.header_size,
            "machine_type": str(header.machine_type).split(".")[1],
            "nr_sections": header.numberof_sections,
            "nr_segments": header.numberof_segments,
            "object_file_version": str(header.object_file_version).split(".")[1],
            "ppc64_flags_list": list(header.ppc64_flags_list),
            "processor_flag": header.processor_flag,
            "program_header_offset": header.program_header_offset,
            "program_header_size": header.program_header_size,
            "section_header_offset": header.section_header_offset,
            "section_header_size": header.section_header_size,
        }

    def get_sections(self):

        sections = []

        for section in self.binary.sections:

            section_data = {
                "name": section.name,
                "type": str(section.type).split(".")[1],
                "virtual_address": section.virtual_address,
                "size": section.size,
                "flags": section.flags,
                "entropy": section.entropy,
            }
            sections.append(section_data)

        return sections

    def get_segments(self):

        segments = []

        for segment in self.binary.segments:

            segment_data = {
                "type": str(segment.type).split(".")[1],
                "physical_address": segment.physical_address,
                "physical_size": segment.physical_size,
                "virtual_address": segment.virtual_address,
                "virtual_size": segment.virtual_size,
                "flags": str(segment.flags).split(".")[1],
            }
            segments.append(segment_data)

        return segments

    def get_symbols(self, dynamic=False):

        symbols = []

        symbol_iterator = (
            self.binary.dynamic_symbols if dynamic else self.binary.static_symbols
        )

        for symbol in symbol_iterator:

            symbol_data = {
                "name": symbol.name,
                "demangled_name": symbol.demangled_name,
                "type": str(symbol.type).split(".")[1],
                "size": symbol.size,
                "exported": symbol.exported,
                "imported": symbol.imported,
                "is_function": symbol.is_function,
                "is_variable": symbol.is_variable,
            }

            symbols.append(symbol_data)

        return symbols


#!/usr/bin/python3
""" city module"""
from models.base_model import BaseModel, Base
from os import getenv
from sqlalchemy import Column, String, ForeignKey
from sqlalchemy.orm import relationship

class City(BaseModel, Base):
    """ The city class, contains state ID and name """
    if getenv('HBNB_TYPE_STORAGE') == "db":
        __tablename__ = "cities"
        name = Column(String(128), nullable=False)
        state_id = Column(String(60), ForeignKey("states.id"), nullable=False)
        places = relationship("Place", backref='cities')
    else:
        state_id = ""
        name = ""

    def __init__(self, *args, **kwargs):
        """initializes City"""
        super().__init__(*args, **kwargs)


from django.contrib import admin
from django.urls import path, include
from .views import index, kategoriolustur, dersolustur, kategoriler, kategori_ayrinti,ders_ayrinti

app_name = 'dersler'

urlpatterns = [
    path('', index, name='index'),
    path('yeni-kategori/', kategoriolustur, name='kategoriolustur'),
    path('yeni-ders/', dersolustur, name='dersolustur'),
    path('kategoriler/', kategoriler, name='kategoriler'),
    path('kategori/<int:kategori_id>', kategori_ayrinti, name='kategori_ayrinti'),
    path('ders/<int:ders_id>', ders_ayrinti, name='ders_ayrinti'),

]


from django.contrib import admin
from .models import *

# Register your models here.
admin.site.register(SchoolProfile)
admin.site.register(StaffProfile)
@admin.register(User)
class UserAdmin(admin.ModelAdmin):
	list_display = ['login_id', 'email']
	search_fields = ['login_id', 'email']


"""
Pickem ETL
Author: Gabe Baduqui

Retrieve location data from Geocode.maps forward geocode API
"""
import requests

def get_city_name(location_name: str):
    """Function that extracts city name from a given location string
       Accepts `location_name`: String
       Returns `city`: String"""
    try:
        city = location_name.split(', ')[0]
    except:
        city = None
    return city

def get_state_name(location_name: dict):
    """Function that extracts state name from a given location string
       Accepts `location_name`: String
       Returns `state`: String"""
    try:
        state = location_name.split(', ')[1]
    except:
        state = None
    return state

def get_latitude(geocode_record: dict):
    """Function that extracts latitude property from the Geocode API response
       Accepts `geocode_record`: Dictionary (JSON response)
       Returns `latitude`: Number"""
    try:
        lat = geocode_record['lat']
    except:
        lat = None
    return lat

def get_longitude(geocode_record: dict):
    """Function that extracts longitude property from the Geocode API response
       Accepts `geocode_record`: Dictionary (JSON response)
       Returns `longitude`: Number"""
    try:
        long = geocode_record['lat']
    except:
        long = None
    return long

def call_geocode_api(stadium: str, city: str, state: str):
    """Fucntion that makes a GET request to 'https://geocode.maps.co/search?q=' for a given location
       Accepts `stadium`: String, `cit`: String, `state`: String
       Returns `geocode_record`: Dictionary"""
    if stadium is None:
        geocode_api_url = f'https://geocode.maps.co/search?city={city}&state={state}'
    else:
        general_query = stadium.replace(' ', '+')
        if state is None:
            geocode_api_url = f'https://geocode.maps.co/search?q={general_query}'
        else:
            geocode_api_url = f'https://geocode.maps.co/search?q={general_query}&city={city}&state={state}'
    try:
        response = requests.get(geocode_api_url)
        geocode_record = response.json()[0]
    except:
        geocode_record = None
    return geocode_record

def get_location_data(location_id: str, stadium: str, location_name: str, logfile: object):
    """Function that calls the Geocode.maps forward geocode API.
       Accepts `location_id`: String, `stadium`: String, `location_name`: String, `logfile`: File Object
       Returns `location_data`: Dictionary"""
    print(f'~~ Scraping geocode data for {stadium}, {location_name}')
    logfile.write(f'~~ Scraping geocode data for {stadium}, {location_name}\n')

    # Call Forward Geocode API
    city = get_city_name(location_name)
    state = get_state_name(location_name)
    geocode_record = call_geocode_api(stadium, city, state)
    lat = get_latitude(geocode_record)
    lon = get_longitude(geocode_record)

    # Instantiate `location_data` dictionary
    location_data = {
        'location_id': location_id,
        'stadium': stadium,
        'city': city,
        'state': state,
        'latitude': lat,
        'longitude': lon
    }

    return location_data


import numpy as np
import pandas as pd
from typing import Dict

from algotrader.technical import Indicator

class RollingApply(Indicator):
    _slots__ = (
        'length',
        'func'
    )

    def __init__(self, time_series=None, inputs=None, input_keys=None, desc="Rolling Apply", length=0, func=np.std):
        super(RollingApply, self).__init__(time_series=time_series, inputs=inputs, input_keys=input_keys, desc=desc,
                                           length=length)
        self.func = func
        self.length = self.get_int_config("length", 0)

    def _process_update(self, source: str, timestamp: int, data: Dict[str, float]):
        result = {}
        if self.first_input.size() >= self.length:
            sliced = self.first_input.get_by_idx(keys=self.first_input_keys, idx=slice(-self.length, None, None))
            result[Indicator.VALUE] = self.func(sliced)
        else:
            result[Indicator.VALUE] = np.nan

        self.add(timestamp=timestamp, data=result)

class StdDev(RollingApply):
    def __init__(self, time_series=None, inputs=None, input_keys='close', desc="Rolling Standard Deviation", length=30):
        super(StdDev, self).__init__(time_series=time_series, inputs=inputs, input_keys=input_keys, desc=desc,
                                     length=length, func=lambda x: np.std(x, axis=0))

def pd_skew_wrapper(x):
    ts = pd.Series(x)
    return ts.skew()

def pd_kurtosis_wrapper(x):
    ts = pd.Series(x)
    return ts.kurtosis()

def pd_kurt_wrapper(x):
    ts = pd.Series(x)
    return ts.kurt()

class Skew(RollingApply):
    def __init__(self, time_series=None, inputs=None, input_keys='close', desc="Rolling Skew", length=30):
        super(Skew, self).__init__(time_series=time_series, inputs=inputs, input_keys=input_keys, desc=desc,
                                   length=length, func=lambda x: pd_skew_wrapper(x))

class Kurtosis(RollingApply):
    def __init__(self, time_series=None, inputs=None, input_keys='close', desc="Rolling Kurtosis", length=30):
        super(Kurtosis, self).__init__(time_series=time_series, inputs=inputs, input_keys=input_keys, desc=desc,
                                       length=length, func=lambda x: pd_kurtosis_wrapper(x))

class Kurt(RollingApply):
    def __init__(self, time_series=None, inputs=None, input_keys='close', desc="Rolling Kurt", length=30):
        super(Kurt, self).__init__(time_series=time_series, inputs=inputs, input_keys=input_keys, desc=desc,
                                   length=length, func=lambda x: pd_kurt_wrapper(x))

#
#
# from jinja2 import Template
# rollingTmp = Template("""
# class {{className}}(RollingApply):
#     def __init__(self, input, input_key='close', length=30, desc="Rolling {{className}}"):
#         super({{className}}, self).__init__(input,
#                                      func={{func}},
#                                      name=Indicator.get_name({{className}}.__name__, input, input_key, length),
#                                      length=length,
#                                      input_key=input_key, desc=desc)
# """)
#
# print rollingTmp.render({"className": "Skew",
#                          "func" : "lambda x: pd_skew_wrapper(x)"})
#
# print rollingTmp.render({"className": "Kurtosis",
#                          "func" : "lambda x: pd_kurtosis_wrapper(x)"})
#
# print rollingTmp.render({"className": "Kurt",
#                          "func" : "lambda x: pd_kurt_wrapper(x)"})


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jul 24 23:37:15 2018

@author: enriquelazcorretapuigmarti
"""

import os, psutil, sys, hashlib, time

def tiempo_transcurrido(inicio, mensaje='', fin=None):
    if fin is None:
        fin = time.time()
    segundos = fin - inicio

    dias = 0
    horas = 0
    minutos = 0
    ms = int((segundos % 1) * 1000)
    segundos = int(segundos)

    resultado = mensaje

    if segundos >= 86400:
        dias = segundos // 86400
        segundos -= dias * 86400

    if segundos >= 3600:
        horas = segundos // 3600
        segundos -= horas * 3600

    if segundos >= 60:
        minutos = segundos // 60
        segundos -= minutos * 60

    if dias:
        resultado += '{:,}'.format(dias) + ' días '

    if horas:
        resultado += str(horas) + 'h '

    if minutos:
        resultado += str(minutos) + 'm '

    if segundos:
        resultado += str(segundos) + 's '

    if len(resultado):
        if ms:
            resultado += str(ms) + 'ms'
    else:
        resultado += str(ms) + 'ms'

    return resultado

def tamanyo_legible(tamanyo_en_bytes, si=False):
    if si:
        if tamanyo_en_bytes < 1024:
            return '{:}'.format(tamanyo_en_bytes) + 'B'

        unidad = ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB')
        base = 10
        exponente = (3, 6, 9, 12, 15, 18, 21, 24, 27)
    else:
        if tamanyo_en_bytes < 1024:
            return '{:}'.format(tamanyo_en_bytes) + 'B'

        unidad = ('KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB')
        base = 2
        exponente = (10, 20, 30, 40, 50, 60, 70, 80)

    for i in range(7):
        if tamanyo_en_bytes < base ** exponente[i+1]:
            return '{:.1f}'.format(tamanyo_en_bytes / (base ** exponente[i])) \
                    + unidad[i]

    return '{:.1f}'.format(tamanyo_en_bytes / (base ** exponente[7])) + 'YB'

def memoria_objeto(objeto, mensaje=''):
    return tamanyo_legible(sys.getsizeof(objeto)) + mensaje

#TODO Con memoria_objeto() no debería ser necesaria memoria_dataset()
#TODO Aunque memoria_objeto() es mucho más lento (22sg frente a 18sg al usar
#     datasetclasificacion-inplace con todos los archivos), HACER MÁS PRUEBAS.
def memoria_dataset(pandas_dataframe, mensaje=''):
    ram = 0
    for columna in pandas_dataframe.memory_usage():
        ram += columna
    return tamanyo_legible(ram) + mensaje

def memoria_proceso(mensaje=' RAM usada por el proceso'):
    pid = os.getpid()
    py = psutil.Process(pid)
    return tamanyo_legible(py.memory_info()[0]) + mensaje

def sha1_archivo(archivo):
    try:
        with open(archivo, 'rb') as file:
            return hashlib.sha1(file.read()).hexdigest()
    except:
        return 'No he podido abrir el archivo ' + archivo

def md5_archivo(archivo):
    try:
        with open(archivo, 'rb') as file:
            return hashlib.md5(file.read()).hexdigest()
    except:
        return 'No he podido abrir el archivo ' + archivo

if __name__ == '__main__':
    print(tiempo_transcurrido(10000.1236))
    t0 = time.time()
    time.sleep(2)
    print(tiempo_transcurrido(inicio=t0))
    t1 = time.time()
    print(tiempo_transcurrido(inicio=t0, fin=t1))


import os

from setuptools import find_namespace_packages, setup

here = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(here, 'VERSION')) as f:
    VERSION = f.read().strip()

with open(os.path.join(here, 'README.md')) as f:
    README = f.read()

requires = [
    'bcrypt',
    'fastapi',
    'uvicorn',
    'passlib',
    'python-dotenv',

    'pyjwt',
    'cryptography',

    # http client stuff
    # see: https://docs.aiohttp.org/en/v2.3.4/
    'aiohttp',
    'cchardet',
    'aiodns',
    # another one
    'httpx',
]

standalone_require = [
]

tests_require = [
    'mypy>=0.760',
    'pytest',
    'requests',
    'vcrpy',
]

setup(name='pansen-castlabs',
      version=VERSION,
      description='castlabs challenge',
      long_description=README,
      classifiers=[
          "Programming Language :: Python",
          "Programming Language :: Python :: 3 :: Only",
          "Programming Language :: Python :: 3.7",
          "Programming Language :: Python :: 3.8",
          "Framework :: FastAPI",
          "Topic :: Internet :: WWW/HTTP",
          "Topic :: Internet :: WWW/HTTP :: ASGI :: Application"
      ],
      keywords="web services",
      author='pansen',
      author_email='andi@zerotired.com',
      url='',
      packages=find_namespace_packages(include=['pansen.*']),
      include_package_data=True,
      # http://pythonhosted.org/distribute/setuptools.html#namespace-packages
      namespace_packages=['pansen'],
      zip_safe=False,
      extras_require={
          'testing': tests_require + standalone_require,
          'standalone': standalone_require,
      },
      install_requires=requires,
      entry_points={
          'console_scripts': [
              'pansen_castlabs = pansen.castlabs.main:run'
          ],
      },
      )


#!/usr/bin/env python
# coding: utf-8

# En linux
# export NUITRACK_HOME="/usr/etc/nuitrack"
# export LD_LIBRARY_PATH={$LD_LIBRARY_PATH}:"/usr/local/lib/nuitrack"

# In[1]:

from PyNuitrack import py_nuitrack
import cv2
from datetime import datetime
from itertools import cycle
import numpy as np
import pandas as pd
import math
import tensorflow as tf
import joblib
import os
import time
import keras
from keras.models import model_from_json

# In[2]:

# Configuracion de camara
camaraNombre = 'Astra'
camaraNumeroSerie = 'AS3M32300AV'
# configurar colores cv2
modes = ["depth", "color"] #defecto del sdk en modo color, deshabilitar la cámara principal en conf windows
mode = modes[1]
tiempoDeEsperaPorCiclo = 33/1000 # en segundos 28fps
pathDeTrabajo='./'

Posiciones=['A','B','C','D','E','Desconocido']
Ang=pd.DataFrame(columns =['A1','A2','A3','A4','A5','A7','A8','A9','A10','A11'])
#metodo='RN' #Red Neuronal
categoriaActual='Desconocido'
letraAesASCII=97;

#valores para el esqueleto
cuerpo = [1,2,3,4]
brazoDerecho = [5,6,7,8,9]
brazoIzquierdo = [10,11,12,13,14]
piernaDerecha = [4,18,19,20]
piernaIzquierda = [4,15,16,17]
esqueleto = [
            { "extr": cuerpo, "colorPunto":(164, 0, 0), "colorLine":(164, 0, 0)},
            { "extr": brazoDerecho, "colorPunto":(0, 150, 0), "colorLine":(0, 150, 0)},
            { "extr": brazoIzquierdo, "colorPunto":(0, 150, 0), "colorLine":(0, 150, 0)},
            { "extr": piernaDerecha, "colorPunto":(0, 0, 150), "colorLine":(0, 0, 150)},
            { "extr": piernaIzquierda, "colorPunto":(0, 0, 150), "colorLine":(0, 0, 150)},
            ]
#Valor de prediccion
out1 = [len(Posiciones)-1];
out2 = [len(Posiciones)-1];
out3 = [len(Posiciones)-1];

# In[3]:

if not os.path.exists(pathDeTrabajo):
    print('No existe el directorio de trabajo')
    exit

#Cargando modelo RN
if os.path.isfile(pathDeTrabajo+'RNN_Arquitactura.json'):  # Si existe
    model = model_from_json(open(pathDeTrabajo+'RNN_Arquitactura.json').read())
    model.load_weights(pathDeTrabajo+'RNN_modelo.h5')
    model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])
    model.summary()
else:
    print('El archivo: '+pathDeTrabajo+'RNN_Arquitactura.json no existe')
    exit

#Cargando modelo RF
if os.path.isfile(pathDeTrabajo+'RandomForest.pkl'):  # Si existe
    modelRF = joblib.load(pathDeTrabajo+"RandomForest.pkl")
else:
    print('El archivo: '+pathDeTrabajo+'RandomForest.pkl no existe')
    exit

#Cargando modelo GB
if os.path.isfile(pathDeTrabajo+'GradientBoosting.pkl'):  # Si existe
    modelGB = joblib.load(pathDeTrabajo+"GradientBoosting.pkl")
else:
    print('El archivo: '+pathDeTrabajo+'GradientBoosting.pkl no existe')
    exit

if os.path.isfile(pathDeTrabajo + 'Datos.csv'):  # Si existe
    Dang = pd.read_csv(pathDeTrabajo + 'Datos.csv', sep=';', index_col=[0])
    if Dang.shape[0] >0:
        N = Dang.iloc[-1, 0] + 1
    else:
        N=1
else:  # No existe el archivo
    N = 1
    if not os.path.exists(pathDeTrabajo+'Captura'):
    	os.makedirs(pathDeTrabajo+'Captura')
    Dang = pd.DataFrame(
        columns=['N', 'Fecha', 'A1', 'A2', 'A3', 'A4', 'A5', 'A7', 'A8', 'A9', 'A10', 'A11',
                 'RN','RF','GB','Postura_correcta'])

# In[11]:

def draw_skeleton(image):
    global out1,out2,out3
    for skel in data.skeletons:
        for extremidadDibujo in esqueleto:
            for i in range(0,len(extremidadDibujo.get("extr"))):
                p1 = (round(skel[extremidadDibujo.get("extr")[i]].projection[0]), round(skel[extremidadDibujo.get("extr")[i]].projection[1]))
                #Graficar recuadro y texto de las predicciones
                if extremidadDibujo.get("extr")[i]==1:
                    cv2.rectangle(image, (10,10), (60,60),(0,0,0), 50)
                    cv2.putText(image,"RN: "+Posiciones[int(out1[0])], (1,20), cv2.FONT_ITALIC,0.7, (255,255,255), 2, bottomLeftOrigin = False)
                    cv2.putText(image,"RF: "+Posiciones[int(out2[0])], (1,45), cv2.FONT_ITALIC,0.7, (255,255,255), 2, bottomLeftOrigin = False)
                    cv2.putText(image,"GB: "+Posiciones[int(out3[0])], (1,70), cv2.FONT_ITALIC,0.7, (255,255,255), 2, bottomLeftOrigin = False)
                cv2.circle(image, p1, 5, extremidadDibujo.get("colorPunto"), -1)
                if i+1 < len(extremidadDibujo.get("extr")):
                    p2 = (round(skel[extremidadDibujo.get("extr")[i+1]].projection[0]), round(skel[extremidadDibujo.get("extr")[i+1]].projection[1]))
                    if p2 == (0,0):
                        continue
                    cv2.line(image,p1,p2,extremidadDibujo.get("colorLine"),2)

    #cv2.putText(image,categoriaActual, (10,10), cv2.FONT_HERSHEY_SIMPLEX, 25, (255,255,255), 10, bottomLeftOrigin = False)

# In[5]:

#Captura imagen de la camara y el esqueleto
def captura():
    nuitrack.update()
    global data
    data = nuitrack.get_skeleton()
    data_instance = nuitrack.get_instance()
    img_depth = nuitrack.get_depth_data()
    if img_depth.size:
        cv2.normalize(img_depth, img_depth, 0, 255, cv2.NORM_MINMAX)
        img_depth = np.array(cv2.cvtColor(img_depth, cv2.COLOR_GRAY2RGB), dtype=np.uint8)
        if mode == "depth":
            draw_skeleton(img_depth)
            cv2.imshow('Image', img_depth)
            return (img_depth)
        if mode == "color":
            img_color = nuitrack.get_color_data()
            draw_skeleton(img_color)
            if img_color.size:
                cv2.imshow('Image', img_color)
                return (img_color)

# In[6]:

def angulo(P1,P2): #[Punto a medir,Punto referencia]
    c=P1-P2;
    angle=math.atan2(c[1],c[0])
    return(math.degrees(angle))

# In[7]:

#Calculo los Angulos
def calculos():
    global Puntos
    global metodo
    for skeleton in data.skeletons: #Separo los puntos en varias variables
        #print(skeleton.user_id)
        head=skeleton.head   #print(skeleton.head)
        neck=skeleton.neck   #print(skeleton.neck)
        torso=skeleton.torso #print(skeleton.torso)
        waist=skeleton.waist #print(skeleton.waist)
        #lado izquierdo
        left_collar=skeleton.left_collar
        left_shoulder=skeleton.left_shoulder
        left_elbow=skeleton.left_elbow
        left_wrist=skeleton.left_wrist
        left_hand=skeleton.left_hand
        left_hip=skeleton.left_hip
        left_knee=skeleton.left_knee
        left_ankle=skeleton.left_ankle
        #lado izquierdo
        right_collar=skeleton.right_collar
        right_shoulder=skeleton.right_shoulder
        right_elbow=skeleton.right_elbow
        right_wrist=skeleton.right_wrist
        right_hand=skeleton.right_hand
        right_hip=skeleton.right_hip
        right_knee=skeleton.right_knee
        right_ankle=skeleton.right_ankle

        #Lateral
        #Angulos del torso
        A1=90-angulo(head.real[[0,1]],neck.real[[0,1]])      #cabeza-cuello
        A2=90-angulo(neck.real[[0,1]],torso.real[[0,1]])     #cuello-torso
        A3=90-angulo(torso.real[[0,1]],waist.real[[0,1]])    #torso-cintura
        #Angulo del brazo
        A4=angulo(right_shoulder.real[[0,1]],right_elbow.real[[0,1]])-angulo(right_wrist.real[[0,1]],right_elbow.real[[0,1]])    #Hombro-codo-muñeca
        #Angulo de la pierna
        A5=angulo(waist.real[[0,1]],right_hip.real[[0,1]])-angulo(right_knee.real[[0,1]],right_hip.real[[0,1]])    #Cintura-cadera-rodilla
        #Angulo de la rodilla
        #A6=angulo(right_hip.real[[0,1]],right_knee.real[[0,1]])-angulo(right_knee.real[[0,1]],right_ankle.real[[0,1]])    #Cadera-rodilla-tobillo

        #Frontal
        #Angulos del torso
        A7=90-angulo(head.real[[2,1]],torso.real[[2,1]])      #cabeza-torso
        A8=90-angulo(torso.real[[2,1]],waist.real[[2,1]])    #torso-cintura
        #Angulos del brazo
        A9=90-angulo(right_shoulder.real[[2,1]],right_elbow.real[[2,1]])    #Hombro-codo Derecha
        A10=90-angulo(left_shoulder.real[[2,1]],left_elbow.real[[2,1]])    #Hombro-codo Izquierda
        #Angulos de la pierna
        A11=360-abs(angulo(right_knee.real[[2,1]],waist.real[[2,1]]))+abs(angulo(left_knee.real[[2,1]],waist.real[[2,1]]))      #Cadera_derecha-Cintura-Cadera Izquierda
        #A12=90-angulo(right_knee.real[[2,1]],right_ankle.real[[2,1]])    #Rodilla-tobillo Derecha
        #A13=90-angulo(left_knee.real[[2,1]],left_ankle.real[[2,1]])    #Rodilla-tobillo Izquierda
        return [A1, A2, A3, A4, A5, A7, A8,A9,A10, A11]
    return None
#///////////////////////////////////////////////////////

# Predice usando los 3 modelos
def predecir(Angulos):
    global categoriaActual
    global Ang
    global out1,out2,out3

    Ang=pd.DataFrame([Angulos],columns=Ang.columns)
    # Red neuronal
    NN=model.predict(abs(Ang[0:1].values),verbose=0)
    out=np.where(100*NN== np.amax(100*NN))
    out1=out[1]
    # Random Forest
    out2=modelRF.predict(abs(Ang[0:1]))
    # Gradient
    out3=modelGB.predict(abs(Ang[0:1]))

def guardar(categoria):
    global N
    global foto
    global muestra
    global Dang
    nombre = 'Imagen_' + str(N)
    if not os.path.exists(pathDeTrabajo+'clase_'+categoria):
        os.makedirs(pathDeTrabajo+'clase_'+categoria)
    cv2.imwrite(pathDeTrabajo+'clase_'+categoria+'/' + nombre + '.jpg', foto)
    [A1, A2, A3, A4, A5, A7, A8,A9,A10, A11]=muestra
    Dang= pd.concat([Dang, pd.DataFrame([[N, datetime.now(), A1, A2, A3, A4, A5, A7, A8,A9,A10, A11, Posiciones[int(out1[0])],Posiciones[int(out2[0])],Posiciones[int(out3[0])],categoria]],columns=Dang.columns, index=[nombre])])
    Dang.to_csv(pathDeTrabajo + 'Datos.csv', sep=';')
    N = N + 1

# In[8]:

# Inicializa la camara
nuitrack = py_nuitrack.Nuitrack()
nuitrack.init()
devices = nuitrack.get_device_list()
for dev in devices:
    if dev.get_name() == camaraNombre and camaraNumeroSerie == dev.get_serial_number():
        nuitrack.set_device(dev)
        print(dev.get_name(), dev.get_serial_number())
nuitrack.create_modules()
nuitrack.run()
nuitrack.set_config_value("Faces.ToUse", "true")#coordenada de la cabeza centro de la cabeza ---false->nariz
nuitrack.set_config_value("DepthProvider.Depth2ColorRegistration", "true")

# In[9]:

# Bucle principal
while 1:
    try:
        key = cv2.waitKey(1)

        if key == 27: #Presiona Esc para salir y guardar
            break

        for i in range(0,len(Posiciones)): # Recorre las categorias creadas
            if key == i+letraAesASCII:  # verifica si la tecla presionada corresponde al ASCII de la categoria
                guardar(Posiciones[i])

        foto=captura()
        muestra=calculos()
        if muestra is not None:
            predecir(muestra)

    except Exception as e:
        print('Ocurrio un error:')
        print(str(e))
        time.sleep(2000) #tiempo de espera en caso de falla
    finally:
        time.sleep(tiempoDeEsperaPorCiclo) #tiempo de espera en seg-> 28fps

cv2.destroyAllWindows()

# In[10]:

nuitrack.release()

# In[ ]:

# In[ ]:


#!/usr/bin/env python

import sys
# add current directory to path
sys.path.insert(0, '')

from sa import _version, repo as rp, versioning, docker_api
import os, argparse, git

def main():
    # initialize argparser
    parser = argparse.ArgumentParser()
    # initialize sub argument parsers
    subparsers = parser.add_subparsers(dest='command')

    parser.add_argument('--version', action='version',
                    version='%(prog)s {version}'.format(version=_version.__version__))

    parser.add_argument('-gu','--gitusername',
                    dest='git_username',
                    type=str,
                    help='git username to use during commits, defaults to \'Jenkins\'')
    parser.add_argument('-ge','--gituseremail',
                    dest='git_useremail',
                    type=str,
                    help='git email to use during commits, defaults to \'jenkins@noreply.com\'')

    # git
    parser_git = subparsers.add_parser('git')
    subparser_git = parser_git.add_subparsers(dest='git_subcommand')
    # git push
    git_subparser_push = subparser_git.add_parser('push')
    add_required_creds_args(git_subparser_push)
    add_required_repo_args(git_subparser_push)
    # git checkprevcommit
    git_subparser_checkprevcommit = subparsers.add_parser('checkprevcommit')
    add_required_repo_args(git_subparser_checkprevcommit)

    # version
    parser_version = subparsers.add_parser('version')
    subparser_version = parser_version.add_subparsers(dest='version_subcommand')
    # version get
    version_subparser_get = subparser_version.add_parser('get')
    add_optional_docker_arg(version_subparser_get)
    add_required_files_args(version_subparser_get)
    # version bump
    version_subparser_bump = subparser_version.add_parser('bump')
    add_required_repo_args(version_subparser_bump)
    add_optional_docker_arg(version_subparser_bump)
    add_required_files_args(version_subparser_bump)
    # TODO add flag to allow for custom provided versioning script

    # docker
    parser_docker = subparsers.add_parser('docker')
    subparser_docker = parser_docker.add_subparsers(dest='docker_subcommand')
    # docker build
    docker_subparser_build = subparser_docker.add_parser('build')
    add_required_files_args(docker_subparser_build)
    # docker geninfo
    #TODO: generate sample docker info json file

    # Process CLI arguments
    options = parser.parse_args()

    if options.command == 'git':

        if options.git_subcommand == 'push':
            rp.push(get_git_repo(options.repo_dir), options.username, options.password)
            exit()
        elif options.git_subcommand == 'checkprevcommit':
            #TODO: implement function
            rp.check_prev_commit(get_git_repo(options.repo_dir))

    elif options.command == 'version':

        if options.version_subcommand == 'bump':
            # Process all file paths provided and only include the valid files
            rp.version_bump(get_git_repo(options.repo_dir), get_files(options.files), options)
            exit()
        elif options.version_subcommand == 'get':
            # Process all files paths provided and only include the valid files
            version_info = versioning.version_get(get_files(options.files), options)
            for file,version in version_info:
                print(''.join([file,' ', version]))
            exit()

    elif options.command == 'docker':

        if options.docker_subcommand == 'build':
            docker_api.build(get_files(options.files))
        elif options.docker_subcommand == 'push':
            docker_api.push()

def get_git_repo(repo_dir):
    # get/validate absolute path of provided repository directory
    abs_repo_path = ""
    if repo_dir != None:
        if os.path.isdir(repo_dir):
            abs_repo_path = os.path.abspath(repo_dir)
        else:
            # raise argparse.ArgumentTypeError(f"readable_dir:{repo_dir} is not a valid path")
            print(f'readable_dir:{abs_repo_path} is not a valid path')
            exit(1)

    # validate provided path is a git repository and setup for use
    if (rp.is_git_repo(abs_repo_path)):
        repo = git.Repo(abs_repo_path)
        rp.reattach_head(repo)
    else:
        # raise Exception(f"{repo_dir} is not a valid git repository")
        print(f'{abs_repo_path} is not a valid git repository')
        exit(1)
    return repo

def get_files(infiles):
    files = []
    if infiles != None:
        for f in infiles:
            if os.path.isfile(f):
                files.append(os.path.abspath(f))

    if (len(files) == 0):
        print('no files provided, use \'-h|--help\' for more details'.format())
        exit(1)
    return files

def add_required_files_args(parser):
    parser.add_argument(
                    dest='files',
                    type=str,
                    nargs='*',
                    help='list of all files to be to be operated on')

def add_optional_docker_arg(parser):
    parser.add_argument('-d','--docker',
                    action='store_true',
                    help='indicate if provided files are docker-info.json files for versioning')

def add_required_creds_args(parser):
    parser.add_argument('-u','--username',
                    dest='username',
                    required=True,
                    help='username for logging into service')
    parser.add_argument('-p','--password',
                    dest='password',
                    required=True,
                    help='password for logging into service')

def add_required_repo_args(parser):
    parser.add_argument('-r','--repo',
                    dest='repo_dir',
                    type=str,
                    required=True,
                    help='specific path to git project files')

if __name__ == '__main__':
    sys.exit(main())


# %%
import numpy as np
import random

# namedtuple:   factory function for creating tuple subclasses with named fields
# deque:        list-like container with fast appends and pops on either end
from collections import namedtuple, deque

# Custom DQN model set up in model.py
from model import QNetwork

# Import torch framework
import torch
import torch.nn.functional as F
import torch.optim as optim

# %%
# Set hyperparameters
BUFFER_SIZE = int(100000)   # Replay memory size
BATCH_SIZE = 64             # Mini-batch size
GAMMA = 0.99                # Discount factor
TAU = 0.001                 # For soft update of target parameters
LR = 0.0001                 # Learning rate alpha
UPDATE_EVERY = 4            # How often to update the network

# Check if we have CUDA enabled device i.e. a GPU, else use the CPU
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# Define DQN agent
class Agent():
    '''Agent to interact with the environment and learn
    '''

    def __init__(self, state_size, action_size, seed):
        self.state_size = state_size
        self.action_size = action_size
        self.seed = random.seed(seed)

        ### Make Q-network, both the local and the target network ###
        self.qnetwork_local = QNetwork(state_size, action_size, seed)
        self.qnetwork_target = QNetwork(state_size, action_size, seed)

        ### Set optimizer for gradient descent ###
        # Select the parameters of the local qnetwork we are training on.
        # Weights will later be copied to the target qnetwork
        self.optimizer = optim.Adam(params=self.qnetwork_local.parameters(), lr=LR)

        ### Initialize replay memory ###
        self.replay_memory = ReplayMemory(buffer_size=BUFFER_SIZE, batch_size=BATCH_SIZE)

        ### Initialize time step (for updating every UPDATE_EVERY steps)###
        self.t_step = 0

    def step(self, state, action, reward, next_state, done):
        ### Save experience in memory ###
        self.replay_memory.add(state, action, reward, next_state, done)

        ### Learn every UPDATE_EVERY time steps ###
        self.t_step += 1
        # If remainder of division is 0, then learn from experience replay memory
        if self.t_step % UPDATE_EVERY == 0:
            # If we have collected enough experience in our memory i.e. more
            # than the mini-batch size, then call the self.learn() function
            if len(self.replay_memory) > BATCH_SIZE:
                experiences = self.replay_memory.sample()
                self.learn(experiences, GAMMA)

    def act(self, state, eps=0.0):
        """Return actions for the given state and policy (e.g. ε-greedy)

        Arguments:
            state {[numpy array]} -- Current state
            eps {float} -- Epsilon for ε-greedy policy (default: {0.0})
        """
        # Convert 'state' numpy array to pytorch tensor using the current device
        # i.e. GPU or CPU.
        # unsqueeze() returns a new tensor with a dimension of size one
        # inserted at the specified position.
        state = torch.from_numpy(state).float().unsqueeze(0).to(device)

        # Set the module in evaluation mode.
        self.qnetwork_local.eval()
        with torch.no_grad():
            # Evaluate the Q-network with the current state
            # Call a forward pass with the current state to get q-values for
            # that state.
            action_values = self.qnetwork_local(state)

        # Set the module in training mode.
        self.qnetwork_local.train()

        # ε-greedy action selection
        if random.random() > eps:
            # Get action with the max q-value
            # .cpu(): Move all model parameters and buffers to the CPU
            return np.argmax(action_values.cpu().data.numpy())
        else:
            return random.choice(np.arange(self.action_size))

    def learn(self, experiences, gamma):
        """Update value parameters using given batch of experience tuples

        Arguments:
            experiences {Tupple(torch.Tensor)} -- Tuple of (s, a, r, s', done) tuples
            gamma {float} -- Discount factor
        """
        # Experiences, mini-batch of 64
        states, actions, rewards, next_states, dones = experiences

        ### Get max predicted Q-values (for next states) from target model ###
        # .detach(): returns a new Variable that does not back-propagate to
        # whatever detach() was called on. Creates a tensor that shares storage
        # with tensor that does not require grad.
        # .max(): return the max value of a tensor on a given dimension (dim)
        Q_targets_next = self.qnetwork_target(next_states).detach().max(1)[0].unsqueeze(1)
        # size([4, 1])
        # the tensors below are [64, x]
        ### Computer Q-targets for current states ###
        Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))

        ### Get expected Q-values from local model ###
        # .gather(): Gathers values along an axis specified by dim.
        Q_expected = self.qnetwork_local(states).gather(1, actions)

        ### Compute loss ###
        loss = F.mse_loss(Q_expected, Q_targets)
        # Minimize loss
        # Clear the gradients so they are not accumulated
        self.optimizer.zero_grad()
        loss.backward()
        # Perform a single optimization step.
        self.optimizer.step()

        ### Update target network ###
        self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU)

    def soft_update(self, local_model, target_model, tau):
        """Soft update model parameters

        Arguments:
            local_model {PyTorch model} -- Copy weights from
            target_model {PyTorch model} -- Copy weights to
            tau {float} -- Interpolation parameter
        """
        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
            target_param.data.copy_(tau*local_param.data + (1.0-tau) * target_param.data)

# Define replay memory class
class ReplayMemory:
    def __init__(self, buffer_size, batch_size):
        """Initialise a replay memory buffer
        """
        self.memory = deque(maxlen=buffer_size)
        self.batch_size = batch_size
        self.experience = namedtuple("Experience", field_names=["state", "action", "reward", "next_state", "done"])

    def add(self, state, action, reward, next_state, done):
        """Add new experience to the replay memory buffer

        Arguments:
            state {[type]} -- Current state
            action {[type]} -- Current action
            reward {[type]} -- Reward for taking action in current state
            next_state {[type]} -- Next state after taking action
            done {bool} -- Whether episode is done (terminal state)
        """
        new_experience = self.experience(state, action, reward, next_state, done)
        self.memory.append(new_experience)

    def sample(self):
        """Sample a random mini-batch from the experience replay buffer
        """
        sampled_experiences = random.sample(self.memory, k=self.batch_size)

        # Convert the sampled experiences to torch tensors on the GPU or CPU,
        # depending on what 'device' is. 'device' is set at the beggining of
        # the script.
        # The '.to()' method of Tensors and Modules can be used to easily move
        # objects to different devices (instead of having to call cpu() or
        # cuda() based on the context).
        states = torch.from_numpy(np.vstack([e.state for e in sampled_experiences if e is not None])).float().to(device)
        actions = torch.from_numpy(np.vstack([e.action for e in sampled_experiences if e is not None])).long().to(device)
        rewards = torch.from_numpy(np.vstack([e.reward for e in sampled_experiences if e is not None])).float().to(device)
        next_states = torch.from_numpy(np.vstack([e.next_state for e in sampled_experiences if e is not None])).float().to(device)
        dones = torch.from_numpy(np.vstack([e.done for e in sampled_experiences if e is not None]).astype(np.uint8)).float().to(device)

        # Return tuple of random sampled experiences
        return (states, actions, rewards, next_states, dones)

    def __len__(self):
        """Return the length of the replay memory

        Returns:
            [int] -- Length of the current replay memory
        """
        return len(self.memory)

Project 1 - Navigation with DQN Agent/agent.py


from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters.state import StatesGroup, State
from aiogram import types, Dispatcher
from create_bot import dp
from aiogram.types import KeyboardButton, InlineKeyboardButton, ReplyKeyboardRemove
from dbase_bot import database
from aiogram.dispatcher.filters import Text

class FsmPrizes(StatesGroup):
    prizes = State()

async def prizes(message: types.Message):
    keyboard = types.ReplyKeyboardMarkup(resize_keyboard=True).add(
    KeyboardButton('Ежемесячные призы!')).add(
    KeyboardButton('Годовые призы!'))
    await FsmPrizes.prizes.set()
    await message.answer('Мы разыгрываем призы каждый месяц! А также супер-призы среди всех отзывов в конце года!',
                         reply_markup=keyboard)

async def month_prizes(message: types.Message):
    keyboard = types.ReplyKeyboardMarkup(resize_keyboard=True).add(
    KeyboardButton('Годовые призы!')).add(
    KeyboardButton('Отмена'))
    await message.answer('Первый приз: 2000 \u20BD\n'
                         'Второй приз: 1000 \u20BD\n'
                         'Третий приз: 500 \u20BD', reply_markup=keyboard)

async def years_prizes(message: types.Message):
    get_prizes = await database.view_prizes()
    keyboard = types.ReplyKeyboardMarkup(resize_keyboard=True).add(
    KeyboardButton('Ежемесячные призы!')).add(
    KeyboardButton('Отмена'))
    await message.answer('В конце года разыграем следующие призы:', reply_markup=keyboard)
    for i in get_prizes:
        await message.answer_photo(i["pics"], f'{i["description"]}')

def register_handlers_prizes(dp: Dispatcher):
    dp.register_message_handler(prizes, lambda message: 'Разыгрываемые' in message.text, state=None)
    dp.register_message_handler(prizes, lambda message: 'Разыгрываемые' in message.text, state='*')
    dp.register_message_handler(month_prizes, lambda message: 'Ежемесячные' in message.text, state=FsmPrizes.prizes)
    dp.register_message_handler(years_prizes, lambda message: 'Годовые' in message.text, state=FsmPrizes.prizes)


from django.shortcuts import render, redirect
from django.contrib.auth.forms import UserCreationForm
from django.contrib import messages
from django.contrib.auth import logout as auth_logout
from users.forms import UserForm , UserUpdateForm
from django.contrib.auth.models import User
from django.contrib.auth.decorators import login_required

def signup(request):
    if request.method =='POST':
        form = UserForm(request.POST)
        messages.error(request,str(form.errors))
        if form.is_valid():
            form.save()
            username = form.cleaned_data.get('username')
            messages.success(request,f'Your account is created! You can now log in')
            return redirect('login')
    else:
        form = UserForm()
    return render(request,'users/signup.html', {'form':form})

def logout(request):
    auth_logout(request)
    return render(request,'users/logout.html')

def userSettings(request):
    if request.method =='POST':
        u_form = UserUpdateForm(request.POST, instance=request.user)
        if u_form.is_valid():
            u_form.save()
            messages.success(request,f'Your Profile has been updated!')
            return redirect('settings')
    else:
        u_form = UserUpdateForm(instance=request.user)

    context = {
        'u_form': u_form,
    }
    return render(request,'users/userSettings.html',context)

def profile(request):
    return render(request,'users/profile.html')

def deleteAccount(request):
    if request.method =='POST':
        user=User.objects.get(username=request.user.username)
        print(user)
        user.delete()
        return redirect('login')
    return render(request,'users/deleteAccount.html')


# coding: utf-8
from __future__ import (absolute_import, division, print_function, unicode_literals)

import boto3

def lambda_handler(event, context):

    deviceId = event['params']['deviceId']

    dynamodb = boto3.resource('dynamodb')
    certTable = dynamodb.Table('device-certs')

    getCerts = certTable.get_item(
        Key={
            'deviceId': deviceId
        }
    )
    print("getCerts output: ", getCerts['ResponseMetadata'])

    return {"deviceCredentials":
        {"deviceId": getCerts['Item']['deviceId'],
        "certificatePem": getCerts['Item']['certificatePem'],
        "publicKey": getCerts['Item']['publicKey'],
        "privateKey": getCerts['Item']['privateKey']
    }}


import re
from typing import Dict, Set, Tuple, Optional

def parse_extras(requirements_path: str) -> Dict[str, Set[str]]:
    """Parse over the requirements.txt file to find extras requested.

    Args:
        requirements_path: The filepath for the requirements.txt file to parse.

    Returns:
         A dictionary mapping the requirement name to a set of extras requested.
    """

    extras_requested = {}
    with open(requirements_path, "r") as requirements:
        # Merge all backslash line continuations so we parse each requirement as a single line.
        for line in requirements.read().replace("\\\n", "").split("\n"):
            requirement, extras = _parse_requirement_for_extra(line)
            if requirement and extras:
                extras_requested[requirement] = extras

    return extras_requested

def _parse_requirement_for_extra(
    requirement: str,
) -> Tuple[Optional[str], Optional[Set[str]]]:
    """Given a requirement string, returns the requirement name and set of extras, if extras specified.
    Else, returns (None, None)
    """

    # https://www.python.org/dev/peps/pep-0508/#grammar
    extras_pattern = re.compile(
        r"^\s*([0-9A-Za-z][0-9A-Za-z_.\-]*)\s*\[\s*([0-9A-Za-z][0-9A-Za-z_.\-]*(?:\s*,\s*[0-9A-Za-z][0-9A-Za-z_.\-]*)*)\s*\]"
    )

    matches = extras_pattern.match(requirement)
    if matches:
        return (
            matches.group(1),
            {extra.strip() for extra in matches.group(2).split(",")},
        )

    return None, None


import requests
from bs4 import BeautifulSoup
import csv
import io
import os

for file in os.listdir("/Users/novin/Desktop/Extractor"):
    if file.endswith(".html"):
        quotes = []  # a list to store quotes
        f = open(file, "r", encoding="utf-8")
        soup = BeautifulSoup(f.read(), 'html.parser')
        rows = soup.findAll('div', {"class": 'display_question question multiple_choice_question'})
        name = soup.select('div.displaying h1')[0].text
        for row in rows:
            questions = row.findAll('div', {"class": 'question_text user_content enhanced'})
            # print(len(questions))
            question = {}
            for q in questions:
                question = q.text
            options = row.select('div.answer.answer_for_')
            opts = []
            for opt in options:
                option = {'text': opt.findAll('div', {"class": 'answer_text'})[0].text,
                          'is_ans': 'correct_answer' in opt.get('class')}
                opts.append(option)
            quotes.append({'question': question, 'options': opts})

        opt_header = ['
الف)
',
                      '
ب)
',
                      '
ج)
',
                      '
د)
']

        filename = 'All_Questions.html'
        with io.open(filename, "a", encoding="utf-8") as f:
            f.write("  "
                    ""
                    "\n
")
            f.write("

" + name + "
\n

")
            for quote in quotes:
                f.write("
\n
")
                # print([quote['name'], quote['uni'], quote['date']])
                f.write("{0}\n".format(quote['question']))
                i = 0
                for opt in quote['options']:
                    if opt['is_ans']:
                        f.write("
")
                    f.write('
'+opt_header[i] + opt['text'] + "\n
")
                    i += 1
                    if opt['is_ans']:
                        f.write("
")
                f.write("
\n")
            f.write("
  ")


import secrets
import json
from pathlib import Path

def random_key(L):
    '''
    生成随机密钥，返回整数
    '''
    bytes_key = secrets.token_bytes(nbytes=L)
    int_key = int.from_bytes(bytes_key, byteorder='big') # 大端存储
    # int_key = int.from_bytes(bytes_key, byteorder='little') # 小端存储
    return int_key
# print(random_key(10))
def Encryption(string):
    '''
    将字符串参数编码为字节串并转换为整数，与一个随机密钥进行异或运算生成
    '''
    string_bytes = str.encode(string)
    int_string = int.from_bytes(string_bytes, byteorder='big')
    random_k = random_key(len(string_bytes))
    secret = int_string ^ random_k
    return {'key':random_k, 'secret':secret}

# random_k,secret = Encryption('abc')

def Decryption(random_k,secret):
    '''
    将密文和密钥异或，计算密文长度，返回字符串
    '''
    int_message = secret ^ random_k
    byte_message = int_message.to_bytes((int_message.bit_length() + 7) // 8,byteorder='big')
    str_message = bytes.decode(byte_message)
    return str(str_message)

string1 = "路漫漫其修远兮，吾将上下而求索。"
string2 = "Never put off until tomorrow what may be done today."

# # 选择要加解密的字符串
stringx = string1
print(Encryption(stringx))
Encryption_pair = Encryption(stringx)
print(Decryption(Encryption_pair['key'],Encryption_pair['secret']))

stringx = string2
print(Encryption(stringx))
Encryption_pair = Encryption(stringx)
print(Decryption(Encryption_pair['key'],Encryption_pair['secret']))

# path = 'key.txt'
def Encryption_file(path='key.txt'):
    '''
    加密文件，得到key和密文
    '''
    path = Path(path) # path of random_key
    message_path = 'message.txt'
    secret_path_new = 'secret_new.txt'

    with open(path,'wt',encoding='utf-8') as fkey, open(message_path,'rt',encoding='utf-8') as fmessage, open(secret_path_new,'wt',encoding='utf-8') as fsecret:
        Encryptiontxt = Encryption(fmessage.read())
        secret1, key1 = Encryptiontxt['secret'], Encryptiontxt['key']
        # print(Encryptiontxt)
        json.dump(key1,fkey)
        json.dump(secret1,fsecret)

def Decryption_file():
    '''
    解密文件，得到明文
    '''
    key_path = 'key.txt'
    message_path_new = 'message_new.txt'
    secret_path = 'secret_new.txt'

    with open(key_path,'rt',encoding='utf-8') as fkey, open(message_path_new,'wt',encoding='utf-8') as fmessage, open(secret_path,'rt',encoding='utf-8') as fsecret:
        Decryptiontxt = Decryption(json.load(fkey),json.load(fsecret))
        fmessage.write(Decryptiontxt)

# Encryption_file()
# Decryption_file()


# -*- coding: utf-8 -*-
import scrapy
from pyquery import PyQuery
from page_read.json_read import *

class JobSpider(scrapy.Spider):
    name = 'Job'
    allowed_domains = ['51job.com']
    urlInfo = read_json_file("config/startUrl.json")
    start_urls = []
    start_urls.append(urlInfo["start"])
    for i in range(2, urlInfo["pageNumber"]):
        start_urls.append(urlInfo["default"].format(i))

    print(start_urls)

    def parse(self, response):
        jpy = PyQuery(response.text)
        items = jpy('.el').items()
        for item in items:
            position = item('.t1 > span > a').text()
            positionUrl = item('.t1 > span > a').attr('href')
            company = item('span.t2 > a').text()
            companyUrl = item('span.t2 > a').attr('href')
            address = item('.t3').text()
            salary = item('.t4').text()
            updateTime = item('.t5').text()

            if position:
                info = {
                    "position": position,
                    "company": company,
                    "positionUrl": response.urljoin(positionUrl),
                    "companyUrl": response.urljoin(companyUrl),
                    "address": address,
                    "salary": salary,
                    "updateTime": updateTime
                }
                yield scrapy.Request(info["positionUrl"], meta={"info": info}, callback=self.parse_second)

    def parse_second(self, response):
        info = response.meta["info"]
        jpy = PyQuery(response.text)
        info["information"] = "|".join(jpy('.msg.ltype').text().split("   |  "))
        info["requirement"] = jpy('.sp4').text()
        info["others"] = jpy('.t2').text()
        info["workAddress"] = jpy(
            'body > div.tCompanyPage > div.tCompany_center.clearfix > div.tCompany_main > div:nth-child(3) > div > p').text().replace(
            "上班地址：", "")
        info["companyInfo"] = jpy(
            'body > div.tCompanyPage > div.tCompany_center.clearfix > div.tCompany_main > div:nth-child(4) > div').text()
        info["positionInfo"] = jpy(
            'body > div.tCompanyPage > div.tCompany_center.clearfix > div.tCompany_main > div:nth-child(2) > div').text()
        return info


import os
from pathlib import Path
from typing import Any, Dict, Union
import numpy as np

def isdicom(path: Union[str, Path]) -> bool:
    """ Judge whether a given file is a valid dicom.

    Args:
        path: given file path.

    Returns:
        True if given path is a valid dicom, otherwise False.
    """
    if not os.path.isfile(path):
        return False

    # read preamble and magic code
    with open(path, "rb") as f:
        header = f.read(132)

    if not header:
        return False

    # magic code of a dicom file should be "DICM"
    return False if header[128:132] != b"DICM" else True

def get_reorient_image(vtd: Dict[str, Any]) -> np.ndarray:
    return np.flip(
        vtd["image_zyx"],
        np.where(vtd["direction_zyx"] < 0)[0]
    )