#start --

def solution(companies, applicants):
    is_change = True
    company_list = dict()
    applicant_list = dict()
    left = dict()
    for company in companies:
        raw_data = company.split(" ")
        company_list[raw_data[0]] = {"prefer":list(raw_data[1]),
                                     "count":int(raw_data[2]),
                                     "applicant":dict(),
                                     "approve":dict()}
    for applicant in applicants:
        raw_data = applicant.split(" ")
        applicant_list[raw_data[0]] = {"prefer":raw_data[1],
                                       "left_chance":int(raw_data[2]),
                                       "count": int(raw_data[2])}
        left[raw_data[0]] = True

    while len(left) > 0:
        complete_appliance = list()
        for applicant in left.keys():
            if  applicant_list[applicant]["left_chance"] <= 0:
                complete_appliance.append(applicant)
                continue
            appliance_order = applicant_list[applicant]["count"] - applicant_list[applicant]["left_chance"]
            company = applicant_list[applicant]["prefer"][appliance_order]
            applicant_list[applicant]["left_chance"] -= 1
            if applicant in company_list[company]["prefer"]:
                company_list[company]["applicant"][applicant] = False

        for applicant in complete_appliance:
            if left.get(applicant) != None:
                left.pop(applicant)

        if len(left.keys()) <= 0:
            break

        approve_list = list()
        for company in company_list.keys():
            prev_all_approve = set(company_list[company]["applicant"].keys())
            company_list[company]["approve"] = dict()
            for prefer_applicant in company_list[company]["prefer"]:
                if len(company_list[company]["approve"]) >= company_list[company]["count"]:
                    break
                if company_list[company]["applicant"].get(prefer_applicant) != None:
                    company_list[company]["approve"][prefer_applicant] = True
                    approve_list.append(prefer_applicant)

            current_approve = set(company_list[company]["approve"].keys())
            left_applicants = prev_all_approve - current_approve
            while len(left_applicants) > 0:
                left_applicant = left_applicants.pop()
                left[left_applicant] = True
        for approve in approve_list:
            if left.get(approve) != None:
                left.pop(approve)

    result = []
    for company in company_list.keys():
        applicant = sorted(list(company_list[company]["approve"].keys()))
        result.append("{0}_{1}".format(company, ''.join(applicant)))
    return result

if __name__ == "__main__":
    print(solution(["A fabdec 2", "B cebdfa 2", "C ecfadb 2"], ["a BAC 1", "b BAC 3", "c BCA 2", "d ABC 3", "e BCA 3", "f ABC 2"]))
    print(solution(["A abc 2", "B abc 1"], ["a AB 1", "b AB 1", "c AB 1"]))


import cv2
import numpy as np

kernel = [(-1,-2), (0,-2), (1,-2),
          (-2,-1), (-1,-1), (0,-1), (1,-1), (2,-1),
          (-2,0), (-1,0), (0,0), (1,0), (2,0),
          (-2,1), (-1,1), (0,1), (1,1), (2,1),
          (-1,2), (0,2), (-1,2)]

def dilation(img):
    r, c = img.shape
    d_img = np.zeros(img.shape, dtype=np.uint8)

    for i in range(r):
        for j in range(c):
            if img[i][j] != 0:
                maximum = 0
                for x, y in kernel:
                    if 0 <= i+x < r and 0 <= j+y < c:
                        maximum = max(maximum, img[i+x][j+y])
                for x, y in kernel:
                    if 0 <= i+x < r and 0 <= j+y < c:
                        d_img[i+x][j+y] = maximum
    return d_img

def erosion(img):
    r, c = img.shape
    e_img = np.zeros(img.shape, dtype=np.uint8)

    for i in range(r):
        for j in range(c):
            set_to_min = True
            minimum = 255
            for x, y in kernel:
                if 0 <= i+x < r and 0 <= j+y < c:
                    minimum = min(minimum, img[i+x][j+y])
                    if img[i+x][j+y] == 0:
                        set_to_min = False
                        break
                else:
                    set_to_min = False
                    break

            if set_to_min:
                e_img[i][j] = minimum
    return e_img

def open(img):
    o_img = dilation(erosion(img))

    return o_img

def close(img):
    c_img = erosion(dilation(img))

    return c_img

if __name__=='__main__':

    image = cv2.imread('lena.bmp', cv2.IMREAD_GRAYSCALE)

    dilation_image = dilation(image)
    cv2.imshow('Dilation Image', dilation_image)
    cv2.imwrite('Dilation Image.jpg', dilation_image)

    erosion_image = erosion(image)
    cv2.imshow('Erosion Image', erosion_image)
    cv2.imwrite('Erosion Image.jpg', erosion_image)

    open_image = open(image)
    cv2.imshow('Open Image', open_image)
    cv2.imwrite('Open Image.jpg', open_image)

    close_image = close(image)
    cv2.imshow('Close Image', close_image)
    cv2.imwrite('Close Image.jpg', close_image)

    cv2.waitKey(0)


#Assembly encoder by Eric Bauer

###Please use TrungLe's code for HW4 ###
#When writing this code I did not know support for tags was needed,
#this code does NOT support tags, and TrungLe's does
##I apologize for the error.

#Supports add, addi, and jump instructions
#Currently only supports immediates written in decimal.
import numpy as np

class converter:
    def __init__(self, infile):
        #read file and split lines into an array
        lines = open(infile,'r').read().split('\n')
        #split each line into command elements
        self.comms=[]
        for x in range(np.size(lines)):
            self.comms.append(lines[x].split(' '))

    def run(self):
        out=open('mc.txt','w')
        for instr in self.comms:
            line=0
            #remove extraneous characters
            for x in range(len(instr)):
                instr[x]=instr[x].replace('$','')
                instr[x]=instr[x].replace(',','')
            #encode arguments and opcode based on the command
            if instr[0]=='add':
                op=0
                func=0x20
                rd=int(instr[1])
                rs=int(instr[2])
                rt=int(instr[3])
                imm=0
            elif instr[0]=='addi':
                op=0x08
                func=0
                rd=0
                rt=int(instr[1])
                rs=int(instr[2])
                imm=int(instr[3])

            elif instr[0]=='jump':
                op=0x02
                func=0
                rt=0
                rs=0
                rd=0
                imm=int(instr[1])>>2

            line+=(op<<26)
            line+=(rs<<21)
            line+=(rt<<16)
            line+=(rd<<11)
            line+=imm
            line+=func
            out.write(format(line,'032b')+'\n')
        out.close()

#file = "mips1.asm"
thing=converter('mips1.asm')
thing.run()
#lines = open(file,'r').read().split('\n')
#    #split each line into command elements
#instr=[]
#for x in range(np.size(lines)):
#    instr.append(lines[x].split(' '))


# inplace sorting
x = [5,2,8,5,1]
x.sort()
print(x) # [1, 2, 5, 5, 8]

# sorting by creating new list
y = [5,2,8,5,1]
k = sorted(y)
print(k) # [1, 2, 5, 5, 8]

# reverse sorting
l = [5,1,7,3,6]
l.sort(reverse=True)
print(l) # [7, 6, 5, 3, 1]


# Definition for the class: Car
class Car(object):
	# initialize your class attributes
	carMaxPrice = 0
	carMPG = 0
	carType = ""
	carName = ""

	def __init__(self, name):
		self.carName = name

	# All the setter methods for this class
	# Set Car Type
	def setCarType(self, typeOption):
		carTypeDict = {"1": "Convertible", "2": "Hybrid", "3" : "Composite"}
		if carTypeDict.get(typeOption):
			self.carType = carTypeDict[typeOption]
		else:
			self.carType = -1

	# Set Price
	def setPrice(self, price_range):
		priceDict = { "1" : 40000, "2" : 50000 ,"3" : 100000}
		if priceDict.get(price_range):
			priceRange = priceDict[price_range]
			self.carMaxPrice = priceRange
		else:
			self.carMaxPrice = -1

	# Set MPG
	def setMPG(self, mpg):
		mpgDict = { "1" : 20 ,"2" : 30, "3" : 40}
		if mpgDict.get(mpg):
			self.carMPG = mpgDict[mpg]
		else:
			self.carMPG = -1

	# All the getter methods for this class
	# Get name
	def getName(self):
		return self.carName

	# Get MPG
	def getMPG(self):
		return self.carMPG

	# Get Price
	def getPrice(self):
		return self.carMaxPrice

	# Get Car Type
	def getCarType(self):
		return self.carType

def carInventory(name, type, maxprice, mpg):
	found = 0
	typesDict = {
		"Convertible" : ["Ford Mustang", "Mercedes SLK"],
		"Hybrid" : ["Honda Civic", "Toyota Prius"],
		"Composite" : ["Chevrolet Volt", "Tesla Model S"]
	}

	carDetailsDict = {
		"Ford Mustang" : [25000, 25, ["silver","blue","yellow","red"]],
		"Mercedes SLK" : [48000, 30, ["silver","white","black"]],
		"Honda Civic" : [30000, 45, ["slate grey","white"]],
		"Toyota Prius" : [30000, 45, ["baby blue","dark grey"]],
		"Chevrolet Volt" : [35000, 100, ["white","maroon"]],
		"Tesla Model S" : [68000, 100, ["black","deep blue"]],
	}
	carList = typesDict.get(type)
	for car in carList:
		details = carDetailsDict[car]
		if details[0] <= maxprice:
			if details[1] >= mpg:
				found += 1
				print ("\n\nMatch found for: ", name)
				print ("Car Make: ", car)
				print ("MSRP USD: ", details[0])
				print ("MPG: ", details[1])
				print ("Available colors: ", details[2])
	if found == 0:
		print ("Search criteria should be updated, no matches found for: ", name)

# Main starts here
print ('\n\n #### WELCOME!! to the NON-SLEAZY virtual car shop! ####\n\n')

# This will be used in a print statement to provide options to the user
# on car types he/she can pick from
car_options = """
Please pick the type of car you are interested in:
1. Convertible
2. Hybrid
3. Composite (Electricity and Gas)
"""

# This will be used in a print statement to provide options to the user on
# price ranges
price_ranges = """
1. < 40K
2. < 50K
3. < 100K
"""

# This section will be used to take a color input on your car
miles_per_gallon = """
1. 20 mpg
2. 30 mpg
3. 40 mpg
"""
carName = input("What would you like to call your car: ")
carType = input(car_options)
carPriceRange = input(price_ranges)
carMPG = input(miles_per_gallon)

# Instantiate your car from the "Car" class
myCar = Car(carName)

# Set the type of car and the price range for the car
myCar.setCarType(carType)
myCar.setPrice(carPriceRange)
myCar.setMPG(carMPG)

# Get the car name, price range and type of car
myCarName = myCar.getName()
myCarType = myCar.getCarType()
myCarMax = myCar.getPrice()
myCarMpg = myCar.getMPG()
#

# Error Checking Code
if myCarType == -1:
	print("Invalid type of car specified, exiting...")
	exit(1)

if myCarMax == 0 or myCarMax == -1:
	print("Invalid Price range specified for your car, exiting...")
	exit(1)

if myCarMpg == 0 or myCarMpg == -1:
	print("Invalid selection for MPG, exiting..")
	exit(1)

#print ("My car is called: ", myCarName)
#print ("Car type selected: ", myCarType)
#print ("Maximum price range specified is: ", myCarMax)
#print ("Miles per gallon for %s is:  %s mpg." %(myCarName, myCarMpg))

carInventory(myCarName, myCarType, myCarMax, myCarMpg)


# -*- coding: cp1251 -*-
"""
Потокозащищённый словарик с поддержкой слоёв, где
из одного потока можно и читать и писать, а из других потоков - только читать (методы get_th_store,get_th_all,get_th_layer_ids,get_th_status)
Use:
th_dic['db']={}
th_dic['db'][1]=2
также см. тесты внизу файла
"""

import thread, threading
import cp_utils as cpu
import copy

#Внутренний механизм словаря в данном случае потокозащищенный при добавлении ключей Thi:
#а подключи db: и т.д. - нет, но к ним идет обращение 2-мя способами (каждый поток пишет лишь в свою ветку):
#1.из одного потока, поэтому читатели и писатели одного потока не мешают друг другу и их не нужно защищаить блокировками
#2.из другого потока при вызове get_th_store (читатель), поэтому нужно защитить методы-писатели(__setitem__ и __delitem__) и этот метод одной блокировкой
#Итого: защищаем блокировками все методы, куда передаётся thread_id + __setitem__ и __delitem__, кроме _dic по текущему потоку.

class _ThDictMultiLayer(object):
    """"Хранилище данных потоков в разрезе слоёв.

    Старая структура
    _tss = {Th1: {'db': {}, 'tr_cnt': {}, 'id_top_system': {}, ...},
            Th2: {'db': {}, 'tr_cnt': {}, 'id_top_system': {}, ...},
            Th3: {'db': {}, 'tr_cnt': {}, 'id_top_system': {}, ...}
           }

    Новая структура
    _tss = {
            Th1: {-1:     {'db': {}, 'tr_cnt': {}, 'id_top_system': {}, ...}, //системный слой хранит данные неавторизованных пользователей
                  Layer1: {'db': {}, 'tr_cnt': {}, 'id_top_system': {}, ...},
                  ...
                  LayerN: {'db': {}, 'tr_cnt': {}, 'id_top_system': {}, ...}
                 },
            ...
            ThN: {-1:     {'db': {}, 'tr_cnt': {}, 'id_top_system': {}, ...}, //системный слой
                  Layer1: {'db': {}, 'tr_cnt': {}, 'id_top_system': {}, ...},
                  ...
                  LayerN: {'db': {}, 'tr_cnt': {}, 'id_top_system': {}, ...}
                 }
           }
    """

    _tss = {}

    def __init__(self):
        """Конструктор. Создаёт один объект блокировки на все потоки и слои."""
        self.Lock = threading.RLock() # вложенность блокировок используется в get_th_status

    # Методы, работающие с разными потоками и слоями

    def _all(self):
        """Возвращает ссылку на весь словарик данных класса.
        ТОЛЬКО ДЛЯ ВНУТРЕННЕГО ИСПОЛЬЗОВАНИЯ (тестирования). Непотокобезопасный!
        """
        return _ThDictMultiLayer._tss

    def _get_th_all(self, thread_id):
        """Возвращает ссылку на всю структуру слоёв и их данных по потоку ДЛЯ ВНУТРЕННЕГО ИСПОЛЬЗОВАНИЯ (непотокобезопасный).
        Если thread_id is None, используется текущий поток.
        Если запрошена отсутствующая структура по потоку, она создаётся.
        """
        if thread_id is None:
            thread_id = thread.get_ident()

        try:
            tss = _ThDictMultiLayer._tss[thread_id]
        except KeyError:
            tss = _ThDictMultiLayer._tss[thread_id] = {}
        return tss

    def get_th_all(self, thread_id=None):
        """Потокозащищенная версия метода _get_th_all для использования вне класса.
        Возвращает КОПИЮ всей структуры слоёв и их данных по потоку.
        Если thread_id is None, используется текущий поток.

        Вызов метода замедляет общую производительность веб-сервера (поскольку захватывает блокировку и выполняет копирование большого объема данных), если
        вместе с его вызовом выполняется хотя бы одно из 3-х условий:
        1.параметр hot_conf.ini sleep_on_connect != 0 (db.sleepOnConnect вызывает get_th_store)
        2.идет активная фаза коннектов из потоков при старте сервера (заполняются ключи данных потока вызовом __setitem__)
        3.используется интерфейс "Управление веб-сервером" (вызов get_th_store из интерфейса непосредственно и через db.getCntActiveTrans)
        4.веб-сервер останавливается (db.db_connectClose вызывает get_th_store)
        """
        if thread_id is None:
            thread_id = thread.get_ident()

        self.Lock.acquire()
        try:
            try:
                tss = _ThDictMultiLayer._tss[thread_id]
            except KeyError:
                tss = _ThDictMultiLayer._tss[thread_id] = {}
            tss_copy = copy.copy(tss)   #deepcopy не копирует объекты kinterbasdb
        finally:
            self.Lock.release()

        return tss_copy

    def _dic(self, thread_id=None, layer_id=None): # def _dic(_get_ident=thread.get_ident) make thread.get_ident a local var
        """Возвращает ссылку на словарик данных, определяемый потоком и слоем ДЛЯ ВНУТРЕННЕГО ИСПОЛЬЗОВАНИЯ (непотокобезопасный) в виде:
              {'db': {}, 'tr_cnt': {}, 'id_top_system': {}, 'cnt_reconnects': ..., 'in_trans': ..., 'last_sql': ..., 'trans_start': ..., 'trans_end': ...}

          Методы-писатели, а также читатели данных других потоков обрамоляют его блокировкой.

          Если thread_id is None, используется текущий поток, если layer_id is None - текущий слой.
          Если пользователь не авторизовался и запрошен текущий слой - используется слой -1.
          Если запрошена отсутствующая структура по потоку или слою, она создаётся.

          при коннектах/реконнектах заполняются
          th_dic['db']           ={id_top_system1: connect1,   id_top_system2: connect2} # словарь открытых потоком соединенией по топ-системам
          th_dic['tr_cnt']       ={id_top_system1: count1,     id_top_system2: count2} # количество транзакций, стартованных потоком с момента последнего коннекта/реконнекта, по базам данных топ-систем
          th_dic['id_top_system']={id_system1: id_top_system1, id_system2: id_top_system2} # топ-система для каждой подсистемы - храним в каждом потоке, чтоб не защищать блокировками читателей, и в каждом слое, если в будущем можно будет NO_MAP подсистем по слоям
          th_dic['cnt_reconnects']=cnt_reconnects # к-во реконнектов к базе DBKURS, совершённых данным потоком в данном слое

          при обработке транзакций заполняются
          th_dic['in_trans']=0 или 1 - пишут также Direct-ф-ции (по любым базам, кроме DBENGINE)
          th_dic['last_sql']='select * from ...' - пишут также Direct-ф-ции (по любым базам, кроме DBENGINE)
          th_dic['trans_start']=time_beg - пишут также Direct-ф-ции (по любым базам, кроме DBENGINE)
          th_dic['trans_end']=time_end - пишут также Direct-ф-ции (по любым базам, кроме DBENGINE)
        """

        if layer_id is None:
            layer_id = cpu.getUserVar('layer_id')
        if layer_id is None:
            layer_id = -1

        th_all = self._get_th_all(thread_id)
        try:
            tss = th_all[layer_id]
        except KeyError:
            tss = th_all[layer_id] = {}
        return tss

    def get_th_store(self, thread_id, key, layer_id=None):
        """Возвращает КОПИИ данных по потоку, слою и ключу внутри слоя.
        Потокозащищенный метод для использования вне класса.
        Если thread_id is None, используется текущий поток, если layer_id is None - текущий слой.
        Если запрошена отсутствующая структура, она создаётся.
        """
        #self.Lock.acquire()
        #try:
        #    return self._dic(thread_id=thread_id, layer_id=layer_id).get(key) # обезопасиваем чтение данных по ключам другого потока
        #finally:
        #    self.Lock.release()

        if layer_id is None:
            layer_id = cpu.getUserVar('layer_id')
        if layer_id is None:
            layer_id = -1

        self.Lock.acquire()
        try:
            th_all = self._get_th_all(thread_id)
            try:
                tss = th_all[layer_id]
            except KeyError:
                tss = th_all[layer_id] = {}
            val_copy = copy.copy(tss.get(key)) #deepcopy не копирует объекты kinterbasdb
        finally:
            self.Lock.release()

        return val_copy

    def get_th_layer_ids(self, thread_id=None):
        """Потокозащищенный метод для использования как внутри, так и вовне класса.
        Возвращает список layer_id по потоку.
        Если thread_id is None, используется текущий поток.
        Если словарь слоёв по потоку отсутствует, он создаётся.
        """
        if thread_id is None:
            thread_id = thread.get_ident()

        self.Lock.acquire()
        try:
            try:
                tss = _ThDictMultiLayer._tss[thread_id]
            except KeyError:
                tss = _ThDictMultiLayer._tss[thread_id] = {}
            keys = tss.keys()
        finally:
            self.Lock.release()

        return keys

    # Метод мониторинга транзакций по всем слоям потока.
    # Вынесен сюда (в сам класс, а не в место вызова в интерфейсе управления веб-сервером)
    # для ускорения работы, чтоб избежать копирования
    # всего словаря слоёв по потоку и использовать быстрый доступ к этим данным
    # через вызов _dic

    def get_th_status(self, thread_id=None):
        """Мониторинг транзакций по всем слоям потока.
        Потокозащищенный метод для использования вне класса.

        Сканирует данные всех слоёв потока и возвращает кортеж
        (layer_id, in_trans, trans_start, trans_end, last_sql), где
        возвращаемые параметры выбираются из одного из слоёв потока.

        Выбор слоя зависит от активности транзакций в нём (ключ словаря слоя 'in_trans'),
        и начала транзакций по этому слою (ключ 'trans_start'):
        -если найден слой, в словаре данных которого значение ключа 'in_trans'=1, то выбирается этот слой
        -иначе выбирается слой c максмимальным значением ключа 'trans_start'
        (поскольку поток не начинает обрабатывать новый запрос(слой), пока не завершит предыдущий, то всё равно, сравнивать ли значения ключей 'trans_start' или 'trans_end')

        Если thread_id is None, используется текущий поток.
        Если запрошена отсутствующая структура по потоку, она создаётся.
        """

        layer_id_found = None
        in_trans = 0
        last_trans_start = 0
        last_trans_end = 0
        last_last_sql = ''

        self.Lock.acquire()
        try:
            for layer_id in self.get_th_layer_ids(thread_id=thread_id):
                dic = self._dic(thread_id=thread_id, layer_id=layer_id)
                in_trans = dic.get('in_trans')
                trans_start = dic.get('trans_start')
                trans_end = dic.get('trans_end')
                last_sql = dic.get('last_sql')

                if in_trans:
                    layer_id_found = layer_id
                    last_trans_start = trans_start
                    last_trans_end = trans_end
                    last_last_sql = last_sql
                    break

                if trans_start >= last_trans_start:
                    layer_id_found = layer_id
                    last_trans_start = trans_start
                    last_trans_end = trans_end
                    last_last_sql = last_sql
            return layer_id_found, in_trans, last_trans_start, last_trans_end, last_last_sql
        finally:
            self.Lock.release()

    # Методы, работающие с текущим потоком и слоем

    def __getitem__(self, key):
        return self._dic()[key]

    def get(self, key, default=None):
        return self._dic().get(key, default)

    def items(self):
        # for item in myset.items():
        return self._dic().items()

    def iteritems(self):
        # for item in myset.iteritems():
        return self._dic().iteritems()

    def has_key(self, key):
        return self._dic().has_key(key)

    def __setitem__(self, key, value):
        self.Lock.acquire()
        try:
            self._dic()[key] = value
        finally:
            self.Lock.release()

    def __delitem__(self, key):
        self.Lock.acquire()
        try:
            del self._dic()[key]
        finally:
            self.Lock.release()

    def __contains__(self, key):
        return key in self._dic()

    def __repr__(self):
        return repr(self._dic())

    def __str__(self):
        return str(self._dic())

    def __len__(self):
        return len(self._dic())

    def __cmp__(self, other):
        return cmp(self._dic(), other._dic())

#th_dic=_ThDictMultiLayer()

"""
print "Test 1:"
print
try:
    print "th_dic['db']=", th_dic['db']
except KeyError, e:
    print " ... th_dic['db']: exception-", str(e)
print "'db' in th_dic=", 'db' in th_dic
print "th_dic.has_key('db')=", th_dic.has_key('db')
print "th_dic.get('db')=", th_dic.get('db')
print
print "set th_dic['db']={1:2}"
th_dic['db']={1:2}
print
print "th_dic['db']=", th_dic['db']
print "'db' in th_dic=", 'db' in th_dic
print "th_dic.get('db')=", th_dic.get('db')
print "th_dic.has_key('db')=", th_dic.has_key('db')
print
print "Test 2:"
print
db=th_dic['db']
#db[1]=2
th_dic['db'][1]=2
print 'db.has_key(1)=', db.has_key(1)
print 'db.has_key(2)=', db.has_key(2)
print 'db[1]=', db[1]
try:
    print 'db[2]=', db[2]
except KeyError, e:
    print " ... db[2]: exception-", str(e)
print 'db.get(1)=', db.get(1)
print 'db.get(2)=', db.get(2)
print "1 in th_dic['db']", 1 in th_dic['db']
print '1 in db', 1 in db
print '2 in db', 2 in db
print 'db before del db[1]=', db
del db[1]
print 'db after del db[1]=', db
"""

"""
print
print "Test 3:"
print
#start 12 threads
#1,2 thread - adds key 'db' and del its
#3,4 thread - modifys key 'db'
import time

SLEEP = 0

def th12(t_index):
    while True:
        if th_dic.has_key('db'):
            del th_dic['db']
        if not th_dic.has_key('db'):
            th_dic['db']={}
        th_dic['db'][1]=t_index
        time.sleep(SLEEP)
        if th_dic['db'][1]!=t_index:
            print "BOK12 !"
        #print "th12, t_index=%s, th_dic['db'][1]=%s" % (t_index, th_dic['db'][1])
        #th_dic['db'][1]=random.random()
        del th_dic['db'][1]

def th34(t_index):
    while True:
        th_dic['db']={}
        th_dic['db'][1]=t_index
        #print "----, t_index=%s, th_dic['db'][1]=%s" % (t_index, th_dic['db'][1])
        time.sleep(SLEEP)
        if th_dic['db'][1]!=t_index:
            print "BOK34 !"

def th56(t_index, otherident):
    while True:
        th_dic.get_th_all(thread_id=otherident)
        time.sleep(SLEEP)

def th78(t_index, otherident):
    while True:
        th_dic.get_th_store(thread_id=otherident, key='db', layer_id=None) #тот же слой
        time.sleep(SLEEP)

def th910(t_index, otherident):
    while True:
        th_dic.get_th_layer_ids(thread_id=otherident)
        time.sleep(SLEEP)

def th1112(t_index, otherident):
    while True:
        th_dic.get_th_status(thread_id=otherident)
        time.sleep(SLEEP)

p1 = threading.Thread(target=th12, name="t1", args=[1])
p2 = threading.Thread(target=th12, name="t2", args=[2])
p3 = threading.Thread(target=th34, name="t3", args=[3])
p4 = threading.Thread(target=th34, name="t4", args=[4])
p5 = threading.Thread(target=th56, name="t5", args=[5, p1.ident])
p6 = threading.Thread(target=th56, name="t6", args=[6, p3.ident])
p7 = threading.Thread(target=th78, name="t7", args=[7, p1.ident])
p8 = threading.Thread(target=th78, name="t8", args=[8, p3.ident])
p9 = threading.Thread(target=th910, name="t9", args=[9, p1.ident])
p10 = threading.Thread(target=th910, name="t10", args=[10, p3.ident])
p11 = threading.Thread(target=th1112, name="t11", args=[11, p1.ident])
p12 = threading.Thread(target=th1112, name="t12", args=[12, p3.ident])

p1.start()
p2.start()
p3.start()
p4.start()
p5.start()
p6.start()
p7.start()
p8.start()
p9.start()
p10.start()
p11.start()
p12.start()
"""


# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-x86_64/egg/botcore/config/config.py
# Compiled at: 2018-04-01 06:36:57
# Size of source mod 2**32: 265 bytes
import json

def load_config(config_file='config/config.json'):
    try:
        with open(config_file) as (json_data_file):
            config = json.load(json_data_file)
        return config
    except Exception:
        raise Exception('Error when load config')


import tkinter as tk
from tkinter import *
from random import seed
from random import random
from PIL import ImageGrab
class Application(tk.Frame):
    def __init__(self, master=None):
        super().__init__(master)

        self.color = 'red'

        self.master = master
        self.pack()
        self.create_widgets()

    def create_widgets(self):
        self.canvas_panel = tk.PanedWindow(self)

        self.canvas_label = tk.LabelFrame(self.canvas_panel, text='Canvas', width=1500, height=1000)

        self.canvas = tk.Canvas(self.canvas_label, bg="white", width=3000, height=1000)

        #self.draw(self.canvas,self.rSlide,self.gSlide,self.bSlide)

        self.canvas_panel.add(self.canvas_label)

        self.save_panel = tk.PanedWindow(self)
        self.save_button = tk.Button(self.save_panel)
        self.save_button["text"] = "Save"
        self.save_button["command"] = self.save

        self.save_panel.add(self.save_button)

        self.quit = tk.Button(self, text="Quit", fg="red",
                              command=self.master.destroy)

        self.save_panel.add(self.quit)
        self.save_panel.pack(side="bottom")

#        self.slider_panel = tk.PanedWindow(self)
        self.slider_label = tk.LabelFrame(self.canvas_panel, text='Options', width=500, height=100)
        self.slider_panel = tk.PanedWindow(self.slider_label)
        self.slider_panel.add(self.slider_label)

        self.update_button = tk.Button(self.slider_panel)
        self.update_button["text"] = "Update"
        self.update_button["command"] = self.update
        self.slider_panel.add(self.update_button)

        #xrSlide
        self.xrSlide = Scale(self.slider_label, from_=1,to=20,orient=HORIZONTAL)
        self.xrSlide.pack()
        self.xrLabel = Label(self.slider_label, text="xrNoise")
        self.xrLabel.pack()
        #yrSlide
        self.yrSlide = Scale(self.slider_label, from_=1,to=20,orient=HORIZONTAL)
        self.yrSlide.pack()
        self.yrLabel = Label(self.slider_label, text="yrNoise")
        self.yrLabel.pack()
        #xbSlide
        self.xbSlide = Scale(self.slider_label, from_=1,to=20,orient=HORIZONTAL)
        self.xbSlide.pack()
        self.xbLabel = Label(self.slider_label, text="xbNoise")
        self.xbLabel.pack()
        #ybSlide
        self.ybSlide = Scale(self.slider_label, from_=1,to=20,orient=HORIZONTAL)
        self.ybSlide.pack()
        self.ybLabel = Label(self.slider_label, text="ybNoise")
        self.ybLabel.pack()
        #xgSlide
        self.xgSlide = Scale(self.slider_label, from_=1,to=20,orient=HORIZONTAL)
        self.xgSlide.pack()
        self.xgLabel = Label(self.slider_label, text="xgNoise")
        self.xgLabel.pack()
        #ygSlide
        self.ygSlide = Scale(self.slider_label, from_=1,to=20,orient=HORIZONTAL)
        self.ygSlide.pack()
        self.ygLabel = Label(self.slider_label, text="ygNoise")
        self.ygLabel.pack()
        #gridSlide
        self.gridSlide = Scale(self.slider_label, from_=5,to=32,orient=HORIZONTAL)
        self.gridSlide.pack()
        self.gridLabel = Label(self.slider_label, text="gridSize")
        self.gridLabel.pack()
        #strokeSlide
        self.strokeSlide = Scale(self.slider_label, from_=2,to=4,orient=HORIZONTAL)
        self.strokeSlide.pack()
        self.strokeLabel = Label(self.slider_label, text="strokeDiv")
        self.strokeLabel.pack()
        #blackground
        self.blackground = IntVar()
        self.check = Checkbutton(self.slider_label, text="for dark mode", variable=self.blackground)
        self.check.pack()

        self.slider_panel.pack(side="top")

        self.canvas_panel.add(self.slider_label)
        self.canvas_panel.pack(side="top")

        #self.slider_panel.pack(side="right")

    def hexColor(self, rgb):
        return "#%02x%02x%02x" % rgb

    def save(self):
        #print("Saved (not really)")
        #self.color = 'green'
        #self.draw(self.canvas,self.rSlide,self.gSlide,self.bSlide)
        x=root.winfo_rootx()+self.canvas.winfo_x()
        y=root.winfo_rooty()+self.canvas.winfo_y()
        x1 = x+self.canvas.winfo_width()
        y1=y+self.canvas.winfo_height()
        ImageGrab.grab().crop((x+3,y+3,x1-3,y1-3)).save("catcrossing.png")
    def darkmode(self):
        if self.blackground.get() == 1:
            self.canvas["bg"] = "black"
        else:
            self.canvas["bg"] = "white"

    def update(self):
        self.darkmode()
        self.draw(self.canvas,self.xrSlide,self.yrSlide,self.xbSlide,self.ybSlide,self.xgSlide,self.ygSlide,self.gridSlide,self.strokeSlide)

    def noise(self, y, x):
        n = y * x / 100.0

        n = n - int(n)

        if n > 1:
            print(str(y) + " " + str(x))

        return n

    def draw(self, canvas, xrNoise, yrNoise, xbNoise, ybNoise, xgNoise, ygNoise, gridSize, strokeDiv):
        canvas.delete("all")
        noiseScale = 0.1
        width = 1500
        height = 1000
        x = 0
        seed(1)
        xrNoiseSet = xrNoise.get()
        xbNoiseSet = xbNoise.get()
        xgNoiseSet = xgNoise.get()

        while x <= width + gridSize.get():
            yrNoiseStart = yrNoise.get()
            ybNoiseStart = ybNoise.get()
            ygNoiseStart = ygNoise.get()
            yrNoiseSet = yrNoiseStart
            ybNoiseSet = ybNoiseStart
            ygNoiseSet = ygNoiseStart
            y = 0
            while y <= height + gridSize.get():
                red = int(255 * self.noise(yrNoiseSet, xrNoiseSet))
                blue = int(255 * self.noise(ybNoiseSet, xbNoiseSet))
                green = int(255 * self.noise(ygNoiseSet, xgNoiseSet))

                if(random() > 0.5):
                    canvas.create_line(x,y,x+gridSize.get(),y+gridSize.get(), fill=self.hexColor((red,blue,green)), width = gridSize.get()/strokeDiv.get(),smooth=True,splinesteps=24)
                else:
                    canvas.create_line(x,y+gridSize.get(),x+gridSize.get(),y, fill=self.hexColor((red,blue,green)), width = gridSize.get()/strokeDiv.get(),smooth=True,splinesteps=24)
                y += gridSize.get()

                yrNoiseSet += noiseScale
                ybNoiseSet += noiseScale
                ygNoiseSet += noiseScale

            x += gridSize.get()

            xrNoiseSet += noiseScale
            xbNoiseSet += noiseScale
            xgNoiseSet += noiseScale

        canvas.pack()

root = tk.Tk()
app = Application(master=root)
app.mainloop()


import string, sys

def getAllWords(filepath):
	flr = open(filepath, 'r')
	dicts = []
	for i in xrange(27):
		dicts.append({})

	for line in flr.readlines():
		line = line[:-1]
		linesplit = line.split(' ')
		w = linesplit[0]
		idx = int(linesplit[1])

		didx = 26
		try:
			lett = w[0]
		except:
			continue
		if lett in string.ascii_lowercase:
			didx = ord(lett) - 97
		dicts[didx][w] = idx

	return dicts

def getIDF(filepath):
	flr = open(filepath, 'r')
	dicts = []
	for i in xrange(27):
		dicts.append({})

	for line in flr.readlines():
		line = line[:-1]
		linesplit = line.split(' ')
		w = linesplit[0]
		idfval = float(linesplit[1])

		didx = 26
		try:
			lett = w[0]
		except:
			continue
		if lett in string.ascii_lowercase:
			didx = ord(lett) - 97
		dicts[didx][w] = idfval

	return dicts

if __name__ == '__main__':
	flr = open(sys.argv[1], 'r')
	idfdicts = getIDF(sys.argv[2])
	globaldicts = getAllWords(sys.argv[3])

	flw = open('matrix.txt', 'w')
	sno = 0
	for line in flr.readlines():
		line = line[:-1]
		if (line == '<s>') or (line == '</s>'):
			if line == '</s>':
				sno = sno + 1
			continue

		linesplit = line.split(' ')
		w = linesplit[0]
		val = float(linesplit[1])

		try:
			lett = w[0]
		except:
			continue
		didx = 26
		if lett in string.ascii_lowercase:
			didx = ord(lett) - 97

		try:
			idfval = idfdicts[didx][w]
			val = val * idfval

			row = globaldicts[didx][w]
			col = sno
		except:
			print("problem occured with " + w)
			continue

		flw.write(str(row) + " " + str(col) + " " + str(val) + "\n")
	flr.close()


"""
Given 2 lists a and b. Each element is a pair of integers where the first integer represents the unique id and the second integer represents a value.
Your task is to find an element from a and an element form b such that the sum of their values is less or equal to target and as close to target as possible.
Return a list of ids of selected elements. If no pair is possible, return an empty list.

Example 1:

Input:
a = [[1, 2], [2, 4], [3, 6]]
b = [[1, 2]]
target = 7

Output: [[2, 1]]

Explanation:
There are only three combinations [1, 1], [2, 1], and [3, 1], which have a total sum of 4, 6 and 8, respectively.
Since 6 is the largest sum that does not exceed 7, [2, 1] is the optimal pair.

Example 2:

Input:
a = [[1, 3], [2, 5], [3, 7], [4, 10]]
b = [[1, 2], [2, 3], [3, 4], [4, 5]]
target = 10

Output: [[2, 4], [3, 2]]

Explanation:
There are two pairs possible. Element with id = 2 from the list `a` has a value 5, and element with id = 4 from the list `b` also has a value 5.
Combined, they add up to 10. Similarily, element with id = 3 from `a` has a value 7, and element with id = 2 from `b` has a value 3.
These also add up to 10. Therefore, the optimal pairs are [2, 4] and [3, 2].

"""

import unittest

class Solution(object):

    def function(self, arr1, arr2, target):

        arr1 = sorted(arr1, key = lambda x : x[1])
        arr2 = sorted(arr2, key = lambda x : x[1])

        left, right = 0, len(arr2) - 1
        diff = float('inf')
        res = []

        while left < len(arr1) and right >= 0:
            id1, val1 = arr1[left]
            id2, val2 = arr2[right]
            new_diff = target - val1 - val2
            if new_diff == diff:
                res.append([id1, id2])
            elif 0 <= new_diff < diff:
                diff = new_diff
                res.clear()
                res.append([id1, id2])
            if new_diff > 0:
                left += 1
            else:
                right -= 1

        return res

class Test(unittest.TestCase):

    s = Solution()

    def test1(self):
        a = [[1, 2], [2, 4], [3, 6]]
        b = [[1, 2]]
        target = 7
        self.assertEqual(self.s.function(a, b, target), [[2, 1]])

    def test2(self):
        a = [[1, 3], [2, 5], [3, 7], [4, 10]]
        b = [[1, 2], [2, 3], [3, 4], [4, 5]]
        target = 10
        self.assertEqual(self.s.function(a, b, target), [[2, 4], [3, 2]])

    def test3(self):
        a = [[1, 8], [2, 7], [3, 14]]
        b = [[1, 5], [2, 10], [3, 14]]
        target = 20
        self.assertEqual(self.s.function(a, b, target), [[3, 1]])

    def test4(self):
        a = [[1, 8], [2, 15], [3, 9]]
        b = [[1, 8], [2, 11], [3, 12]]
        target = 20

        self.assertEqual(self.s.function(a, b, target), [[1, 3], [3, 2]])

if __name__ == '__main__':
    unittest.main()


#!/bin/bash
##############################################################
# File Name:
# Version: V1.0
# Author: shujingping
# Organization: https://www.cnblogs.com/jinggs/
# Created Time : 2020-05-13 19:36:17
# Description:一个http等的服务端
##############################################################
# coding:utf-8

import socket

from multiprocessing import Process

def handle_client(client_socket):
    """
    处理客户端请求
    """
    request_data = client_socket.recv(2048)
    print("request data: %s" % request_data)
    # 构造响应数据
    response_start_line = "HTTP/1.1 200 OK\r\n"
    response_headers = "Server: My server\r\n"
    response_body = "
Python HTTP Test
"
    response = response_start_line + response_headers + "\r\n" + response_body

    # 向客户端返回响应数据
    client_socket.send(bytes(response, "utf-8"))

    print('向客户端返回响应成功，内容是：%s' % response)
    # 关闭客户端连接
    client_socket.close()

if __name__ == "__main__":
    server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    server_socket.bind(("", 8000))
    server_socket.listen(128)

    print('程序已启动，监听8000端口')
    while True:
        client_socket, client_address = server_socket.accept()
        print("[%s, %s]用户连接上了" % (client_socket,client_address))
        handle_client_process = Process(target=handle_client, args=(client_socket,))
        handle_client_process.start()
        client_socket.close()


# 创建从mysql全量拉取数据到hdfs的job任务
# 执行一次，将job任务创建出来即可

import os

table_list = ['user',
              'user_extend',
              'user_addr',
              'code_category'
              ]

def cre():
    for i in table_list:
        os.system("sqoop job --create bap_" +
                  i +
                  " -- import --connect jdbc:mysql://192.168.163.21:3"
                  "306/qfbap_ods --driver com.mysql.jdbc.Driver "
                  "--username root "
                  "--password 123456 "
                  "--table " + i +
                  " --delete-target-dir "
                  "--target-dir /qfbap/ods/tmp/ods_" + i +
                  "--fields-terminated-by '\001'"
                  )

if __name__ == '__main__':
    cre()


import numpy as np
import pandas as pd
import tensorflow as tf

data = pd.read_csv("data/iris.csv", header=None)

species = list(data[4].unique())
data[5] = data[4].map(lambda x: np.eye(len(species))[species.index(x)])
data[6] = data[4]
mapping = {'Iris-setosa': 1, 'Iris-virginica': 2, 'Iris-versicolor': 3}
data = data.replace({6: mapping})

train_data_1 = data[[0, 1, 2, 3]].as_matrix().tolist()
train_label_1 = [[x] for x in data[6].as_matrix().tolist()]
train_data_2 = data[[0, 1, 2, 3]].as_matrix().tolist()
train_label_2 = data[5].as_matrix().tolist()

train_data = train_data_1
train_label = train_label_2

input_features = len(train_data[0])
output_layer_nodes = len(train_label[0])
epoch = 10000
display_step = epoch / 10
learning_rate = 0.001
hidden_layers = 2
hidden_layer_nodes = 4

x = tf.placeholder(dtype=tf.float32, shape=[None, input_features], name="input_placeholder")
y = tf.placeholder(dtype=tf.float32, shape=[None, output_layer_nodes], name="input_placeholder")

def model_def(x, weights, bias):
    layer1 = tf.nn.sigmoid(tf.add(tf.matmul(x, weights['w1']), bias['b1']))
    layer2 = tf.nn.sigmoid(tf.add(tf.matmul(layer1, weights['w2']), bias['b2']))
    layer3 = tf.nn.softmax(tf.add(tf.matmul(layer2, weights['out']), bias['out']))
    return layer3

weights = {
    'w1': tf.Variable(tf.random_normal([input_features, hidden_layer_nodes])),
    'w2': tf.Variable(tf.random_normal([hidden_layer_nodes, hidden_layer_nodes])),
    'out': tf.Variable(tf.random_normal([hidden_layer_nodes, output_layer_nodes]))
}
bias = {
    'b1': tf.Variable(tf.random_normal([hidden_layer_nodes])),
    'b2': tf.Variable(tf.random_normal([hidden_layer_nodes])),
    'out': tf.Variable(tf.random_normal([output_layer_nodes]))
}

model = model_def(x, weights, bias)
cost = -tf.reduce_sum(y * tf.log(model))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)  # Adam Optimizer

init = tf.initialize_all_variables()

def accuracy_simple(pred_data, labelled_data):
    acc = 0
    for p, a in list(zip(pred_data, labelled_data)):
        a = a.tolist()
        p = [1.0 if (x >= 0.5) else 0.0 for x in p]
        if p == a:
            acc += 1
    return acc / len(pred_data) * 100

def accuracy_complex(pred_data, labelled_data):
    acc = 0
    for p, a in list(zip(pred_data, labelled_data)):
        if [p.index(max(p))] == a:
            acc += 1

    return acc / len(pred_data) * 100

with tf.Session() as sess:
    sess.run(init)
    for i in range(0, epoch):
        c, m, _ = sess.run([cost, model, optimizer], feed_dict={x: train_data, y: train_label})
        if i % display_step == 0:
            print("Cost:" + str(c) + " Accuracy:" + str(accuracy_simple(m.tolist(), train_label)))


import os
import argparse
import smtplib
import zipfile
import tempfile
from email import encoders
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart

sender='115cs0231@gmail.com'
recipient='115cs0221@gmail.com'

zf = tempfile.TemporaryFile(prefix='mail', suffix='.zip')
zip = zipfile.ZipFile(zf, 'w')
#print ("Zipping current dir: %s" %os.getcwd())
for file_name in os.listdir(os.getcwd()):
        zip.write(file_name)
zip.close()
zf.seek(0)

print ("Creating email message...")
email_msg = MIMEMultipart()
email_msg['Subject'] = 'File from path %s' %os.getcwd()
email_msg['To'] = ', '.join(recipient)
email_msg['From'] = sender
email_msg.preamble = 'Testing email from Python.\n'
msg = MIMEBase('application', 'zip')
msg.set_payload(zf.read())
encoders.encode_base64(msg)
msg.add_header('Content-Disposition', 'attachment',
           filename=os.getcwd()[-1] + '.zip')
email_msg.attach(msg)
email_msg = email_msg.as_string()

print ("Sending email message...")
try:
        smtp = smtplib.SMTP('localhost')
        smtp.set_debuglevel(1)
        smtp.sendmail(sender, recipient, email_msg)
except Exception as e:
        print ("Error: %s" %str(e))
finally:
        smtp.close()


from __future__ import print_function
import argparse
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
import numpy as np
import pickle

class MimicNet(nn.Module):
    def __init__(self, args):
        super(MimicNet, self).__init__()
        try:
            self.activation = F.__dict__[args.activation]
        except KeyError:
            raise KeyError('torch functional module doesnt implement the specified activation')
        self._mimic = True
        self.conv1 = nn.Conv2d(1, 20, 5, 1)
        self.conv2 = nn.Conv2d(20, 50, 5, 1)
        self.fc1 = nn.Linear(4 * 4 * 50, 500)
        self.fc2 = nn.Linear(500, 10)

    def forward(self, x):
        x = self.conv1(x)  # [batch x 20 x 24 x 24]
        conv_1 = x.view(-1, 20 * 24 * 24)
        x = self.activation(x)
        x = F.max_pool2d(x, 2, 2)
        x = self.conv2(x)  # [batch x 50 x 8 x 8]
        conv_2 = x.view(-1, 50 * 8 * 8)
        x = self.activation(x)
        x = F.max_pool2d(x, 2, 2)
        x = x.view(-1, 4 * 4 * 50)
        x = self.fc1(x)    # [batch x 500]
        fc_1 = x
        x = self.activation(x)
        out = self.fc2(x)  # [batch x 10]
        if self._mimic:
            return torch.cat([conv_1.view(-1, 20 * 24 * 24), conv_2.view(-1, 50 * 8 * 8), fc_1], dim=1), out
        else:
            return out

    def mimic(self):
        self._mimic = True

    def classify(self):
        self._mimic = False

class Net(nn.Module):
    def __init__(self, args):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5, 1)
        self.conv2 = nn.Conv2d(20, 50, 5, 1)
        self.fc1 = nn.Linear(4 * 4 * 50, 500)
        self.fc2 = nn.Linear(500, 10)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.max_pool2d(x, 2, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2, 2)
        x = x.view(-1, 4 * 4 * 50)
        x = F.relu(self.fc1(x))
        out = self.fc2(x)
        return out

val_accs = []
val_losses = []

def train_mimic(args, model, mimic_model, device, train_loader, test_loader, optimizer, epoch):
    model.train()
    model.mimic()
    losses = []
    for batch_idx, (data, _) in enumerate(train_loader):
        if batch_idx % args.test_freq == 0:
            model.classify()
            test(args, model, device, test_loader)
            model.mimic()
        data = data.to(device)
        optimizer.zero_grad()
        with torch.no_grad():
            target_activations, target_output = mimic_model(data)
        activations, output = model(data)
        # get output  loss
        if args.bce_loss:
            target_output = F.sigmoid(target_output)
            output_loss = F.binary_cross_entropy_with_logits(output, target_output)
        else:
            # manually compute cross entropy between distributions encoded in logits
            output = F.log_softmax(output)
            target_output = F.softmax(target_output)
            output_loss = -torch.sum(output * target_output) / output.shape[0]
        loss = output_loss
        if args.mimic_activations:
            # get activation loss
            if args.mse_loss:
                activation_loss = F.mse_loss(activations, target_activations)
            else:
                activations = F.sigmoid(activations)
                target_activations = F.sigmoid(target_activations)
                activation_loss = F.binary_cross_entropy(activations, target_activations)
            loss += activation_loss
        loss.backward()
        optimizer.step()
        if batch_idx % args.log_interval == 0:
            losses += [loss.item()]
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                       100. * batch_idx / len(train_loader), loss.item()))
    model.classify()
    return losses

batch_idx = 0

def train(args, model, device, train_loader, test_loader, optimizer, epoch):
    global batch_idx
    model.train()
    losses = []
    for _, (data, target) in enumerate(train_loader):
        if args.test_freq is not None and batch_idx % args.test_freq == 0:
            test(args, model, device, test_loader)
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)
        if args.bce_loss:
            new_target = torch.zeros((target.shape[0], 10))
            new_target[np.arange(target.shape[0]), target] = 1.
            loss = F.binary_cross_entropy_with_logits(output, new_target)
        else:
            loss = F.cross_entropy(output, target)
        loss.backward()
        optimizer.step()
        if batch_idx % args.log_interval == 0:
            losses += [loss.item()]
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                       100. * batch_idx / len(train_loader), loss.item()))
        batch_idx += 1
    return losses

def test(args, model, device, test_loader):
    global val_accs, val_losses
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            output = F.log_softmax(output, dim=1)
            test_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss
            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)
    test_acc = 100. * correct / len(test_loader.dataset)

    val_accs += [test_acc]
    val_losses += [test_loss]

    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset), test_acc))

    model.train()
    return test_loss, test_acc

def save_file(args, model):
    # save to different path depending on args
    path = args.experiment_id + '/' + args.experiment_id + '.pt'
    torch.save(model.state_dict(), path)

def main():
    global val_accs, val_losses
    parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
    parser.add_argument('--experiment-id', type=str, default='0')
    # Mimic Settings
    parser.add_argument('--activation', type=str, choices=['relu', 'sigmoid'], default='relu')
    parser.add_argument('--mse-loss', action='store_true', help='Formulate MSE Loss on activations? Otherwise'
                                                                'loss is formulated from passing raw layer output'
                                                                'through a sigmoid')
    parser.add_argument('--baseline', action='store_false', dest='mimic')
    parser.add_argument('--only-mimic-output', action='store_false', dest='mimic_activations')
    parser.add_argument('--mimic-model', type=str, default=None,
                        help='The model to mimic output of. If none then train on ground truth')
    parser.add_argument('--bce-loss', action='store_true',
                        help='Whether to train output on BCE Loss (both for mimicking and classification)')
    # Statistics settings
    parser.add_argument('--test-freq', type=int, default=None, help="Frequency of validating training model")
    # Training settings
    parser.add_argument('--batch-size', type=int, default=64, metavar='N',
                        help='input batch size for training (default: 64)')
    parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N',
                        help='input batch size for testing (default: 1000)')
    parser.add_argument('--epochs', type=int, default=10, metavar='N',
                        help='number of epochs to train (default: 10)')
    parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
                        help='learning rate (default: 0.01)')
    parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
                        help='SGD momentum (default: 0.5)')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='disables CUDA training')
    parser.add_argument('--seed', type=int, default=1, metavar='S',
                        help='random seed (default: 1)')
    parser.add_argument('--log-interval', type=int, default=10, metavar='N',
                        help='how many batches to wait before logging training status')

    parser.add_argument('--save-model', action='store_true', default=False,
                        help='For Saving the current Model')
    args = parser.parse_args()
    use_cuda = not args.no_cuda and torch.cuda.is_available()
    if args.mimic_model is not None:
        assert args.mimic, "Must use MimicModel for mimic training. Pass --mimic"

    torch.manual_seed(args.seed)

    device = torch.device("cuda" if use_cuda else "cpu")

    kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
    train_loader = torch.utils.data.DataLoader(
        datasets.MNIST('../data', train=True, download=True,
                       transform=transforms.Compose([
                           transforms.ToTensor(),
                           transforms.Normalize((0.1307,), (0.3081,))
                       ])),
        batch_size=args.batch_size, shuffle=True, **kwargs)
    test_loader = torch.utils.data.DataLoader(
        datasets.MNIST('../data', train=False, transform=transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize((0.1307,), (0.3081,))
        ])),
        batch_size=args.test_batch_size, shuffle=True, **kwargs)

    if args.mimic:
        model = MimicNet(args).to(device)
    else:
        model = Net(args).to(device)
    optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)

    model.classify()
    if args.mimic_model is not None:
        mimic_model = MimicNet(args).to(device)
        mimic_model.load_state_dict(torch.load(args.mimic_model))

    train_losses = []
    for epoch in range(1, args.epochs + 1):
        if args.mimic_model is not None:
            train_losses += train_mimic(args, model, mimic_model, device, train_loader, test_loader, optimizer, epoch)
        else:
            train_losses += train(args, model, device, train_loader, test_loader, optimizer, epoch)
        if args.test_freq is None:
            val_loss, val_accuracy = test(args, model, device, test_loader)
            val_losses += [val_loss]
            val_accs += [val_accuracy]

    with open('%s/%s-train-loss.pkl' % (args.experiment_id, args.experiment_id), 'wb+') as f:
        pickle.dump(train_losses, f)
    with open('%s/%s-val-accuracy.pkl' % (args.experiment_id, args.experiment_id), 'wb+') as f:
        pickle.dump(val_accs, f)
    with open('%s/%s-val-loss.pkl' % (args.experiment_id, args.experiment_id), 'wb+') as f:
        pickle.dump(val_losses, f)

    if args.save_model:
        save_file(args, model)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-

from django.conf.urls.defaults import *
from guestbook.views import IndexView, SignView, UpdateView, DeleteView
from guestbook.api import Greetings, Greeting

urlpatterns = patterns('',
	url(r'^$', IndexView.as_view(), name='index', ),
	url(r'^sign/$', SignView.as_view(), name='sign'),
	url(r'^update/$', UpdateView.as_view(), name='update'),
	url(r'^delete/(?P<guestbook_id>[0-9].*)/(?P<guestbook_name>.*)/$', DeleteView.as_view(), name='delete'),

	## greeting apis
	url(r'^api/v1/(?P<guestbook_name>.*)/greetings/$', Greetings.as_view(),
		name='greeting_list'),
	url(r'^api/v1/(?P<guestbook_name>.*)/(?P<greeting_id>[0-9].*)/$', Greeting.as_view(),
		name='greeting_detail'),
)


from django.contrib import admin
from treebeard.admin import TreeAdmin

from . import forms, models

@admin.register(models.LiteratureAssessment)
class LiteratureAssessmentAdmin(admin.ModelAdmin):
    form = forms.LiteratureAssessmentForm
    readonly_fields = ("assessment", "topic_tsne_refresh_requested", "topic_tsne_last_refresh")

    list_display = (
        "assessment",
        "extraction_tag",
        "topic_tsne_data",
        "topic_tsne_refresh_requested",
        "topic_tsne_last_refresh",
    )

@admin.register(models.ReferenceFilterTag)
class ReferenceFilterTagAdmin(TreeAdmin):
    pass

@admin.register(models.Search)
class SearchAdmin(admin.ModelAdmin):
    list_display = (
        "id",
        "assessment",
        "search_type",
        "source",
        "title",
        "import_file",
        "created",
    )
    list_filter = (
        "search_type",
        "source",
        "assessment",
    )
    search_fields = ("title",)

@admin.register(models.Identifiers)
class IdentifiersAdmin(admin.ModelAdmin):
    list_display = ("unique_id", "database")
    list_filter = ("database",)
    search_fields = ("unique_id",)

@admin.register(models.Reference)
class ReferenceAdmin(admin.ModelAdmin):
    list_display = (
        "id",
        "assessment",
        "title",
        "authors",
        "year",
        "journal",
        "full_text_url",
        "created",
    )
    list_filter = ("year", ("assessment", admin.RelatedOnlyFieldListFilter))
    raw_id_fields = ("searches", "identifiers")
    search_fields = ("title", "authors", "year")


#!/usr/bin/env python
import __builtin__
from os import *
from os.path import *
from traceback import *
from signal import *
import subprocess
from subprocess import *
from argquote import *
from isinstance import *
from public import *
from publish import *
from rm import *
from temp import *
from tolist import *
from write import *

publish(CalledProcessError, PIPE)

class CalledProcessError(CalledProcessError):
    def __str__(self):
        # return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
        return "Command '%s' returned non-zero exit status %d\n%s" % (self.cmd,self.returncode,self.output)

@public
def Popen(args, bufsize=0, executable=None, stdin=None, stdout=PIPE, stderr=PIPE, preexec_fn=None, close_fds=True, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0,echo=False,kill=True,rstrip=True):
    temp = None
    pid = None
    try:
        if stdin is not None and istext(stdin):
            temp = tempfile()
            write(temp,stdin)
            stdin = __builtin__.open(temp)
        stdoutpath = False
        if stdout and istext(stdout) and exists(stdout):
            stdout = __builtin__.open(stdout,"w")
            stdoutpath = True
        if echo:
            quoted = map(argquote,tolist(args))
            print(" ".join(quoted))
        if islist(args):
            for i,arg in enumerate(args):
                if not isint(arg):
                    args[i] = str(arg)
        else:
            if not istext(args):
                args = unicode(args)
        if not preexec_fn:
            preexec_fn = setsid
        if not env:
            env = environ.copy()
        process = subprocess.Popen(args,
            bufsize = bufsize,
            executable = executable,
            stdin = stdin,
            stderr=stderr,
            stdout=stdout,
            preexec_fn=preexec_fn,
            close_fds=close_fds,
            shell=shell,
            cwd=cwd,
            env=env,
            universal_newlines=universal_newlines,
            startupinfo=startupinfo#,
            #creationflags=creationflags
        )
        pid = process.pid
        stdoutdata, stderrdata = process.communicate()
        if rstrip and stdoutdata:
            stdoutdata = stdoutdata.rstrip()
        if rstrip and stderrdata:
            stderrdata = stderrdata.rstrip()
        returncode = process.returncode
        if stdoutpath:
            stdout.flush()
        if returncode==1:
            if stderrdata is not None:
                err = stderrdata
            else:
                err = stdoutdata
            raise CalledProcessError(returncode,args,err)
        if returncode!=0:
            raise CalledProcessError(returncode,args,stderrdata)
        return stdoutdata
    finally:
        try:
            if pid:
                killpg(pid,SIGTERM)
        except:
            pass
            #print(format_exc())
        rm(temp)

#print Popen(["ls","~"],shell=True)
#args = ["python","/Users/nordmenss/git/osascript.py/setup.py","register"]
#cwd = "/Users/nordmenss/git/osascript.py"
#print Popen(args,cwd=cwd)
#from os.path import *
#from temp import *
#temp = tempfile()
#img="/Users/nordmenss/git/git-init.finder-toolbar/img/git.png"

#Popen(["DeRez","-only","icns",img],stdout=temp)
#print getsize(temp)


import os

import gym

from rl.agents.vpg_gae import VPGGAE
from rl.loops import EpisodeTrainLoop
from rl.metrics import AverageReturn, AverageEpisodeLength
from zoo.cartpole.core import PolicyNetwork, ValueFunctionNetwork
from zoo.utils import parse_args, get_output_dirs, evaluate_policy

if __name__ == '__main__':
    args = parse_args()
    ckpt_dir, log_dir = get_output_dirs(os.path.dirname(__file__), 'vpg_gae', args)

    env = gym.make('CartPole-v0')
    policy_fn = lambda: PolicyNetwork(env.observation_space.shape, env.action_space.n)
    vf_fn = lambda: ValueFunctionNetwork(env.observation_space.shape)
    agent = VPGGAE(
        env=env,
        policy_fn=policy_fn,
        vf_fn=vf_fn,
        lr_policy=1e-3,
        lr_vf=1e-3,
        gamma=0.98,
        lambda_=0.96,
        vf_update_iterations=20,
        replay_buffer_size=250 * 2,
        policy_update_batch_size=256,
        vf_update_batch_size=256,
    )
    train_loop = EpisodeTrainLoop(
        agent=agent,
        n_episodes=1000,
        max_episode_length=250,
        ckpt_dir=ckpt_dir,
        log_dir=log_dir,
        ckpt_every=100,
        log_every=10,
        update_every=2,
        metrics=[AverageReturn(2), AverageEpisodeLength(2)]
    )

    if args.mode == 'train':
        train_loop.run()
    if args.mode == 'evaluate':
        evaluate_policy(agent.env, agent.policy)


#!/usr/bin/env python
"""
Test case for Rucio WMCore Service class
"""
from __future__ import print_function, division, absolute_import

import os

from rucio.client import Client as testClient

from WMCore.Services.Rucio.Rucio import Rucio, validateMetaData, RUCIO_VALID_PROJECT
from WMQuality.Emulators.EmulatedUnitTestCase import EmulatedUnitTestCase

DSET = "/SingleElectron/Run2017F-17Nov2017-v1/MINIAOD"
BLOCK = "/SingleElectron/Run2017F-17Nov2017-v1/MINIAOD#f924e248-e029-11e7-aa2a-02163e01b396"

class RucioTest(EmulatedUnitTestCase):
    """
    Unit tests for Rucio Service module
    """

    def __init__(self, methodName='runTest'):
        # TODO figure out what's going on with CRIC mock
        super(RucioTest, self).__init__(methodName=methodName, mockCRIC=False)

        self.acct = "wma_test"

        # HACK: do not verify the SSL certificate because docker images
        # do not contain the CA certificate bundle
        # Relying on the config file in the jenkins infrastructure is a PITA
        # so let's make sure to pass all the necessary arguments
        self.creds = {"client_cert": os.getenv("X509_USER_CERT", "Unknown"),
                      "client_key": os.getenv("X509_USER_KEY", "Unknown")}

        self.defaultArgs = {"host": 'http://cmsrucio-int.cern.ch',
                            "auth_host": 'https://cmsrucio-auth-int.cern.ch',
                            "auth_type": "x509", "account": self.acct,
                            "ca_cert": False, "timeout": 30, "request_retries": 3,
                            "creds": self.creds}

    def setUp(self):
        """
        Setup for unit tests
        """
        super(RucioTest, self).setUp()

        self.myRucio = Rucio(self.acct,
                             hostUrl=self.defaultArgs['host'],
                             authUrl=self.defaultArgs['auth_host'],
                             configDict=self.defaultArgs)

        self.client = testClient(rucio_host=self.defaultArgs['host'],
                                 auth_host=self.defaultArgs['auth_host'],
                                 account=self.acct,
                                 ca_cert=self.defaultArgs['ca_cert'],
                                 auth_type=self.defaultArgs['auth_type'],
                                 creds=self.defaultArgs['creds'],
                                 timeout=self.defaultArgs['timeout'])

    def tearDown(self):
        """
        Nothing to be done for this case
        """
        pass

    def testConfig(self):
        """
        Test service attributes and the override mechanism
        """
        for key in self.defaultArgs:
            self.assertEqual(getattr(self.myRucio.cli, key), self.defaultArgs[key])
        self.assertTrue(getattr(self.myRucio.cli, "user_agent").startswith("wmcore-client/"))
        self.assertTrue(getattr(self.client, "user_agent").startswith("rucio-clients/"))

        newParams = {"host": 'http://cmsrucio-int.cern.ch',
                     "auth_host": 'https://cmsrucio-auth-int.cern.ch',
                     "auth_type": "x509", "account": self.acct,
                     "ca_cert": False, "timeout": 5, "phedexCompatible": False}
        newKeys = newParams.keys()
        newKeys.remove("phedexCompatible")

        rucio = Rucio(newParams['account'], hostUrl=newParams['host'],
                      authUrl=newParams['auth_host'], configDict=newParams)

        self.assertEqual(getattr(rucio, "phedexCompat"), False)
        for key in newKeys:
            self.assertEqual(getattr(rucio.cli, key), newParams[key])

    def testGetAccount(self):
        """
        Test whether we can fetch data about a specific rucio account
        """
        res = self.client.get_account(self.acct)
        res2 = self.myRucio.getAccount(self.acct)
        self.assertEqual(res['account'], self.acct)
        self.assertEqual(res['status'], "ACTIVE")
        self.assertEqual(res['account_type'], "USER")
        self.assertTrue({"status", "account", "account_type"}.issubset(set(res2.keys())))
        self.assertTrue({self.acct, "ACTIVE", "USER"}.issubset(set(res2.values())))

    def testGetAccountUsage(self):
        """
        Test whether we can fetch data about a specific rucio account
        """
        # test against a specific RSE
        res = list(self.client.get_local_account_usage(self.acct, rse="T1_US_FNAL_Disk"))
        res2 = self.myRucio.getAccountUsage(self.acct, rse="T1_US_FNAL_Disk")
        self.assertEqual(res[0]["rse"], "T1_US_FNAL_Disk")
        self.assertTrue(res[0]["files"] >= 1)
        self.assertTrue(res[0]["bytes"] >= 1000)
        self.assertTrue("bytes_remaining" in res[0])
        self.assertTrue(res[0]["bytes_limit"] >= 0)
        self.assertEqual(res, res2)

        # test against all RSEs
        res = list(self.client.get_local_account_usage(self.acct))
        res2 = self.myRucio.getAccountUsage(self.acct)
        self.assertTrue(len(res) > 1)
        self.assertTrue(len(res2) > 1)
        # I have manually created a rule for this account, so it will be there...
        self.assertEqual(res, res2)

        # now test against an account that either does not exist or that we cannot access
        res = self.myRucio.getAccountUsage("admin")
        self.assertIsNone(res)

    # @attr('integration')
    def testWhoAmI(self):
        """
        Test user mapping information from the request headers
        """
        res = dict(self.client.whoami())
        res2 = dict(self.myRucio.whoAmI())
        self.assertTrue({"status", "account"}.issubset(set(res.keys())))
        self.assertTrue(set(res.keys()) == set(res2.keys()))

    def testPing(self):
        """
        Tests server ping
        """
        res = self.client.ping()
        res2 = self.myRucio.pingServer()
        self.assertTrue("version" in res)
        self.assertItemsEqual(res, res2)

    def testGetBlocksInContainer(self):
        """
        Test `getBlocksInContainer` method, the ability to retrieve blocks
        inside a container.
        """
        # test a CMS dataset that does not exist
        res = self.myRucio.getBlocksInContainer("Alan")
        self.assertEqual(res, [])

        # provide a CMS block instead of a dataset
        res = self.myRucio.getBlocksInContainer(BLOCK)
        self.assertEqual(res, [])

        # finally provide a real CMS dataset
        res = self.myRucio.getBlocksInContainer(DSET)
        self.assertTrue(len(res) >= len([BLOCK]))
        self.assertIn(BLOCK, res)

    def testGetReplicaInfoForBlocks(self):
        """
        Test `getReplicaInfoForBlocks` method, the ability to retrieve replica
        locations provided a dataset or block. Same output as PhEDEx.
        """
        res = self.myRucio.getReplicaInfoForBlocks(block=BLOCK)
        self.assertEqual(len(res['phedex']['block']), 1)
        block = res['phedex']['block'].pop()
        self.assertEqual(block['name'], BLOCK)
        replicas = [item['node'] for item in block['replica']]
        self.assertTrue(len(replicas) > 0)

        # same test, but providing a dataset as input (which has 4 blocks)
        res = self.myRucio.getReplicaInfoForBlocks(dataset=DSET)
        self.assertTrue(len(res['phedex']['block']) >= 1)  # at this very moment, there are 11 replicas
        blocks = [item['name'] for item in res['phedex']['block']]
        self.assertTrue(BLOCK in blocks)
        for item in res['phedex']['block']:
            self.assertTrue(len(item['replica']) > 0)

    def testGetReplicaInfoForBlocksRucio(self):
        """
        Test `getReplicaInfoForBlocks` method, however not using
        the output compatibility with PhEDEx
        """
        theseArgs = self.defaultArgs.copy()
        theseArgs['phedexCompatible'] = False
        myRucio = Rucio(self.acct,
                        hostUrl=theseArgs['host'],
                        authUrl=theseArgs['auth_host'],
                        configDict=theseArgs)

        res = myRucio.getReplicaInfoForBlocks(dataset=DSET)
        self.assertTrue(isinstance(res, list))
        self.assertTrue(len(res) >= 1)  # at this very moment, there are 11 replicas
        blocks = [item['name'] for item in res]
        self.assertTrue(BLOCK in blocks)
        for item in res:
            self.assertTrue(len(item['replica']) > 0)

    def testGetPFN(self):
        """
        Test `getPFN` method
        """
        self.assertRaises(NotImplementedError, self.myRucio.getPFN)

    def testListContent(self):
        """
        Test `listContent` method, to list content of a given DID
        """
        # listing blocks for a dataset
        res = self.myRucio.listContent(DSET)
        self.assertTrue(len(res) > 10)
        self.assertEqual(res[0]["type"], "DATASET")

        # listing files for a block
        res = self.myRucio.listContent(BLOCK)
        self.assertTrue(len(res) > 10)
        self.assertEqual(res[0]["type"], "FILE")

        res = self.myRucio.listContent("/Primary/ProcStr-v1/tier")
        self.assertItemsEqual(res, [])

    def testListDataRules(self):
        """
        Test `listContent` method
        """
        res = self.myRucio.listDataRules(DSET)
        self.assertItemsEqual(res, [])

    def testGetRule(self):
        """
        Test `getRule` method
        """
        # Badly formatted rule id, raises/catches a general exception
        res = self.myRucio.getRule("blah")
        self.assertItemsEqual(res, {})

        # Properly formatted rule, but inexistent id
        res = self.myRucio.getRule("1d6ea1d916d5492e81b1bb30ed4aebc0")
        self.assertItemsEqual(res, {})

        # Properly formatted rule, rule manually created
        res = self.myRucio.getRule("1d6ea1d916d5492e81b1bb30ed4aebc1")
        self.assertTrue(res)

    def testMetaDataValidation(self):
        """
        Test the `validateMetaData` validation function
        """
        for thisProj in RUCIO_VALID_PROJECT:
            response = validateMetaData("any_DID_name", dict(project=thisProj), self.myRucio.logger)
            self.assertTrue(response)

        # test with no "project" meta data at all
        response = validateMetaData("any_DID_name", dict(), self.myRucio.logger)
        self.assertTrue(response)

        # now an invalid "project" meta data
        response = validateMetaData("any_DID_name", dict(project="mistake"), self.myRucio.logger)
        self.assertFalse(response)


import math
import numpy as np
import random as rn
import tensorflow as tf

def DNN_model(x, y_actual, feature_length, hidden_layer, keep_prob, learning_rate):
    y_deep = x
    for i in range(0, len(hidden_layer)):
        if i is 0:
            row = feature_length
        else:
            row = hidden_layer[i - 1]
        w = tf.Variable(tf.random_normal(shape=[row, hidden_layer[i]], mean=0, stddev=0.1))
        bias = tf.Variable(tf.random_normal(shape=[hidden_layer[i]], mean=0, stddev=0.01))
        y_deep = tf.nn.relu(tf.add(tf.matmul(y_deep, w), bias))
        y_deep = tf.nn.dropout(y_deep, keep_prob)
    w_output = tf.Variable(
        tf.random_normal(shape=[hidden_layer[len(hidden_layer) - 1], 1], mean=0, stddev=0.1))
    bias_output = tf.Variable(tf.random_normal(shape=[1], mean=0, stddev=0.01))
    y_predict = tf.sigmoid(tf.add(tf.matmul(y_deep, w_output), bias_output), name="y_predict")

    loss = tf.losses.log_loss(y_actual, y_predict)
    mean_loss = tf.reduce_mean(loss, name="mean_loss")
    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-8)
    global_step = tf.train.get_or_create_global_step()
    train = optimizer.minimize(mean_loss, global_step=global_step)
    return y_predict, mean_loss, train, global_step

def main(_):
    # 数据预处理
    data = np.loadtxt("", dtype=float)
    train_data = data[0:math.ceil(len(data) * 0.6)]
    test_data = data[math.ceil(len(data) * 0.6):len(data)]
    batch_size = 100

    cluster = tf.train.ClusterSpec({"ps": FLAGS.strps_hosts.split(","), "worker": FLAGS.strwork_hosts.split(",")})
    server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index)
    if FLAGS.job_name == "ps":
        server.join()
    elif FLAGS.job_name == "worker":
        with tf.device(tf.train.replica_device_setter(worker_device="/job:worker/task:%d" % FLAGS.task_index,
                                                      cluster=cluster)):
            # 特征长度
            feature_length = 16
            x = tf.placeholder(tf.float32, [None, feature_length], name="x_input")
            y_actual = tf.placeholder(tf.float32, [None, 1], name="y_actual")
            keep_prob = tf.placeholder(tf.float32, name="keep_prob")
            y_predict, mean_loss, train, global_step = DNN_model(x, y_actual, feature_length, [15, 10, 5], keep_prob,
                                                                 0.001)
            auc_reset, auc_value = tf.metrics.auc(y_actual, y_predict)

            hooks = [tf.train.StopAtStepHook(last_step=500)]
            checkpoint_dir = ""
            sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
            with tf.train.MonitoredTrainingSession(server.target,
                                                   is_chief=(FLAGS.task_index == 0),
                                                   hooks=hooks,
                                                   checkpoint_dir=checkpoint_dir,
                                                   save_checkpoint_secs=2,
                                                   config=sess_config) as sess:
                while not sess.should_stop():
                    sample_flag = rn.randint(0, train_data.shape[0] - batch_size)
                    train_input = train_data[sample_flag: sample_flag + batch_size][:, 1:]
                    train_label = train_data[sample_flag: (sample_flag + batch_size)][:, 0].reshape(batch_size, 1)
                    feed_dict_train_sample = {x: train_input, y_actual: train_label, keep_prob: 0.5}
                    _, step, _, train_auc_value, train_mean_loss = sess.run(
                        [train, global_step, auc_reset, auc_value, mean_loss],
                        feed_dict=feed_dict_train_sample)
                    if (step % 50 == 0):
                        # 训练样本
                        print("step = ", step, "train_auc_value=", train_auc_value, "train_mean_loss=", train_mean_loss,
                              end=" ")
                        # 测试样本
                        feed_dict_test_data = {x: test_data[:, 1:],
                                               y_actual: test_data[:, 0].reshape(test_data.shape[0], 1),
                                               keep_prob: 1.0}
                        _, test_auc_value, test_mean_loss = sess.run([auc_reset, auc_value, mean_loss],
                                                                     feed_dict=feed_dict_test_data)
                        print("test_auc_value = ", test_auc_value, "test_mean_loss = ", test_mean_loss)

if __name__ == "__main__":
    FLAGS = tf.app.flags.FLAGS
    tf.app.flags.DEFINE_string("strps_hosts", "localhost:2222,localhost:2223", "参数服务器")
    tf.app.flags.DEFINE_string("strwork_hosts", "localhost:2224,localhost:2225", "工作服务器")
    tf.app.flags.DEFINE_string("job_name", "ps", "参数服务器或者工作服务器")
    tf.app.flags.DEFINE_integer("task_index", 1, "job的task索引")
    tf.app.run()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
This file defines the semantic class used in our parser for BEL statements.
"""

import collections
import math
import os
import pprint
import requests

import yaml

from bel_lang.exceptions import *

###################
# SEMANTICS CLASS #
###################

class BELSemantics(object):
    def __init__(self, version='2_0_0'):

        # load the dict from yaml file here

        current_directory = os.path.dirname(__file__)
        path_to_yaml_file = '{}/versions/bel_v{}.yaml'.format(current_directory, version)

        yaml_file = open(path_to_yaml_file, 'r').read()
        yaml_dict = yaml.load(yaml_file)

        self.abbre_to_name = self.abbreviations_to_names(yaml_dict)
        self.name_to_abbre = self.names_to_abbreviations(yaml_dict)
        self.function_signatures = self.get_function_signatures(yaml_dict)
        self.relationships = self.get_relationships(yaml_dict)

        self.term_store_endpoint = 'https://api.bel.bio/v1/terms/{}'

    #######################
    # SEMANTIC RULE NAMES #
    #######################

    def function(self, ast):

        fn_given = ast.get('function', None)
        args_given = ast.get('function_args', None)

        self.check_valid_signature(fn_given, args_given)

        # print('Function found: {}().'.format(fn_given))
        # print('arguments: ')
        # pprint(args_given, indent=2)
        # print('\n')

        # for arg in args_given:
        #     if arg:  # if the given argument list is not empty
        #         if 'm_function' in arg:
        #             m_fn = arg['m_function']
        #             # print('modifier function: {}'.format(m_fn))
        #             self.check_function_modifier(fn, m_fn)
        #
        #     else:  # if the list is empty, move to the next arg
        #         continue

        return ast

    def modifier_function(self, ast):
        # mfn = ast['m_function']
        # print('Modifier function found: {}().'.format(mfn))
        return ast

    def relationship(self, ast):
        # r_given = ast.get('relation', None)s
        self.check_valid_relationship(ast)

        return ast

    def namespace_arg(self, ast):
        self.check_signature_params(ast)

        return ast

    def string_arg(self, ast):
        # str_arg = ast['str_arg']
        # print('String argument found: {}.'.format(str_arg))
        return ast

    ##################################
    # SEMANTIC CHECKING STAGE 1 OF 3 #
    ##################################

    def get_function_signatures(self, yaml_dict):
        """
        This is stage 1 of semantic checking. It has one argument:

        1. yaml_dict: This is the dictionary parsed from the given YAML file defined by the user.

        We are focused on the function_signature key found inside yaml_dict, as that gives us the information we need to
        build our dictionary-based signature definitions. For each signature, we store both a required_params and a
        optional_params OrderedDict(). This serves two purpose: we need to keep track of each param type's optional and
        multiple booleans in a key/value pair, but also need to keep the dict ordered for the required parameters.
        """

        signature_dict = {}

        for func in yaml_dict.get('function_signatures', []):  # for each function in the yaml function signatures...
            f_name = func.get('name', 'unknown')  # get the name of the function in yaml
            f_sigs = func.get('signatures', [])  # get the signatures of the function
            signature_dict[f_name] = f_sigs  # set function name as key, and yaml's f_sigs as the value for this key

            for valid_sig in f_sigs:  # for each signature from signatures...
                params = valid_sig.get('parameters', None)  # get the parameter types of this signature
                required_params = collections.OrderedDict()  # required param types list (must be ordered)
                optional_params = collections.OrderedDict()  # optional param types list (does not need to be ordered)

                for par in params:  # for each type in parameter types
                    par_type = par.get('type', 'Unknown')  # get the type name
                    optional = par.get('optional', False)  # get the optional boolean
                    multiple = par.get('multiple', False)  # get the multiple boolean

                    # param is REQUIRED AND SINGULAR
                    # set the value as the number of times this type can appear
                    if not optional and not multiple:
                        required_params[par_type] = required_params.get(par_type, 0) + 1

                    # param is REQUIRED AND MULTIPLE
                    # set the value to infinity as we'll always have 1 or more of this type
                    elif not optional and multiple:
                        required_params[par_type] = math.inf

                    # param is OPTIONAL AND SINGULAR
                    # set the value as the number of times this type can appear
                    elif optional and not multiple:
                        optional_params[par_type] = optional_params.get(par_type, 0) + 1

                    # param is OPTIONAL AND MULTIPLE
                    # set the value to infinity as we'll always have 0 or more of this type
                    else:
                        optional_params[par_type] = math.inf

                # add these two OrderedDicts as key/values within valid_sig so we can refer to them later on
                valid_sig['required'] = required_params
                valid_sig['optional'] = optional_params

            # clone the value of f_name into its respective abbreviation key
            # e.g. signature_dict[activity] == signature_dict[act]; signature_dict[pathology] == signature_dict[path]
            signature_dict[self.name_to_abbre[f_name]] = f_sigs

        return signature_dict

    ##################################
    # SEMANTIC CHECKING STAGE 2 OF 3 #
    ##################################

    def check_valid_signature(self, fn_given, args_given):
        """
        This is stage 2 of semantic checking. It has three arguments:

        1. fn_given: This is the given function from the BEL statement.

        For example, consider the following BEL statement:

        tloc(p(HGNC:SGK1), fromLoc(MESHCS:Cytoplasm), toLoc(MESHCS:"Cell Nucleus")) association bp(GOBP:"cell cycle")

        The functions present above are p(), tloc(), and bp(), and thus this BEL statement will require three calls to
        check_valid_signature(), once for each of the present BEL functions.

        2. args_given: These are the arguments given with each respective fn_given from above in the form of an AST.
        3. function_signatures: The signatures defined from stage 1 above.

        The purpose of check_valid_signature is to check that the function given from the BEL statement matches one of
        the function signatures defined from stage 1. Specifically, it first checks if the parameter types (Entity,
        Modifier, Function, String, etc.), their optionality, and their multiplicity match with the args_given. Then,
        it checks to see if each given parameter is valid for their respective defined parameter type. In the example
        above, tloc()'s arguments match the signature definition [Function, Modifier, Modifier], but we also have to
        check that p() is a valid value for tloc()'s Function parameter, fromLoc() is a valid value for tloc()'s
        Modifier parameter, etc.
        """

        # these are specific functions to ignore in case their signatures are wrong or need reworking
        # if fn_given in ['activity']:
        #     return

        func_sig_detected = self.detect_function_signature(fn_given, args_given)

        if func_sig_detected is not None:
            # print('\t\033[91mIdentified signature:\033[0m {}'.format(func_sig_detected))
            return True
        else:
            # print('\t\033[91mNo function signature detected.\033[0m')
            return False

    def detect_function_signature(self, fn_given, args_given):

        msg_or_sig = None

        # check if at least one parameter is given
        if not args_given:

            raise ParameterMissing(fn_given)

        valid_sigs = self.function_signatures.get(fn_given, [])
        given_param_list = self.args_to_given_sig_list(args_given)

        for v_sig in valid_sigs:

            valid, msg_or_sig = self.check_valid_sig_given_args(v_sig, given_param_list)
            if valid:  # if valid is true, return message back
                return msg_or_sig

        # this exception will be raised if valid is False
        raise InvalidParameter(msg_or_sig)

    def check_valid_sig_given_args(self, v_sig, given_param_list):

        required = v_sig.get('required', [])
        optional = v_sig.get('optional', [])

        idx = 0  # keeps track of our given parameter index we're currently checking

        required_count = len(required)  # number of required param TYPES (not total, as types can have multiples)
        given_param_count = len(given_param_list)  # number of params given

        if given_param_count < required_count:
            exception_msg = '{} parameter type(s) are required. {} given.'.format(required_count, given_param_count)
            return False, exception_msg

        for required_param_type in required:
            if required[required_param_type] == 1:  # singular parameter
                if required_param_type == given_param_list[idx]:  # if the given parameter at the index == required type
                    idx += 1
                    continue  # continue to the next required parameter type
                else:  # given parameter at index is not a required type. set matched to False and break out of loop
                    exception_msg = 'A required \"{}\" type was expected, but \"{}\" type was given instead.'.format(
                        required_param_type, given_param_list[idx])
                    return False, exception_msg
            else:  # this parameter type can have multiples - loop until end or diff type
                while idx < given_param_count:
                    if required_param_type == given_param_list[idx]:
                        idx += 1
                    else:  # break the loop because we encountered a parameter that is not our multiple param type
                        break

        if idx < given_param_count:  # there are still optional parameters given that have not been accounted for
            remaining_params = given_param_list[idx:]
            for remain in remaining_params:
                if remain in optional:

                    # check occurences of this optional param is less than the max allowed
                    if remaining_params.count(remain) <= optional[remain]:
                        continue
                    else:
                        exception_msg = 'Too many optional parameters of type \"{}\" in statement. ' \
                                        'Does not match any function signatures.'.format(remain)
                        return False, exception_msg

                else:
                    exception_msg = '\"{}\" is not a valid optional parameter type for this function.'.format(remain)
                    return False, exception_msg

        return True, v_sig

    ##################################
    # SEMANTIC CHECKING STAGE 3 OF 3 #
    ##################################

    def check_signature_params(self, ast):
        # Stage 3 of 3: check each of the parameters in the function call to see if they are valid
        ns_arg = ast.get('ns_arg', {})

        ns = ns_arg.get('nspace', '')
        nsv = ns_arg.get('ns_value', '')

        term = '{}:{}'.format(ns, nsv)
        r_url = self.term_store_endpoint.format(term)

        r = requests.get(r_url)
        if r.status_code == 404:
            print('WARNING: {} is not a valid namespace in your TermStore API!'.format(term))

        #  can't check this until TermStore is ready

        return

    ##############################
    # VALID RELATIONSHIP CHECKER #
    ##############################

    def check_valid_relationship(self, given):
        # checks against all relationships in YAML to make sure the given one is valid

        if given not in self.relationships:
            raise InvalidRelationship(given)

        return

    ###############################
    # SEMANTIC HELPER DEFINITIONS #
    ###############################

    def abbreviations_to_names(self, yaml_dict):
        # uses YAML to translate from abbreviated name to actual full-length name

        abbreviations = {}

        for fn in yaml_dict.get('functions', []):
            f_name = fn.get('name', 'unknown')
            f_abbr = fn.get('abbreviation', 'unknown')
            abbreviations[f_abbr] = f_name

        for m_fn in yaml_dict.get('modifier_functions', []):
            mf_name = m_fn.get('name', 'unknown')
            mf_abbr = m_fn.get('abbreviation', 'unknown')
            abbreviations[mf_abbr] = mf_name

        return abbreviations

    def names_to_abbreviations(self, yaml_dict):
        # uses YAML to translate from full-length name to abbreviated name.

        names = {}

        for fn in yaml_dict.get('functions', []):
            f_name = fn.get('name', 'unknown')
            f_abbr = fn.get('abbreviation', 'unknown')
            names[f_name] = f_abbr

        for m_fn in yaml_dict.get('modifier_functions', []):
            mf_name = m_fn.get('name', 'unknown')
            mf_abbr = m_fn.get('abbreviation', 'unknown')
            names[mf_name] = mf_abbr

        return names

    def args_to_given_sig_list(self, args_given):
        signature_list = []

        for arg in args_given:
            for arg_type in arg:
                if arg_type == 'ns_arg':
                    signature_list.append('Entity')
                elif arg_type == 'function':
                    signature_list.append('Function')
                elif arg_type == 'm_function':
                    signature_list.append('Modifier')
                elif arg_type == 'str_arg':
                    signature_list.append('String')
                else:  # else could simply be an argument of a function, we don't need to include this in signature
                    pass

        return signature_list

    def get_relationships(self, yaml_dict):
        """
        Retrieves all valid relationships specified in the YAML and returns a list of unique relationships.
        """

        relationships = set()

        for relationship in yaml_dict.get('relationships', []):
            r_name = relationship.get('name', None)
            r_abbr = relationship.get('abbreviation', None)

            relationships.add(r_name)
            relationships.add(r_abbr)

        return list(relationships)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Mar 16 13:27:47 2019

@author: igor
"""

#conda install numpy scipy scikit-learn pandas
#pip install deap update_checker tqdm stopit
#pip install xgboost
#pip install tpot
def tpot_test(conf):
    from tpot import TPOTRegressor
    from sklearn.model_selection import train_test_split
    from sklearn.model_selection import TimeSeriesSplit

    p.load_config(conf)
    ds = dl.load_price_data()
    ds = add_features(ds)

    X = ds[p.feature_list][:-1]
    y = ds['DR'].shift(-1)[:-1]

    # Split Train and Test
    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8, test_size=0.2)

    tpot = TPOTRegressor(n_jobs=-1, verbosity=2, max_time_mins=60, cv=TimeSeriesSplit(n_splits=3))

    tpot.fit(X_train, y_train)
    print(tpot.score(X_test, y_test))
    tpot.export('./tpot_out.py')

tpot_test('ETHUSDNN')