import os
import sys
import socket
import struct
import numpy as np
from utils.model_info import ModelInfo
import cv2
def read_image(filename, resize_height, resize_width,normalization=False):
    '''
       读取图片数据,默认返回的是uint8,[0,255]
       :param filename:
       :param resize_height:
       :param resize_width:
       :param normalization:是否归一化到[0.,1.0]
       :return: 返回的图片数据
       '''

    bgr_image = cv2.imread(filename)
    if len(bgr_image.shape) == 2:  # 若是灰度图则转为三通道
        print("Warning:gray image", filename)
        bgr_image = cv2.cvtColor(bgr_image, cv2.COLOR_GRAY2BGR)

    rgb_image = cv2.cvtColor(bgr_image, cv2.COLOR_BGR2RGB)  # 将BGR转为RGB
    # show_image(filename,rgb_image)
    # rgb_image=Image.open(filename)
    if resize_height > 0 and resize_width > 0:
        rgb_image = cv2.resize(rgb_image, (resize_width, resize_height))
    rgb_image = np.asanyarray(rgb_image)
    if normalization:
        # 不能写成:rgb_image=rgb_image/255
        rgb_image = (rgb_image-128) / 127.0
    # show_image("src resize image",image)
    return rgb_image

def deal_input_image(model_name,img_str,normalization=False):
    # get the shape of the input data
    #print("=========",model_name)
    model_info = ModelInfo()
    shape = model_info.get_input_info(model_name)[0]
    resize_height = shape[0]
    resize_width = shape[1]
    # 2. convert the type of the image from string to numpy
    nparr = np.fromstring(img_str, np.uint8)
    img_np = cv2.imdecode(nparr, cv2.IMREAD_COLOR)

    # 3. conver the channel of image from BGR to RGB
    rgb_image = cv2.cvtColor(img_np, cv2.COLOR_BGR2RGB)
    if resize_height>0 and resize_width>0:
        rgb_image=cv2.resize(rgb_image,(resize_width,resize_height))
    # 4. change the dtype
    rgb_image=np.asanyarray(rgb_image)
    if normalization:
        # 不能写成:rgb_image=rgb_image/255
        rgb_image=(rgb_image-128)/127.0
    return rgb_image

class SocketCommunication:
   def recvall(self, conn, n):
      # Helper function to recv n bytes or return None if EOF is hit
      data = b''
      while len(data) < n:
         packet = conn.recv(n - len(data))
         if not packet:
            return None
         data += packet
      return data

   def send_data(self,conn, content):
      # Prefix each message with a 4-byte length (network byte order)
      # Send data to the server
      content = bytes(content,encoding="utf-8")
      msg = struct.pack('>I', len(content)) + content
      conn.send(msg)

   def recv_data(self,conn):
      # Receive data from the server
      # Return result(an numpy array)
      resp_len = self.recvall(conn, 4)
      if not resp_len:
         return None
      resp_len = struct.unpack('>I', resp_len)[0]
      if not resp_len:
         return None
      result = self.recvall(conn, resp_len)
      result = result.decode("utf8")
      return result

class FileOperation:
    def get_module_config(self):
        """
        read system config info
        :return: dict
        """
        fileOpr = FileOperation()
        config = fileOpr.readFile("./utils/system_config", "r")
        #print(config)
        config = eval(config)
        return config
    # read data from the file
    # openModel could be r or rb
    def printLogFile(self, filePath):
        self.createFile(filePath)
        sys.stdout = open(filePath, "a+")

    def readFile(self, filePath, openMode):
        result = ""
        if os.path.exists(filePath):
            with open(filePath, openMode,encoding='UTF-8') as f:
                result = f.read()
                f.close()
        else:
            print(filePath, "does not exist")
        return result

    def isFileExist(self, filePath):
        result = True
        if not os.path.exists(filePath):
            result = False
        return result

    def listFile(self,dirPath):
        """
        list the name of all the files under this dirPath
        :param dirPath
        :return: the file's name under this dir
        """
        file_list = None
        if os.path.isdir(dirPath):
            file_list = os.listdir(dirPath)
        return file_list

    def createFile(self, filePath):
        if self.isFileExist(filePath):
            return
        else:
            filePath = str(filePath)
            foldPath = filePath[:filePath.rindex("/")]
            # print("foldpath => ",foldPath)
            if not os.path.isdir(foldPath):
                os.makedirs(foldPath)
            with open(filePath, "w+") as f:
                f.write('')
                f.close()

    # write data into a file
    # writeMode could be w  w+ wb, and it first checks whether the file exists
    # if it not exists, create the file,the put into the content.
    def writeFile(self, filePath, writeMode, content):
        self.createFile(filePath)
        # print("filePath = >", filePath)
        with open(filePath, writeMode) as f:
            f.write(content)
            f.close()


#!/usr/bin/python3
# @FileName    :demo02_response.py
# @Time        :2020/6/9 下午10:53
# @Author      :ABC
# @Description : flask中的响应对象相关demo:实现原理——>都是在make_response()方法上做了二次封装
import json

from flask import Flask, make_response, jsonify

app = Flask(__name__)
# 设置接口响应体不以ASCII格式展示，以中文展示。
app.config['JSON_AS_ASCII'] = False

@app.route('/')
def index():
    # response_data = json.dumps({"username": "小花花"})
    # print('response_data:', response_data)
    # print('type(response_data):', type(response_data))

    # 手动构造一个响应头方法(1)：
    # return json.dumps({"username": "小花花"}), 203, {"content-type": "application/json"}

    # 手动构造一个响应头方法(2)：make_response()方法
    # response_data = make_response(json.dumps({"哈拉少儿": "哈撒给"}), {"content-type": "application/json"})
    # return response_data

    # 使用jsonify()方法自动将字典格式化为json格式，自动设置content-type=application/json
    response_data = jsonify({"name": "小花花", "age": 21, "gender": "女"})
    # 自定义响应状态码为207
    response_data.status = '207'
    return response_data

if __name__ == '__main__':
    app.run(debug=True)


# coding: utf-8
"""
    @author: zhangjk
    @file: 7.py
    @date: 2020-02-28
    说明：堆，队列
"""

# 堆
def f1():
    stack = [3, 4, 5]
    stack.append(6)
    stack.append(7)
    print(stack)
    s = stack.pop()
    print(s)
    print(stack)
    s = stack.pop()
    print(s)
    s = stack.pop()
    print(s)
    print(stack)

# 队列
def f2():
    from collections import deque
    queue = deque(["Eric", "John", "Michael"])
    queue.append("Terry")           # Terry arrives
    queue.append("Graham")          # Graham arrives
    q = queue.popleft()                 # The first to arrive now leaves
    print(q)
    q = queue.popleft()                 # The second to arrive now leaves
    print(q)
    print(queue)                           # Remaining queue in order of arrival
    q1 = queue.pop()
    print(q1)
    print(queue)

def f3():
    from queue import LifoQueue
    queue = LifoQueue()
    queue.put('Java')
    queue.put('Python')
    queue.put('C')
    print(queue)
    while not queue.empty():
        print(queue.get())

def f4():
    from queue import SimpleQueue
    queue = SimpleQueue()
    queue.put('Java')
    queue.put('Python')
    queue.put('C')
    print(queue)
    while not queue.empty():
        print(queue.get())

def f5():
    from queue import PriorityQueue
    queue = PriorityQueue()
    queue.put('Java')
    queue.put('Python')
    queue.put('C')
    print(queue)
    while not queue.empty():
        print(queue.get())

def f6():
    from queue import Queue
    queue = Queue()
    queue.put('Java')
    queue.put('Python')
    queue.put('C')
    print(queue)
    while not queue.empty():
        print(queue.get())

# f1()
# f2()
# f3()
# f4()
# f5()
f6()


# -*- coding: utf-8 -*-
import mock

from . import sitemap
import time

class TestPageData(object):
    """
    Test `sitemap.PageData`.
    """
    def test_init(self):
        url = 'http://demo.com/'
        page = sitemap.PageData(url)
        assert url == page.url

    def test_assets(self):
        page = sitemap.PageData('')
        assert page.assets == set()
        values = ['demo', 'lorem']
        page.assets = values
        assert set(values) == page.assets

    def test_connections(self):
        page = sitemap.PageData('')
        assert page.connections == set()
        values = ['demo', 'lorem']
        page.connections = values
        assert set(values) == page.connections

class TestParsePageProcess(object):
    """
    Test `sitemap.ParsePageProcess`.
    """
    def test_run_url_already_visited(self):
        url = 'http://demo.com/page1'
        queue = mock.Mock()
        queue.get = mock.Mock(return_value=url)
        queue.task_done = mock.Mock()
        visited = {url: False}

        p = sitemap.ParsePageProcess(queue, 'http://demo.com', visited, [])
        p.start()
        time.sleep(0.1)
        p.terminate()
        assert not visited[url]
        # assert queue.task_done.call_count == 1

    def test_parse_page_assets(self):
        html = """
        Duis mollis, est non commodo luctus, nisi erat porttitor ligula, eget lacinia odio sem nec.
        est http://demo.com/static/assets/demo/styles.css ligula, eget lacinia odio sem nec.
        Duis mollis, est non commodo luctus, nisi erat porttitor ligula, eget lacinia odio sem nec.
        lorem /app.js Tellus Sem Venenatis
        Quam Euismod Tellus Sem Venenatis
        Tellus Sem Venenatis /static/last.js Tellus Sem Venenatis
        """
        p = sitemap.ParsePageProcess(None, 'http://demo.com', {}, [])
        assets, connections = p._parse_page(html)

        assert len(connections) == 0
        assert len(assets) == 3
        assert set(assets) == set([
            'http://demo.com/static/assets/demo/styles.css',
            'http://demo.com/app.js',
            'http://demo.com/static/last.js',
        ])

    def test_parse_page_connections(self):
        html = """
        Duis mollis, est non commodo luctus, nisi erat porttitor ligula, eget lacinia odio sem nec.
        link lorem
        Duis mollis, est non commodo luctus, nisi erat porttitor ligula, eget lacinia odio sem nec.
        erat porttitor ligula Link Sem Venenatis
        Quam Euismod Tellus Sem Venenatis /test
        erat porttitor ligula Link
        """
        p = sitemap.ParsePageProcess(None, 'http://demo.com', {}, [])
        assets, connections = p._parse_page(html)

        assert len(assets) == 0
        assert len(connections) == 3
        assert set(connections) == set([
            'http://demo.com/lorem-ipsum-link',
            'http://demo.com/link78/',
            'http://demo.com/demo/link/2',
        ])


def robot():
    robotscore = 0
        #5th quest for the user
    print("\nWe need to get this robot to move\n")

    print("\n1: y,z axis\n2: x,y axis\n3: x,z axis\n")
    Quest5 = input("\nTo get the robot to move to the sides, then which of the axis's are the right ones?\n")

    while Quest5 != '2':
        print("\nDummy! thats not how a robot works!\n")
        Quest5 = input("\n You better get it right this time!\n")
        robotscore -= 1
    else:
        print("\nLet's move forward\n")
        robotscore += 1

    print("\nIn order to get the item, with the robotarm, you need to calculate alittle\n")

    #6th quest for the user
    print("\nIf the robot is 2m tall, and you need to grab a thing that is 4m away, how long does your robot arm have to be?\n")
    Quest6 = input("\nYou have to find the correct lenght with x,xx\n")

    while Quest6 != '4,47':
        print("\nThat is not correct at all, try again\n")
        Quest6 = input("\nYou have to find the correct lenght with x,xx\n")
        robotscore -= 1
    else:
        print("\nCorrect, you have reached the item\n")
        robotscore += 1

    return robotscore


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
#
#
import sys
import os

from engine import log
log = log.init_log("main")
log.important('=== KXKM - DNC PLAYER ===')

def set_python_path(depth=0):
    f = sys._getframe(1)
    fname = f.f_code.co_filename
    fpath = os.path.dirname(os.path.abspath(fname))
    PythonPath = os.path.join(fpath, "../".join(["" for x in xrange(depth+1)]))
    sys.path.append(PythonPath)
set_python_path(depth=1)

import time
import application
from engine.setting import settings
from engine.log import set_default_log_by_settings
set_default_log_by_settings(settings)                   # Set default log level and output via settings

try:
    application.init()
    application.start()

    try:
        while application.POWEROFF is False:
            time.sleep(1)
    except KeyboardInterrupt:
        log.debug("KeyboardException received")
        application.POWEROFF = 1
    except Exception as e:
        log.exception("EXITING after exception in MAIN : \n"+log.show_exception(e))

except Exception as e:
    log.exception(log.show_exception(e))
    log.error(e)

application.stop()

log.important('=== EXIT : '+str(application.POWEROFF)+' ===' )
os._exit(application.POWEROFF)


import matplotlib.pyplot as plt

def plotCurve(finers, soil_distributions, roundRadius, maximums, unit):
	finers.append(0)
	soil_distributions.append(0)
	roundRadius.insert(0, maximums[0])

	real_radius = []
	for radius in roundRadius:
		real_radius.append(radius * unit)

	plt.title("Particle-size distribution")
	plt.plot(real_radius, finers, linestyle = 'dashed', label='2D model')
	plt.plot(real_radius, soil_distributions, linestyle = 'dotted', label='target soil data')
	plt.xscale('log',base=10)
	plt.xlabel("particle size")
	plt.ylabel("finer percent")
	plt.legend()
	plt.show()
	print(finers, soil_distributions, roundRadius)


"""
Given a string of round, curly, and square opening and closing brackets, return whether the brackets are balanced(well-formed)

"""

def balance(s):
    stack = []

    for char in s:
        if char in ["(", "[", "{"]:
            stack.append(char)
        else:
            if not stack:
                return False

            if (char == ")" and stack[-1] != "(") or \
               (char == "]" and stack[-1] != "[") or \
               (char == "}" and stack[-1] != "{") :
                return False
            stack.pop()

    return len(stack) == 0

"""
Runtime is 0(N) and Space is 0(N)

"""


#!/usr/bin/env python

from sensor_msgs.msg import LaserScan
from geometry_msgs.msg import Twist
import rospy

class distanceEStopNode(object):
  def __init__(self):
    rospy.init_node('emergencyStop')
    self.r = rospy.Rate(5)
    rospy.Subscriber('/scan', LaserScan, self.process_scan)
    self.publisher = rospy.Publisher('/cmd_vel', Twist, queue_size = 10)
    self.sees_obstacle = False

  def process_scan(self, m):
  	self.sees_obstacle = False
  	for distance in m.ranges:
  		if distance < .5 and distance != 0.0:
  			self.sees_obstacle = True

  def run(self):
    go_command = Twist()
    go_command.linear.x = .1
    stop_command = Twist()
    stop_command.linear.x = 0
    while not rospy.is_shutdown():
      if self.sees_obstacle:
        self.publisher.publish(stop_command)
      else:
        self.publisher.publish(go_command)
      self.r.sleep()

if __name__ == '__main__':
  node = distanceEStopNode()
  node.run()


#!/usr/bin/env python3
# Copyright (c) The Libra Core Contributors
# SPDX-License-Identifier: Apache-2.0
from .cli import execute_cmd_with_text_output

# length of git hash prefix
SHORT_LEN = 7
TAG_PREFIX = "master_"
LBT_SHORT_LEN = 8
LBT_TAG_PREFIX = "land_"

def _fetch(remote: str = "origin") -> None:
    execute_cmd_with_text_output(
        ["git", "fetch", remote],
        err="Something wrong with git. Are you in the right dir?",
    )

def latest_testnet_tag() -> str:
    """Get the image tag based on the latest revision in branch"""
    _fetch()
    short_rev = execute_cmd_with_text_output(
        ["git", "log", "origin/testnet", '--format=format:"%h"', "-1"]
    ).strip('"')[:SHORT_LEN]
    return TAG_PREFIX + short_rev

def prev_tag(n: int) -> str:
    """Get the previous nth commit's image tag"""
    _fetch()
    short_rev = execute_cmd_with_text_output(
        ["git", "rev-parse", f"--short={LBT_SHORT_LEN}", f"HEAD~{n}"]
    )
    return LBT_TAG_PREFIX + short_rev


#!/usr/bin/python3
# coding:utf-8

import threading
import os
import time
import requests
from bs4 import BeautifulSoup as b4
from queue import Queue
from threading import Lock

class TakeData(threading.Thread):

    def __init__(self):
        threading.Thread.__init__(self)
        self.url = "http://www.jdlingyu.mobi/37315/"
        self.header = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0'}

    def htmlRsp(self):
        rsp = requests.get(self.url, headers=self.header).text
        return rsp

    def run(self):
        htmSoup = b4(self.htmlRsp(), "html5lib")
        for i in htmSoup.find_all("div", {"id": "content-innerText"})[0].p.children:
            imgUrl = i['src']
           # print(imgUrl)
            imgSrc.put(imgUrl)
        # print("队列大小:%s" % queue.qsize())
        lockA.release()

class Process(threading.Thread):

    def __init__(self):
        threading.Thread.__init__(self)
        self.header = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0'}

    def downloadFile(self, src):
        header = requests.head(src)
        rsp = requests.get(src, headers=self.header, stream=True)
        size = header.headers['Content-Length']
        p = 0
        fileName = src.split("/")[4]
        print("正在下载==＞>%s" % fileName)
        print("文件大小: %.2f MB" % (int(size)/1024/1024))
        with open(fileName, "wb") as f:
            for i in rsp.iter_content(chunk_size=1024):
                p += len(i)
                f.write(i)
                done = int(50 * p / int(size))
            print("[%s%s] %s%%" % ('\033[1;032;40m#\033[0m' * done, '' * (50 - done), done+done), end='\r')

    def run(self):
        if lockA.acquire():
            print("共有图片: %s 张" % imgSrc.qsize())
            while True:
                if not imgSrc.empty():
                    # print("已获取==＞%s" % imgSrc.get()) # 提前get也是神奇
                    self.downloadFile(imgSrc.get())
                    print("\n队列内剩余图片: %s 张" % imgSrc.qsize())
                else:
                    break
            # lockA.release()

if __name__ == '__main__':
    os.chdir("./img")
    lockA = Lock()
    lockA.acquire()
    imgSrc = Queue()

    sr = time.time()
    t1 = TakeData()
    t1.start()
    t2 = Process()
    t2.start()
    t1.join()
    t2.join()
    ed = time.time()
    print("\n共耗时：%.2fs" % (ed - sr))


# Author: Dimitrios Economou
# tet.py
# A simple Tetris clone done in Python using the pygame library.

# INITIALIZATION
import pygame, math, sys, random, time
from pygame.locals import *

# The board is the "play area" where the pieces live and die.
class Board(object):
    START_ZONE_HEIGHT = 4
    NEXT_X = 600
    NEXT_y = 50

    def __init__(self, DISPLAYSURF, BOARD_COLOR, BOARD_X, BOARD_Y, GRID_WIDTH, GRID_HEIGHT, BLOCK_SIZE):
        self.DISPLAYSURF = DISPLAYSURF
        self.BOARD_COLOR = BOARD_COLOR
        self.BOARD_X = BOARD_X
        self.BOARD_Y = BOARD_Y
        self.GRID_WIDTH = GRID_WIDTH
        self.GRID_HEIGHT = GRID_HEIGHT
        self.BLOCK_SIZE = BLOCK_SIZE
        self.fallingPiece = None
        self.spawnPiece()
        self.fallenPieces = [] # all of the pieces that have already fallen

    def spawnPiece(self):
        #self.fallingPiece = self.nextPiece
        #self.nextPiece = Piece()
        pass

    def draw(self):
        pygame.draw.rect(self.DISPLAYSURF, self.BOARD_COLOR, (self.BOARD_X, self.BOARD_Y,
            self.BLOCK_SIZE*self.GRID_WIDTH, self.BLOCK_SIZE*self.GRID_HEIGHT))

        for piece in self.fallenPieces:
            self.DISPLAYSURF.blit(piece.pieceType.redBlockImage, (0,0))

# There are seven piece shapes consisting of four filled squares: T, S, Z, L, J, I, O. You can guess what they are.
class Piece(object):
    def __init__(self, DISPLAYSURF):
        self.DISPLAYSURF = DISPLAYSURF
        self.x = 0
        self.y = 0
        self.pieceType = PieceType.randomPieceType()

    def draw(self, surfacePos):
        self.DISPLAYSURF.blit(self.pieceType.redBlockImage, surfacePos)

class PieceType(object):
    # Each piece type is given by an image of its block and its block positions with respect to the board origin (top left) for a given orientation.
    # I need to add the other bloack images. The red piece type will just be a single block for testing purposes.
    # The localPiecePosByOrientation gives local coordinates for each of the blocks of a given piece type with respect to the board origin
    # and of a given orientation.
    def __init__(self, redBlockImage, localPiecePosByOrientation):
        self.redBlockImage = redBlockImage
        self.localPiecePosByOrientation = localPiecePosByOrientation

    @staticmethod
    def initializePieceTypes(redBlockImage):
        red = redBlockImage
        PieceType.TYPES = [PieceType(red, (0, 0))]

    @staticmethod
    def randomPieceType():
        return random.choice(PieceType.TYPES)

class InfoDisplay(object):
    SCORE = 0

    def __init__(self, DISPLAYSURF):
        #self.scoreLabel
        pass

class Input(object):
    pass

class GameStep(object):
    pass

class Game(object):
    def __init__(self, board):
        self.board = board

    def draw(self):
        self.board.draw()

############ MAIN PROGRAM ############
def main():
    # colors
    WHITE = (255,255,255)
    BLACK = (0,0,0)
    GREEN = (0,255,0)

    # surface display
    FPS = 30
    WINDOW_WIDTH = 640
    WINDOW_HEIGHT = 480
    BGCOLOR = WHITE
    FPSCLOCK = pygame.time.Clock()

    # board constants
    # GRID_WIDTH is the number of horizontal grid blocks. BOARD_X and BOARD_Y give the position of the upper right corner of the board.
    #
    BOARD_COLOR = BLACK
    BOARD_X = 40
    BOARD_Y = 20
    GRID_WIDTH = 10
    GRID_HEIGHT = 20
    BLOCK_SIZE = 22

    # load resources
    redBlockImage = pygame.image.load('red_block.png')

    # initialize the game state
    pygame.init() # should this go here?
    pygame.display.set_caption('Tet')
    DISPLAYSURF = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT))
    DISPLAYSURF.fill(BGCOLOR)
    PieceType.initializePieceTypes(redBlockImage)
    board = Board(DISPLAYSURF, BOARD_COLOR, BOARD_X, BOARD_Y, GRID_WIDTH, GRID_HEIGHT, BLOCK_SIZE)
    game = Game(board)

    # main game loop
    while True:
        DISPLAYSURF.fill(BGCOLOR)
        piece = Piece(DISPLAYSURF)
        board.fallenPieces.append(piece)
        game.draw()
        for event in pygame.event.get():
            if event.type == QUIT:
                pygame.quit()
                sys.exit()
        pygame.display.update()
        FPSCLOCK.tick(FPS) # prevents game from running too fast

if __name__ == "__main__":
    main()


# Licensed under a 3-clause BSD style license - see LICENSE.rst
"""
Core functionality of ctapipe
"""

from .component import Component, TelescopeComponent, non_abstract_children
from .container import Container, Field, DeprecatedField, Map, FieldValidationError
from .provenance import Provenance, get_module_version
from .tool import Tool, ToolConfigurationError, run_tool
from .qualityquery import QualityQuery, QualityCriteriaError

__all__ = [
    "Component",
    "TelescopeComponent",
    "Container",
    "Tool",
    "Field",
    "DeprecatedField",
    "Map",
    "Provenance",
    "ToolConfigurationError",
    "non_abstract_children",
    "get_module_version",
    "run_tool",
    "QualityQuery",
    "QualityCriteriaError",
    "FieldValidationError"
]


from time import time

if __name__ == "__main__":
    ns = [10000, 100000, 1000000, 10000000, 100000000]

    for n in ns:
        s = "x" * n
        ls = ["x" for i in range(n)]

        # 文字列のスライスの実行時間を出力
        start = time()
        s[(n // 2) :]
        end = time()
        print(f"string slice n={n}: {end-start} s")

        # リストのスライスの実行時間を出力
        start = time()
        ls[(n // 2) :]
        end = time()
        print(f"list slice n={n}: {end-start} s")


import tensorflow as tf
import numpy as np

def get_file_lists(data_dir):
    import glob

    train_list = glob.glob(data_dir + '/' + 'Training_Set-*')
    valid_list = glob.glob(data_dir + '/' + 'Validation_Set-*')
    if len(train_list) == 0 and \
                    len(valid_list) == 0:
        raise IOError('No files found at specified path!')
    return train_list, valid_list

def _parse_record(raw_record, is_training, code_len, set_len):
    """Parse a set from `value`."""
    keys_to_features = {
      'set/code': tf.FixedLenFeature([code_len * set_len], dtype=tf.float32, default_value=np.zeros(code_len * set_len)),
      'set/file': tf.FixedLenFeature((), tf.string, default_value='Dog.jpeg,Dog.jpeg,Dog.jpeg,'),
      'set/classes':tf.FixedLenFeature([set_len], dtype=tf.int64, default_value=np.zeros(set_len)),
      'set/num_unique': tf.FixedLenFeature([1], dtype=tf.int64, default_value=np.zeros(1)),
      'set/code_size': tf.FixedLenFeature([1], dtype=tf.int64, default_value=np.zeros(1)),
      'set/set_size': tf.FixedLenFeature([1], dtype=tf.int64, default_value=np.zeros(1))
    }

    parsed = tf.parse_single_example(raw_record, keys_to_features)
    code_size = parsed['set/code_size']
    set_size = parsed['set/set_size']

    code = tf.cast(
        tf.reshape(parsed['set/code'], shape=[set_len, code_len]),
        dtype=tf.float16)

    classes = tf.cast(
        tf.reshape(parsed['set/classes'], shape=[set_len]),
        dtype=tf.int16)

    #make file into
    files = parsed['set/file']
    #files = files.split(',')[0:set_size]

    uniques = tf.cast(
        tf.reshape(parsed['set/num_unique'], shape=[]),
        dtype=tf.int32)

    return code, classes, files, uniques

def get_set_batch(is_training, filenames, batch_size, num_epochs=1, num_parallel_calls=1):
    dataset = tf.data.TFRecordDataset(filenames)

    if is_training:
        dataset = dataset.shuffle(buffer_size=1500)

    dataset = dataset.map(lambda value: parse_record(value, is_training),
                          num_parallel_calls=num_parallel_calls)
    dataset = dataset.shuffle(buffer_size=10000)
    dataset = dataset.batch(batch_size)
    dataset = dataset.repeat(num_epochs)
    iterator = dataset.make_one_shot_iterator()

    features, labels = iterator.get_next()

    return features, labels

def build_set_iterator(is_training, filenames, batch_size, num_epochs=1000, num_parallel_calls=12):
    dataset = tf.data.TFRecordDataset(filenames)

    if is_training:
        dataset = dataset.shuffle(buffer_size=1500)

    dataset = dataset.map(lambda value: _parse_record(value, is_training),
                            num_parallel_calls=num_parallel_calls)
    dataset = dataset.shuffle(buffer_size=10000)
    dataset = dataset.batch(batch_size)
    dataset = dataset.repeat(num_epochs)
    iterator = dataset.make_initializable_iterator()
    return iterator

def build_set_dataset(is_training, filenames, code_size=2048, set_size=3,batch_size=30, num_epochs=1000, num_parallel_calls=12):
    dataset = tf.data.TFRecordDataset(filenames)

    if is_training:
        dataset = dataset.shuffle(buffer_size=1500)

    dataset = dataset.map(lambda value: _parse_record(value, is_training, code_size, set_size),
                            num_parallel_calls=num_parallel_calls)
    dataset = dataset.shuffle(buffer_size=10000)
    dataset = dataset.batch(batch_size)
    dataset = dataset.repeat(num_epochs)
    return dataset


import pandas as pd
import numpy as np
from sklearn.preprocessing import normalize
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score
from sklearn.externals import joblib
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn import svm
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import classification_report,confusion_matrix
from sklearn.decomposition import PCA
from concurrent.futures import ProcessPoolExecutor,wait
from concurrent import futures
import traceback
import multiprocessing as mp
from multiprocessing import Manager
from sklearn.neighbors import NearestNeighbors
from sklearn.neighbors import KNeighborsClassifier
import time
import os
from sklearn import svm

from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.metrics import precision_recall_fscore_support as score

global flag_standard_scaling
global flag_pipeline_to_seclect_model
global flag_create_pca_data

class ProcessManager:
    def __init__(self):
        self.processes = []

    def add_process(self, fn, args,k,output):
        self.processes.append((fn,args,k,output))

    def launch(self, max_workers=4):

        futs = []
        ex = ProcessPoolExecutor(max_workers=max_workers)
        for p in self.processes:
            try:
                fut = ex.submit(p[0], p[1],p[2],p[3])
                futs.append(fut)
            except:
                print(traceback.print_exc())
        return futs

def get_train_data(chrm,training_labels):
    # load the labels
    #get without NA data

    d = pd.read_csv('/data2/han_lab/sravani/Data/without_NA/' + chrm + '_withoutNA.csv')
    d = d.set_index('Composite')
    dT = d.T  # transpose it
    print(chrm+" : TCGA withoutNA Shape:" + str(dT.shape))

    # drop probes with mean <0.2
    dT = dT[dT.columns[dT.mean(axis=0) > 0.2]]
    print(chrm+" : TCGA after removing row means <0.2 Shape:" + str(dT.shape))

    # merge dfT and labels
    data = pd.concat([dT, training_labels], axis=1)
    print(chrm+" : Training shape(after adding labels):"+ str(data.shape))

    # remove samples with type count <100
    temp = data.groupby(['type']).size().reset_index()
    filter = temp[temp[0] > 100]
    train_filter = data[data['type'].isin(filter['type'])]
    print(chrm + " : Train Shape (after removing samples with count(type) <100):"+ str(train_filter.shape))

    # drop few uneccesary columns
    X = train_filter.reset_index().drop(columns=['Barcode', 'Label', 'Barcode1', 'TSS', 'Center', 'Plate', 'type'])
    y = train_filter.reset_index()['type']
    probes=[i for i in X.columns]

    return X,y,probes,list(filter['type'])

def get_independent_data(chrm,idata_labels,probes,req_labels):
    IData_path = '/data2/han_lab/sravani/Data/IndependentData/per_chromosome/'
    file = IData_path + chrm + '.csv'
    df = pd.read_csv(file, sep=',')
    df1 = df.drop(columns=['chr.' + str(i) for i in range(1, 21)])
    df1 = df1.drop(columns=['chr'])
    df2 = df1[df1[u'composite'].isin(probes)]

    df2 = df2.set_index([u'composite'])
    test = df2.T
    x=idata_labels[idata_labels.index.isin(test.index)]
    data = pd.concat([test, x], axis=1,sort=False)
    print (chrm+" : Independent data shape:"+ str(data.shape))
    data = data.rename(columns={"TCGA Label ": "type"})
    data1 = data[data['type'].isin(req_labels)]
    print(chrm + " : Independent data shape (after removing labels not considered in training):" + str(data1.shape))
    '''
    #display rows with Null values
    df1 = data[data.isnull().any(axis=1)]
    #find if any lables are missing
    temp=data[data.isnull().any(axis=1)]
    null_index=temp[temp['TCGA Label '].isnull()].index.tolist()
    idata_labels.ix[null_index]
    '''
    X = data1.reset_index().drop(columns=['type', 'Tissue Type', 'index'])
    y = data1.reset_index()['type']
    idx=[i for i in probes]
    X=X[idx]
    barcodes = data1.reset_index().rename(columns={'index': 'barcode'})

    return X,y,barcodes['barcode']

def perform_standard_scalling(X):
    # perform standard scaling
    scaler = StandardScaler()
    scaler.fit(X)
    X1 = scaler.transform(X)
    X1 = pd.DataFrame(X1,columns=X.columns)
    return X1

def fill_missing_mean(df):
    if df.isnull().values.any():

        #fill missing values with column means( probe mean)
        return df.fillna(df.mean())

def impute_missing_data(chrm,indices, train, test):

    print("=====================================================================")
    print(chrm+" Imputing...")
    print(chrm+": Recieved Train shape: " + str(train.shape) + " Test shape:" + str(test.shape))
    # get same cols in test and train
    # print(test.iloc[0:5,0:5])
    testT = test.T
    testT = testT.reset_index()
    # print(testT.iloc[0:5,0:5])
    test2 = testT[testT[u'index'].isin(train.columns)]
    test2 = test2.set_index('index')
    test2 = test2.T
    #test2.to_csv('/data2/han_lab/sravani/before_impute.csv', sep=',')
    print(chrm+": With Required Train shape: " + str(train.shape) + " Test shape:" + str(test2.shape))
    train = train.reset_index()
    impute = pd.DataFrame(columns=test2.columns)
    for i in range(indices.shape[0]):
        #print("=====================================")
        #print(test2.index[i])
        sum_missing = test2.iloc[i, :].isnull().values.sum()
        #print(chrm+": Index"+str(test2.index[i])+" - Number of missing in test:"+str(test2.iloc[i, :].isnull().values.sum()))
        if sum_missing > 0:
            rows = list(indices[i])
            temp = train.loc[rows, :]
            #temp = temp.set_index('Barcode1')
            # print(temp)

            col_means = temp.mean()
            # print(col_means[u'cg22934785'])
            # temp_means = pd.DataFrame(0, index=np.arange(len(temp)), columns=temp.columns)
            # for col in temp_means.columns:
            #    temp_means[col]=train[col].mean()

            # test2[test2.isnull()] = temp_means

            x = test2.iloc[i, :]
            # x.to_csv('/data2/han_lab/sravani/impute1.csv',sep=',')
            #print(chrm+" :imputing..."+str(test2.index[i]))
            x[x.isnull()] = col_means
            # x.to_csv('/data2/han_lab/sravani/impute2.csv', sep=',')

            #print(chrm+": Number of missing in test after imputing:"+str(x.isnull().values.sum()))
            impute = impute.append(x)
            #print(chrm+": Impute size:"+str(impute.shape))

        else:
            impute = impute.append(test2.iloc[i, :])
    #print(chrm+': writing to file: /data2/han_lab/sravani/Data/IndependentData/imputed/'+chrm+'_imputed_test.csv')
    #impute.to_csv('/data2/han_lab/sravani/Data/IndependentData/imputed/'+chrm+'_imputed.csv', sep=',')
    return test2,impute

def get_nearest_neighbours(chrm,k,train_X,test_X):
    # remove missing values from test
    test_withoutNA = test_X.dropna(axis='columns')
    print(chrm + ": Test Shape (after removing missing):" +str(test_withoutNA.shape))

    # extract common columns in test and production
    trainT = train_X.T
    x = trainT[trainT.index.isin(test_withoutNA.columns)]
    train_comm_X = x.T  # has common columns

    print("\n" + chrm + ": Train Shape (only with common columns):"+str(train_comm_X.shape))

    testT = test_withoutNA.T
    test_comm_X = testT[testT.index.isin(train_comm_X.columns)].T  # has common columns
    print(chrm + ": Test shape (only with common columns):"+str(test_comm_X.shape))

    # apply PCA
    print("\n" + chrm + ": Imputing - Applying PCA on Train....")
    model = PCA(n_components=100)
    model.fit(train_comm_X)
    train_X_trans = model.transform(train_comm_X)
    print(chrm + ": Imputing - Train PCA shape:"+str(train_X_trans.shape))

    print("\n" + chrm + ": Imputing - Applying PCA on Test...")
    #model1 = PCA(n_components=100)
    #model1.fit(test_comm_X)
    test_X_trans = model.transform(test_comm_X)
    print(chrm + ": Imputing - Test PCA shape:"+str(test_X_trans.shape))

    # apply KNN
    print("\n" + chrm + ": Imputing - Applying nearest neighbours...")
    nbrs = NearestNeighbors(n_neighbors=k, algorithm='ball_tree').fit(train_X_trans)
    #print(nbrs)
    distances, indices = nbrs.kneighbors(test_X_trans)
    print(indices.shape)
    print(chrm + ": Imputing - Indices shape:" +str(indices.shape))

    return indices

def do_PCA_subtask_with_mean_impute(chrm, k,output):
    print("===========================================================================")
    print("chromosome:", chrm)
    print(chrm + " : loading training labels...")
    training_labels = pd.read_csv('/data2/han_lab/sravani/Data/patientToCancerLabels_final.csv', sep=',')
    training_labels = training_labels.set_index('Barcode1')

    # probes : those probes considered for PCA
    print(chrm + " : Getting training data...")
    train_X, train_y, probes ,req_labels= get_train_data(chrm, training_labels)

    print("")

    print(chrm + " : Loading independent data labels..")
    idata_labels = pd.read_csv('/data2/han_lab/sravani/Data/IndependentData/independent_data_labels.csv', sep=',')
    idata_labels = idata_labels.set_index('Sample ID')

    print(chrm + " : Getting independent data..")
    idata_X,idata_y,barcodes = get_independent_data(chrm,idata_labels,probes,req_labels)
    '''

    print(chrm + " : Getting independent imputed data..")
    idata_X, idata_y = get_imputed_data(chrm)
    '''
    if idata_X.shape[1] == train_X.shape[1]:
        if idata_X.isnull().values.any():
            print(chrm + " : Independent Data has missing values..")
            print(chrm + " : Filling missing values with mean..")
            idata_X_new = fill_missing_mean(idata_X)
            print(chrm + " : Fitting training data to model..")

            model = PCA(n_components=100)
            model.fit(train_X)

            print("")
            print(chrm + " : Transforming TCGA data according to model..")
            train_X_trans = model.transform(train_X)
            train_X_trans = pd.DataFrame(train_X_trans)
            train_X_trans = train_X_trans.add_prefix(str(chrm + '_PC_'))

            print("")
            print(chrm + " : Transforming independent data according to model..")
            idata_X_trans = model.transform(idata_X_new)
            idata_X_trans = pd.DataFrame(idata_X_trans)  # convert numpy array to dataframe
            idata_X_trans = idata_X_trans.add_prefix(str(chrm + '_'))  # give column names

            print(chrm + " : Output size (before):", len(output))
            output[chrm] = {'training': train_X_trans,
                            't_labels': train_y,
                            'independent': idata_X_trans,
                            'i_labels': idata_y,
                            'i_barcodes': barcodes}
            print(chrm + " : Data saved in shared variable")
            print(chrm + " : Output size (after):", len(output))
    else:
        print(chrm + " : Number of probes mismatch in Independent Data and train data.")

def do_PCA_subtask_with_kNN_impute(chrm,k,output):
    print(chrm + " : loading training labels...")
    training_labels = pd.read_csv('/data2/han_lab/sravani/Data/patientToCancerLabels_final.csv', sep=',')
    training_labels = training_labels.set_index('Barcode1')

    # probes : those probes considered for PCA
    print(chrm + " : Getting training data...")
    train_X, train_y, probes, req_labels = get_train_data(chrm, training_labels)

    if flag_standard_scaling:
        #perform standard scalling
        print(chrm+" : standard scalling on train..")
        train_X=perform_standard_scalling(train_X)

    print(chrm + " : Loading independent data labels..")
    idata_labels = pd.read_csv('/data2/han_lab/sravani/Data/IndependentData/independent_data_labels.csv', sep=',')
    idata_labels = idata_labels.set_index('Sample ID')

    print(chrm + " : Getting independent data..")
    idata_X, idata_y,barcodes = get_independent_data(chrm, idata_labels,probes, req_labels) #if probes= all , all probes are considered

    print(chrm+" : Finding nearest neighbours..")
    indices=get_nearest_neighbours(chrm,k,train_X,idata_X)
    #impute the missing values with the mean of neighbours
    test_req, test_imp_X = impute_missing_data(chrm, indices, train_X, idata_X)
    print(chrm+" : Missing values in imputed data:" + str(test_imp_X.isnull().values.sum()))
    print("\n"+chrm+" : Chromosome " + chrm + " imputation finished...!")

    if flag_standard_scaling:
    # perform standard scalling
        print(chrm + " : standard scalling on test..")
        test_imp_X= perform_standard_scalling(test_imp_X)

    if train_X.shape[1]==test_imp_X.shape[1]:
        print(chrm + " : Fitting training data to model..")

        model = PCA(n_components=100)
        model.fit(train_X)

        print(chrm + " : Transforming TCGA data according to model..")
        train_X_trans = model.transform(train_X)
        train_X_trans = pd.DataFrame(train_X_trans)
        train_X_trans = train_X_trans.add_prefix(str(chrm + '_PC_'))

        print("")
        print(chrm + " : Transforming independent data according to model..")
        idata_X_trans = model.transform(test_imp_X)
        idata_X_trans = pd.DataFrame(idata_X_trans)  # convert numpy array to dataframe
        idata_X_trans = idata_X_trans.add_prefix(str(chrm + '_'))  # give column names

        print(chrm + " : Output size (before):"+ str(len(output)))
        output[chrm] = {'training': train_X_trans,
                        't_labels': train_y,
                        'independent': idata_X_trans,
                        'i_labels': idata_y,
                        'i_barcodes':barcodes}
        print(chrm + " : Data saved in shared variable")
        print("===============================================================================")
        print(chrm + " : Output size (after):" + str(len(output)))

    else:
        print(chrm+" : Number of probes mismatch in Independent Data and train data.")

def create_PCA_data_v3():
    '''
    pca performed on training after removing labels with count(type)<100
    does mean imputtion for missing values
    does multiprocessing
    :return:
    '''
    #if __name__==__main__():
    chromosomes = ['chr'+str(i) for i in range(1,23)]

    merge_tdata=None #holds PCA on training data for 22 chromosomes
    merge_idata=None #holds PCA on independent data for 22 chromosomes

    max_workers = 10
    pm = ProcessManager()
    m = Manager()
    output=m.dict()
    for chrm in chromosomes:
        pm.add_process(do_PCA_subtask_with_mean_impute, chrm,k,output)

    futs=pm.launch(max_workers=max_workers)
    wait(futs, return_when=futures.ALL_COMPLETED)
    print("Output:",len(output))
    for key,val in output.items():
        if merge_tdata is None:
            merge_tdata = val['training']
            print(key+" : training - After merge:" + str(merge_tdata.shape))
            merge_tdata = pd.concat([merge_tdata, val['t_labels']], axis=1)
            print(key+" : training - After adding labels:" + str(merge_tdata.shape))
        else:
            print(key+" : training - Merging with other chromosomes..")
            merge_tdata = pd.concat([merge_tdata, val['training']], axis=1)  # merge with other chromosomes
            print(key+" : training - After merge:" + str(merge_tdata.shape))
        if merge_idata is None:
            merge_idata = val['independent']
            print(key+" : Independent - After merge:" + str(merge_idata.shape))
            merge_idata = pd.concat([merge_idata, val['i_labels']], axis=1)
            print(key+" : Independent - After adding labels:" + str(merge_idata.shape))
        else:
            print(key+" : Independent - Merging with other chromosomes..")
            merge_idata = pd.concat([merge_idata, val['independent']], axis=1)  # merge with other chromosomes
            print(key+" : Independent - After merge:" + str(merge_idata.shape))

    print("Process complete")
    print("Writing merged data to file...")
    merge_idata.to_csv('/data2/han_lab/sravani/PCA/IData_combined_PCA_data_v3.csv',index=False)
    merge_tdata.to_csv('/data2/han_lab/sravani/PCA/TCGA_combined_PCA_data_v3.csv',index=False)

    return merge_tdata,merge_idata

def create_PCA_data_v4(k):
    '''
    missing value imputation with KNN data
    :return:
    '''
    #if __name__==__main__():
    chromosomes = ['chr'+str(i) for i in range(1,23)]

    merge_tdata=None #holds PCA on training data for 22 chromosomes
    merge_idata=None #holds PCA on independent data for 22 chromosomes

    max_workers = 10
    pm = ProcessManager()
    m = Manager()
    output=m.dict()
    for chrm in chromosomes:
        pm.add_process(do_PCA_subtask_with_kNN_impute, chrm,k,output)

    futs=pm.launch(max_workers=max_workers)
    print(wait(futs, return_when=futures.ALL_COMPLETED))
    print("Output:",len(output))
    for key,val in output.items():
        if merge_tdata is None:
            merge_tdata = val['training']
            print(key+" : training - After merge:" + str(merge_tdata.shape))
            merge_tdata = pd.concat([merge_tdata, val['t_labels']], axis=1)
            print(key+" : training - After adding labels:" + str(merge_tdata.shape))
        else:
            print(key+" : training - Merging with other chromosomes..")
            merge_tdata = pd.concat([merge_tdata, val['training']], axis=1)  # merge with other chromosomes
            print(key+" : training - After merge:" + str(merge_tdata.shape))
        if merge_idata is None:
            merge_idata = val['independent']
            print(key+" : Independent - After merge:" + str(merge_idata.shape))
            merge_idata = pd.concat([merge_idata, val['i_labels'],val['i_barcodes']], axis=1)
            print(key+" : Independent - After adding labels:" + str(merge_idata.shape))
        else:
            print(key+" : Independent - Merging with other chromosomes..")
            merge_idata = pd.concat([merge_idata, val['independent']], axis=1)  # merge with other chromosomes
            print(key+" : Independent - After merge:" + str(merge_idata.shape))

    print("Process complete")
    print("Writing merged data to file...")
    merge_idata.to_csv('/data2/han_lab/sravani/PCA/IData_combined_PCA_data_k'+str(k)+'_v4.csv', index=False)
    merge_tdata.to_csv('/data2/han_lab/sravani/PCA/TCGA_combined_PCA_data_k'+str(k)+'_v4.csv', index=False)

    return merge_tdata,merge_idata

def get_PCA_data_v3():
    idata = pd.read_csv('/data2/han_lab/sravani/PCA/IData_combined_PCA_data_v3.csv', sep=',')
    tcga = pd.read_csv('/data2/han_lab/sravani/PCA/TCGA_combined_PCA_data_v3.csv', sep=',')
    return tcga, idata

def get_PCA_data_v4(k):
    idata=pd.read_csv('/data2/han_lab/sravani/PCA/IData_combined_PCA_data_k' + str(k) + '_v4.csv', sep=',')
    tcga=pd.read_csv('/data2/han_lab/sravani/PCA/TCGA_combined_PCA_data_k' + str(k) + '_v4.csv',sep=',')
    return tcga,idata

def pipeline_model_selection(X_train,y_train,X_test,y_test):
    models_details={}
    #pipe_dt = Pipeline([('scl', StandardScaler()),('clf', DecisionTreeClassifier(random_state=42))])
    pipe_dt = Pipeline([('clf', DecisionTreeClassifier(random_state=42))])
    #pipe_rf = Pipeline([('scl', StandardScaler()),('clf', RandomForestClassifier(random_state=42))])
    pipe_rf = Pipeline([('clf', RandomForestClassifier(random_state=42))])
    #pipe_svm = Pipeline([('scl', StandardScaler()),('clf', svm.SVC(random_state=42))])
    pipe_svm = Pipeline([('clf', svm.SVC(random_state=42))])
    #pipe_linear_svm = Pipeline([('scl', StandardScaler()),('clf', svm.SVC(random_state=42))])
    pipe_linear_svm = Pipeline([('clf', svm.SVC(random_state=42))])

    # Set grid search params
    param_range = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    param_range_fl = [1.0, 0.5, 0.1]

    grid_params_dt = [{'clf__criterion': ['gini', 'entropy'],
                       'clf__min_samples_leaf': param_range,
                       'clf__max_depth': param_range,
                       'clf__min_samples_split': param_range[1:]}]

    grid_params_rf = [{'clf__criterion': ['gini', 'entropy'],
                       'clf__min_samples_leaf': param_range,
                       'clf__max_depth': param_range,
                       'clf__min_samples_split': param_range[1:]}]

    #grid_params_svm = [{'clf__kernel': ['linear', 'rbf'],
    #                   'clf__C': param_range}]

    grid_params_svm = [{'clf__kernel': ['linear'],
                        'clf__C': [1,2]}]

    grid_params_Linear_svm = [{'clf__C': [1,2]}]

    # Construct grid searches
    jobs = -1

    gs_dt = GridSearchCV(estimator=pipe_dt,
                         param_grid=grid_params_dt,
                         scoring='accuracy',
                         cv=10,
                         n_jobs=jobs,
                         verbose=10)

    gs_rf = GridSearchCV(estimator=pipe_rf,
                         param_grid=grid_params_rf,
                         scoring='accuracy',
                         cv=10,
                         n_jobs=jobs,
                         verbose=10)

    gs_svm = GridSearchCV(estimator=pipe_svm,
                          param_grid=grid_params_svm,
                          scoring='accuracy',
                          cv=10,
                          n_jobs=jobs,
                          verbose=10)

    gs_linear_svm = GridSearchCV(estimator=pipe_linear_svm,
                                 param_grid=grid_params_Linear_svm,
                                 scoring='accuracy',
                                 cv=10,
                                 n_jobs=jobs,
                                 verbose=10)

    # List of pipelines for ease of iteration
    #grids = [gs_dt, gs_rf, gs_svm, gs_linear_svm]
    grids = [gs_svm, gs_linear_svm]
    # grids=[gs_dt]
    # Dictionary of pipelines and classifier types for ease of reference
    #grid_dict = {0: 'Decision Trees', 1: 'Random Forest',2: 'Support Vector Machine', 3: 'Linear Support Vector Machine'}

    grid_dict={0: 'Support Vector Machine', 1: 'Linear Support Vector Machine' }
    # Fit the grid search objects
    print('Performing model optimizations...')
    best_acc = 0.0
    best_clf = 0
    best_gs = ''
    for idx, gs in enumerate(grids):
        print('\nEstimator: %s' % grid_dict[idx])
        # Fit grid search
        gs.fit(X_train, y_train)
        # Best params
        print('Best params: %s' % gs.best_params_)
        # Best training data accuracy
        print('Best training accuracy: %.3f' % gs.best_score_)
        # Predict on test data with best params
        y_pred = gs.predict(X_test)
        # Test data accuracy of model with best params
        print('Test set accuracy score for best params: %.3f ' % accuracy_score(y_test, y_pred))
        # Track best (highest test accuracy) model
        models_details[grid_dict[idx]]={'best_params':str(gs.best_params_),
                                        'best_training_accuracy':str(gs.best_score_),
                                        'test_accuracy':str(accuracy_score(y_test, y_pred))}

        with open('/data2/han_lab/sravani/PCA/pipeline_results_scaling.txt', 'a') as f:
            f.write('\n\n\nEstimator: ' + str(grid_dict[idx]) +
                    '\nBest params: ' + str(gs.best_params_) +
                    '\nBest training accuracy: ' + str(gs.best_score_) +
                    '\nTest set accuracy score for best params: ' + str(accuracy_score(y_test, y_pred)) +
                    '\nModel filename: /data2/han_lab/sravani/PCA/' + '_'.join(
                grid_dict[idx].split()) + '_model.pkl')
        dump_file = '/data2/han_lab/sravani/PCA/' + '_'.join(grid_dict[idx].split()) + '_model.pkl'
        joblib.dump(best_gs, dump_file, compress=1)
        if accuracy_score(y_test, y_pred) > best_acc:
            best_acc = accuracy_score(y_test, y_pred)
            best_gs = gs
            best_clf = idx
    print('\nClassifier with best test set accuracy: %s' % grid_dict[best_clf])

    # Save best grid search pipeline to file
    dump_file = '/data2/han_lab/sravani/PCA/best_gs_pipeline.pkl'
    joblib.dump(best_gs, dump_file, compress=1)
    print('\nSaved %s grid search pipeline to file: %s' % (grid_dict[best_clf], dump_file))

    print("\nClassification Report by best model:")
    y_pred = best_gs.predict(X_test)
    report = classification_report(y_test, y_pred)
    print(report)

    with open('/data2/han_lab/sravani/PCA/best_pipeline_results_scalling.txt','w') as f:
        f.write('\n\n\nEstimator: ' + str(grid_dict[best_clf]) +
                '\nBest params: ' + str(best_gs.best_params_) +
                '\nBest training accuracy: ' + str(best_gs.best_score_) +
                '\nTest set accuracy score for best params: ' + str(accuracy_score(y_test, y_pred)) +
                '\nModel filename: /data2/han_lab/sravani/PCA' + '_'.join(
            grid_dict[best_clf].split()) + '_model.pkl'+
                '\nClassification Result: \n'+ str(report))

    best_params=str(best_gs.best_params_)
    model_name=grid_dict[best_clf]
    return best_gs,best_params,model_name,models_details

def get_weighted_precision_recall_fscore_support(actual,predicted):
    precision, recall, fscore, support = score(actual,predicted, average='weighted',labels=np.unique(actual))
    return precision,recall,fscore,support

def process(tests,k,file,test_name,predicted_labels_file):
    # tcga_pca,idata_pca=get_PCA_data()

    if test=='mean_impute' and flag_create_pca_data:
        tcga_pca, idata_pca = create_PCA_data_v3()
        idata_X = idata_pca.drop(columns=['type'])
        idata_y = idata_pca['type']
    if test=='kNN_impute'and flag_create_pca_data:
        tcga_pca, idata_pca = create_PCA_data_v4(k)
        idata_X = idata_pca.drop(columns=['type','barcode'])
        idata_y = idata_pca['type']

    if test=='mean_impute' and not flag_create_pca_data:
        tcga_pca, idata_pca = get_PCA_data_v3()
        idata_X = idata_pca.drop(columns=['type'])
        idata_y = idata_pca['type']
    if test=='kNN_impute' and not flag_create_pca_data:
        tcga_pca, idata_pca = get_PCA_data_v4(k)
        idata_X = idata_pca.drop(columns=['type', 'barcode'])
        idata_y = idata_pca['type']

    X = tcga_pca.drop(columns=['type'])
    y = tcga_pca['type']

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    print(idata_pca.columns)

    model=None
    if flag_pipeline_to_seclect_model:
        model, best_params, model_name,models_details = pipeline_model_selection(X_train, y_train, X_test, y_test)
        print("model:",model)
        print("best_params:",best_params)
        print("model_name:",model_name)
        print("model_details:",models_details)

        for algo,details in models_details.items():
            print("\n",algo)
            print(details)

        print("\n Best parameters:")
        print(best_params)
        print(model_name)
        model.fit(X_train, y_train)

        # fitting on test
        y_pred = model.predict(X_test)
        report = classification_report(y_test, y_pred)
        print(report)

        accuracy0 = accuracy_score(y_test, y_pred)
        print("Accuracy:", accuracy0)

        idata_y_pred = model.predict(idata_X)
        idata_report = classification_report(idata_y, idata_y_pred)
        print(idata_report)

        accuracy = accuracy_score(idata_y, idata_y_pred)
        print("Accuracy:", accuracy)

        model_name = 'Support Vector Machine'
        best_params = str({'clf__C': 1, 'clf__kernel': 'linear'})

        temp = pd.DataFrame()
        temp['actual'] = idata_y
        temp['predicted'] = idata_y_pred
        temp['barcode'] = idata_pca['barcode']
        temp.to_csv(predicted_labels_file, sep=',')

        with open(file, 'a') as f:
            f.write("\n\n==========================================================================================")
            f.write("\n" + test_name)
            f.write("\nTraining Dataset\nBest model:" + model_name + '\nModel Parameters:' + best_params)
            f.write("\nClassification report on test data:\n")
            f.write(str(report))
            f.write('\n\nAccuracy:' + str(accuracy0))

            f.write("\n\n\nIndependent Dataset\nBest model:" + model_name + '\nModel Parameters:' + best_params)
            f.write("\nClassification report on independent data:\n")
            f.write(str(idata_report))
            f.write('\n\nAccuracy:' + str(accuracy))

    else:

        #from sklearn import svm
        #model = svm.SVC(C=1, kernel='linear', random_state=42)

        algorithm = {'svm': {'name': 'SVM',
                             'best': svm.SVC(C=1, kernel='linear', random_state=42)},
                     'random_forest': {'name': 'random_forest',
                                       'best': RandomForestClassifier(random_state=42, criterion='entropy',
                                                                      max_depth=10, min_samples_leaf=2,
                                                                      min_samples_split=8)},
                     'decision_trees': {'name': 'decision_trees',
                                        'best': DecisionTreeClassifier(criterion='entropy', max_depth=10,
                                                                       min_samples_leaf=6, min_samples_split=2,
                                                                       random_state=42)}
                     }

        for key, val in algorithm.items():
            model = val['best']
            model.fit(X_train, y_train)
            train_y_pred = model.predict(X_train)
            test_y_pred = model.predict(X_test)
            idata_y_pred = model.predict(idata_X)
            idata_report = classification_report(idata_y, idata_y_pred)
            print(idata_report)
            precision, recall, fscore, support = get_weighted_precision_recall_fscore_support(y_test, test_y_pred)
            i_precision, i_recall, i_fscore, i_support = get_weighted_precision_recall_fscore_support(idata_y, idata_y_pred)
            print(precision, recall, fscore, support)
            print(i_precision, i_recall, i_fscore, i_support)
            with open('/data2/han_lab/sravani/PCA/PCA_' + val['name'] + '_results.txt', 'w') as f:
                f.write("\nIndependent Dataset\nBest model:" + val['name'] + '\nModel Parameters:' + str(val['best']))
                f.write("\nClassification report on independent data:\n")
                f.write(str(idata_report))
                f.write('\n\nAccuracy:' + str(accuracy_score(idata_y, idata_y_pred)))
                f.write('\n\n\nEstimator: ' + str(val['name']) +
                        '\nBest params: ' + str(val['best']) +
                        '\nBest training accuracy: ' + str(accuracy_score(train_y_pred, y_train)) +
                        '\nTest set accuracy score for best params: ' + str(accuracy_score(test_y_pred, y_test)) +
                        '\nTest precision:' + str(precision) +
                        '\nTest recall:' + str(recall) +
                        '\nTest fscore:' + str(fscore) +
                        '\nTest support:' + str(support) +
                        '\n\n\n\nIndependent accuracy:' + str(accuracy_score(idata_y, idata_y_pred)) +
                        '\nIndependent precision:' + str(i_precision) +
                        '\nIndependent recall:' + str(i_recall) +
                        '\nIndependent fscore:' + str(i_fscore) +
                        '\nIndependent support:' + str(i_support) +
                        '\nClassification report:' + str(classification_report(idata_y, idata_y_pred))
                        )

            temp = pd.DataFrame()
            temp['actual'] = idata_y
            temp['predicted'] = idata_y_pred
            #temp.index = idata_pca.index
            temp.to_csv('/data2/han_lab/sravani/PCA/PCA_' + val['name'] + '_labels_predicted.csv')

def process_for_running_pipeline(tests,k,file,test_name):
    # tcga_pca,idata_pca=get_PCA_data()

    if test=='mean_impute':
        tcga_pca, idata_pca = create_PCA_data_v3()
    if test=='kNN_impute':
        tcga_pca, idata_pca = create_PCA_data_v4(k)

    X = tcga_pca.drop(columns=['type'])
    y = tcga_pca['type']

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    model=None
    print("Running pipeline to select model...")
    model, best_params, model_name,models_details = pipeline_model_selection(X_train, y_train, X_test, y_test)

    for algo,details in models_details.items():
        print("\n",algo)
        print(details)

    print("\n Best parameters:")
    print(best_params)
    print(model_name)

t1=time.time()
#tests=['mean_impute','kNN_impute']
tests=['kNN_impute']
k=[1]
results_file='/data2/han_lab/sravani/PCA/PCA_k_1_results.txt'
predicted_labels_file='/data2/han_lab/sravani/PCA/labels_predicted_k_1.csv'

flag_standard_scaling=False
flag_pipeline_to_seclect_model=False #true to run pipeline, false works on best model
flag_create_pca_data=False #flase reads data from file
for test in tests:

    if test == 'mean_impute':
        test_name = 'Missing Value imputation in independent dataset with mean values'
        process(tests, k, results_file,test_name,predicted_labels_file)
    if test == 'kNN_impute':
        for val in k:
            test_name = 'Missing Value imputation in independent dataset with kNN where k='+str(val)
            process(tests, val, results_file,test_name,predicted_labels_file)

print("time take:",time.time()-t1)

'''
import os
cmd = 'scp '+predicted_labels_file+' gannavar@bobby.cs.unlv.edu://home/gannavar/UNLV_Thesis/'
os.system(cmd)
cmd = 'scp '+results_file+' gannavar@bobby.cs.unlv.edu://home/gannavar/UNLV_Thesis/'
os.system(cmd)
'''
'''
import subprocess
p = subprocess.Popen(["scp", "my_file.txt", "username@server:path"])
sts = os.waitpid(p.pid, 0)
'''

'''

'''

import os
file='/data2/han_lab/sravani/PCA/PCA_SVM_results.txt'
cmd1 = 'scp '+file+' gannavar@bobby.cs.unlv.edu://home/gannavar/UNLV_Thesis/'
file2='/data2/han_lab/sravani/PCA/PCA_SVM_labels_predicted.csv'
cmd2 = 'scp '+file2+' gannavar@bobby.cs.unlv.edu://home/gannavar/UNLV_Thesis/'
os.system(cmd1)
os.system(cmd2)

file='/data2/han_lab/sravani/PCA/PCA_random_forest_results.txt'
cmd1 = 'scp '+file+' gannavar@bobby.cs.unlv.edu://home/gannavar/UNLV_Thesis/'
file2='/data2/han_lab/sravani/PCA/PCA_random_forest_labels_predicted.csv'
cmd2 = 'scp '+file2+' gannavar@bobby.cs.unlv.edu://home/gannavar/UNLV_Thesis/'
os.system(cmd1)
os.system(cmd2)

file='/data2/han_lab/sravani/PCA/PCA_decision_trees_results.txt'
cmd1 = 'scp '+file+' gannavar@bobby.cs.unlv.edu://home/gannavar/UNLV_Thesis/'
file2='/data2/han_lab/sravani/PCA/PCA_decision_trees_labels_predicted.csv'
cmd2 = 'scp '+file2+' gannavar@bobby.cs.unlv.edu://home/gannavar/UNLV_Thesis/'
os.system(cmd1)
os.system(cmd2)


from django.contrib import admin
from .models import Knowledge, Category, Rule, Factor, Weight, HistoryData

@admin.register(Knowledge)
class KnowledgeAdmin(admin.ModelAdmin):
    list_display = ['mark', 'detail', 'category']
    search_fields = ['mark', 'detail']

@admin.register(Rule)
class RuleAdmin(admin.ModelAdmin):
    list_display = ['expression', 'conclusion', 'certainty_factor', 'certainty_range']
    list_filter = ['relationship']

@admin.register(HistoryData)
class HistoryDataAdmin(admin.ModelAdmin):
    list_display = ['date', 'temperature', 'max_temp', 'min_temp',
                    'humidity', 'rainfall', 'visibility',
                    'wind_speed', 'max_speed', 'min_speed']
    fieldsets = [
        (None,          {'fields': ['date']}),
        ('temperature', {'fields': ['temperature', 'max_temp', 'min_temp']}),
        (None,          {'fields': ['humidity']}),
        (None,          {'fields': ['rainfall']}),
        (None,          {'fields': ['visibility']}),
        ('wind speed', {'fields': ['wind_speed', 'max_speed', 'min_speed']})
    ]
    list_filter = ['date']
    search_fields = ['date']
    ordering = ['-date']

admin.site.register(Category)
admin.site.register(Factor)
admin.site.register(Weight)


import logging
import xml.dom.minidom
import os

import gobject
import telepathy
import telepathy.client

from sugar.presence import presenceservice
from sugar.presence.tubeconn import TubeConnection
from sugar import util

import utils
import serialize
import constants
from instance import Instance
from recordtube import RecordTube
from recorded import Recorded

logger = logging.getLogger('collab')

class RecordCollab(object):
    def __init__(self, activity_obj, model):
        self.activity = activity_obj
        self.model = model
        self._tube = None
        self._collab_timeout = 10000

    def set_activity_shared(self):
        self._setup()
        self._tubes_channel.OfferDBusTube(constants.DBUS_SERVICE, {})

    def share_recd(self, recd):
        if not self._tube:
            return
        xmlstr = serialize.getRecdXmlMeshString(recd)
        self._tube.notifyBudsOfNewRecd(Instance.keyHashPrintable, xmlstr)

    def joined(self):
        if not self.activity.get_shared_activity():
            return
        self._setup()
        self._tubes_channel.ListTubes(reply_handler=self._list_tubes_reply_cb, error_handler=self._list_tubes_error_cb)

    def request_download(self, recd):
        if recd.meshDownloading:
            logger.debug("meshInitRoundRobin: we are in midst of downloading this file...")
            return

        # start with who took the photo
        recd.triedMeshBuddies = []
        recd.triedMeshBuddies.append(Instance.keyHashPrintable)
        self._req_recd_from_buddy(recd, recd.recorderHash, recd.recorderName)

    def _list_tubes_reply_cb(self, tubes):
        for tube_info in tubes:
            self._new_tube_cb(*tube_info)

    @staticmethod
    def _list_tubes_error_cb(e):
        logger.error('ListTubes() failed: %s', e)

    def _setup(self):
        # sets up the tubes...
        if not self.activity.get_shared_activity():
            logger.error('_setup: Failed to share or join activity')
            return

        pservice = presenceservice.get_instance()
        try:
            name, path = pservice.get_preferred_connection()
            self._connection = telepathy.client.Connection(name, path)
        except:
            logger.error('_setup: Failed to get_preferred_connection')

        # Work out what our room is called and whether we have Tubes already
        bus_name, conn_path, channel_paths = self.activity._shared_activity.get_channels()
        room = None
        tubes_chan = None
        text_chan = None
        for channel_path in channel_paths:
            channel = telepathy.client.Channel(bus_name, channel_path)
            htype, handle = channel.GetHandle()
            if htype == telepathy.HANDLE_TYPE_ROOM:
                logger.debug('Found our room: it has handle#%d "%s"', handle, self._connection.InspectHandles(htype, [handle])[0])
                room = handle
                ctype = channel.GetChannelType()
                if ctype == telepathy.CHANNEL_TYPE_TUBES:
                    logger.debug('Found our Tubes channel at %s', channel_path)
                    tubes_chan = channel
                elif ctype == telepathy.CHANNEL_TYPE_TEXT:
                    logger.debug('Found our Text channel at %s', channel_path)
                    text_chan = channel

        if not room:
            logger.error("Presence service didn't create a room")
            return
        if not text_chan:
            logger.error("Presence service didn't create a text channel")
            return

        # Make sure we have a Tubes channel - PS doesn't yet provide one
        if not tubes_chan:
            logger.debug("Didn't find our Tubes channel, requesting one...")
            tubes_chan = self._connection.request_channel(telepathy.CHANNEL_TYPE_TUBES, telepathy.HANDLE_TYPE_ROOM, room, True)

        self._tubes_channel = tubes_chan[telepathy.CHANNEL_TYPE_TUBES]
        self._text_channel = text_chan[telepathy.CHANNEL_INTERFACE_GROUP]

        self._tubes_channel.connect_to_signal('NewTube', self._new_tube_cb)

    def _new_tube_cb(self, id, initiator, type, service, params, state):
        logger.debug('New tube: ID=%d initator=%d type=%d service=%s params=%r state=%d', id, initiator, type, service, params, state)
        if type != telepathy.TUBE_TYPE_DBUS or service != constants.DBUS_SERVICE:
            return

        if state == telepathy.TUBE_STATE_LOCAL_PENDING:
            self._tubes_channel.AcceptDBusTube(id)
        tube_connection = TubeConnection(self._connection, self._tubes_channel, id, group_iface=self._text_channel)
        self._tube = RecordTube(tube_connection)
        self._tube.connect("new-recd", self._new_recd_cb)
        self._tube.connect("recd-request", self._recd_request_cb)
        self._tube.connect("recd-bits-arrived", self._recd_bits_arrived_cb)
        self._tube.connect("recd-unavailable", self._recd_unavailable_cb)

    def _new_recd_cb(self, remote_object, recorder, xmlstr):
        logger.debug('new_recd_cb')
        dom = None
        try:
            dom = xml.dom.minidom.parseString(xmlstr)
        except:
            logger.error('Unable to parse mesh xml')
        if not dom:
            return

        recd = Recorded()
        recd = serialize.fillRecdFromNode(recd, dom.documentElement)
        if not recd:
            logger.debug('_newRecdCb: recd is None. Unable to parse XML')
            return

        logger.debug('_newRecdCb: adding new recd thumb')
        recd.buddy = True
        recd.downloadedFromBuddy = False
        self.model.add_recd(recd)

    def _req_recd_from_buddy(self, recd, sender, nick):
        recd.triedMeshBuddies.append(sender)
        recd.meshDownloadingFrom = sender
        recd.meshDownloadingFromNick = nick
        recd.meshDownloadingProgress = False
        recd.meshDownloading = True
        recd.meshDownlodingPercent = 0.0
        self.activity.update_download_progress(recd)
        recd.meshReqCallbackId = gobject.timeout_add(self._collab_timeout, self._check_recd_request, recd)
        self._tube.requestRecdBits(Instance.keyHashPrintable, sender, recd.mediaMd5)

    def _next_round_robin_buddy(self, recd):
        logger.debug('meshNextRoundRobinBuddy')
        if recd.meshReqCallbackId:
            gobject.source_remove(recd.meshReqCallbackId)
            recd.meshReqCallbackId = 0

        # delete any stub of a partially downloaded file
        path = recd.getMediaFilepath()
        if path and os.path.exists(path):
            os.remove(path)

        good_buddy_obj = None
        buds = self.activity._shared_activity.get_joined_buddies()
        for buddy_obj in buds:
            buddy = util.sha_data(buddy_obj.props.key)
            buddy = util.printable_hash(buddy)
            if recd.triedMeshBuddies.count(buddy) > 0:
                logger.debug('mnrrb: weve already tried bud ' + buddy_obj.props.nick)
            else:
                logger.debug('mnrrb: ask next buddy: ' + buddy_obj.props.nick)
                good_buddy_obj = buddy_obj
                break

        if good_buddy_obj:
            buddy = util.sha_data(good_buddy_obj.props.key)
            buddy = util.printable_hash(buddy)
            self._req_recd_from_buddy(recd, buddy, good_buddy_obj.props.nick)
        else:
            logger.debug('weve tried all buddies here, and no one has this recd')
            recd.meshDownloading = False
            recd.triedMeshBuddies = []
            recd.triedMeshBuddies.append(Instance.keyHashPrintable)
            self.activity.update_download_progress(recd)

    def _recd_request_cb(self, remote_object, remote_person, md5sum):
        #if we are here, it is because someone has been told we have what they want.
        #we need to send them that thing, whatever that thing is
        recd = self.model.get_recd_by_md5(md5sum)
        if not recd:
            logger.debug('_recdRequestCb: we dont have the recd they asked for')
            self._tube.unavailableRecd(md5sum, Instance.keyHashPrintable, remote_person)
            return

        if recd.deleted:
            logger.debug('_recdRequestCb: we have the recd, but it has been deleted, so we wont share')
            self._tube.unavailableRecd(md5sum, Instance.keyHashPrintable, remote_person)
            return

        if recd.buddy and not recd.downloadedFromBuddy:
            logger.debug('_recdRequestCb: we have an incomplete recd, so we wont share')
            self._tube.unavailableRecd(md5sum, Instance.keyHashPrintable, remote_person)
            return

        recd.meshUploading = True
        path = recd.getMediaFilepath()

        if recd.type == constants.TYPE_AUDIO:
            audioImgFilepath = recd.getAudioImageFilepath()

            dest_path = os.path.join(Instance.instancePath, "audioBundle")
            dest_path = utils.getUniqueFilepath(dest_path, 0)
            cmd = "cat " + path + " " + audioImgFilepath + " > " + dest_path
            logger.debug(cmd)
            os.system(cmd)
            path = dest_path

        self._tube.broadcastRecd(recd.mediaMd5, path, remote_person)
        recd.meshUploading = False
        #if you were deleted while uploading, now throw away those bits now
        if recd.deleted:
            recd.doDeleteRecorded(recd)

    def _check_recd_request(self, recd):
        #todo: add category for "not active activity, so go ahead and delete"

        if recd.downloadedFromBuddy:
            logger.debug('_meshCheckOnRecdRequest: recdRequesting.downloadedFromBuddy')
            if recd.meshReqCallbackId:
                gobject.source_remove(recd.meshReqCallbackId)
                recd.meshReqCallbackId = 0
            return False
        if recd.deleted:
            logger.debug('_meshCheckOnRecdRequest: recdRequesting.deleted')
            if recd.meshReqCallbackId:
                gobject.source_remove(recd.meshReqCallbackId)
                recd.meshReqCallbackId = 0
            return False
        if recd.meshDownloadingProgress:
            logger.debug('_meshCheckOnRecdRequest: recdRequesting.meshDownloadingProgress')
            #we've received some bits since last we checked, so keep waiting...  they'll all get here eventually!
            recd.meshDownloadingProgress = False
            return True
        else:
            logger.debug('_meshCheckOnRecdRequest: ! recdRequesting.meshDownloadingProgress')
            #that buddy we asked info from isn't responding; next buddy!
            #self.meshNextRoundRobinBuddy( recdRequesting )
            gobject.idle_add(self._next_round_robin_buddy, recd)
            return False

    def _recd_bits_arrived_cb(self, remote_object, md5sum, part, num_parts, bytes, sender):
        recd = self.model.get_recd_by_md5(md5sum)
        if not recd:
            logger.debug('_recdBitsArrivedCb: thx 4 yr bits, but we dont even have that photo')
            return
        if recd.deleted:
            logger.debug('_recdBitsArrivedCb: thx 4 yr bits, but we deleted that photo')
            return
        if recd.downloadedFromBuddy:
            logger.debug('_recdBitsArrivedCb: weve already downloadedFromBuddy')
            return
        if not recd.buddy:
            logger.debug('_recdBitsArrivedCb: uh, we took this photo, so dont need your bits')
            return
        if recd.meshDownloadingFrom != sender:
            logger.debug('_recdBitsArrivedCb: wrong bits ' + str(sender) + ", exp:" + str(recd.meshDownloadingFrom))
            return

        #update that we've heard back about this, reset the timeout
        gobject.source_remove(recd.meshReqCallbackId)
        recd.meshReqCallbackId = gobject.timeout_add(self._collab_timeout, self._check_recd_request, recd)

        #update the progress bar
        recd.meshDownlodingPercent = (part+0.0)/(num_parts+0.0)
        recd.meshDownloadingProgress = True
        self.activity.update_download_progress(recd)
        open(recd.getMediaFilepath(), 'a+').write(bytes)

        if part > num_parts:
            logger.error('More parts than required have arrived')
            return
        if part != num_parts:
            return

        logger.debug('Finished receiving %s' % recd.title)
        gobject.source_remove(recd.meshReqCallbackId)
        recd.meshReqCallbackId = 0
        recd.meshDownloading = False
        recd.meshDownlodingPercent = 1.0
        recd.downloadedFromBuddy = True
        if recd.type == constants.TYPE_AUDIO:
            path = recd.getMediaFilepath()
            bundle_path = os.path.join(Instance.instancePath, "audioBundle")
            bundle_path = utils.getUniqueFilepath(bundle_path, 0)

            cmd = "split -a 1 -b " + str(recd.mediaBytes) + " " + path + " " + bundle_path
            logger.debug(cmd)
            os.system(cmd)

            bundle_name = os.path.basename(bundle_path)
            media_filename = bundle_name + "a"
            media_path = os.path.join(Instance.instancePath, media_filename)
            media_path_ext = os.path.join(Instance.instancePath, media_filename+".ogg")
            os.rename(media_path, media_path_ext)
            audio_image_name = bundle_name + "b"
            audio_image_path = os.path.join(Instance.instancePath, audio_image_name)
            audio_image_path_ext = os.path.join(Instance.instancePath, audio_image_name+".png")
            os.rename(audio_image_path, audio_image_path_ext)

            recd.mediaFilename = os.path.basename(media_path_ext)
            recd.audioImageFilename = os.path.basename(audio_image_path_ext)

        self.activity.remote_recd_available(recd)

    def _recd_unavailable_cb(self, remote_object, md5sum, sender):
        logger.debug('_recdUnavailableCb: sux, we want to see that photo')
        recd = self.model.get_recd_by_md5(md5sum)
        if not recd:
            logger.debug('_recdUnavailableCb: actually, we dont even know about that one..')
            return
        if recd.deleted:
            logger.debug('_recdUnavailableCb: actually, since we asked, we deleted.')
            return
        if not recd.buddy:
            logger.debug('_recdUnavailableCb: uh, odd, we took that photo and have it already.')
            return
        if recd.downloadedFromBuddy:
            logger.debug('_recdUnavailableCb: we already downloaded it...  you might have been slow responding.')
            return
        if recd.meshDownloadingFrom != sender:
            logger.debug('_recdUnavailableCb: we arent asking you for a copy now.  slow response, pbly.')
            return


# ninty nine problems
# source; https://wiki.python.org/moin/ProblemSets/99%20Prolog%20Problems%20Solutions

# Problem 11: Modified run-length encoding

l1 = ['a', 'a', 'a', 'b', 'c']
l2 = ['a', 'b', 'c', 'c', 'c']
l3 = ['a', 'b', 'c', 'c', 'd', 'e', 'f', 'f', 'g', 'h', 'i', 'i', 'j', 'j', 'j']

def main():

    list_list = [l1, l2, l3]
    for lst in list_list:
        encoded_list = encode(lst)
        print(lst, '\n', encoded_list)

    return

def encode(lst: list) -> list:
    encoded_list = []

    for i1 in lst:
        count = 0
        for i2 in lst:
            if i1 == i2:
                count += 1
        new_val = str(count) + i1
        if new_val in encoded_list:
            continue
        if count == 1:
            encoded_list.append(str(i1))
        else:
            encoded_list.append(str(count) + i1)

    return encoded_list

if __name__ == "__main__":
    try:
        main()
    except Exception as ex:
        print(ex)


from datetime import datetime, timedelta
from typing import Optional

from flask import jsonify
from urllib3.exceptions import LocationParseError
from urllib3.util import parse_url
from webargs import fields
from webargs.flaskparser import use_args

from src.blueprints import prompt
from src.core import database, helpers
from src.core.auth_helpers import authorize_route
from src.core.helpers import media
from src.core.models.v1.Prompt import Prompt

def prompt_yesterday_exists(given_prompt: Prompt) -> Optional[datetime]:
    yesterday_date = given_prompt.date - timedelta(1)
    if database.prompt.get_by_date(helpers.format_datetime_iso(yesterday_date)):
        return yesterday_date
    return None

def prompt_tomorrow_exists(given_prompt: Prompt) -> Optional[datetime]:
    tomorrow_date = given_prompt.date + timedelta(1)
    if database.prompt.get_by_date(helpers.format_datetime_iso(tomorrow_date)):
        return tomorrow_date
    return None

def __is_valid_url(url: str) -> bool:
    """Attempt to determine if a URL is valid."""
    try:
        parse_url(url)
        return True
    except LocationParseError:
        return False

@prompt.route("/", methods=["GET"])
@use_args({"date": fields.DateTime()}, location="query")
def get(args: dict):
    """Get a Prompt, either the latest or from a specific date."""
    # Hitting the endpoint without a date returns the latest prompt(s)
    if "date" not in args:
        prompts = database.prompt.get_latest()

    # We want the prompt from a particular day
    else:
        # Format the date in the proper format before fetching
        date = helpers.format_datetime_iso(args["date"])

        # A prompt for that date doesn't exist
        prompts = database.prompt.get_by_date(date, date_range=False)
        if not prompts:
            return helpers.make_error_response(
                404, f"No prompt exists for date {date}!"
            )

    # Find out if we have a prompt for tomorrow or yesterday
    for day_prompt in prompts:
        day_prompt["previous_day"] = prompt_yesterday_exists(day_prompt)
        day_prompt["next_day"] = prompt_tomorrow_exists(day_prompt)
    return jsonify(prompts)

@authorize_route
@prompt.route("/", methods=["POST"])
@use_args(
    {
        "id": fields.Str(required=True),
        "uid": fields.Str(required=True),
        "date": fields.DateTime(required=True),
        "word": fields.Str(required=True),
        "content": fields.Str(required=True),
        "media": fields.Str(missing=None, allow_none=True),
        "is_duplicate_date": fields.Bool(required=False, missing=False),
    },
    location="json",
)
def post(args: dict):
    """Create a new Prompt."""
    # Format the date in the proper format before writing
    args["date"] = helpers.format_datetime_iso(args["date"])

    # If we've not recieved an explict flag that this a duplicate data
    # (which can occur because there happened to more >1 prompt for the day),
    # we need to enforce a one-prompt-a-day constraint
    if not args["is_duplicate_date"]:
        # Don't create a prompt if it already exists
        if database.prompt.exists(pid="", date=args["date"]):
            return helpers.make_error_response(
                422, f"A prompt for {args['date']} already exists!"
            )

    # Download the given media
    media_result = True
    if args["media"] is not None and __is_valid_url(args["media"]):
        media_url = args["media"]
        media_result = media.move(media.download(args["id"], media_url))

        # Extract the media URL
        args["media"] = media.saved_name(args["id"], media_url)

    # Write the prompt to the database
    db_result = database.prompt.create(args)

    # Return the proper status depending on adding result
    if db_result and media_result:
        return helpers.make_response(201)
    return helpers.make_error_response(422, "Unable to record new Prompt!")

@authorize_route
@prompt.route("/", methods=["PUT"])
@use_args({"id": fields.Str(required=True)}, location="query")
@use_args(
    {
        "date": fields.DateTime(required=True),
        "word": fields.Str(required=True),
        "content": fields.Str(required=True),
        "media": fields.Str(missing=None, allow_none=True),
        "media_replace": fields.Bool(required=False),
    },
    location="json",
)
def put(query_args: dict, json_args: dict):
    """Update an existing Prompt."""
    # Merge the two args dicts into a single dict for easier use
    args = {**query_args, **json_args}

    # The prompt needs to exist first
    if not database.prompt.exists(pid=args["id"], date=""):
        msg = "The prompt ID '{}' does not exist!".format(args["id"])
        return helpers.make_error_response(404, msg)

    # If media is set to nothing, we want to delete it
    if args["media"] is None:
        media.delete(args["id"])

    # We want to replace the existng media
    elif (
        args["media"] is not None
        and __is_valid_url(args["media"])
        and args["media_replace"]
    ):
        # Start by deleting the old media
        media.delete(args["id"])

        # Download the new media
        media.move(media.download(args["id"], args["media"]))

        # Set the new media file name. It's not likely to be different
        # but better safe than sorry here
        args["media"] = media.saved_name(args["id"], args["media"])

    # Format the date in the proper format
    args["date"] = helpers.format_datetime_iso(args["date"])

    # Finally, save all this to the database
    database.prompt.update(args)
    return helpers.make_response(204)

@prompt.route("/", methods=["DELETE"])
@use_args({"id": fields.Str(required=True)}, location="query")
def delete(args: dict):
    """Delete an existing Prompt."""
    # Going to mimic SQL's behavior and pretend
    # we deleted something even if we didn't
    media.delete(args["id"])
    database.prompt.delete(args["id"])
    return helpers.make_response(204)


# -*- coding: utf-8 -*-

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from scrapy.http import Request
from scrapy.selector import Selector
from scrapy import log
import urlparse
import re
from ifeng.items import  IfengItem
import datetime
from urllib import unquote
from time import sleep

class IfengSpider(BaseSpider):

	name = 'sina_spider'
	start_urls = [
		"http://roll.mil.news.sina.com.cn/col/zgjq/index.shtml"
	]
	allowed_domains = ['sina.com.cn']

	def parse(self, response):
		urls = response.xpath('//ul[@class="linkNews"]/li/a/@href').extract()
		for url in urls:
			url = urlparse.urljoin(response.url, url)
			#self.log('Found category url: %s' %url)
			yield Request(url, callback= self.parseItem)

	def parseItem(self,response):
		#sel = Selector(response)
		item = IfengItem()
		item['title'] = response.xpath('//h1[@id="artibodyTitle"]/text()').extract()
		#item['content'] = response.xpath('//div[@id="main_content"]/text()').extract()
		item['content'] = self.getContent(response,'//div[@id="artibody"]/div[@class="img_wrapper"]','//div[@id="artibody"]/p')
		item['image_urls'] = response.xpath('//div[@id="artibody"]//img/@src').extract()
		item['litpic'] = self.image_list
		if item['content']:
			return item

	def getContent(self,response, xp1, xp2):
		content  = ''
		image_html = ''
		self.image_list = []
		image_divs = response.xpath(xp1)
		if len(image_divs):
			for image_div in image_divs:
				image_div = image_div.extract()
				image_dir = self.getImageDir()
				image_file_name = self.getImageFileName(image_div)
				if image_file_name:
					full_file_name = image_dir + image_file_name
					self.image_list.append(full_file_name)
					image_div = re.sub('"http[^"]*"',full_file_name,image_div)
				image_html += image_div
		paragraphs =  response.xpath(xp2)
		for p in paragraphs:
			p = p.extract()
			p = re.sub('</?a[^>]*>','',p)
			p = re.sub('.*','',p)
			content += p
		return image_html + content

	def getImageFileName(self,image_div):
		img = re.search(']* src="([^"]*)"',image_div)
		if img:
			img_url = img.group(1)
			us = img_url.split('/')[3:]
			image_file_name = '_'.join(us)
			return image_file_name
		else:
			return ''

	def getImageDir(self):
		date = datetime.datetime.now().strftime("%Y%m%d")
		image_dir = "/uploads/allimg/%s/"%date
		return image_dir


# chat/urls.py
from django.urls import path

from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('chat', views.chat, name='chat'),
    path('activities', views.activities, name='activities'),
    path('days', views.days, name='days'),
    path('lovetracker', views.lovetracker, name='lovetracker'),
    path('share', views.share, name='share'),
    path('summary', views.summary, name='summary'),
    path('topics', views.topics, name='topics'),
    path('onetopic', views.onetopic, name='onetopic'),
    path('daysinput', views.daysinput, name='daysinput'),
    path('activitiesinput', views.activitiesinput, name='activitiesinput'),
    path('settings', views.settings, name='settings'),
    path('profile', views.profile, name='profile'),
    path('about_us', views.about_us, name='about_us'),
    path('randomTopics_one', views.randomTopics_one, name='randomTopics_one'),
    path('randomTopics_two', views.randomTopics_two, name='randomTopics_two'),
    path('randomTopics', views.randomTopics, name='randomTopics'),
    path('trackerinput', views.trackerinput, name='trackerinput'),
    path('dark', views.dark, name='dark'),
    path('simple', views.simple, name='simple'),
    path('chinese', views.chinese, name='chinese'),

]


# letter dist. test

import random
import string

def showRandomLetters(n):

    letters = []
    number = []
    alph = string.ascii_uppercase

    for i in range(n):

        letters.append(random.choice(string.ascii_uppercase))

    for i in range(len(string.ascii_uppercase)):

        number.append(letters.count(alph[i]))

        number[i] = round(number[i] / n, 3)

    for i in range(len(string.ascii_uppercase)):

        print("{0}: {1}%".format(alph[i], number[i]))

    print('Total %: {0}%'.format(sum(number)))

def getRandomLetterDistribution(n):

    maxVowels = random.randint(5,8)

    print(maxVowels)

    letterList = []

    finished = False

    letterProb = [8.167, 9.658999999999999, 12.440999999999999,
                  16.694, 29.396, 31.624000000000002, 33.639,
                  39.733000000000004, 46.699000000000005, 46.852000000000004,
                  47.624, 51.649, 54.055, 60.804, 68.311, 70.24000000000001,
                  70.33500000000001, 76.322, 82.649, 91.705, 94.463,
                  95.44099999999999, 97.80099999999999, 97.951, 99.925, 99.999]

    lowLimit = 0
    hiLimit = 1
    vowels = 0

    while len(letterList) < n:
        finished = False
        lowLimit = 0
        hiLimit = 1

        x = random.choice(range(100)) / 100

        while not finished:

            if lowLimit == 0 and x < (letterProb[lowLimit] / 100):
                if not vowels >= maxVowels:
                    letterList.append('A')
                    #print('A')
                    vowels = vowels + 1
                    letter = 'A'
                else:
                    print('-----------------Nope------------------')
                finished = True

            elif (letterProb[lowLimit] / 100) <= x < (
                                            letterProb[hiLimit] / 100):

                letter = string.ascii_uppercase[hiLimit]
                letterList.append(letter)

                if letter == 'Q':
                    letterList.append('U')
                if letter in ['A', 'E', 'I', 'O', 'U']:
                    vowels = vowels + 1
                    if vowels >= maxVowels:
                        print('----------------Nope----------------', letter)
                        del letterList[-1]

                finished = True

            else:

                lowLimit = lowLimit + 1
                hiLimit = hiLimit + 1
        print("{0:02}, {1:02}, {2:4.2}, {3:5.2f}, {4}".format(lowLimit, hiLimit, letterProb[hiLimit], x, letter))

    return letterList

def main():

    print('What would you like to do?')
    print('Type "a" to see a random letter selection, or "b" to get a')
    ans = input('random letter distribution suitable for word games. ')
    n = int(input('How many letters would you like the program to retrieve? '))

    if ans == 'a':
        showRandomLetters(n)
    elif ans == 'b':
        letterList = getRandomLetterDistribution(n)
        print(letterList)
    else:
        print('Sorry, the input was not correct. Please type either "a" or "b" without'
              'quotation marks. Please run the program again.')

if __name__ == '__main__':
    main()

##letterProbs = [8.167, 1.492, 2.782, 4.253, 12.702, 2.228, 2.015, 6.094,
##               6.966, 0.153, 0.772, 4.025, 2.406, 6.749, 7.507, 1.929,
##               0.095, 5.987, 6.327, 9.056, 2.758, 0.978, 2.360, 0.150,
##               1.974, 0.074]

##    for i in range(len(letterProbs)):
##        number.append(sum(letterProbs[0:i + 1]))
##    print(number)

##A   8.167%     F   2.228%     K   0.772%     P   1.929%     U   2.758%     Z   0.074%
##B   1.492%     G   2.015%     L   4.025%     Q   0.095%     V   0.978%
##C   2.782%     H   6.094%     M   2.406%     R   5.987%     W   2.360%
##D   4.253%     I   6.966%     N   6.749%     S   6.327%     X   0.150%
##E  12.702%     J   0.153%     O   7.507%     T   9.056%     Y   1.974%


# coding:utf8
__author__ = 'bluesli'

dict = {
    'shanghai':{
        'pudong1':['xiaopudong1','xiaopudong2','xiaopudong3'],
        'pudong2':['xiaopudong1','xiaopudong2','xiaopudong3'],
        'pudong3':['xiaopudong1','xiaopudong2','xiaopudong3']
    },
    'sichuan':{
        'chengdu':['cheng1','cheng2','cheng3'],
        'guangyuan':['guang1','guang','guang3'],
        'guangyuan2':['guang1','guang','guang3'],
    },
    'beijing':{
        'chaoyang':['chao1','chao2','chaor3'],
        'tianan':['tian1','tian3w','tian3'],
        'sanli':['san1','san2','san3']
    }
}

# provinces = dict.keys()
# for province in provinces:
#     print(province)
#
#     mayors = dict[province].keys()
#     for mayor in mayors:
#         print(mayor)
#

print(dict['shanghai']['pudong1'])

provinces = dict.keys()
for province in provinces:
    print(province)
province = input("请输入省份")
if province in provinces:
    mayors= dict[province].keys()
    for mayor in mayors:
        print(mayor)
    mayor = input('请输入市名称')
    if mayor in mayors:
        countys = dict[province][mayor]
        for county in countys:
            print(county)
        county = input('请输入县')
        if county in countys:
            print("Bingo")


# Collin Pearce 55% TLE
gcdDp = {}
def gcd(a, b):
    A, B = a, b
    if (a, b) in gcdDp:
        return gcdDp[(a, b)]
    while b > 0:
        if (a, b) in gcdDp:
            gcdDp[(A, B)] = gcdDp[(a, b)]
            return gcdDp[(a, b)]
        a %= b
        if a == 0:
            gcdDp[(A, B)] = b
            return b
        b %= a
    gcdDp[(A, B)] = a
    return a

N, K = [int(x) for x in input().split()]

gcds = {}

for num in [int(x) for x in input().split()]:
    new = False
    if num not in gcds:
        gcds[num] = 1
        new = True
    if gcds[num] > 1 or new:
        gcds[num] = 1
        for key in list(gcds.keys()):
            newVal = gcd(key, num)
            if gcds[key] <= K:
                gcds.setdefault(newVal, gcds[key] + 1)
                if gcds[newVal] > gcds[key] + 1: gcds[newVal] = gcds[key] + 1
print(len(gcds))

# create a gcds dict. Add new values to dict with values of '1' go through other k/v in dict
# if other value + 1 <= K, and this is less than what is stored at gcd(num, otherkey) store there
# output length of gcd


import numpy as np
import non_neuron as nu
import function
import random

# f_FPC_unit: a one unit of f-PFC,2
# rmm_layer_unitの複製0519

W = np.array([[0, 1], [0.917, 0]])

class non_layer_unit:
    def __init__(self):
        self.random_del = 1
        self.dt = 0.01
        self.Tau = 10
        self.W = None
        self.b = None
        self.V = None
        self.S = None
        self.si = None
        self.one_layer = nu.neuron()
        self.one_layer.def_parameter(
            self.Tau, self.dt, self.random_del)
        self.function = function.function()
        self.function.def_parameter(
            self.Tau, self.dt, self.random_del)
        self.mode = None

        self.Loss = None
        self.dout = None

        self.lr = 0.01

    def set_lr(self, lr):
        self.lr = lr

    def loss(self, t):
        #print("S:", self.S)
        #print("t:", t)
        self.Loss = 1/2 * np.sum((self.S - t)**2)
        #print("Loss", self.Loss)

    def def_parameter(self, Tau, dt, random_del):
        self.random_del = random_del
        self.dt = dt
        self.Tau = Tau
        self.one_layer.def_parameter(
            self.Tau, self.dt, self.random_del)
        # print("**f=pfc")
        # print(self.random_del)

    def initiate(self, v, b, w, mode=None):
        self.W = w
        self.V = v
        self.S = self.function.sigmoid(v)
        self.b = b
        self.mode = mode

    def sgd(self, dW, db):
        #print("W1", self.W[0])
        self.W -= self.lr * dW
        self.b -= self.lr * db
        #print("W2", self.W[0])
        #print("lr", self.lr)

    def forward(self, si):
        self.si = si
        self.V, self.S = self.one_layer.forward(
            self.V, si, self.W, self.b)
        #print("S", self.S)
        return self.V, self.S

    def forward_dry(self, si):
        self.si = si
        self.V, self.S = self.one_layer.forward_dry(
            self.V, si, self.W, self.b)
        #print("S*", self.S)
        return self.V, self.S

    def backward(self, t):
        dy = self.function.sigmoid_back(self.S, self.Loss)
        dW = np.dot(self.si.T, dy)
        dx = np.dot(dy, self.W.T)
        self.sgd(dW, dy)

    def output_loss(self, t):
        self.dout = self.S - t
        return self.dout[0][0]

    def printS(self):
        print(self.S)


#!/usr/bin/python3
"""This module handles all default RestFul for state object"""

from models import storage
from api.v1.views import app_views
from flask import jsonify, abort, request
from models.state import State

@app_views.route('/states', methods=['GET'], strict_slashes=False)
def retrieve_all_states():
    """ Retrieves the list of all State """
    all_states = []
    for value in storage.all("State").values():
        all_states.append(value.to_dict())
    return jsonify(all_states)

@app_views.route('/states/<state_id>', methods=['GET'], strict_slashes=False)
def retrieve_state_id(state_id):
    """ Retrieves a State object """
    state = (storage.get('State', state_id))
    if state:
        return jsonify(state.to_dict())
    else:
        abort(404)

@app_views.route('/states/<state_id>', methods=['DELETE'],
                 strict_slashes=False)
def Delete_state(state_id):
    """Deletes a State object"""
    state = storage.get('State', state_id)
    if state:
        storage.delete(state)
        storage.save()
        return jsonify({}), 200
    else:
        abort(404)

@app_views.route('/states', methods=['POST'], strict_slashes=False)
def Post_state():
    """ Create a State object"""
    if not request.get_json():
        return jsonify({"error": "Not a JSON"}), 400
    if 'name' not in request.get_json():
        return jsonify({"error": "Missing name"}), 400
    instance = State(**request.get_json())
    instance.save()
    return jsonify(instance.to_dict()), 201

@app_views.route('/states/<state_id>', methods=['PUT'], strict_slashes=False)
def update_state(state_id):
    """ update state """
    keys = ['id', 'created_at', 'updated_at']
    state = storage.get('State', state_id)
    if state is None:
        abort(404)
    if not request.get_json():
        return jsonify({"error": "Not a JSON"}), 400
    for key, value in request.get_json().items():
        if key in keys:
            pass
        else:
            setattr(state, key, value)
    state.save()
    return jsonify(state.to_dict()), 200